From 43c7959d85ba2001e9f74c8f9cd7415290260022 Mon Sep 17 00:00:00 2001 From: mirsal Date: Fri, 12 Jun 2026 07:15:54 +0000 Subject: [PATCH] Rework prometheus blackbox monitoring configuration * Use scrape_config_files correctly * Do not separate prometheus and blackbox exporter nodes * Add a http_2xx_selfsigned module for self-signed certs * Monitor missing services * Switch from CRLF to LF in prometheus.config.j2 --- ansible/inventory | 6 +- ansible/playbooks/monitoring.yml | 14 +-- .../templates/blackbox.config.j2 | 5 + .../prometheus/templates/prometheus.config.j2 | 106 +++++------------- .../prometheus/templates/scrape-blackbox.yml | 28 ++++- .../prometheus/templates/scrape-main.yml | 1 + 6 files changed, 68 insertions(+), 92 deletions(-) diff --git a/ansible/inventory b/ansible/inventory index 6010cc0..d0edb5c 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -22,6 +22,6 @@ all: reverse-proxy.labolyon.dn42: wiki.labolyon.dn42: zigbee2mqtt.labolyon.dn42: -# blackbox: -# hosts: -# monitoring: + monitoring: + hosts: + monitoring.labolyon.dn42: diff --git a/ansible/playbooks/monitoring.yml b/ansible/playbooks/monitoring.yml index ae873c1..247c854 100644 --- a/ansible/playbooks/monitoring.yml +++ b/ansible/playbooks/monitoring.yml @@ -12,21 +12,11 @@ become: yes roles: - prometheus + - blackbox-exporter vars: - blackbox: false scrape_files: - /etc/prometheus/scrape-main.yml - -#- name: Install and configure Prometheus node - Blackbox nodes -# hosts: blackbox -# become: yes -# roles: -# - prometheus -# - blackbox-exporter -# vars: -# blackbox: true -# scrape_files: -# - /etc/prometheus/scrape-blackbox.yml + - /etc/prometheus/scrape-blackbox.yml - name: Install and configure nodes hosts: all diff --git a/ansible/roles/blackbox-exporter/templates/blackbox.config.j2 b/ansible/roles/blackbox-exporter/templates/blackbox.config.j2 index 907c6d1..9207802 100644 --- a/ansible/roles/blackbox-exporter/templates/blackbox.config.j2 +++ b/ansible/roles/blackbox-exporter/templates/blackbox.config.j2 @@ -8,6 +8,11 @@ modules: preferred_ip_protocol: "ip4" ip_protocol_fallback: true {% endif %} + http_2xx_selfsigned: + prober: http + http: + tls_config: + insecure_skip_verify: true http_post_2xx: prober: http {% if inventory_hostname == 'check.home.arpa' %} diff --git a/ansible/roles/prometheus/templates/prometheus.config.j2 b/ansible/roles/prometheus/templates/prometheus.config.j2 index 6161d5f..b44b4f1 100644 --- a/ansible/roles/prometheus/templates/prometheus.config.j2 +++ b/ansible/roles/prometheus/templates/prometheus.config.j2 @@ -1,75 +1,31 @@ -{{ ansible_managed | comment }} - -# Global configuration -global: - scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. - evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. - # scrape_timeout is set to the global default (10s). - -# Alertmanager configuration -alerting: - alertmanagers: - - static_configs: - - targets: - - localhost:9093 - -# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. -rule_files: - - "{{ alertmanager_rules }}/prometheus.rules.yml" - - "{{ alertmanager_rules }}/blackbox.rules.yml" - - "{{ alertmanager_rules }}/hosts.rules.yml" - -# A scrape configuration containing exactly one endpoint to scrape: - -#scrape_config_files: -#{% for item in scrape_files %} -# - "{{ item }}" -#{% endfor %} - -scrape_configs: - - job_name: "prometheus" - static_configs: - - targets: ["localhost:9090"] - -{% if blackbox %} - - job_name: "blackbox" - metrics_path: /probe - params: - module: [http_2xx] - scrape_interval: 5s - static_configs: - - targets: - - https://labolyon.fr - - https://mail.labolyon.fr - - https://wiki.labolyon.fr - - https://git.labolyon.fr - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: target - - target_label: __address__ - replacement: 127.0.0.1:9115 # The blackbox exporter's real hostname:port. -{% else %} - - job_name: "linux-nodes" - static_configs: - - targets: - - ansible.labolyon.dn42:9100 - - dn42-router.labolyon.dn42:9100 - - dns.labolyon.dn42:9100 - - git.labolyon.dn42:9100 - - i2p.labolyon.dn42:9100 - - labolyon-fr.labolyon.dn42:9100 - - lolix-ixpman.labolyon.dn42:9100 - - lolix-rs1.labolyon.dn42:9100 - - lolmox.labolyon.dn42:9100 - - lolmox2.labolyon.dn42:9100 - - mail.labolyon.dn42:9100 - - matrix.labolyon.dn42:9100 - - monitoring.labolyon.dn42:9100 - - mosquitto.labolyon.dn42:9100 - - radius.labolyon.dn42:9100 - - reverse-proxy.labolyon.dn42:9100 - - wiki.labolyon.dn42:9100 - - zigbee2mqtt.labolyon.dn42:9100 -{% endif %} +{{ ansible_managed | comment }} + +# Global configuration +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: + - localhost:9093 + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: + - "{{ alertmanager_rules }}/prometheus.rules.yml" + - "{{ alertmanager_rules }}/blackbox.rules.yml" + - "{{ alertmanager_rules }}/hosts.rules.yml" + +# A scrape configuration containing exactly one endpoint to scrape: +scrape_config_files: +{% for item in scrape_files %} + - "{{ item }}" +{% endfor %} + +scrape_configs: + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] diff --git a/ansible/roles/prometheus/templates/scrape-blackbox.yml b/ansible/roles/prometheus/templates/scrape-blackbox.yml index 91445e4..4391cb0 100644 --- a/ansible/roles/prometheus/templates/scrape-blackbox.yml +++ b/ansible/roles/prometheus/templates/scrape-blackbox.yml @@ -1,4 +1,8 @@ - - job_name: "blackbox" +# managed by ansible + +scrape_configs: + + - job_name: "blackbox-public-https" metrics_path: /probe params: module: [http_2xx] @@ -9,10 +13,30 @@ - https://mail.labolyon.fr - https://wiki.labolyon.fr - https://git.labolyon.fr + - https://sso.labolyon.fr + - https://chat.labolyon.fr + - https://grafana.labolyon.fr relabel_configs: - source_labels: [__address__] target_label: __param_target - source_labels: [__param_target] - target_label: target + target_label: instance + - target_label: __address__ + replacement: 127.0.0.1:9115 # The blackbox exporter's real hostname:port. + + - job_name: "blackbox-internal-https" + metrics_path: /probe + params: + module: [http_2xx_selfsigned] + scrape_interval: 5s + static_configs: + - targets: + - https://octoprint.labolyon.dn42/login/ + - https://stickers.labolyon.dn42 + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance - target_label: __address__ replacement: 127.0.0.1:9115 # The blackbox exporter's real hostname:port. diff --git a/ansible/roles/prometheus/templates/scrape-main.yml b/ansible/roles/prometheus/templates/scrape-main.yml index 2ad9fff..9f77baa 100644 --- a/ansible/roles/prometheus/templates/scrape-main.yml +++ b/ansible/roles/prometheus/templates/scrape-main.yml @@ -1,3 +1,4 @@ +scrape_configs: - job_name: "linux-nodes" static_configs: - targets: