From 43c7959d85ba2001e9f74c8f9cd7415290260022 Mon Sep 17 00:00:00 2001 From: mirsal Date: Fri, 12 Jun 2026 07:15:54 +0000 Subject: [PATCH 1/3] Rework prometheus blackbox monitoring configuration * Use scrape_config_files correctly * Do not separate prometheus and blackbox exporter nodes * Add a http_2xx_selfsigned module for self-signed certs * Monitor missing services * Switch from CRLF to LF in prometheus.config.j2 --- ansible/inventory | 6 +- ansible/playbooks/monitoring.yml | 14 +-- .../templates/blackbox.config.j2 | 5 + .../prometheus/templates/prometheus.config.j2 | 106 +++++------------- .../prometheus/templates/scrape-blackbox.yml | 28 ++++- .../prometheus/templates/scrape-main.yml | 1 + 6 files changed, 68 insertions(+), 92 deletions(-) diff --git a/ansible/inventory b/ansible/inventory index 6010cc0..d0edb5c 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -22,6 +22,6 @@ all: reverse-proxy.labolyon.dn42: wiki.labolyon.dn42: zigbee2mqtt.labolyon.dn42: -# blackbox: -# hosts: -# monitoring: + monitoring: + hosts: + monitoring.labolyon.dn42: diff --git a/ansible/playbooks/monitoring.yml b/ansible/playbooks/monitoring.yml index ae873c1..247c854 100644 --- a/ansible/playbooks/monitoring.yml +++ b/ansible/playbooks/monitoring.yml @@ -12,21 +12,11 @@ become: yes roles: - prometheus + - blackbox-exporter vars: - blackbox: false scrape_files: - /etc/prometheus/scrape-main.yml - -#- name: Install and configure Prometheus node - Blackbox nodes -# hosts: blackbox -# become: yes -# roles: -# - prometheus -# - blackbox-exporter -# vars: -# blackbox: true -# scrape_files: -# - /etc/prometheus/scrape-blackbox.yml + - /etc/prometheus/scrape-blackbox.yml - name: Install and configure nodes hosts: all diff --git a/ansible/roles/blackbox-exporter/templates/blackbox.config.j2 b/ansible/roles/blackbox-exporter/templates/blackbox.config.j2 index 907c6d1..9207802 100644 --- a/ansible/roles/blackbox-exporter/templates/blackbox.config.j2 +++ b/ansible/roles/blackbox-exporter/templates/blackbox.config.j2 @@ -8,6 +8,11 @@ modules: preferred_ip_protocol: "ip4" ip_protocol_fallback: true {% endif %} + http_2xx_selfsigned: + prober: http + http: + tls_config: + insecure_skip_verify: true http_post_2xx: prober: http {% if inventory_hostname == 'check.home.arpa' %} diff --git a/ansible/roles/prometheus/templates/prometheus.config.j2 b/ansible/roles/prometheus/templates/prometheus.config.j2 index 6161d5f..b44b4f1 100644 --- a/ansible/roles/prometheus/templates/prometheus.config.j2 +++ b/ansible/roles/prometheus/templates/prometheus.config.j2 @@ -1,75 +1,31 @@ -{{ ansible_managed | comment }} - -# Global configuration -global: - scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. - evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. - # scrape_timeout is set to the global default (10s). - -# Alertmanager configuration -alerting: - alertmanagers: - - static_configs: - - targets: - - localhost:9093 - -# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. -rule_files: - - "{{ alertmanager_rules }}/prometheus.rules.yml" - - "{{ alertmanager_rules }}/blackbox.rules.yml" - - "{{ alertmanager_rules }}/hosts.rules.yml" - -# A scrape configuration containing exactly one endpoint to scrape: - -#scrape_config_files: -#{% for item in scrape_files %} -# - "{{ item }}" -#{% endfor %} - -scrape_configs: - - job_name: "prometheus" - static_configs: - - targets: ["localhost:9090"] - -{% if blackbox %} - - job_name: "blackbox" - metrics_path: /probe - params: - module: [http_2xx] - scrape_interval: 5s - static_configs: - - targets: - - https://labolyon.fr - - https://mail.labolyon.fr - - https://wiki.labolyon.fr - - https://git.labolyon.fr - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: target - - target_label: __address__ - replacement: 127.0.0.1:9115 # The blackbox exporter's real hostname:port. -{% else %} - - job_name: "linux-nodes" - static_configs: - - targets: - - ansible.labolyon.dn42:9100 - - dn42-router.labolyon.dn42:9100 - - dns.labolyon.dn42:9100 - - git.labolyon.dn42:9100 - - i2p.labolyon.dn42:9100 - - labolyon-fr.labolyon.dn42:9100 - - lolix-ixpman.labolyon.dn42:9100 - - lolix-rs1.labolyon.dn42:9100 - - lolmox.labolyon.dn42:9100 - - lolmox2.labolyon.dn42:9100 - - mail.labolyon.dn42:9100 - - matrix.labolyon.dn42:9100 - - monitoring.labolyon.dn42:9100 - - mosquitto.labolyon.dn42:9100 - - radius.labolyon.dn42:9100 - - reverse-proxy.labolyon.dn42:9100 - - wiki.labolyon.dn42:9100 - - zigbee2mqtt.labolyon.dn42:9100 -{% endif %} +{{ ansible_managed | comment }} + +# Global configuration +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: + - localhost:9093 + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: + - "{{ alertmanager_rules }}/prometheus.rules.yml" + - "{{ alertmanager_rules }}/blackbox.rules.yml" + - "{{ alertmanager_rules }}/hosts.rules.yml" + +# A scrape configuration containing exactly one endpoint to scrape: +scrape_config_files: +{% for item in scrape_files %} + - "{{ item }}" +{% endfor %} + +scrape_configs: + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] diff --git a/ansible/roles/prometheus/templates/scrape-blackbox.yml b/ansible/roles/prometheus/templates/scrape-blackbox.yml index 91445e4..4391cb0 100644 --- a/ansible/roles/prometheus/templates/scrape-blackbox.yml +++ b/ansible/roles/prometheus/templates/scrape-blackbox.yml @@ -1,4 +1,8 @@ - - job_name: "blackbox" +# managed by ansible + +scrape_configs: + + - job_name: "blackbox-public-https" metrics_path: /probe params: module: [http_2xx] @@ -9,10 +13,30 @@ - https://mail.labolyon.fr - https://wiki.labolyon.fr - https://git.labolyon.fr + - https://sso.labolyon.fr + - https://chat.labolyon.fr + - https://grafana.labolyon.fr relabel_configs: - source_labels: [__address__] target_label: __param_target - source_labels: [__param_target] - target_label: target + target_label: instance + - target_label: __address__ + replacement: 127.0.0.1:9115 # The blackbox exporter's real hostname:port. + + - job_name: "blackbox-internal-https" + metrics_path: /probe + params: + module: [http_2xx_selfsigned] + scrape_interval: 5s + static_configs: + - targets: + - https://octoprint.labolyon.dn42/login/ + - https://stickers.labolyon.dn42 + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance - target_label: __address__ replacement: 127.0.0.1:9115 # The blackbox exporter's real hostname:port. diff --git a/ansible/roles/prometheus/templates/scrape-main.yml b/ansible/roles/prometheus/templates/scrape-main.yml index 2ad9fff..9f77baa 100644 --- a/ansible/roles/prometheus/templates/scrape-main.yml +++ b/ansible/roles/prometheus/templates/scrape-main.yml @@ -1,3 +1,4 @@ +scrape_configs: - job_name: "linux-nodes" static_configs: - targets: From 006f1747783ac6653fa3b2594c6b2df19c2668b3 Mon Sep 17 00:00:00 2001 From: mirsal Date: Fri, 12 Jun 2026 07:33:21 +0000 Subject: [PATCH 2/3] roles: blackbox-exporter: templates: blackbox.config.j2: switch to LF --- .../templates/blackbox.config.j2 | 130 +++++++++--------- 1 file changed, 65 insertions(+), 65 deletions(-) diff --git a/ansible/roles/blackbox-exporter/templates/blackbox.config.j2 b/ansible/roles/blackbox-exporter/templates/blackbox.config.j2 index 9207802..e2a691f 100644 --- a/ansible/roles/blackbox-exporter/templates/blackbox.config.j2 +++ b/ansible/roles/blackbox-exporter/templates/blackbox.config.j2 @@ -1,65 +1,65 @@ -{{ ansible_managed | comment }} - -modules: - http_2xx: - prober: http -{% if inventory_hostname == 'check.home.arpa' %} - http: - preferred_ip_protocol: "ip4" - ip_protocol_fallback: true -{% endif %} - http_2xx_selfsigned: - prober: http - http: - tls_config: - insecure_skip_verify: true - http_post_2xx: - prober: http -{% if inventory_hostname == 'check.home.arpa' %} - http: - method: POST - preferred_ip_protocol: "ip4" - ip_protocol_fallback: true -{% endif %} - tcp_connect: - prober: tcp - smtp_check: - prober: tcp - timeout: 5s - tcp: -{% if inventory_hostname == 'check.home.arpa' %} - preferred_ip_protocol: "ip4" - ip_protocol_fallback: true -{% endif %} - query_response: - - expect: "^220(.*)ESMTP(.*)$" - - send: "EHLO prober\r" - - expect: "^250-STARTTLS" - - send: "STARTTLS\r" - - expect: "^220" - - starttls: true - - send: "EHLO prober\r" - - expect: "^250-AUTH" - - send: "QUIT\r" - imap_check: - prober: tcp - timeout: 5s - tcp: -{% if inventory_hostname == 'check.home.arpa' %} - preferred_ip_protocol: "ip4" - ip_protocol_fallback: true -{% endif %} - query_response: - - expect: "OK.*STARTTLS" - - send: ". STARTTLS" - - expect: "OK" - - starttls: true - - send: ". capability" - - expect: "CAPABILITY IMAP4rev1" - icmp: - prober: icmp -{% if inventory_hostname == 'check.home.arpa' %} - icmp: - preferred_ip_protocol: "ip4" - ip_protocol_fallback: true -{% endif %} +{{ ansible_managed | comment }} + +modules: + http_2xx: + prober: http +{% if inventory_hostname == 'check.home.arpa' %} + http: + preferred_ip_protocol: "ip4" + ip_protocol_fallback: true +{% endif %} + http_2xx_selfsigned: + prober: http + http: + tls_config: + insecure_skip_verify: true + http_post_2xx: + prober: http +{% if inventory_hostname == 'check.home.arpa' %} + http: + method: POST + preferred_ip_protocol: "ip4" + ip_protocol_fallback: true +{% endif %} + tcp_connect: + prober: tcp + smtp_check: + prober: tcp + timeout: 5s + tcp: +{% if inventory_hostname == 'check.home.arpa' %} + preferred_ip_protocol: "ip4" + ip_protocol_fallback: true +{% endif %} + query_response: + - expect: "^220(.*)ESMTP(.*)$" + - send: "EHLO prober\r" + - expect: "^250-STARTTLS" + - send: "STARTTLS\r" + - expect: "^220" + - starttls: true + - send: "EHLO prober\r" + - expect: "^250-AUTH" + - send: "QUIT\r" + imap_check: + prober: tcp + timeout: 5s + tcp: +{% if inventory_hostname == 'check.home.arpa' %} + preferred_ip_protocol: "ip4" + ip_protocol_fallback: true +{% endif %} + query_response: + - expect: "OK.*STARTTLS" + - send: ". STARTTLS" + - expect: "OK" + - starttls: true + - send: ". capability" + - expect: "CAPABILITY IMAP4rev1" + icmp: + prober: icmp +{% if inventory_hostname == 'check.home.arpa' %} + icmp: + preferred_ip_protocol: "ip4" + ip_protocol_fallback: true +{% endif %} From 18852dbc2f8e6ca143c646f227b30fda0b97df90 Mon Sep 17 00:00:00 2001 From: mirsal Date: Fri, 12 Jun 2026 07:42:37 +0000 Subject: [PATCH 3/3] roles: prometheus: scrape-blackbox: Monitor webmail.labolyon.fr --- ansible/roles/prometheus/templates/scrape-blackbox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/prometheus/templates/scrape-blackbox.yml b/ansible/roles/prometheus/templates/scrape-blackbox.yml index 4391cb0..4f354a2 100644 --- a/ansible/roles/prometheus/templates/scrape-blackbox.yml +++ b/ansible/roles/prometheus/templates/scrape-blackbox.yml @@ -10,7 +10,7 @@ scrape_configs: static_configs: - targets: - https://labolyon.fr - - https://mail.labolyon.fr + - https://webmail.labolyon.fr - https://wiki.labolyon.fr - https://git.labolyon.fr - https://sso.labolyon.fr