Merge pull request 'Rework blackbox monitoring' (#2) from rework-blackbox-monitoring into main

Reviewed-on: #2

ansible/inventory

@@ -22,6 +22,6 @@ all:
         reverse-proxy.labolyon.dn42:
         wiki.labolyon.dn42:
         zigbee2mqtt.labolyon.dn42:
-#    blackbox:
+    monitoring:
-#      hosts:
+      hosts:
-#        monitoring:
+        monitoring.labolyon.dn42:

ansible/playbooks/monitoring.yml

@@ -12,21 +12,11 @@
   become: yes
   roles:
     - prometheus
+    - blackbox-exporter
   vars:
-    blackbox: false
     scrape_files:
       - /etc/prometheus/scrape-main.yml
-
+      - /etc/prometheus/scrape-blackbox.yml
-#- name: Install and configure Prometheus node - Blackbox nodes
-#  hosts: blackbox
-#  become: yes
-#  roles:
-#    - prometheus
-#    - blackbox-exporter
-#  vars:
-#    blackbox: true
-#    scrape_files:
-#      - /etc/prometheus/scrape-blackbox.yml
 
 - name: Install and configure nodes
   hosts: all

ansible/roles/blackbox-exporter/templates/blackbox.config.j2

@@ -1,60 +1,65 @@
-{{ ansible_managed | comment }}
+{{ ansible_managed | comment }}
-
+
-modules:
+modules:
-  http_2xx:
+  http_2xx:
-    prober: http
+    prober: http
-{% if inventory_hostname == 'check.home.arpa' %}
+{% if inventory_hostname == 'check.home.arpa' %}
-    http:
+    http:
-      preferred_ip_protocol: "ip4"
+      preferred_ip_protocol: "ip4"
-      ip_protocol_fallback: true
+      ip_protocol_fallback: true
-{% endif %}
+{% endif %}
-  http_post_2xx:
+  http_2xx_selfsigned:
-    prober: http
+    prober: http
-{% if inventory_hostname == 'check.home.arpa' %}
+    http:
-    http:
+      tls_config:
-      method: POST
+        insecure_skip_verify: true
-      preferred_ip_protocol: "ip4"
+  http_post_2xx:
-      ip_protocol_fallback: true
+    prober: http
-{% endif %}
+{% if inventory_hostname == 'check.home.arpa' %}
-  tcp_connect:
+    http:
-    prober: tcp
+      method: POST
-  smtp_check:
+      preferred_ip_protocol: "ip4"
-    prober: tcp
+      ip_protocol_fallback: true
-    timeout: 5s
+{% endif %}
-    tcp:
+  tcp_connect:
-{% if inventory_hostname == 'check.home.arpa' %}
+    prober: tcp
-      preferred_ip_protocol: "ip4"
+  smtp_check:
-      ip_protocol_fallback: true
+    prober: tcp
-{% endif %}
+    timeout: 5s
-      query_response:
+    tcp:
-        - expect: "^220(.*)ESMTP(.*)$"
+{% if inventory_hostname == 'check.home.arpa' %}
-        - send: "EHLO prober\r"
+      preferred_ip_protocol: "ip4"
-        - expect: "^250-STARTTLS"
+      ip_protocol_fallback: true
-        - send: "STARTTLS\r"
+{% endif %}
-        - expect: "^220"
+      query_response:
-        - starttls: true
+        - expect: "^220(.*)ESMTP(.*)$"
-        - send: "EHLO prober\r"
+        - send: "EHLO prober\r"
-        - expect: "^250-AUTH"
+        - expect: "^250-STARTTLS"
-        - send: "QUIT\r"
+        - send: "STARTTLS\r"
-  imap_check:
+        - expect: "^220"
-    prober: tcp
+        - starttls: true
-    timeout: 5s
+        - send: "EHLO prober\r"
-    tcp:
+        - expect: "^250-AUTH"
-{% if inventory_hostname == 'check.home.arpa' %}
+        - send: "QUIT\r"
-      preferred_ip_protocol: "ip4"
+  imap_check:
-      ip_protocol_fallback: true
+    prober: tcp
-{% endif %}
+    timeout: 5s
-      query_response:
+    tcp:
-        - expect: "OK.*STARTTLS"
+{% if inventory_hostname == 'check.home.arpa' %}
-        - send: ". STARTTLS"
+      preferred_ip_protocol: "ip4"
-        - expect: "OK"
+      ip_protocol_fallback: true
-        - starttls: true
+{% endif %}
-        - send: ". capability"
+      query_response:
-        - expect: "CAPABILITY IMAP4rev1"
+        - expect: "OK.*STARTTLS"
-  icmp:
+        - send: ". STARTTLS"
-    prober: icmp
+        - expect: "OK"
-{% if inventory_hostname == 'check.home.arpa' %}
+        - starttls: true
-    icmp:
+        - send: ". capability"
-      preferred_ip_protocol: "ip4"
+        - expect: "CAPABILITY IMAP4rev1"
-      ip_protocol_fallback: true
+  icmp:
-{% endif %}
+    prober: icmp
+{% if inventory_hostname == 'check.home.arpa' %}
+    icmp:
+      preferred_ip_protocol: "ip4"
+      ip_protocol_fallback: true
+{% endif %}

ansible/roles/prometheus/templates/prometheus.config.j2

@@ -1,75 +1,31 @@
-{{ ansible_managed | comment }}
+{{ ansible_managed | comment }}
-
+
-# Global configuration
+# Global configuration
-global:
+global:
-  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
-  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
-  # scrape_timeout is set to the global default (10s).
+  # scrape_timeout is set to the global default (10s).
-
+
-# Alertmanager configuration
+# Alertmanager configuration
-alerting:
+alerting:
-  alertmanagers:
+  alertmanagers:
-    - static_configs:
+    - static_configs:
-        - targets:
+        - targets:
-           - localhost:9093
+           - localhost:9093
-
+
-# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
-rule_files:
+rule_files:
-   - "{{ alertmanager_rules }}/prometheus.rules.yml"
+   - "{{ alertmanager_rules }}/prometheus.rules.yml"
-   - "{{ alertmanager_rules }}/blackbox.rules.yml"
+   - "{{ alertmanager_rules }}/blackbox.rules.yml"
-   - "{{ alertmanager_rules }}/hosts.rules.yml"
+   - "{{ alertmanager_rules }}/hosts.rules.yml"
-
+
-# A scrape configuration containing exactly one endpoint to scrape:
+# A scrape configuration containing exactly one endpoint to scrape:
-
+scrape_config_files:
-#scrape_config_files:
+{% for item in scrape_files %}
-#{% for item in scrape_files %}
+  - "{{ item }}"
-#  - "{{ item }}"
+{% endfor %}
-#{% endfor %}
+
-
+scrape_configs:
-scrape_configs:
+  - job_name: "prometheus"
-  - job_name: "prometheus"
+    static_configs:
-    static_configs:
+      - targets: ["localhost:9090"]
-      - targets: ["localhost:9090"]
-
-{% if blackbox %}
-  - job_name: "blackbox"
-    metrics_path: /probe
-    params:
-      module: [http_2xx]
-    scrape_interval: 5s
-    static_configs:
-      - targets:
-          - https://labolyon.fr
-          - https://mail.labolyon.fr
-          - https://wiki.labolyon.fr
-          - https://git.labolyon.fr
-    relabel_configs:
-       - source_labels: [__address__]
-         target_label: __param_target
-       - source_labels: [__param_target]
-         target_label: target
-       - target_label: __address__
-         replacement: 127.0.0.1:9115  # The blackbox exporter's real hostname:port.
-{% else %}
-  - job_name: "linux-nodes"
-    static_configs:
-      - targets:
-         - ansible.labolyon.dn42:9100
-         - dn42-router.labolyon.dn42:9100
-         - dns.labolyon.dn42:9100
-         - git.labolyon.dn42:9100
-         - i2p.labolyon.dn42:9100
-         - labolyon-fr.labolyon.dn42:9100
-         - lolix-ixpman.labolyon.dn42:9100
-         - lolix-rs1.labolyon.dn42:9100
-         - lolmox.labolyon.dn42:9100
-         - lolmox2.labolyon.dn42:9100
-         - mail.labolyon.dn42:9100
-         - matrix.labolyon.dn42:9100
-         - monitoring.labolyon.dn42:9100
-         - mosquitto.labolyon.dn42:9100
-         - radius.labolyon.dn42:9100
-         - reverse-proxy.labolyon.dn42:9100
-         - wiki.labolyon.dn42:9100
-         - zigbee2mqtt.labolyon.dn42:9100
-{% endif %}

ansible/roles/prometheus/templates/scrape-blackbox.yml

@@ -1,4 +1,8 @@
-  - job_name: "blackbox"
+# managed by ansible
+
+scrape_configs:
+
+  - job_name: "blackbox-public-https"
     metrics_path: /probe
     params:
       module: [http_2xx]
@@ -6,13 +10,33 @@
     static_configs:
       - targets:
           - https://labolyon.fr
-          - https://mail.labolyon.fr
+          - https://webmail.labolyon.fr
           - https://wiki.labolyon.fr
           - https://git.labolyon.fr
+          - https://sso.labolyon.fr
+          - https://chat.labolyon.fr
+          - https://grafana.labolyon.fr
     relabel_configs:
        - source_labels: [__address__]
          target_label: __param_target
        - source_labels: [__param_target]
-         target_label: target
+         target_label: instance
+       - target_label: __address__
+         replacement: 127.0.0.1:9115  # The blackbox exporter's real hostname:port.
+
+  - job_name: "blackbox-internal-https"
+    metrics_path: /probe
+    params:
+      module: [http_2xx_selfsigned]
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+          - https://octoprint.labolyon.dn42/login/
+          - https://stickers.labolyon.dn42
+    relabel_configs:
+       - source_labels: [__address__]
+         target_label: __param_target
+       - source_labels: [__param_target]
+         target_label: instance
        - target_label: __address__
          replacement: 127.0.0.1:9115  # The blackbox exporter's real hostname:port.

ansible/roles/prometheus/templates/scrape-main.yml

@@ -1,3 +1,4 @@
+scrape_configs:
   - job_name: "linux-nodes"
     static_configs:
       - targets: