Merge pull request 'roles: dns-internal: nsd: Update IP addresses for zola.vinishor.xyz' (#5) from dns-secondaire-vinishor into main

Reviewed-on: #5

ansible/inventory

@@ -9,7 +9,7 @@ all:
         git.labolyon.dn42:
         i2p.labolyon.dn42:
         labolyon-fr.labolyon.dn42:
-        lolix-ixpman.labolyon.dn42:
+#        lolix-ixpman.labolyon.dn42:
         lolix-rs1.labolyon.dn42:
         lolmox.labolyon.dn42:
         lolmox2.labolyon.dn42:
@@ -22,6 +22,6 @@ all:
         reverse-proxy.labolyon.dn42:
         wiki.labolyon.dn42:
         zigbee2mqtt.labolyon.dn42:
-#    blackbox:
+    monitoring:
-#      hosts:
+      hosts:
-#        monitoring:
+        monitoring.labolyon.dn42:

ansible/playbooks/monitoring.yml

@@ -12,21 +12,11 @@
   become: yes
   roles:
     - prometheus
+    - blackbox-exporter
   vars:
-    blackbox: false
     scrape_files:
       - /etc/prometheus/scrape-main.yml
-
+      - /etc/prometheus/scrape-blackbox.yml
-#- name: Install and configure Prometheus node - Blackbox nodes
-#  hosts: blackbox
-#  become: yes
-#  roles:
-#    - prometheus
-#    - blackbox-exporter
-#  vars:
-#    blackbox: true
-#    scrape_files:
-#      - /etc/prometheus/scrape-blackbox.yml
 
 - name: Install and configure nodes
   hosts: all

ansible/roles/blackbox-exporter/templates/blackbox.config.j2

@@ -8,6 +8,11 @@ modules:
       preferred_ip_protocol: "ip4"
       ip_protocol_fallback: true
 {% endif %}
+  http_2xx_selfsigned:
+    prober: http
+    http:
+      tls_config:
+        insecure_skip_verify: true
   http_post_2xx:
     prober: http
 {% if inventory_hostname == 'check.home.arpa' %}

ansible/roles/dns-internal/templates/nsd.conf

@@ -31,10 +31,10 @@ zone:
   name: "labolyon.fr"
   zonefile: "labolyon.fr.zone"
   ## zola.vinishor.xyz
-  notify: 2a0e:e701:11e7:cafe:3737::1 clesecrete
+  notify: 2001:41d0:fc32:d710:3737::1 clesecrete
-  provide-xfr: 2a0e:e701:11e7:cafe:3737::1 clesecrete
+  provide-xfr: 2001:41d0:fc32:d710:3737::1 clesecrete
-  notify: 45.13.105.181 clesecrete
+  notify: 109.190.187.14 clesecrete
-  provide-xfr: 45.13.105.181 clesecrete
+  provide-xfr: 109.190.187.14 clesecrete
   ## ns1.1312.zone
   notify: 2a03:b0c0:2:d0::12f0:4001 dns_labolyon_fr_to_ns1_1312_zone
   provide-xfr: 2a03:b0c0:2:d0::12f0:4001 dns_labolyon_fr_to_ns1_1312_zone

ansible/roles/prometheus/tasks/main.yml

@@ -20,6 +20,14 @@
   notify:
     - restart prometheus
 
+- name: Add alertmanager configuration
+  tags: alertmanager-configure
+  ansible.builtin.template:
+    src: alertmanager.yml.j2
+    dest: '{{ alertmanager_config }}'
+  notify:
+    - restart alertmanager
+
 - name: Create folder for rules definition
   ansible.builtin.file:
     path: /etc/prometheus/rules

ansible/roles/prometheus/templates/alertmanager.yml.j2

@@ -0,0 +1,67 @@
+{{ ansible_managed | comment }}
+# See https://prometheus.io/docs/alerting/configuration/ for documentation.
+
+global:
+  # The smarthost and SMTP sender used for mail notifications.
+  smtp_smarthost: 'mail.labolyon.fr:587'
+  smtp_from: 'alerts@labolyon.fr'
+  smtp_auth_username: 'alerts@labolyon.fr'
+  smtp_auth_password: {{ lookup('community.general.passwordstore', 'monitoring/e-mail/alerts@labolyon.fr')}}
+
+# The directory from which notification templates are read.
+templates:
+- '/etc/prometheus/alertmanager_templates/*.tmpl'
+
+# The root route on which each incoming alert enters.
+route:
+  # The labels by which incoming alerts are grouped together. For example,
+  # multiple alerts coming in for cluster=A and alertname=LatencyHigh would
+  # be batched into a single group.
+  #
+  # To aggregate by all possible labels use '...' as the sole label name.
+  # This effectively disables aggregation entirely, passing through all
+  # alerts as-is. This is unlikely to be what you want, unless you have
+  # a very low alert volume or your upstream notification system performs
+  # its own grouping. Example: group_by: [...]
+  group_by: ['alertname', 'cluster', 'service']
+
+  # When a new group of alerts is created by an incoming alert, wait at
+  # least 'group_wait' to send the initial notification.
+  # This way ensures that you get multiple alerts for the same group that start
+  # firing shortly after another are batched together on the first
+  # notification.
+  group_wait: 30s
+
+  # When the first notification was sent, wait 'group_interval' to send a batch
+  # of new alerts that started firing for that group.
+  group_interval: 5m
+
+  # If an alert has successfully been sent, wait 'repeat_interval' to
+  # resend them.
+  repeat_interval: 3h
+
+  # A default receiver
+  receiver: all-admins-email
+
+  # All the above attributes are inherited by all child routes and can
+  # overwritten on each.
+
+# Inhibition rules allow to mute a set of alerts given that another alert is
+# firing.
+# We use this to mute any warning-level notifications if the same alert is
+# already critical.
+inhibit_rules:
+  - source_matchers: [severity="critical"]
+    target_matchers: [severity="warning"]
+    # Apply inhibition if the alertname is the same.
+    # CAUTION:
+    #   If all label names listed in `equal` are missing
+    #   from both the source and target alerts,
+    #   the inhibition rule will apply!
+    equal: [alertname, cluster, service]
+
+receivers:
+  - name: 'all-admins-email'
+    email_configs:
+      - to: 'mirsal@mirsal.fr,mirsal@labolyon.fr,vinishor@vinishor.xyz'
+        send_resolved: true

ansible/roles/prometheus/templates/prometheus.config.j2

@@ -20,56 +20,12 @@ rule_files:
    - "{{ alertmanager_rules }}/hosts.rules.yml"
 
 # A scrape configuration containing exactly one endpoint to scrape:
-
+scrape_config_files:
-#scrape_config_files:
+{% for item in scrape_files %}
-#{% for item in scrape_files %}
+  - "{{ item }}"
-#  - "{{ item }}"
+{% endfor %}
-#{% endfor %}
 
 scrape_configs:
   - job_name: "prometheus"
     static_configs:
       - targets: ["localhost:9090"]
-
-{% if blackbox %}
-  - job_name: "blackbox"
-    metrics_path: /probe
-    params:
-      module: [http_2xx]
-    scrape_interval: 5s
-    static_configs:
-      - targets:
-          - https://labolyon.fr
-          - https://mail.labolyon.fr
-          - https://wiki.labolyon.fr
-          - https://git.labolyon.fr
-    relabel_configs:
-       - source_labels: [__address__]
-         target_label: __param_target
-       - source_labels: [__param_target]
-         target_label: target
-       - target_label: __address__
-         replacement: 127.0.0.1:9115  # The blackbox exporter's real hostname:port.
-{% else %}
-  - job_name: "linux-nodes"
-    static_configs:
-      - targets:
-         - ansible.labolyon.dn42:9100
-         - dn42-router.labolyon.dn42:9100
-         - dns.labolyon.dn42:9100
-         - git.labolyon.dn42:9100
-         - i2p.labolyon.dn42:9100
-         - labolyon-fr.labolyon.dn42:9100
-         - lolix-ixpman.labolyon.dn42:9100
-         - lolix-rs1.labolyon.dn42:9100
-         - lolmox.labolyon.dn42:9100
-         - lolmox2.labolyon.dn42:9100
-         - mail.labolyon.dn42:9100
-         - matrix.labolyon.dn42:9100
-         - monitoring.labolyon.dn42:9100
-         - mosquitto.labolyon.dn42:9100
-         - radius.labolyon.dn42:9100
-         - reverse-proxy.labolyon.dn42:9100
-         - wiki.labolyon.dn42:9100
-         - zigbee2mqtt.labolyon.dn42:9100
-{% endif %}

ansible/roles/prometheus/templates/scrape-blackbox.yml

@@ -1,4 +1,8 @@
-  - job_name: "blackbox"
+# managed by ansible
+
+scrape_configs:
+
+  - job_name: "blackbox-public-https"
     metrics_path: /probe
     params:
       module: [http_2xx]
@@ -6,13 +10,33 @@
     static_configs:
       - targets:
           - https://labolyon.fr
-          - https://mail.labolyon.fr
+          - https://webmail.labolyon.fr
           - https://wiki.labolyon.fr
           - https://git.labolyon.fr
+          - https://sso.labolyon.fr
+          - https://chat.labolyon.fr
+          - https://grafana.labolyon.fr
     relabel_configs:
        - source_labels: [__address__]
          target_label: __param_target
        - source_labels: [__param_target]
-         target_label: target
+         target_label: instance
+       - target_label: __address__
+         replacement: 127.0.0.1:9115  # The blackbox exporter's real hostname:port.
+
+  - job_name: "blackbox-internal-https"
+    metrics_path: /probe
+    params:
+      module: [http_2xx_selfsigned]
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+          - https://octoprint.labolyon.dn42/login/
+          - https://stickers.labolyon.dn42
+    relabel_configs:
+       - source_labels: [__address__]
+         target_label: __param_target
+       - source_labels: [__param_target]
+         target_label: instance
        - target_label: __address__
          replacement: 127.0.0.1:9115  # The blackbox exporter's real hostname:port.

ansible/roles/prometheus/templates/scrape-main.yml

@@ -1,3 +1,4 @@
+scrape_configs:
   - job_name: "linux-nodes"
     static_configs:
       - targets:
@@ -7,7 +8,7 @@
          - git.labolyon.dn42:9100
          - i2p.labolyon.dn42:9100
          - labolyon-fr.labolyon.dn42:9100
-         - lolix-ixpman.labolyon.dn42:9100
+#         - lolix-ixpman.labolyon.dn42:9100
          - lolix-rs1.labolyon.dn42:9100
          - lolmox.labolyon.dn42:9100
          - lolmox2.labolyon.dn42:9100
@@ -19,3 +20,8 @@
          - reverse-proxy.labolyon.dn42:9100
          - wiki.labolyon.dn42:9100
          - zigbee2mqtt.labolyon.dn42:9100
+    metric_relabel_configs:
+      - source_labels: [__name__, state]
+        separator: '_'
+        regex: node_systemd_unit_state_[^f].*
+        action: drop