roles: grafana: Add grafana configuration

ansible/inventory

@@ -22,6 +22,6 @@ all:
         reverse-proxy.labolyon.dn42:
         wiki.labolyon.dn42:
         zigbee2mqtt.labolyon.dn42:
-    monitoring:
+#    blackbox:
-      hosts:
+#      hosts:
-        monitoring.labolyon.dn42:
+#        monitoring:

ansible/playbooks/monitoring.yml

@@ -12,11 +12,21 @@
   become: yes
   roles:
     - prometheus
-    - blackbox-exporter
   vars:
+    blackbox: false
     scrape_files:
       - /etc/prometheus/scrape-main.yml
-      - /etc/prometheus/scrape-blackbox.yml
+
+#- name: Install and configure Prometheus node - Blackbox nodes
+#  hosts: blackbox
+#  become: yes
+#  roles:
+#    - prometheus
+#    - blackbox-exporter
+#  vars:
+#    blackbox: true
+#    scrape_files:
+#      - /etc/prometheus/scrape-blackbox.yml
 
 - name: Install and configure nodes
   hosts: all

ansible/roles/blackbox-exporter/templates/blackbox.config.j2

@@ -1,65 +1,60 @@
-{{ ansible_managed | comment }}
+{{ ansible_managed | comment }}
-
+
-modules:
+modules:
-  http_2xx:
+  http_2xx:
-    prober: http
+    prober: http
-{% if inventory_hostname == 'check.home.arpa' %}
+{% if inventory_hostname == 'check.home.arpa' %}
-    http:
+    http:
-      preferred_ip_protocol: "ip4"
+      preferred_ip_protocol: "ip4"
-      ip_protocol_fallback: true
+      ip_protocol_fallback: true
-{% endif %}
+{% endif %}
-  http_2xx_selfsigned:
+  http_post_2xx:
-    prober: http
+    prober: http
-    http:
+{% if inventory_hostname == 'check.home.arpa' %}
-      tls_config:
+    http:
-        insecure_skip_verify: true
+      method: POST
-  http_post_2xx:
+      preferred_ip_protocol: "ip4"
-    prober: http
+      ip_protocol_fallback: true
-{% if inventory_hostname == 'check.home.arpa' %}
+{% endif %}
-    http:
+  tcp_connect:
-      method: POST
+    prober: tcp
-      preferred_ip_protocol: "ip4"
+  smtp_check:
-      ip_protocol_fallback: true
+    prober: tcp
-{% endif %}
+    timeout: 5s
-  tcp_connect:
+    tcp:
-    prober: tcp
+{% if inventory_hostname == 'check.home.arpa' %}
-  smtp_check:
+      preferred_ip_protocol: "ip4"
-    prober: tcp
+      ip_protocol_fallback: true
-    timeout: 5s
+{% endif %}
-    tcp:
+      query_response:
-{% if inventory_hostname == 'check.home.arpa' %}
+        - expect: "^220(.*)ESMTP(.*)$"
-      preferred_ip_protocol: "ip4"
+        - send: "EHLO prober\r"
-      ip_protocol_fallback: true
+        - expect: "^250-STARTTLS"
-{% endif %}
+        - send: "STARTTLS\r"
-      query_response:
+        - expect: "^220"
-        - expect: "^220(.*)ESMTP(.*)$"
+        - starttls: true
-        - send: "EHLO prober\r"
+        - send: "EHLO prober\r"
-        - expect: "^250-STARTTLS"
+        - expect: "^250-AUTH"
-        - send: "STARTTLS\r"
+        - send: "QUIT\r"
-        - expect: "^220"
+  imap_check:
-        - starttls: true
+    prober: tcp
-        - send: "EHLO prober\r"
+    timeout: 5s
-        - expect: "^250-AUTH"
+    tcp:
-        - send: "QUIT\r"
+{% if inventory_hostname == 'check.home.arpa' %}
-  imap_check:
+      preferred_ip_protocol: "ip4"
-    prober: tcp
+      ip_protocol_fallback: true
-    timeout: 5s
+{% endif %}
-    tcp:
+      query_response:
-{% if inventory_hostname == 'check.home.arpa' %}
+        - expect: "OK.*STARTTLS"
-      preferred_ip_protocol: "ip4"
+        - send: ". STARTTLS"
-      ip_protocol_fallback: true
+        - expect: "OK"
-{% endif %}
+        - starttls: true
-      query_response:
+        - send: ". capability"
-        - expect: "OK.*STARTTLS"
+        - expect: "CAPABILITY IMAP4rev1"
-        - send: ". STARTTLS"
+  icmp:
-        - expect: "OK"
+    prober: icmp
-        - starttls: true
+{% if inventory_hostname == 'check.home.arpa' %}
-        - send: ". capability"
+    icmp:
-        - expect: "CAPABILITY IMAP4rev1"
+      preferred_ip_protocol: "ip4"
-  icmp:
+      ip_protocol_fallback: true
-    prober: icmp
+{% endif %}
-{% if inventory_hostname == 'check.home.arpa' %}
-    icmp:
-      preferred_ip_protocol: "ip4"
-      ip_protocol_fallback: true
-{% endif %}

ansible/roles/grafana/handlers/main.yml

@@ -0,0 +1,5 @@
+## Restart grafana on changes
+- name: restart grafana
+  ansible.builtin.service:
+    name: 'grafana-server'
+    state: restarted

ansible/roles/grafana/tasks/main.yml

@@ -1,14 +1,21 @@
----
+---
-
+
-- name: Install Grafana
+- name: Install Grafana
-  tags: grafana-install
+  tags: grafana-install
-  ansible.builtin.package:
+  ansible.builtin.package:
-    name: 'grafana'
+    name: 'grafana'
-    state: latest
+    state: latest
-
+
-- name: Enable Grafana service
+- name: Add grafana configuration
-  tags: grafana-enable
+  tags: grafana-configure
-  ansible.builtin.service:
+  ansible.builtin.template:
-    name: 'grafana-server'
+    src: grafana.ini.j2
-    state: started
+    dest: '/etc/grafana/grafana.ini'
-    enabled: yes
+  notify: restart grafana
+
+- name: Enable Grafana service
+  tags: grafana-enable
+  ansible.builtin.service:
+    name: 'grafana-server'
+    state: started
+    enabled: yes

ansible/roles/prometheus/tasks/main.yml

@@ -1,70 +1,62 @@
----
+---
-
+
-- name: Load variables
+- name: Load variables
-  include_vars: "{{ ansible_os_family|lower }}.yml"
+  include_vars: "{{ ansible_os_family|lower }}.yml"
-
+
-- name: Install Prometheus
+- name: Install Prometheus
-  tags: prometheus-install
+  tags: prometheus-install
-  ansible.builtin.package:
+  ansible.builtin.package:
-    name: '{{ prometheus_package }}'
+    name: '{{ prometheus_package }}'
-    state: latest
+    state: latest
-
+
-- name: Add scrape configuration
+- name: Add scrape configuration
-  tags: prometheus-scrape-configure
+  tags: prometheus-scrape-configure
-  ansible.builtin.copy:
+  ansible.builtin.copy:
-    src: "{{ item.src }}"
+    src: "{{ item.src }}"
-    dest: "/etc/prometheus/{{ item.dest }}"
+    dest: "/etc/prometheus/{{ item.dest }}"
-  loop:
+  loop:
-    - { src: '../templates/scrape-main.yml', dest: 'scrape-main.yml' }
+    - { src: '../templates/scrape-main.yml', dest: 'scrape-main.yml' }
-    - { src: '../templates/scrape-blackbox.yml', dest: 'scrape-blackbox.yml' }
+    - { src: '../templates/scrape-blackbox.yml', dest: 'scrape-blackbox.yml' }
-  notify:
+  notify:
-    - restart prometheus
+    - restart prometheus
-
+
-- name: Add alertmanager configuration
+- name: Create folder for rules definition
-  tags: alertmanager-configure
+  ansible.builtin.file:
-  ansible.builtin.template:
+    path: /etc/prometheus/rules
-    src: alertmanager.yml.j2
+    state: directory
-    dest: '{{ alertmanager_config }}'
+    mode: 0755
-  notify:
+
-    - restart alertmanager
+- name: Add rules configuration
-
+  tags: alertmanager-rules-configure
-- name: Create folder for rules definition
+  ansible.builtin.copy:
-  ansible.builtin.file:
+    src: "{{ item.src }}"
-    path: /etc/prometheus/rules
+    dest: "{{ alertmanager_rules }}/{{ item.dest }}"
-    state: directory
+  loop:
-    mode: 0755
+    - { src: '../templates/hosts.rules', dest: 'hosts.rules.yml' }
-
+    - { src: '../templates/prometheus.rules', dest: 'prometheus.rules.yml' }
-- name: Add rules configuration
+    - { src: '../templates/blackbox.rules', dest: 'blackbox.rules.yml' }
-  tags: alertmanager-rules-configure
+  notify:
-  ansible.builtin.copy:
+    - restart prometheus
-    src: "{{ item.src }}"
+    - restart alertmanager
-    dest: "{{ alertmanager_rules }}/{{ item.dest }}"
+
-  loop:
+- name: Enable Prometheus configuration
-    - { src: '../templates/hosts.rules', dest: 'hosts.rules.yml' }
+  tags: prometheus-configure
-    - { src: '../templates/prometheus.rules', dest: 'prometheus.rules.yml' }
+  ansible.builtin.template:
-    - { src: '../templates/blackbox.rules', dest: 'blackbox.rules.yml' }
+    src: prometheus.config.j2
-  notify:
+    dest: '{{ prometheus_config }}'
-    - restart prometheus
+    validate: promtool check config %s
-    - restart alertmanager
+  notify: restart prometheus
-
+
-- name: Enable Prometheus configuration
+- name: Enable Prometheus service
-  tags: prometheus-configure
+  tags: prometheus-enable
-  ansible.builtin.template:
+  ansible.builtin.service:
-    src: prometheus.config.j2
+    name: '{{ prometheus_service }}'
-    dest: '{{ prometheus_config }}'
+    state: started
-    validate: promtool check config %s
+    enabled: yes
-  notify: restart prometheus
+
-
+- name: Enable Alertmanager service
-- name: Enable Prometheus service
+  tags: alertmanager-enable
-  tags: prometheus-enable
+  ansible.builtin.service:
-  ansible.builtin.service:
+    name: '{{ alertmanager_service }}'
-    name: '{{ prometheus_service }}'
+    state: started
-    state: started
+    enabled: yes
-    enabled: yes
-
-- name: Enable Alertmanager service
-  tags: alertmanager-enable
-  ansible.builtin.service:
-    name: '{{ alertmanager_service }}'
-    state: started
-    enabled: yes

ansible/roles/prometheus/templates/alertmanager.yml.j2

@@ -1,66 +0,0 @@
-{{ ansible_managed | comment }}
-# See https://prometheus.io/docs/alerting/configuration/ for documentation.
-
-global:
-  # The smarthost and SMTP sender used for mail notifications.
-  smtp_smarthost: 'mail.labolyon.fr:587'
-  smtp_from: 'alerts@labolyon.fr'
-  smtp_auth_username: 'alerts@labolyon.fr'
-  smtp_auth_password: {{ lookup('community.general.passwordstore', 'monitoring/e-mail/alerts@labolyon.fr')}}
-
-# The directory from which notification templates are read.
-templates:
-- '/etc/prometheus/alertmanager_templates/*.tmpl'
-
-# The root route on which each incoming alert enters.
-route:
-  # The labels by which incoming alerts are grouped together. For example,
-  # multiple alerts coming in for cluster=A and alertname=LatencyHigh would
-  # be batched into a single group.
-  #
-  # To aggregate by all possible labels use '...' as the sole label name.
-  # This effectively disables aggregation entirely, passing through all
-  # alerts as-is. This is unlikely to be what you want, unless you have
-  # a very low alert volume or your upstream notification system performs
-  # its own grouping. Example: group_by: [...]
-  group_by: ['alertname', 'cluster', 'service']
-
-  # When a new group of alerts is created by an incoming alert, wait at
-  # least 'group_wait' to send the initial notification.
-  # This way ensures that you get multiple alerts for the same group that start
-  # firing shortly after another are batched together on the first
-  # notification.
-  group_wait: 30s
-
-  # When the first notification was sent, wait 'group_interval' to send a batch
-  # of new alerts that started firing for that group.
-  group_interval: 5m
-
-  # If an alert has successfully been sent, wait 'repeat_interval' to
-  # resend them.
-  repeat_interval: 3h
-
-  # A default receiver
-  receiver: all-admins-email
-
-  # All the above attributes are inherited by all child routes and can
-  # overwritten on each.
-
-# Inhibition rules allow to mute a set of alerts given that another alert is
-# firing.
-# We use this to mute any warning-level notifications if the same alert is
-# already critical.
-inhibit_rules:
-  - source_matchers: [severity="critical"]
-    target_matchers: [severity="warning"]
-    # Apply inhibition if the alertname is the same.
-    # CAUTION:
-    #   If all label names listed in `equal` are missing
-    #   from both the source and target alerts,
-    #   the inhibition rule will apply!
-    equal: [alertname, cluster, service]
-
-receivers:
-  - name: 'all-admins-email'
-    email_configs:
-      - to: 'mirsal@mirsal.fr'

ansible/roles/prometheus/templates/prometheus.config.j2

@@ -1,31 +1,75 @@
-{{ ansible_managed | comment }}
+{{ ansible_managed | comment }}
-
+
-# Global configuration
+# Global configuration
-global:
+global:
-  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
-  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
-  # scrape_timeout is set to the global default (10s).
+  # scrape_timeout is set to the global default (10s).
-
+
-# Alertmanager configuration
+# Alertmanager configuration
-alerting:
+alerting:
-  alertmanagers:
+  alertmanagers:
-    - static_configs:
+    - static_configs:
-        - targets:
+        - targets:
-           - localhost:9093
+           - localhost:9093
-
+
-# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
-rule_files:
+rule_files:
-   - "{{ alertmanager_rules }}/prometheus.rules.yml"
+   - "{{ alertmanager_rules }}/prometheus.rules.yml"
-   - "{{ alertmanager_rules }}/blackbox.rules.yml"
+   - "{{ alertmanager_rules }}/blackbox.rules.yml"
-   - "{{ alertmanager_rules }}/hosts.rules.yml"
+   - "{{ alertmanager_rules }}/hosts.rules.yml"
-
+
-# A scrape configuration containing exactly one endpoint to scrape:
+# A scrape configuration containing exactly one endpoint to scrape:
-scrape_config_files:
+
-{% for item in scrape_files %}
+#scrape_config_files:
-  - "{{ item }}"
+#{% for item in scrape_files %}
-{% endfor %}
+#  - "{{ item }}"
-
+#{% endfor %}
-scrape_configs:
+
-  - job_name: "prometheus"
+scrape_configs:
-    static_configs:
+  - job_name: "prometheus"
-      - targets: ["localhost:9090"]
+    static_configs:
+      - targets: ["localhost:9090"]
+
+{% if blackbox %}
+  - job_name: "blackbox"
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+          - https://labolyon.fr
+          - https://mail.labolyon.fr
+          - https://wiki.labolyon.fr
+          - https://git.labolyon.fr
+    relabel_configs:
+       - source_labels: [__address__]
+         target_label: __param_target
+       - source_labels: [__param_target]
+         target_label: target
+       - target_label: __address__
+         replacement: 127.0.0.1:9115  # The blackbox exporter's real hostname:port.
+{% else %}
+  - job_name: "linux-nodes"
+    static_configs:
+      - targets:
+         - ansible.labolyon.dn42:9100
+         - dn42-router.labolyon.dn42:9100
+         - dns.labolyon.dn42:9100
+         - git.labolyon.dn42:9100
+         - i2p.labolyon.dn42:9100
+         - labolyon-fr.labolyon.dn42:9100
+         - lolix-ixpman.labolyon.dn42:9100
+         - lolix-rs1.labolyon.dn42:9100
+         - lolmox.labolyon.dn42:9100
+         - lolmox2.labolyon.dn42:9100
+         - mail.labolyon.dn42:9100
+         - matrix.labolyon.dn42:9100
+         - monitoring.labolyon.dn42:9100
+         - mosquitto.labolyon.dn42:9100
+         - radius.labolyon.dn42:9100
+         - reverse-proxy.labolyon.dn42:9100
+         - wiki.labolyon.dn42:9100
+         - zigbee2mqtt.labolyon.dn42:9100
+{% endif %}

ansible/roles/prometheus/templates/scrape-blackbox.yml

@@ -1,8 +1,4 @@
-# managed by ansible
+  - job_name: "blackbox"
-
-scrape_configs:
-
-  - job_name: "blackbox-public-https"
     metrics_path: /probe
     params:
       module: [http_2xx]
@@ -10,33 +6,13 @@ scrape_configs:
     static_configs:
       - targets:
           - https://labolyon.fr
-          - https://webmail.labolyon.fr
+          - https://mail.labolyon.fr
           - https://wiki.labolyon.fr
           - https://git.labolyon.fr
-          - https://sso.labolyon.fr
-          - https://chat.labolyon.fr
-          - https://grafana.labolyon.fr
     relabel_configs:
        - source_labels: [__address__]
          target_label: __param_target
        - source_labels: [__param_target]
-         target_label: instance
+         target_label: target
-       - target_label: __address__
-         replacement: 127.0.0.1:9115  # The blackbox exporter's real hostname:port.
-
-  - job_name: "blackbox-internal-https"
-    metrics_path: /probe
-    params:
-      module: [http_2xx_selfsigned]
-    scrape_interval: 5s
-    static_configs:
-      - targets:
-          - https://octoprint.labolyon.dn42/login/
-          - https://stickers.labolyon.dn42
-    relabel_configs:
-       - source_labels: [__address__]
-         target_label: __param_target
-       - source_labels: [__param_target]
-         target_label: instance
        - target_label: __address__
          replacement: 127.0.0.1:9115  # The blackbox exporter's real hostname:port.

ansible/roles/prometheus/templates/scrape-main.yml

@@ -1,4 +1,3 @@
-scrape_configs:
   - job_name: "linux-nodes"
     static_configs:
       - targets: