feat: add disk exporter module and related configurations

- Introduced a new disk exporter module to monitor disk health using SMART data.
- Updated `apps.tf` to include the disk exporter module with necessary parameters.
- Added local variables for smartctl version and checksum in `config.tf`.
- Enhanced `alertmanager.yml` to handle DiskUnhealthy alerts with specific grouping and dispatch settings.
- Created new Prometheus rules for monitoring disk health, including alerts for unhealthy disks.
- Updated Prometheus configuration to scrape metrics from the new disk exporter.
- Enhanced Telegram notification template to include resolved alerts and improved formatting.
- Updated the auth-exporter module to specify the Python image tag correctly.
- Created the disk-exporter module with Kubernetes resources, including deployment, service account, and config map.
- Added necessary Python scripts and requirements for the disk exporter functionality.

terraform/apps/dev-01/apps.tf

@@ -81,3 +81,12 @@ module "auth_exporter" {
   source            = "../../modules/utils/auth-exporter"
   persistent_folder = "${local.persistent_folder}/monitoring"
 }
+
+module "disk_exporter" {
+  source                  = "../../modules/utils/disk-exporter"
+  persistent_folder       = "${local.persistent_folder}/monitoring"
+  smartctl_static_version = local.smartctl_static_version
+  smartctl_sha256         = local.smartctl_sha256
+  check_frequency         = 600
+  standby_mode            = false
+}

terraform/apps/dev-01/config.tf

@@ -184,6 +184,9 @@ locals {
     auth_password = local.mail_app_api_secret_key
   }
 
+  smartctl_static_version = "8.0-198-g34e7e4975211"
+  smartctl_sha256         = "d7e779664248942bb519b254dcb49cddaec2593e192cc468254c58e64da20840"
+
   telegram_chat_id = "5432313610" # a13labs channel
   telegram_message = file("${path.module}/resources/telegram.tpl")
 

terraform/apps/dev-01/resources/alertmanager.yml

@@ -8,4 +8,16 @@ route:
     - receiver: "telegram"
       matchers:
         - 'alertname="UserLoggedIn"'
+      group_by: ["alertname", "username"] # add username to avoid batching different users
+      group_wait: 0s # immediate dispatch
+      group_interval: 2m # short regroup window
+      repeat_interval: 30m # optional: adjust reminder cadence
+      continue: false
+    - receiver: "telegram"
+      matchers:
+        - 'alertname="DiskUnhealthy"'
+      group_by: ["alertname", "device"] # add device to avoid batching different disks
+      group_wait: 0s # immediate dispatch
+      group_interval: 2m # short regroup window
+      repeat_interval: 30m # optional: adjust reminder cadence
       continue: false

terraform/apps/dev-01/resources/prometheus-rules.yml

@@ -2,9 +2,19 @@ groups:
   - name: auth.rules
     rules:
       - alert: UserLoggedIn
-        expr: increase(auth_sessions_active_by_user[1m]) > 0
+        expr: changes(auth_sessions_active_by_user[1m]) > 0
         labels:
           severity: info
         annotations:
-          summary: "Login: {{ $labels.username }} on {{ $labels.instance }}"
-          description: "User {{ $labels.username }} logged in via {{ $labels.source }} on {{ $labels.instance }}."
+          summary: "Login: {{ $labels.username }} on dev-01"
+          description: "User {{ $labels.username }} logged in via {{ $labels.source }} on dev-01."
+  - name: disk.rules
+    rules:
+      - alert: DiskUnhealthy
+        expr: count(disk_info{})-count(disk_healthy{}) > 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Disk unhealthy: {{ $labels.device }} on dev-01"
+          description: "SMART overall health failed for device {{ $labels.device }} (disk_healthy=0). Investigate immediately."

terraform/apps/dev-01/resources/prometheus.yml

@@ -5,8 +5,19 @@ scrape_configs:
       - targets:
           - "node-exporter.prometheus.svc.cluster.local:9100"
           - "10.19.4.1:9100"
+      - labels:
+          cluster: "dev-01"
   - job_name: auth-exporter
     scrape_interval: 10s
     static_configs:
       - targets:
           - "auth-exporter.prometheus.svc.cluster.local:9100"
+      - labels:
+          cluster: "dev-01"
+  - job_name: disk-exporter
+    scrape_interval: 120s
+    static_configs:
+      - targets:
+          - "disk-exporter.prometheus.svc.cluster.local:9100"
+      - labels:
+          cluster: "dev-01"

terraform/apps/dev-01/resources/telegram.tpl

@@ -1,13 +1,33 @@
-{{/* Always produce a non-empty message. Handles single or multiple firing alerts and missing username. */}}
-{{ if gt (len .Alerts.Firing) 1 -}}
-<b>{{ len .Alerts.Firing }} logins</b> on {{ with .CommonLabels.cluster }}<b>{{ . }}</b>{{ else }}<b>unknown host</b>{{ end }}:
+{{/* Template enhanced to include resolved alerts while using annotations. */}}
+{{ $firingCount := len .Alerts.Firing -}}
+{{ $resolvedCount := len .Alerts.Resolved -}}
+
+{{ if or (gt $firingCount 0) (gt $resolvedCount 0) -}}
+  {{ if and (gt $firingCount 0) (gt $resolvedCount 0) -}}
+<b>{{ $firingCount }} firing</b> / <b>{{ $resolvedCount }} resolved</b>
+{{ if gt $firingCount 0 -}}
+<b>Firing:</b>
 {{- range .Alerts.Firing }}
-• {{ with .Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} — Source: <code>{{ with .Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with .Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
+• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
 {{- end }}
-{{ else if eq (len .Alerts.Firing) 1 -}}
-{{ $a := index .Alerts.Firing 0 -}}
-<b>Login:</b> {{ with $a.Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} on <b>{{ $a.Labels.cluster }}</b>
-Source: <code>{{ with $a.Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with $a.Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
+{{ end -}}
+{{ if gt $resolvedCount 0 -}}
+<b>Resolved:</b>
+{{- range .Alerts.Resolved }}
+• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
+{{- end }}
+{{ end -}}
+  {{ else if gt $firingCount 0 -}}
+<b>{{ $firingCount }} alert{{ if ne $firingCount 1 }}s{{ end }} firing</b>:
+{{- range .Alerts.Firing }}
+• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
+{{- end }}
+  {{ else -}}
+<b>{{ $resolvedCount }} alert{{ if ne $resolvedCount 1 }}s{{ end }} resolved</b>:
+{{- range .Alerts.Resolved }}
+• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
+{{- end }}
+  {{ end -}}
 {{ else -}}
-<b>Login detected</b> — details unavailable.
+<b>No alert data</b> — nothing firing or recently resolved.
 {{ end -}}