feat: add disk exporter module and related configurations

- Introduced a new disk exporter module to monitor disk health using SMART data.
- Updated `apps.tf` to include the disk exporter module with necessary parameters.
- Added local variables for smartctl version and checksum in `config.tf`.
- Enhanced `alertmanager.yml` to handle DiskUnhealthy alerts with specific grouping and dispatch settings.
- Created new Prometheus rules for monitoring disk health, including alerts for unhealthy disks.
- Updated Prometheus configuration to scrape metrics from the new disk exporter.
- Enhanced Telegram notification template to include resolved alerts and improved formatting.
- Updated the auth-exporter module to specify the Python image tag correctly.
- Created the disk-exporter module with Kubernetes resources, including deployment, service account, and config map.
- Added necessary Python scripts and requirements for the disk exporter functionality.

terraform/apps/prod-01/resources/alertmanager.yml

@@ -8,4 +8,8 @@ route:
     - receiver: "telegram"
       matchers:
         - 'alertname="UserLoggedIn"'
+      group_by: ["alertname", "username"] # add username to avoid batching different users
+      group_wait: 0s # immediate dispatch
+      group_interval: 2m # short regroup window
+      repeat_interval: 30m # optional: adjust reminder cadence
       continue: false

terraform/apps/prod-01/resources/prometheus-rules.yml

@@ -6,5 +6,5 @@ groups:
         labels:
           severity: info
         annotations:
-          summary: "Login: {{ $labels.username }} on {{ $labels.instance }}"
-          description: "User {{ $labels.username }} logged in via {{ $labels.source }} on {{ $labels.instance }}."
+          summary: "Login: {{ $labels.username }} on prod-01"
+          description: "User {{ $labels.username }} logged in via {{ $labels.source }} on prod-01."

terraform/apps/prod-01/resources/prometheus.yml

@@ -4,13 +4,19 @@ scrape_configs:
     static_configs:
       - targets:
           - "node-exporter.prometheus.svc.cluster.local:9100"
+        labels:
+          cluster: "prod-01"
   - job_name: fail2ban
     scrape_interval: 60s
     static_configs:
       - targets:
           - "fail2ban-exporter.prometheus.svc.cluster.local:9100"
+        labels:
+          cluster: "prod-01"
   - job_name: auth-exporter
     scrape_interval: 10s
     static_configs:
       - targets:
           - "auth-exporter.prometheus.svc.cluster.local:9100"
+        labels:
+          cluster: "prod-01"

terraform/apps/prod-01/resources/telegram.tpl

@@ -1,28 +1,33 @@
-{{- /* Resolved alerts (if any) */ -}}
-{{- if gt (len .Alerts.Resolved) 1 -}}
-<b>{{ len .Alerts.Resolved }} logins resolved</b> on {{ with .CommonLabels.cluster }}<b>{{ . }}</b>{{ else }}<b>unknown host</b>{{ end }}:
-{{- range .Alerts.Resolved }}
-• {{ with .Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} — Source: <code>{{ with .Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with .Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
-{{- end }}
-{{- else if eq (len .Alerts.Resolved) 1 -}}
-{{ $a := index .Alerts.Resolved 0 -}}
-<b>Resolved login:</b> {{ with $a.Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} on <b>{{ $a.Labels.cluster }}</b>
-Source: <code>{{ with $a.Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with $a.Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
-{{- end -}}
+{{/* Template enhanced to include resolved alerts while using annotations. */}}
+{{ $firingCount := len .Alerts.Firing -}}
+{{ $resolvedCount := len .Alerts.Resolved -}}
 
-{{- /* Firing alerts (original logic) */ -}}
-{{ if gt (len .Alerts.Firing) 1 -}}
-<b>{{ len .Alerts.Firing }} logins</b> on {{ with .CommonLabels.cluster }}<b>{{ . }}</b>{{ else }}<b>unknown host</b>{{ end }}:
+{{ if or (gt $firingCount 0) (gt $resolvedCount 0) -}}
+  {{ if and (gt $firingCount 0) (gt $resolvedCount 0) -}}
+<b>{{ $firingCount }} firing</b> / <b>{{ $resolvedCount }} resolved</b>
+{{ if gt $firingCount 0 -}}
+<b>Firing:</b>
 {{- range .Alerts.Firing }}
-• {{ with .Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} — Source: <code>{{ with .Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with .Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
+• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
 {{- end }}
-{{ else if eq (len .Alerts.Firing) 1 -}}
-{{ $a := index .Alerts.Firing 0 -}}
-<b>Login:</b> {{ with $a.Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} on <b>{{ $a.Labels.cluster }}</b>
-Source: <code>{{ with $a.Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with $a.Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
-{{ else -}}
-{{- /* If there were resolved alerts above this will still show when no firing alerts exist. */ -}}
-{{ if eq (len .Alerts.Resolved) 0 -}}
-<b>Login detected</b> — details unavailable.
 {{ end -}}
+{{ if gt $resolvedCount 0 -}}
+<b>Resolved:</b>
+{{- range .Alerts.Resolved }}
+• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
+{{- end }}
+{{ end -}}
+  {{ else if gt $firingCount 0 -}}
+<b>{{ $firingCount }} alert{{ if ne $firingCount 1 }}s{{ end }} firing</b>:
+{{- range .Alerts.Firing }}
+• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
+{{- end }}
+  {{ else -}}
+<b>{{ $resolvedCount }} alert{{ if ne $resolvedCount 1 }}s{{ end }} resolved</b>:
+{{- range .Alerts.Resolved }}
+• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
+{{- end }}
+  {{ end -}}
+{{ else -}}
+<b>No alert data</b> — nothing firing or recently resolved.
 {{ end -}}