feat: add disk exporter module and related configurations

- Introduced a new disk exporter module to monitor disk health using SMART data.
- Updated `apps.tf` to include the disk exporter module with necessary parameters.
- Added local variables for smartctl version and checksum in `config.tf`.
- Enhanced `alertmanager.yml` to handle DiskUnhealthy alerts with specific grouping and dispatch settings.
- Created new Prometheus rules for monitoring disk health, including alerts for unhealthy disks.
- Updated Prometheus configuration to scrape metrics from the new disk exporter.
- Enhanced Telegram notification template to include resolved alerts and improved formatting.
- Updated the auth-exporter module to specify the Python image tag correctly.
- Created the disk-exporter module with Kubernetes resources, including deployment, service account, and config map.
- Added necessary Python scripts and requirements for the disk exporter functionality.

terraform/apps/dev-01/apps.tf

@@ -81,3 +81,12 @@ module "auth_exporter" {
   source            = "../../modules/utils/auth-exporter"
   persistent_folder = "${local.persistent_folder}/monitoring"
 }
+
+module "disk_exporter" {
+  source                  = "../../modules/utils/disk-exporter"
+  persistent_folder       = "${local.persistent_folder}/monitoring"
+  smartctl_static_version = local.smartctl_static_version
+  smartctl_sha256         = local.smartctl_sha256
+  check_frequency         = 600
+  standby_mode            = false
+}

terraform/apps/dev-01/config.tf

@@ -184,6 +184,9 @@ locals {
     auth_password = local.mail_app_api_secret_key
   }
 
+  smartctl_static_version = "8.0-198-g34e7e4975211"
+  smartctl_sha256         = "d7e779664248942bb519b254dcb49cddaec2593e192cc468254c58e64da20840"
+
   telegram_chat_id = "5432313610" # a13labs channel
   telegram_message = file("${path.module}/resources/telegram.tpl")
 

terraform/apps/dev-01/resources/alertmanager.yml

@@ -8,4 +8,16 @@ route:
     - receiver: "telegram"
       matchers:
         - 'alertname="UserLoggedIn"'
+      group_by: ["alertname", "username"] # add username to avoid batching different users
+      group_wait: 0s # immediate dispatch
+      group_interval: 2m # short regroup window
+      repeat_interval: 30m # optional: adjust reminder cadence
+      continue: false
+    - receiver: "telegram"
+      matchers:
+        - 'alertname="DiskUnhealthy"'
+      group_by: ["alertname", "device"] # add device to avoid batching different disks
+      group_wait: 0s # immediate dispatch
+      group_interval: 2m # short regroup window
+      repeat_interval: 30m # optional: adjust reminder cadence
       continue: false

terraform/apps/dev-01/resources/prometheus-rules.yml

@@ -2,9 +2,19 @@ groups:
   - name: auth.rules
     rules:
       - alert: UserLoggedIn
-        expr: increase(auth_sessions_active_by_user[1m]) > 0
+        expr: changes(auth_sessions_active_by_user[1m]) > 0
         labels:
           severity: info
         annotations:
-          summary: "Login: {{ $labels.username }} on {{ $labels.instance }}"
-          description: "User {{ $labels.username }} logged in via {{ $labels.source }} on {{ $labels.instance }}."
+          summary: "Login: {{ $labels.username }} on dev-01"
+          description: "User {{ $labels.username }} logged in via {{ $labels.source }} on dev-01."
+  - name: disk.rules
+    rules:
+      - alert: DiskUnhealthy
+        expr: count(disk_info{})-count(disk_healthy{}) > 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Disk unhealthy: {{ $labels.device }} on dev-01"
+          description: "SMART overall health failed for device {{ $labels.device }} (disk_healthy=0). Investigate immediately."

terraform/apps/dev-01/resources/prometheus.yml

@@ -5,8 +5,19 @@ scrape_configs:
       - targets:
           - "node-exporter.prometheus.svc.cluster.local:9100"
           - "10.19.4.1:9100"
+      - labels:
+          cluster: "dev-01"
   - job_name: auth-exporter
     scrape_interval: 10s
     static_configs:
       - targets:
           - "auth-exporter.prometheus.svc.cluster.local:9100"
+      - labels:
+          cluster: "dev-01"
+  - job_name: disk-exporter
+    scrape_interval: 120s
+    static_configs:
+      - targets:
+          - "disk-exporter.prometheus.svc.cluster.local:9100"
+      - labels:
+          cluster: "dev-01"

terraform/apps/dev-01/resources/telegram.tpl

@@ -1,13 +1,33 @@
-{{/* Always produce a non-empty message. Handles single or multiple firing alerts and missing username. */}}
-{{ if gt (len .Alerts.Firing) 1 -}}
-<b>{{ len .Alerts.Firing }} logins</b> on {{ with .CommonLabels.cluster }}<b>{{ . }}</b>{{ else }}<b>unknown host</b>{{ end }}:
+{{/* Template enhanced to include resolved alerts while using annotations. */}}
+{{ $firingCount := len .Alerts.Firing -}}
+{{ $resolvedCount := len .Alerts.Resolved -}}
+
+{{ if or (gt $firingCount 0) (gt $resolvedCount 0) -}}
+  {{ if and (gt $firingCount 0) (gt $resolvedCount 0) -}}
+<b>{{ $firingCount }} firing</b> / <b>{{ $resolvedCount }} resolved</b>
+{{ if gt $firingCount 0 -}}
+<b>Firing:</b>
 {{- range .Alerts.Firing }}
-• {{ with .Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} — Source: <code>{{ with .Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with .Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
+• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
 {{- end }}
-{{ else if eq (len .Alerts.Firing) 1 -}}
-{{ $a := index .Alerts.Firing 0 -}}
-<b>Login:</b> {{ with $a.Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} on <b>{{ $a.Labels.cluster }}</b>
-Source: <code>{{ with $a.Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with $a.Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
-{{ else -}}
-<b>Login detected</b> — details unavailable.
+{{ end -}}
+{{ if gt $resolvedCount 0 -}}
+<b>Resolved:</b>
+{{- range .Alerts.Resolved }}
+• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
+{{- end }}
+{{ end -}}
+  {{ else if gt $firingCount 0 -}}
+<b>{{ $firingCount }} alert{{ if ne $firingCount 1 }}s{{ end }} firing</b>:
+{{- range .Alerts.Firing }}
+• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
+{{- end }}
+  {{ else -}}
+<b>{{ $resolvedCount }} alert{{ if ne $resolvedCount 1 }}s{{ end }} resolved</b>:
+{{- range .Alerts.Resolved }}
+• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
+{{- end }}
+  {{ end -}}
+{{ else -}}
+<b>No alert data</b> — nothing firing or recently resolved.
 {{ end -}}

terraform/apps/prod-01/resources/alertmanager.yml

@@ -8,4 +8,8 @@ route:
     - receiver: "telegram"
       matchers:
         - 'alertname="UserLoggedIn"'
+      group_by: ["alertname", "username"] # add username to avoid batching different users
+      group_wait: 0s # immediate dispatch
+      group_interval: 2m # short regroup window
+      repeat_interval: 30m # optional: adjust reminder cadence
       continue: false

terraform/apps/prod-01/resources/prometheus-rules.yml

@@ -6,5 +6,5 @@ groups:
         labels:
           severity: info
         annotations:
-          summary: "Login: {{ $labels.username }} on {{ $labels.instance }}"
-          description: "User {{ $labels.username }} logged in via {{ $labels.source }} on {{ $labels.instance }}."
+          summary: "Login: {{ $labels.username }} on prod-01"
+          description: "User {{ $labels.username }} logged in via {{ $labels.source }} on prod-01."

terraform/apps/prod-01/resources/prometheus.yml

@@ -4,13 +4,19 @@ scrape_configs:
     static_configs:
       - targets:
           - "node-exporter.prometheus.svc.cluster.local:9100"
+        labels:
+          cluster: "prod-01"
   - job_name: fail2ban
     scrape_interval: 60s
     static_configs:
       - targets:
           - "fail2ban-exporter.prometheus.svc.cluster.local:9100"
+        labels:
+          cluster: "prod-01"
   - job_name: auth-exporter
     scrape_interval: 10s
     static_configs:
       - targets:
           - "auth-exporter.prometheus.svc.cluster.local:9100"
+        labels:
+          cluster: "prod-01"

terraform/apps/prod-01/resources/telegram.tpl

@@ -1,28 +1,33 @@
-{{- /* Resolved alerts (if any) */ -}}
-{{- if gt (len .Alerts.Resolved) 1 -}}
-<b>{{ len .Alerts.Resolved }} logins resolved</b> on {{ with .CommonLabels.cluster }}<b>{{ . }}</b>{{ else }}<b>unknown host</b>{{ end }}:
-{{- range .Alerts.Resolved }}
-• {{ with .Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} — Source: <code>{{ with .Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with .Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
-{{- end }}
-{{- else if eq (len .Alerts.Resolved) 1 -}}
-{{ $a := index .Alerts.Resolved 0 -}}
-<b>Resolved login:</b> {{ with $a.Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} on <b>{{ $a.Labels.cluster }}</b>
-Source: <code>{{ with $a.Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with $a.Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
-{{- end -}}
+{{/* Template enhanced to include resolved alerts while using annotations. */}}
+{{ $firingCount := len .Alerts.Firing -}}
+{{ $resolvedCount := len .Alerts.Resolved -}}
 
-{{- /* Firing alerts (original logic) */ -}}
-{{ if gt (len .Alerts.Firing) 1 -}}
-<b>{{ len .Alerts.Firing }} logins</b> on {{ with .CommonLabels.cluster }}<b>{{ . }}</b>{{ else }}<b>unknown host</b>{{ end }}:
+{{ if or (gt $firingCount 0) (gt $resolvedCount 0) -}}
+  {{ if and (gt $firingCount 0) (gt $resolvedCount 0) -}}
+<b>{{ $firingCount }} firing</b> / <b>{{ $resolvedCount }} resolved</b>
+{{ if gt $firingCount 0 -}}
+<b>Firing:</b>
 {{- range .Alerts.Firing }}
-• {{ with .Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} — Source: <code>{{ with .Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with .Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
+• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
+{{- end }}
+{{ end -}}
+{{ if gt $resolvedCount 0 -}}
+<b>Resolved:</b>
+{{- range .Alerts.Resolved }}
+• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
+{{- end }}
+{{ end -}}
+  {{ else if gt $firingCount 0 -}}
+<b>{{ $firingCount }} alert{{ if ne $firingCount 1 }}s{{ end }} firing</b>:
+{{- range .Alerts.Firing }}
+• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
 {{- end }}
-{{ else if eq (len .Alerts.Firing) 1 -}}
-{{ $a := index .Alerts.Firing 0 -}}
-<b>Login:</b> {{ with $a.Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} on <b>{{ $a.Labels.cluster }}</b>
-Source: <code>{{ with $a.Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with $a.Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
   {{ else -}}
-{{- /* If there were resolved alerts above this will still show when no firing alerts exist. */ -}}
-{{ if eq (len .Alerts.Resolved) 0 -}}
-<b>Login detected</b> — details unavailable.
+<b>{{ $resolvedCount }} alert{{ if ne $resolvedCount 1 }}s{{ end }} resolved</b>:
+{{- range .Alerts.Resolved }}
+• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
+{{- end }}
   {{ end -}}
+{{ else -}}
+<b>No alert data</b> — nothing firing or recently resolved.
 {{ end -}}

terraform/modules/utils/auth-exporter/main.tf

@@ -47,7 +47,7 @@ resource "kubernetes_deployment" "auth_exporter" {
         automount_service_account_token = false
         container {
           name    = "auth-exporter"
-          image   = var.tag
+          image   = "python:${var.tag}"
           command = ["sh", "-c", join(" && ", local.commands)]
           port {
             container_port = 9100

terraform/modules/utils/auth-exporter/variables.tf

@@ -1,7 +1,7 @@
 variable "tag" {
   description = "Python image tag"
   type        = string
-  default     = "python:3.12-slim"
+  default     = "3.12-slim"
 }
 
 variable "namespace" {

terraform/modules/utils/disk-exporter/config.tf

@@ -0,0 +1,15 @@
+locals {
+  app_version = var.tag
+  workdir     = "${var.persistent_folder}/disk-exporter"
+  commands = [
+    "python -m venv /tmp/venv",
+    ". /tmp/venv/bin/activate",
+    "pip install -r /exporter/requirements.txt",
+    "python /exporter/exporter.py --export-usernames"
+  ]
+  exporter_script  = file("${path.module}/resources/exporter.py")
+  requirements_txt = file("${path.module}/resources/requirements.txt")
+
+  exporter_checksum     = md5(local.exporter_script)
+  requirements_checksum = md5(local.requirements_txt)
+}

terraform/modules/utils/disk-exporter/main.tf

@@ -0,0 +1,147 @@
+resource "kubernetes_service_account" "disk_exporter" {
+  metadata {
+    name      = "disk-exporter-sa"
+    namespace = var.namespace
+  }
+}
+
+resource "kubernetes_config_map" "disk_exporter" {
+  metadata {
+    name      = "disk-exporter-config"
+    namespace = var.namespace
+  }
+  data = {
+    "exporter.py"      = local.exporter_script
+    "requirements.txt" = local.requirements_txt
+  }
+}
+
+resource "kubernetes_deployment" "disk_exporter" {
+  metadata {
+    name      = "disk-exporter"
+    namespace = var.namespace
+    labels = {
+      app = "disk-exporter"
+    }
+  }
+  spec {
+    replicas = 1
+    selector {
+      match_labels = {
+        app = "disk-exporter"
+      }
+    }
+    strategy {
+      type = "RollingUpdate"
+    }
+    template {
+      metadata {
+        labels = {
+          app = "disk-exporter"
+        }
+        annotations = {
+          "checksum/exporter"     = local.exporter_checksum
+          "checksum/requirements" = local.requirements_checksum
+        }
+      }
+      spec {
+        service_account_name            = kubernetes_service_account.disk_exporter.metadata[0].name
+        automount_service_account_token = false
+
+        container {
+          name    = "disk-exporter"
+          image   = "python:${var.tag}"
+          command = ["sh", "-c", join(" && ", local.commands)]
+
+          env {
+            name  = "SMARTCTL_STATIC_VERSION"
+            value = var.smartctl_static_version
+          }
+          env {
+            name  = "SMARTCTL_SHA256"
+            value = coalesce(var.smartctl_sha256, "")
+          }
+          env {
+            name  = "SMARTCTL_CACHE_BASE"
+            value = "/tmp/smartctl-cache"
+          }
+          env {
+            name  = "CHECK_FREQUENCY"
+            value = tostring(var.check_frequency)
+          }
+          env {
+            name  = "STANDBY_MODE"
+            value = var.standby_mode
+          }
+          env {
+            name  = "EXPORTER_PORT"
+            value = "9100"
+          }
+          env {
+            name  = "DEVICE_ROOT"
+            value = "/dev"
+          }
+          port {
+            container_port = 9100
+            name           = "http-metrics"
+          }
+
+          volume_mount {
+            name       = "exporter-volume"
+            mount_path = "/exporter"
+            read_only  = true
+          }
+
+          volume_mount {
+            name       = "workdir"
+            mount_path = "/tmp"
+          }
+
+          security_context {
+            run_as_user               = 0
+            run_as_group              = 0
+            read_only_root_filesystem = true
+            privileged                = true
+          }
+        }
+
+        volume {
+          name = "exporter-volume"
+          config_map {
+            name = kubernetes_config_map.disk_exporter.metadata[0].name
+          }
+        }
+
+        volume {
+          name = "workdir"
+          host_path {
+            path = local.workdir
+            type = "DirectoryOrCreate"
+          }
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_service" "disk_exporter" {
+  metadata {
+    name      = "disk-exporter"
+    namespace = var.namespace
+    labels = {
+      app = "disk-exporter"
+    }
+  }
+  spec {
+    selector = {
+      app = "disk-exporter"
+    }
+    port {
+      port        = 9100
+      target_port = 9100
+      protocol    = "TCP"
+      name        = "http-metrics"
+    }
+    type = "ClusterIP"
+  }
+}

terraform/modules/utils/disk-exporter/provider.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = "~>1.8"
+  required_providers {
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "2.36.0"
+    }
+  }
+}

terraform/modules/utils/disk-exporter/resources/exporter.py

@@ -0,0 +1,432 @@
+#!/usr/bin/env python3
+import os
+import re
+import json
+import time
+import threading
+import subprocess
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from prometheus_client import start_http_server, Gauge, Counter, Summary
+import stat
+import tarfile
+import hashlib
+import urllib.request
+from io import BytesIO
+import signal   # <-- added
+
+SMARTCTL = os.environ.get("SMARTCTL_PATH", "/usr/sbin/smartctl")
+CHECK_FREQUENCY = int(os.environ.get("CHECK_FREQUENCY", "600"))
+EXPORTER_PORT = int(os.environ.get("EXPORTER_PORT", "9100"))
+STANDBY_MODE = os.environ.get("STANDBY_MODE", "0") == "1"
+
+SMARTCTL_STATIC_VERSION = os.environ.get("SMARTCTL_STATIC_VERSION", "")
+SMARTCTL_SHA256 = os.environ.get("SMARTCTL_SHA256", "")
+SMARTCTL_CACHE_BASE = os.environ.get("SMARTCTL_CACHE_BASE", "/tmp/smartctl-cache")
+SMARTCTL_FORCE_DOWNLOAD = os.environ.get("SMARTCTL_FORCE_DOWNLOAD", "false") == "true"
+
+DEBUG = os.environ.get("DEBUG", "0") == "1"
+SMARTCTL_STATUS_BITS = {
+    0: "Command line did not parse",
+    1: "Device open failed",
+    2: "SMART status check returned 'DISK FAILING'",
+    3: "Prefail Attributes <= threshold",
+    4: "Usage Attributes <= threshold",
+    5: "Self-test log contains errors",
+    6: "Self-test in progress",
+    7: "Device has read SMART data (some internal error if also bit 1?)"
+}
+# -------- Prometheus Metrics --------
+disk_healthy = Gauge('disk_healthy', 'SMART overall health status (1=passed,0=fail)', ['device'])
+disk_temperature = Gauge('disk_temperature', 'Drive temperature (C)', ['device'])
+disk_reallocated_sector_count = Gauge('disk_reallocated_sector_count', 'Reallocated Sector Count', ['device'])
+disk_reallocated_event_count = Gauge('disk_reallocated_event_count', 'Reallocated Event Count', ['device'])
+disk_offline_uncorrectable = Gauge('disk_offline_uncorrectable', 'Offline Uncorrectable Count', ['device'])
+
+disk_smart_attr_raw_value = Gauge('disk_smart_attr_raw_value', 'Raw value of SMART attribute', ['device','id','name'])
+disk_smart_attr_normalized_value = Gauge('disk_smart_attr_normalized_value', 'Normalized value of SMART attribute', ['device','id','name'])
+
+disk_info = Gauge('disk_info', 'Static disk identity info', ['device','model','serial','firmware','transport'])
+disk_last_smart_poll_timestamp = Gauge('disk_last_smart_poll_timestamp', 'Unix timestamp of last successful SMART poll', ['device'])
+
+# NVMe specific
+nvme_critical_warning = Gauge('nvme_critical_warning', 'NVMe critical warning bitmap', ['device'])
+nvme_available_spare = Gauge('nvme_available_spare', 'NVMe available spare (%)', ['device'])
+nvme_available_spare_threshold = Gauge('nvme_available_spare_threshold', 'NVMe available spare threshold (%)', ['device'])
+nvme_percentage_used = Gauge('nvme_percentage_used', 'NVMe percentage used (wear level)', ['device'])
+nvme_media_errors = Gauge('nvme_media_errors', 'NVMe media errors', ['device'])
+nvme_num_err_log_entries = Gauge('nvme_num_err_log_entries', 'NVMe number of error log entries', ['device'])
+nvme_data_units_read = Gauge('nvme_data_units_read', 'NVMe data units read (1000 * 512KiB)', ['device'])
+nvme_data_units_written = Gauge('nvme_data_units_written', 'NVMe data units written (1000 * 512KiB)', ['device'])
+nvme_temperature = Gauge('nvme_temperature', 'NVMe temperature (C)', ['device'])
+
+# mdadm (if any arrays)
+mdraid_degraded = Gauge('mdraid_degraded', 'MD RAID degraded flag (1=degraded)', ['array'])
+mdraid_sync_action = Gauge('mdraid_sync_action', 'MD RAID sync action indicator (idle=0,resync=1,recover=2,check=3,repair=4,reshape=5,unknown=99)', ['array'])
+
+# Exporter operational metrics
+exporter_errors_total = Counter('disk_exporter_errors_total', 'Total errors during SMART collection')
+smart_poll_duration_seconds = Summary('disk_smart_poll_duration_seconds', 'Duration of a full SMART poll cycle')
+smart_commands_total = Counter('disk_smart_commands_total', 'Total smartctl command invocations', ['result'])
+
+# Add exit code metric
+smartctl_exit_code = Gauge('smartctl_exit_code', 'Last smartctl raw exit code', ['device'])
+
+DEVICE_REGEX = re.compile(r'^(sd[a-z]+|nvme\d+n\d+)$')
+LOCK = threading.Lock()
+DEVICES = set()
+DEVICE_ROOT = os.environ.get("DEVICE_ROOT", "/dev")  # NEW: root path for block devices (allows mounting host /dev elsewhere)
+
+# Graceful shutdown event
+STOP_EVENT = threading.Event()  # <-- added
+
+# -------- Utility Functions --------
+def _build_smartctl_url(version: str) -> str:
+    return f"https://github.com/smartmontools/smartmontools-builds/releases/download/smartmontools-{version}/smartmontools-linux-x86_64-static-{version}.tar.gz"
+
+def _version_dir(version: str) -> str:
+    return os.path.join(SMARTCTL_CACHE_BASE, version)
+
+def _cached_binary_path(version: str) -> str:
+    return os.path.join(_version_dir(version), "smartctl")
+
+def _cached_sha_file(version: str) -> str:
+    return os.path.join(_version_dir(version), ".archive_sha256")
+
+def _url_exists(url: str) -> bool:
+    try:
+        req = urllib.request.Request(url, method="HEAD")
+        with urllib.request.urlopen(req, timeout=15):
+            return True
+    except Exception:
+        # Some servers may not allow HEAD; fallback to optimistic True
+        return True
+
+def _validate_cached(version: str) -> bool:
+    bin_path = _cached_binary_path(version)
+    if not (os.path.isfile(bin_path) and os.access(bin_path, os.X_OK)):
+        return False
+    if SMARTCTL_SHA256:
+        sha_file = _cached_sha_file(version)
+        if not os.path.isfile(sha_file):
+            return False
+        recorded = open(sha_file).read().strip()
+        if recorded.lower() != SMARTCTL_SHA256.lower():
+            print(f"Cached smartctl for {version} sha mismatch (have {recorded}, expected {SMARTCTL_SHA256}); will re-download.")
+            return False
+    return True
+
+def _download_and_extract_smartctl(version: str):
+    url = _build_smartctl_url(version)
+    if not _url_exists(url):
+        print(f"URL not reachable (HEAD failed): {url}")
+        return None
+    print(f"Fetching static smartctl archive: {url}")
+    try:
+        with urllib.request.urlopen(url, timeout=120) as r:
+            archive = r.read()
+    except Exception as e:
+        print(f"Download failed: {e}")
+        return None
+    if SMARTCTL_SHA256:
+        digest = hashlib.sha256(archive).hexdigest()
+        if digest.lower() != SMARTCTL_SHA256.lower():
+            print(f"Archive SHA256 mismatch (expected {SMARTCTL_SHA256} got {digest})")
+            return None
+    vdir = _version_dir(version)
+    os.makedirs(vdir, exist_ok=True)
+    try:
+        with tarfile.open(fileobj=BytesIO(archive), mode="r:gz") as tf:
+            member = next((m for m in tf.getmembers() if m.name.endswith("/smartctl") or m.name == "smartctl"), None)
+            if not member:
+                print("smartctl binary not found inside archive")
+                return None
+            tf.extract(member, vdir)
+            extracted = os.path.join(vdir, member.name)
+            final_path = _cached_binary_path(version)
+            if extracted != final_path:
+                os.replace(extracted, final_path)
+            os.chmod(final_path, os.stat(final_path).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+            if SMARTCTL_SHA256:
+                with open(_cached_sha_file(version), "w") as f:
+                    f.write(SMARTCTL_SHA256.lower())
+            print(f"Static smartctl ready: {final_path}")
+            return final_path
+    except Exception as e:
+        print(f"Extraction error: {e}")
+        return None
+
+def ensure_smartctl():
+    global SMARTCTL
+    # If user-specified path works, keep it
+    if os.path.isfile(SMARTCTL) and os.access(SMARTCTL, os.X_OK):
+        return
+    if SMARTCTL_STATIC_VERSION:
+        if not SMARTCTL_FORCE_DOWNLOAD and _validate_cached(SMARTCTL_STATIC_VERSION):
+            SMARTCTL = _cached_binary_path(SMARTCTL_STATIC_VERSION)
+            print(f"Using cached smartctl: {SMARTCTL}")
+            return
+        # Download fresh
+        alt = _download_and_extract_smartctl(SMARTCTL_STATIC_VERSION)
+        if alt:
+            SMARTCTL = alt
+            return
+    # Fallback search
+    for p in ("/usr/bin/smartctl", "/usr/sbin/smartctl"):
+        if os.path.isfile(p) and os.access(p, os.X_OK):
+            SMARTCTL = p
+            return
+
+def sanity_checks():
+    ensure_smartctl()
+    if not os.path.isfile(SMARTCTL):
+        print("smartctl not found. Install smartmontools or set SMARTCTL_STATIC_VERSION. Exiting.")
+        raise SystemExit(1)
+
+def list_block_devices():
+    # lsblk returns both disks & partitions. Filter to TYPE=disk to avoid partitions.
+    try:
+        res = subprocess.run(
+            ["lsblk", "-dn", "-o", "NAME,TYPE"],
+            check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+        )
+    except subprocess.CalledProcessError as e:
+        print(f"lsblk failed: {e.stderr}")
+        exporter_errors_total.inc()
+        return set()
+    devs = set()
+    for line in res.stdout.strip().splitlines():
+        parts = line.split()
+        if len(parts) != 2:
+            continue
+        name, typ = parts
+        if typ != "disk":
+            continue
+        if DEVICE_REGEX.match(name):
+            devs.add(name)
+    return devs
+
+def refresh_devices():
+    global DEVICES
+    current = list_block_devices()
+    with LOCK:
+        added = current - DEVICES
+        removed = DEVICES - current
+        if added:
+            print(f"New devices detected: {', '.join(sorted(added))}")
+        if removed:
+            print(f"Devices removed: {', '.join(sorted(removed))}")
+        DEVICES = current
+
+def _decode_smartctl_exit(code: int) -> str:
+    if code == 0:
+        return "OK"
+    bits = []
+    for bit, desc in SMARTCTL_STATUS_BITS.items():
+        if code & (1 << bit):
+            bits.append(f"{bit}:{desc}")
+    return "; ".join(bits) if bits else f"raw:{code}"
+
+def run_smartctl(device):
+    dev_path = f"{DEVICE_ROOT}/{device}"
+    cmd = [SMARTCTL, "-a", "-j", dev_path]
+    if STANDBY_MODE:
+        cmd.insert(1, "-n")
+        cmd.insert(2, "standby")
+    start = time.time()
+    try:
+        proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+        out = proc.stdout.strip()
+        exit_code = proc.returncode
+        smartctl_exit_code.labels(device).set(exit_code)
+        if not out:
+            if DEBUG:
+                print(f"[DEBUG] {device} smartctl produced no stdout. stderr='{proc.stderr.strip()}' rc={exit_code} bits={_decode_smartctl_exit(exit_code)}")
+            smart_commands_total.labels('empty_output').inc()
+            return None
+        try:
+            data = json.loads(out)
+        except json.JSONDecodeError:
+            if DEBUG:
+                print(f"[DEBUG] {device} JSON parse error. stderr='{proc.stderr.strip()}'")
+            smart_commands_total.labels('json_error').inc()
+            return None
+        if exit_code == 0:
+            smart_commands_total.labels('ok').inc()
+        else:
+            smart_commands_total.labels('nonzero_exit').inc()
+            if DEBUG:
+                print(f"[DEBUG] {device} smartctl non-zero rc={exit_code} bits={_decode_smartctl_exit(exit_code)} stderr='{proc.stderr.strip()}'")
+        # If no model & no SMART sections, log once
+        if DEBUG and not any(k in data for k in ("ata_smart_attributes","nvme_smart_health_information_log")):
+            print(f"[DEBUG] {device} SMART sections missing. Keys: {list(data.keys())}")
+        return data
+    except Exception as e:
+        print(f"smartctl failure {device}: {e}")
+        exporter_errors_total.inc()
+        smart_commands_total.labels('exception').inc()
+        return None
+    finally:
+        _ = time.time() - start  # duration per device if needed
+
+def map_md_sync_action(val):
+    # Map string -> numeric for gauge
+    mapping = {
+        'idle':0,'resync':1,'recover':2,'check':3,'repair':4,'reshape':5
+    }
+    return mapping.get(val, 99)
+
+def collect_mdraid():
+    # Parse /proc/mdstat and sysfs for degraded + sync_action
+    mdstat = "/proc/mdstat"
+    if not os.path.isfile(mdstat):
+        return
+    try:
+        with open(mdstat,'r') as f:
+            text = f.read()
+    except Exception:
+        return
+    arrays = re.findall(r'^(md\d+)\s*:\s*(\w+).*$', text, re.MULTILINE)
+    for name,_ in arrays:
+        base = f"/sys/block/{name}/md"
+        degraded_file = os.path.join(base, "degraded")
+        sync_action_file = os.path.join(base, "sync_action")
+        try:
+            if os.path.isfile(degraded_file):
+                with open(degraded_file) as f:
+                    d = int(f.read().strip())
+                mdraid_degraded.labels(name).set(1 if d > 0 else 0)
+            if os.path.isfile(sync_action_file):
+                with open(sync_action_file) as f:
+                    act = f.read().strip()
+                mdraid_sync_action.labels(name).set(map_md_sync_action(act))
+        except Exception:
+            exporter_errors_total.inc()
+
+def export_ata_attributes(device, data):
+    attrs = data.get("ata_smart_attributes", {}).get("table", [])
+    for a in attrs:
+        attr_id = str(a.get("id"))
+        name = a.get("name") or attr_id
+        raw_val = a.get("raw", {}).get("value")
+        normalized = a.get("value")
+        if raw_val is not None:
+            disk_smart_attr_raw_value.labels(device, attr_id, name).set(raw_val)
+        if normalized is not None:
+            disk_smart_attr_normalized_value.labels(device, attr_id, name).set(normalized)
+        # Legacy named metrics
+        if attr_id == "5":
+            disk_reallocated_sector_count.labels(device).set(raw_val or 0)
+        elif attr_id in ("190","194") and raw_val is not None:
+            disk_temperature.labels(device).set(raw_val)
+        elif attr_id == "196":
+            disk_reallocated_event_count.labels(device).set(raw_val or 0)
+        elif attr_id == "198":
+            disk_offline_uncorrectable.labels(device).set(raw_val or 0)
+
+def export_nvme(device, data):
+    n = data.get("nvme_smart_health_information_log", {})
+    if not n:
+        return
+    # Temps in Kelvin in smartctl JSON? smartctl usually returns Celsius already for 'temperature'
+    temp = n.get("temperature")
+    if temp is not None:
+        nvme_temperature.labels(device).set(temp)
+        disk_temperature.labels(device).set(temp)
+    fields = [
+        ('critical_warning', nvme_critical_warning),
+        ('available_spare', nvme_available_spare),
+        ('available_spare_threshold', nvme_available_spare_threshold),
+        ('percentage_used', nvme_percentage_used),
+        ('media_errors', nvme_media_errors),
+        ('num_err_log_entries', nvme_num_err_log_entries),
+        ('data_units_read', nvme_data_units_read),
+        ('data_units_written', nvme_data_units_written),
+    ]
+    for key, metric in fields:
+        val = n.get(key)
+        if val is not None:
+            metric.labels(device).set(val)
+
+def export_identity(device, data):
+    model = data.get("model_name") or data.get("model_family") or ""
+    serial = data.get("serial_number") or ""
+    firmware = data.get("firmware_version") or ""
+    transport = data.get("device", {}).get("transport") or data.get("interface_name") or ""
+    # Set gauge value 1 (info pattern)
+    disk_info.labels(device, model, serial, firmware, transport).set(1)
+
+def export_health(device, data):
+    # ATA
+    passed = data.get("smart_status", {}).get("passed")
+    if passed is None:
+        # NVMe path: smart_status might appear too
+        passed = data.get("smart_status", {}).get("passed")
+    if passed is not None:
+        disk_healthy.labels(device).set(1 if passed else 0)
+    else:
+        # Unknown -> do not overwrite previous; or set -1
+        disk_healthy.labels(device).set(-1)
+
+def process_device(device):
+    data = run_smartctl(device)
+    if not data:
+        return
+    try:
+        export_identity(device, data)
+        export_health(device, data)
+        if "ata_smart_attributes" in data:
+            export_ata_attributes(device, data)
+        if "nvme_smart_health_information_log" in data:
+            export_nvme(device, data)
+        # Last poll timestamp
+        disk_last_smart_poll_timestamp.labels(device).set(time.time())
+    except Exception as e:
+        print(f"Processing error {device}: {e}")
+        exporter_errors_total.inc()
+
+@smart_poll_duration_seconds.time()
+def poll_cycle():
+    refresh_devices()
+    with LOCK:
+        devs = list(DEVICES)
+    if not devs:
+        return
+    # Parallel collection
+    threads = min(len(devs), 8)
+    with ThreadPoolExecutor(max_workers=threads) as exe:
+        futs = {exe.submit(process_device, d): d for d in devs}
+        for fut in as_completed(futs):
+            _ = fut.result()
+    collect_mdraid()
+
+def main():
+    sanity_checks()
+    start_http_server(EXPORTER_PORT)
+    print(f"Started disk exporter on :{EXPORTER_PORT}, interval={CHECK_FREQUENCY}s")
+
+    def _handle_signal(signum, frame):
+        print(f"Received signal {signum}, initiating shutdown...")
+        STOP_EVENT.set()
+
+    # Register handlers (Kubernetes / Docker send SIGTERM)
+    signal.signal(signal.SIGTERM, _handle_signal)
+    signal.signal(signal.SIGINT, _handle_signal)
+
+    try:
+        while not STOP_EVENT.is_set():
+            start = time.time()
+            poll_cycle()
+            if STOP_EVENT.is_set():
+                break
+            elapsed = time.time() - start
+            sleep_for = max(1, CHECK_FREQUENCY - elapsed)
+            # Sleep but wake early if signal received
+            STOP_EVENT.wait(timeout=sleep_for)
+    finally:
+        print("Exporter shutdown complete.")
+        # If any future cleanup is needed, add here.
+
+if __name__ == "__main__":
+    main()
+
+

terraform/modules/utils/disk-exporter/resources/requirements.txt

@@ -0,0 +1 @@
+prometheus_client==0.23.1

terraform/modules/utils/disk-exporter/variables.tf

@@ -0,0 +1,40 @@
+variable "tag" {
+  description = "Python image tag"
+  type        = string
+  default     = "3.12-slim"
+}
+
+variable "namespace" {
+  description = "Kubernetes namespace to deploy the prometheus exporter"
+  type        = string
+  default     = "prometheus"
+}
+
+variable "persistent_folder" {
+  description = "The path to the persistent folder"
+  type        = string
+}
+
+variable "smartctl_static_version" {
+  description = "The smartctl static version to use"
+  type        = string
+  default     = "8.0-198-g34e7e4975211"
+}
+
+variable "smartctl_sha256" {
+  description = "The smartctl static binary sha256 checksum (optional, leave empty if not used)"
+  type        = string
+  default     = "d7e779664248942bb519b254dcb49cddaec2593e192cc468254c58e64da20840"
+}
+
+variable "check_frequency" {
+  description = "The frequency (in seconds) to check the disk health"
+  type        = number
+  default     = 600
+}
+
+variable "standby_mode" {
+  description = "Enable standby mode"
+  type        = bool
+  default     = false
+}

terraform/modules/utils/prometheus/prometheus.tf

@@ -97,6 +97,7 @@ resource "kubernetes_deployment" "prometheus" {
         annotations = {
           "checksum/config"       = local.prometheus_config_checksum
           "checksum/rules"        = local.prometheus_rules_checksum
+          "checksum/alertmanager" = local.alertmanager_config_checksum
         }
       }