diff --git a/terraform/apps/dev-01/apps.tf b/terraform/apps/dev-01/apps.tf index 2e138ec..2e67da4 100644 --- a/terraform/apps/dev-01/apps.tf +++ b/terraform/apps/dev-01/apps.tf @@ -81,3 +81,12 @@ module "auth_exporter" { source = "../../modules/utils/auth-exporter" persistent_folder = "${local.persistent_folder}/monitoring" } + +module "disk_exporter" { + source = "../../modules/utils/disk-exporter" + persistent_folder = "${local.persistent_folder}/monitoring" + smartctl_static_version = local.smartctl_static_version + smartctl_sha256 = local.smartctl_sha256 + check_frequency = 600 + standby_mode = false +} diff --git a/terraform/apps/dev-01/config.tf b/terraform/apps/dev-01/config.tf index 44df377..29de895 100644 --- a/terraform/apps/dev-01/config.tf +++ b/terraform/apps/dev-01/config.tf @@ -184,6 +184,9 @@ locals { auth_password = local.mail_app_api_secret_key } + smartctl_static_version = "8.0-198-g34e7e4975211" + smartctl_sha256 = "d7e779664248942bb519b254dcb49cddaec2593e192cc468254c58e64da20840" + telegram_chat_id = "5432313610" # a13labs channel telegram_message = file("${path.module}/resources/telegram.tpl") diff --git a/terraform/apps/dev-01/resources/alertmanager.yml b/terraform/apps/dev-01/resources/alertmanager.yml index 320a381..96e501d 100644 --- a/terraform/apps/dev-01/resources/alertmanager.yml +++ b/terraform/apps/dev-01/resources/alertmanager.yml @@ -8,4 +8,16 @@ route: - receiver: "telegram" matchers: - 'alertname="UserLoggedIn"' + group_by: ["alertname", "username"] # add username to avoid batching different users + group_wait: 0s # immediate dispatch + group_interval: 2m # short regroup window + repeat_interval: 30m # optional: adjust reminder cadence + continue: false + - receiver: "telegram" + matchers: + - 'alertname="DiskUnhealthy"' + group_by: ["alertname", "device"] # add device to avoid batching different disks + group_wait: 0s # immediate dispatch + group_interval: 2m # short regroup window + repeat_interval: 30m # optional: adjust reminder cadence continue: false diff --git a/terraform/apps/dev-01/resources/prometheus-rules.yml b/terraform/apps/dev-01/resources/prometheus-rules.yml index 96f975a..7b902d7 100644 --- a/terraform/apps/dev-01/resources/prometheus-rules.yml +++ b/terraform/apps/dev-01/resources/prometheus-rules.yml @@ -2,9 +2,19 @@ groups: - name: auth.rules rules: - alert: UserLoggedIn - expr: increase(auth_sessions_active_by_user[1m]) > 0 + expr: changes(auth_sessions_active_by_user[1m]) > 0 labels: severity: info annotations: - summary: "Login: {{ $labels.username }} on {{ $labels.instance }}" - description: "User {{ $labels.username }} logged in via {{ $labels.source }} on {{ $labels.instance }}." + summary: "Login: {{ $labels.username }} on dev-01" + description: "User {{ $labels.username }} logged in via {{ $labels.source }} on dev-01." + - name: disk.rules + rules: + - alert: DiskUnhealthy + expr: count(disk_info{})-count(disk_healthy{}) > 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Disk unhealthy: {{ $labels.device }} on dev-01" + description: "SMART overall health failed for device {{ $labels.device }} (disk_healthy=0). Investigate immediately." diff --git a/terraform/apps/dev-01/resources/prometheus.yml b/terraform/apps/dev-01/resources/prometheus.yml index 77b2516..24026c4 100644 --- a/terraform/apps/dev-01/resources/prometheus.yml +++ b/terraform/apps/dev-01/resources/prometheus.yml @@ -5,8 +5,19 @@ scrape_configs: - targets: - "node-exporter.prometheus.svc.cluster.local:9100" - "10.19.4.1:9100" + - labels: + cluster: "dev-01" - job_name: auth-exporter scrape_interval: 10s static_configs: - targets: - "auth-exporter.prometheus.svc.cluster.local:9100" + - labels: + cluster: "dev-01" + - job_name: disk-exporter + scrape_interval: 120s + static_configs: + - targets: + - "disk-exporter.prometheus.svc.cluster.local:9100" + - labels: + cluster: "dev-01" diff --git a/terraform/apps/dev-01/resources/telegram.tpl b/terraform/apps/dev-01/resources/telegram.tpl index 88a6606..045f9b2 100644 --- a/terraform/apps/dev-01/resources/telegram.tpl +++ b/terraform/apps/dev-01/resources/telegram.tpl @@ -1,13 +1,33 @@ -{{/* Always produce a non-empty message. Handles single or multiple firing alerts and missing username. */}} -{{ if gt (len .Alerts.Firing) 1 -}} -{{ len .Alerts.Firing }} logins on {{ with .CommonLabels.cluster }}{{ . }}{{ else }}unknown host{{ end }}: +{{/* Template enhanced to include resolved alerts while using annotations. */}} +{{ $firingCount := len .Alerts.Firing -}} +{{ $resolvedCount := len .Alerts.Resolved -}} + +{{ if or (gt $firingCount 0) (gt $resolvedCount 0) -}} + {{ if and (gt $firingCount 0) (gt $resolvedCount 0) -}} +{{ $firingCount }} firing / {{ $resolvedCount }} resolved +{{ if gt $firingCount 0 -}} +Firing: {{- range .Alerts.Firing }} -• {{ with .Labels.username }}{{ . }}{{ else }}unknown{{ end }} — Source: {{ with .Labels.source }}{{ . }}{{ else }}?{{ end }}{{ with .Labels.method }}, Method: {{ . }}{{ else }}, Method: ?{{ end }} +• {{ or .Annotations.summary "No summary" }}{{ with .Annotations.description }} — {{ . }}{{ end }} {{- end }} -{{ else if eq (len .Alerts.Firing) 1 -}} -{{ $a := index .Alerts.Firing 0 -}} -Login: {{ with $a.Labels.username }}{{ . }}{{ else }}unknown{{ end }} on {{ $a.Labels.cluster }} -Source: {{ with $a.Labels.source }}{{ . }}{{ else }}?{{ end }}{{ with $a.Labels.method }}, Method: {{ . }}{{ else }}, Method: ?{{ end }} +{{ end -}} +{{ if gt $resolvedCount 0 -}} +Resolved: +{{- range .Alerts.Resolved }} +• {{ or .Annotations.summary "No summary" }}{{ with .Annotations.description }} — {{ . }}{{ end }} +{{- end }} +{{ end -}} + {{ else if gt $firingCount 0 -}} +{{ $firingCount }} alert{{ if ne $firingCount 1 }}s{{ end }} firing: +{{- range .Alerts.Firing }} +• {{ or .Annotations.summary "No summary" }}{{ with .Annotations.description }} — {{ . }}{{ end }} +{{- end }} + {{ else -}} +{{ $resolvedCount }} alert{{ if ne $resolvedCount 1 }}s{{ end }} resolved: +{{- range .Alerts.Resolved }} +• {{ or .Annotations.summary "No summary" }}{{ with .Annotations.description }} — {{ . }}{{ end }} +{{- end }} + {{ end -}} {{ else -}} -Login detected — details unavailable. +No alert data — nothing firing or recently resolved. {{ end -}} \ No newline at end of file diff --git a/terraform/apps/prod-01/resources/alertmanager.yml b/terraform/apps/prod-01/resources/alertmanager.yml index 320a381..f98d647 100644 --- a/terraform/apps/prod-01/resources/alertmanager.yml +++ b/terraform/apps/prod-01/resources/alertmanager.yml @@ -8,4 +8,8 @@ route: - receiver: "telegram" matchers: - 'alertname="UserLoggedIn"' + group_by: ["alertname", "username"] # add username to avoid batching different users + group_wait: 0s # immediate dispatch + group_interval: 2m # short regroup window + repeat_interval: 30m # optional: adjust reminder cadence continue: false diff --git a/terraform/apps/prod-01/resources/prometheus-rules.yml b/terraform/apps/prod-01/resources/prometheus-rules.yml index 96f975a..b48d51a 100644 --- a/terraform/apps/prod-01/resources/prometheus-rules.yml +++ b/terraform/apps/prod-01/resources/prometheus-rules.yml @@ -6,5 +6,5 @@ groups: labels: severity: info annotations: - summary: "Login: {{ $labels.username }} on {{ $labels.instance }}" - description: "User {{ $labels.username }} logged in via {{ $labels.source }} on {{ $labels.instance }}." + summary: "Login: {{ $labels.username }} on prod-01" + description: "User {{ $labels.username }} logged in via {{ $labels.source }} on prod-01." diff --git a/terraform/apps/prod-01/resources/prometheus.yml b/terraform/apps/prod-01/resources/prometheus.yml index 9e5d501..5942a16 100644 --- a/terraform/apps/prod-01/resources/prometheus.yml +++ b/terraform/apps/prod-01/resources/prometheus.yml @@ -4,13 +4,19 @@ scrape_configs: static_configs: - targets: - "node-exporter.prometheus.svc.cluster.local:9100" + labels: + cluster: "prod-01" - job_name: fail2ban scrape_interval: 60s static_configs: - targets: - "fail2ban-exporter.prometheus.svc.cluster.local:9100" + labels: + cluster: "prod-01" - job_name: auth-exporter scrape_interval: 10s static_configs: - targets: - "auth-exporter.prometheus.svc.cluster.local:9100" + labels: + cluster: "prod-01" diff --git a/terraform/apps/prod-01/resources/telegram.tpl b/terraform/apps/prod-01/resources/telegram.tpl index bdcfbfa..045f9b2 100644 --- a/terraform/apps/prod-01/resources/telegram.tpl +++ b/terraform/apps/prod-01/resources/telegram.tpl @@ -1,28 +1,33 @@ -{{- /* Resolved alerts (if any) */ -}} -{{- if gt (len .Alerts.Resolved) 1 -}} -{{ len .Alerts.Resolved }} logins resolved on {{ with .CommonLabels.cluster }}{{ . }}{{ else }}unknown host{{ end }}: -{{- range .Alerts.Resolved }} -• {{ with .Labels.username }}{{ . }}{{ else }}unknown{{ end }} — Source: {{ with .Labels.source }}{{ . }}{{ else }}?{{ end }}{{ with .Labels.method }}, Method: {{ . }}{{ else }}, Method: ?{{ end }} -{{- end }} -{{- else if eq (len .Alerts.Resolved) 1 -}} -{{ $a := index .Alerts.Resolved 0 -}} -Resolved login: {{ with $a.Labels.username }}{{ . }}{{ else }}unknown{{ end }} on {{ $a.Labels.cluster }} -Source: {{ with $a.Labels.source }}{{ . }}{{ else }}?{{ end }}{{ with $a.Labels.method }}, Method: {{ . }}{{ else }}, Method: ?{{ end }} -{{- end -}} +{{/* Template enhanced to include resolved alerts while using annotations. */}} +{{ $firingCount := len .Alerts.Firing -}} +{{ $resolvedCount := len .Alerts.Resolved -}} -{{- /* Firing alerts (original logic) */ -}} -{{ if gt (len .Alerts.Firing) 1 -}} -{{ len .Alerts.Firing }} logins on {{ with .CommonLabels.cluster }}{{ . }}{{ else }}unknown host{{ end }}: +{{ if or (gt $firingCount 0) (gt $resolvedCount 0) -}} + {{ if and (gt $firingCount 0) (gt $resolvedCount 0) -}} +{{ $firingCount }} firing / {{ $resolvedCount }} resolved +{{ if gt $firingCount 0 -}} +Firing: {{- range .Alerts.Firing }} -• {{ with .Labels.username }}{{ . }}{{ else }}unknown{{ end }} — Source: {{ with .Labels.source }}{{ . }}{{ else }}?{{ end }}{{ with .Labels.method }}, Method: {{ . }}{{ else }}, Method: ?{{ end }} +• {{ or .Annotations.summary "No summary" }}{{ with .Annotations.description }} — {{ . }}{{ end }} {{- end }} -{{ else if eq (len .Alerts.Firing) 1 -}} -{{ $a := index .Alerts.Firing 0 -}} -Login: {{ with $a.Labels.username }}{{ . }}{{ else }}unknown{{ end }} on {{ $a.Labels.cluster }} -Source: {{ with $a.Labels.source }}{{ . }}{{ else }}?{{ end }}{{ with $a.Labels.method }}, Method: {{ . }}{{ else }}, Method: ?{{ end }} -{{ else -}} -{{- /* If there were resolved alerts above this will still show when no firing alerts exist. */ -}} -{{ if eq (len .Alerts.Resolved) 0 -}} -Login detected — details unavailable. {{ end -}} +{{ if gt $resolvedCount 0 -}} +Resolved: +{{- range .Alerts.Resolved }} +• {{ or .Annotations.summary "No summary" }}{{ with .Annotations.description }} — {{ . }}{{ end }} +{{- end }} +{{ end -}} + {{ else if gt $firingCount 0 -}} +{{ $firingCount }} alert{{ if ne $firingCount 1 }}s{{ end }} firing: +{{- range .Alerts.Firing }} +• {{ or .Annotations.summary "No summary" }}{{ with .Annotations.description }} — {{ . }}{{ end }} +{{- end }} + {{ else -}} +{{ $resolvedCount }} alert{{ if ne $resolvedCount 1 }}s{{ end }} resolved: +{{- range .Alerts.Resolved }} +• {{ or .Annotations.summary "No summary" }}{{ with .Annotations.description }} — {{ . }}{{ end }} +{{- end }} + {{ end -}} +{{ else -}} +No alert data — nothing firing or recently resolved. {{ end -}} \ No newline at end of file diff --git a/terraform/modules/utils/auth-exporter/main.tf b/terraform/modules/utils/auth-exporter/main.tf index b0443c8..919c4e2 100644 --- a/terraform/modules/utils/auth-exporter/main.tf +++ b/terraform/modules/utils/auth-exporter/main.tf @@ -47,7 +47,7 @@ resource "kubernetes_deployment" "auth_exporter" { automount_service_account_token = false container { name = "auth-exporter" - image = var.tag + image = "python:${var.tag}" command = ["sh", "-c", join(" && ", local.commands)] port { container_port = 9100 diff --git a/terraform/modules/utils/auth-exporter/variables.tf b/terraform/modules/utils/auth-exporter/variables.tf index 226b1fb..f346f6d 100644 --- a/terraform/modules/utils/auth-exporter/variables.tf +++ b/terraform/modules/utils/auth-exporter/variables.tf @@ -1,7 +1,7 @@ variable "tag" { description = "Python image tag" type = string - default = "python:3.12-slim" + default = "3.12-slim" } variable "namespace" { diff --git a/terraform/modules/utils/disk-exporter/config.tf b/terraform/modules/utils/disk-exporter/config.tf new file mode 100644 index 0000000..1830808 --- /dev/null +++ b/terraform/modules/utils/disk-exporter/config.tf @@ -0,0 +1,15 @@ +locals { + app_version = var.tag + workdir = "${var.persistent_folder}/disk-exporter" + commands = [ + "python -m venv /tmp/venv", + ". /tmp/venv/bin/activate", + "pip install -r /exporter/requirements.txt", + "python /exporter/exporter.py --export-usernames" + ] + exporter_script = file("${path.module}/resources/exporter.py") + requirements_txt = file("${path.module}/resources/requirements.txt") + + exporter_checksum = md5(local.exporter_script) + requirements_checksum = md5(local.requirements_txt) +} diff --git a/terraform/modules/utils/disk-exporter/main.tf b/terraform/modules/utils/disk-exporter/main.tf new file mode 100644 index 0000000..c8e959e --- /dev/null +++ b/terraform/modules/utils/disk-exporter/main.tf @@ -0,0 +1,147 @@ +resource "kubernetes_service_account" "disk_exporter" { + metadata { + name = "disk-exporter-sa" + namespace = var.namespace + } +} + +resource "kubernetes_config_map" "disk_exporter" { + metadata { + name = "disk-exporter-config" + namespace = var.namespace + } + data = { + "exporter.py" = local.exporter_script + "requirements.txt" = local.requirements_txt + } +} + +resource "kubernetes_deployment" "disk_exporter" { + metadata { + name = "disk-exporter" + namespace = var.namespace + labels = { + app = "disk-exporter" + } + } + spec { + replicas = 1 + selector { + match_labels = { + app = "disk-exporter" + } + } + strategy { + type = "RollingUpdate" + } + template { + metadata { + labels = { + app = "disk-exporter" + } + annotations = { + "checksum/exporter" = local.exporter_checksum + "checksum/requirements" = local.requirements_checksum + } + } + spec { + service_account_name = kubernetes_service_account.disk_exporter.metadata[0].name + automount_service_account_token = false + + container { + name = "disk-exporter" + image = "python:${var.tag}" + command = ["sh", "-c", join(" && ", local.commands)] + + env { + name = "SMARTCTL_STATIC_VERSION" + value = var.smartctl_static_version + } + env { + name = "SMARTCTL_SHA256" + value = coalesce(var.smartctl_sha256, "") + } + env { + name = "SMARTCTL_CACHE_BASE" + value = "/tmp/smartctl-cache" + } + env { + name = "CHECK_FREQUENCY" + value = tostring(var.check_frequency) + } + env { + name = "STANDBY_MODE" + value = var.standby_mode + } + env { + name = "EXPORTER_PORT" + value = "9100" + } + env { + name = "DEVICE_ROOT" + value = "/dev" + } + port { + container_port = 9100 + name = "http-metrics" + } + + volume_mount { + name = "exporter-volume" + mount_path = "/exporter" + read_only = true + } + + volume_mount { + name = "workdir" + mount_path = "/tmp" + } + + security_context { + run_as_user = 0 + run_as_group = 0 + read_only_root_filesystem = true + privileged = true + } + } + + volume { + name = "exporter-volume" + config_map { + name = kubernetes_config_map.disk_exporter.metadata[0].name + } + } + + volume { + name = "workdir" + host_path { + path = local.workdir + type = "DirectoryOrCreate" + } + } + } + } + } +} + +resource "kubernetes_service" "disk_exporter" { + metadata { + name = "disk-exporter" + namespace = var.namespace + labels = { + app = "disk-exporter" + } + } + spec { + selector = { + app = "disk-exporter" + } + port { + port = 9100 + target_port = 9100 + protocol = "TCP" + name = "http-metrics" + } + type = "ClusterIP" + } +} diff --git a/terraform/modules/utils/disk-exporter/provider.tf b/terraform/modules/utils/disk-exporter/provider.tf new file mode 100644 index 0000000..2eaa508 --- /dev/null +++ b/terraform/modules/utils/disk-exporter/provider.tf @@ -0,0 +1,9 @@ +terraform { + required_version = "~>1.8" + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.36.0" + } + } +} diff --git a/terraform/modules/utils/disk-exporter/resources/exporter.py b/terraform/modules/utils/disk-exporter/resources/exporter.py new file mode 100644 index 0000000..4c0d2eb --- /dev/null +++ b/terraform/modules/utils/disk-exporter/resources/exporter.py @@ -0,0 +1,432 @@ +#!/usr/bin/env python3 +import os +import re +import json +import time +import threading +import subprocess +from concurrent.futures import ThreadPoolExecutor, as_completed +from prometheus_client import start_http_server, Gauge, Counter, Summary +import stat +import tarfile +import hashlib +import urllib.request +from io import BytesIO +import signal # <-- added + +SMARTCTL = os.environ.get("SMARTCTL_PATH", "/usr/sbin/smartctl") +CHECK_FREQUENCY = int(os.environ.get("CHECK_FREQUENCY", "600")) +EXPORTER_PORT = int(os.environ.get("EXPORTER_PORT", "9100")) +STANDBY_MODE = os.environ.get("STANDBY_MODE", "0") == "1" + +SMARTCTL_STATIC_VERSION = os.environ.get("SMARTCTL_STATIC_VERSION", "") +SMARTCTL_SHA256 = os.environ.get("SMARTCTL_SHA256", "") +SMARTCTL_CACHE_BASE = os.environ.get("SMARTCTL_CACHE_BASE", "/tmp/smartctl-cache") +SMARTCTL_FORCE_DOWNLOAD = os.environ.get("SMARTCTL_FORCE_DOWNLOAD", "false") == "true" + +DEBUG = os.environ.get("DEBUG", "0") == "1" +SMARTCTL_STATUS_BITS = { + 0: "Command line did not parse", + 1: "Device open failed", + 2: "SMART status check returned 'DISK FAILING'", + 3: "Prefail Attributes <= threshold", + 4: "Usage Attributes <= threshold", + 5: "Self-test log contains errors", + 6: "Self-test in progress", + 7: "Device has read SMART data (some internal error if also bit 1?)" +} +# -------- Prometheus Metrics -------- +disk_healthy = Gauge('disk_healthy', 'SMART overall health status (1=passed,0=fail)', ['device']) +disk_temperature = Gauge('disk_temperature', 'Drive temperature (C)', ['device']) +disk_reallocated_sector_count = Gauge('disk_reallocated_sector_count', 'Reallocated Sector Count', ['device']) +disk_reallocated_event_count = Gauge('disk_reallocated_event_count', 'Reallocated Event Count', ['device']) +disk_offline_uncorrectable = Gauge('disk_offline_uncorrectable', 'Offline Uncorrectable Count', ['device']) + +disk_smart_attr_raw_value = Gauge('disk_smart_attr_raw_value', 'Raw value of SMART attribute', ['device','id','name']) +disk_smart_attr_normalized_value = Gauge('disk_smart_attr_normalized_value', 'Normalized value of SMART attribute', ['device','id','name']) + +disk_info = Gauge('disk_info', 'Static disk identity info', ['device','model','serial','firmware','transport']) +disk_last_smart_poll_timestamp = Gauge('disk_last_smart_poll_timestamp', 'Unix timestamp of last successful SMART poll', ['device']) + +# NVMe specific +nvme_critical_warning = Gauge('nvme_critical_warning', 'NVMe critical warning bitmap', ['device']) +nvme_available_spare = Gauge('nvme_available_spare', 'NVMe available spare (%)', ['device']) +nvme_available_spare_threshold = Gauge('nvme_available_spare_threshold', 'NVMe available spare threshold (%)', ['device']) +nvme_percentage_used = Gauge('nvme_percentage_used', 'NVMe percentage used (wear level)', ['device']) +nvme_media_errors = Gauge('nvme_media_errors', 'NVMe media errors', ['device']) +nvme_num_err_log_entries = Gauge('nvme_num_err_log_entries', 'NVMe number of error log entries', ['device']) +nvme_data_units_read = Gauge('nvme_data_units_read', 'NVMe data units read (1000 * 512KiB)', ['device']) +nvme_data_units_written = Gauge('nvme_data_units_written', 'NVMe data units written (1000 * 512KiB)', ['device']) +nvme_temperature = Gauge('nvme_temperature', 'NVMe temperature (C)', ['device']) + +# mdadm (if any arrays) +mdraid_degraded = Gauge('mdraid_degraded', 'MD RAID degraded flag (1=degraded)', ['array']) +mdraid_sync_action = Gauge('mdraid_sync_action', 'MD RAID sync action indicator (idle=0,resync=1,recover=2,check=3,repair=4,reshape=5,unknown=99)', ['array']) + +# Exporter operational metrics +exporter_errors_total = Counter('disk_exporter_errors_total', 'Total errors during SMART collection') +smart_poll_duration_seconds = Summary('disk_smart_poll_duration_seconds', 'Duration of a full SMART poll cycle') +smart_commands_total = Counter('disk_smart_commands_total', 'Total smartctl command invocations', ['result']) + +# Add exit code metric +smartctl_exit_code = Gauge('smartctl_exit_code', 'Last smartctl raw exit code', ['device']) + +DEVICE_REGEX = re.compile(r'^(sd[a-z]+|nvme\d+n\d+)$') +LOCK = threading.Lock() +DEVICES = set() +DEVICE_ROOT = os.environ.get("DEVICE_ROOT", "/dev") # NEW: root path for block devices (allows mounting host /dev elsewhere) + +# Graceful shutdown event +STOP_EVENT = threading.Event() # <-- added + +# -------- Utility Functions -------- +def _build_smartctl_url(version: str) -> str: + return f"https://github.com/smartmontools/smartmontools-builds/releases/download/smartmontools-{version}/smartmontools-linux-x86_64-static-{version}.tar.gz" + +def _version_dir(version: str) -> str: + return os.path.join(SMARTCTL_CACHE_BASE, version) + +def _cached_binary_path(version: str) -> str: + return os.path.join(_version_dir(version), "smartctl") + +def _cached_sha_file(version: str) -> str: + return os.path.join(_version_dir(version), ".archive_sha256") + +def _url_exists(url: str) -> bool: + try: + req = urllib.request.Request(url, method="HEAD") + with urllib.request.urlopen(req, timeout=15): + return True + except Exception: + # Some servers may not allow HEAD; fallback to optimistic True + return True + +def _validate_cached(version: str) -> bool: + bin_path = _cached_binary_path(version) + if not (os.path.isfile(bin_path) and os.access(bin_path, os.X_OK)): + return False + if SMARTCTL_SHA256: + sha_file = _cached_sha_file(version) + if not os.path.isfile(sha_file): + return False + recorded = open(sha_file).read().strip() + if recorded.lower() != SMARTCTL_SHA256.lower(): + print(f"Cached smartctl for {version} sha mismatch (have {recorded}, expected {SMARTCTL_SHA256}); will re-download.") + return False + return True + +def _download_and_extract_smartctl(version: str): + url = _build_smartctl_url(version) + if not _url_exists(url): + print(f"URL not reachable (HEAD failed): {url}") + return None + print(f"Fetching static smartctl archive: {url}") + try: + with urllib.request.urlopen(url, timeout=120) as r: + archive = r.read() + except Exception as e: + print(f"Download failed: {e}") + return None + if SMARTCTL_SHA256: + digest = hashlib.sha256(archive).hexdigest() + if digest.lower() != SMARTCTL_SHA256.lower(): + print(f"Archive SHA256 mismatch (expected {SMARTCTL_SHA256} got {digest})") + return None + vdir = _version_dir(version) + os.makedirs(vdir, exist_ok=True) + try: + with tarfile.open(fileobj=BytesIO(archive), mode="r:gz") as tf: + member = next((m for m in tf.getmembers() if m.name.endswith("/smartctl") or m.name == "smartctl"), None) + if not member: + print("smartctl binary not found inside archive") + return None + tf.extract(member, vdir) + extracted = os.path.join(vdir, member.name) + final_path = _cached_binary_path(version) + if extracted != final_path: + os.replace(extracted, final_path) + os.chmod(final_path, os.stat(final_path).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + if SMARTCTL_SHA256: + with open(_cached_sha_file(version), "w") as f: + f.write(SMARTCTL_SHA256.lower()) + print(f"Static smartctl ready: {final_path}") + return final_path + except Exception as e: + print(f"Extraction error: {e}") + return None + +def ensure_smartctl(): + global SMARTCTL + # If user-specified path works, keep it + if os.path.isfile(SMARTCTL) and os.access(SMARTCTL, os.X_OK): + return + if SMARTCTL_STATIC_VERSION: + if not SMARTCTL_FORCE_DOWNLOAD and _validate_cached(SMARTCTL_STATIC_VERSION): + SMARTCTL = _cached_binary_path(SMARTCTL_STATIC_VERSION) + print(f"Using cached smartctl: {SMARTCTL}") + return + # Download fresh + alt = _download_and_extract_smartctl(SMARTCTL_STATIC_VERSION) + if alt: + SMARTCTL = alt + return + # Fallback search + for p in ("/usr/bin/smartctl", "/usr/sbin/smartctl"): + if os.path.isfile(p) and os.access(p, os.X_OK): + SMARTCTL = p + return + +def sanity_checks(): + ensure_smartctl() + if not os.path.isfile(SMARTCTL): + print("smartctl not found. Install smartmontools or set SMARTCTL_STATIC_VERSION. Exiting.") + raise SystemExit(1) + +def list_block_devices(): + # lsblk returns both disks & partitions. Filter to TYPE=disk to avoid partitions. + try: + res = subprocess.run( + ["lsblk", "-dn", "-o", "NAME,TYPE"], + check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + except subprocess.CalledProcessError as e: + print(f"lsblk failed: {e.stderr}") + exporter_errors_total.inc() + return set() + devs = set() + for line in res.stdout.strip().splitlines(): + parts = line.split() + if len(parts) != 2: + continue + name, typ = parts + if typ != "disk": + continue + if DEVICE_REGEX.match(name): + devs.add(name) + return devs + +def refresh_devices(): + global DEVICES + current = list_block_devices() + with LOCK: + added = current - DEVICES + removed = DEVICES - current + if added: + print(f"New devices detected: {', '.join(sorted(added))}") + if removed: + print(f"Devices removed: {', '.join(sorted(removed))}") + DEVICES = current + +def _decode_smartctl_exit(code: int) -> str: + if code == 0: + return "OK" + bits = [] + for bit, desc in SMARTCTL_STATUS_BITS.items(): + if code & (1 << bit): + bits.append(f"{bit}:{desc}") + return "; ".join(bits) if bits else f"raw:{code}" + +def run_smartctl(device): + dev_path = f"{DEVICE_ROOT}/{device}" + cmd = [SMARTCTL, "-a", "-j", dev_path] + if STANDBY_MODE: + cmd.insert(1, "-n") + cmd.insert(2, "standby") + start = time.time() + try: + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + out = proc.stdout.strip() + exit_code = proc.returncode + smartctl_exit_code.labels(device).set(exit_code) + if not out: + if DEBUG: + print(f"[DEBUG] {device} smartctl produced no stdout. stderr='{proc.stderr.strip()}' rc={exit_code} bits={_decode_smartctl_exit(exit_code)}") + smart_commands_total.labels('empty_output').inc() + return None + try: + data = json.loads(out) + except json.JSONDecodeError: + if DEBUG: + print(f"[DEBUG] {device} JSON parse error. stderr='{proc.stderr.strip()}'") + smart_commands_total.labels('json_error').inc() + return None + if exit_code == 0: + smart_commands_total.labels('ok').inc() + else: + smart_commands_total.labels('nonzero_exit').inc() + if DEBUG: + print(f"[DEBUG] {device} smartctl non-zero rc={exit_code} bits={_decode_smartctl_exit(exit_code)} stderr='{proc.stderr.strip()}'") + # If no model & no SMART sections, log once + if DEBUG and not any(k in data for k in ("ata_smart_attributes","nvme_smart_health_information_log")): + print(f"[DEBUG] {device} SMART sections missing. Keys: {list(data.keys())}") + return data + except Exception as e: + print(f"smartctl failure {device}: {e}") + exporter_errors_total.inc() + smart_commands_total.labels('exception').inc() + return None + finally: + _ = time.time() - start # duration per device if needed + +def map_md_sync_action(val): + # Map string -> numeric for gauge + mapping = { + 'idle':0,'resync':1,'recover':2,'check':3,'repair':4,'reshape':5 + } + return mapping.get(val, 99) + +def collect_mdraid(): + # Parse /proc/mdstat and sysfs for degraded + sync_action + mdstat = "/proc/mdstat" + if not os.path.isfile(mdstat): + return + try: + with open(mdstat,'r') as f: + text = f.read() + except Exception: + return + arrays = re.findall(r'^(md\d+)\s*:\s*(\w+).*$', text, re.MULTILINE) + for name,_ in arrays: + base = f"/sys/block/{name}/md" + degraded_file = os.path.join(base, "degraded") + sync_action_file = os.path.join(base, "sync_action") + try: + if os.path.isfile(degraded_file): + with open(degraded_file) as f: + d = int(f.read().strip()) + mdraid_degraded.labels(name).set(1 if d > 0 else 0) + if os.path.isfile(sync_action_file): + with open(sync_action_file) as f: + act = f.read().strip() + mdraid_sync_action.labels(name).set(map_md_sync_action(act)) + except Exception: + exporter_errors_total.inc() + +def export_ata_attributes(device, data): + attrs = data.get("ata_smart_attributes", {}).get("table", []) + for a in attrs: + attr_id = str(a.get("id")) + name = a.get("name") or attr_id + raw_val = a.get("raw", {}).get("value") + normalized = a.get("value") + if raw_val is not None: + disk_smart_attr_raw_value.labels(device, attr_id, name).set(raw_val) + if normalized is not None: + disk_smart_attr_normalized_value.labels(device, attr_id, name).set(normalized) + # Legacy named metrics + if attr_id == "5": + disk_reallocated_sector_count.labels(device).set(raw_val or 0) + elif attr_id in ("190","194") and raw_val is not None: + disk_temperature.labels(device).set(raw_val) + elif attr_id == "196": + disk_reallocated_event_count.labels(device).set(raw_val or 0) + elif attr_id == "198": + disk_offline_uncorrectable.labels(device).set(raw_val or 0) + +def export_nvme(device, data): + n = data.get("nvme_smart_health_information_log", {}) + if not n: + return + # Temps in Kelvin in smartctl JSON? smartctl usually returns Celsius already for 'temperature' + temp = n.get("temperature") + if temp is not None: + nvme_temperature.labels(device).set(temp) + disk_temperature.labels(device).set(temp) + fields = [ + ('critical_warning', nvme_critical_warning), + ('available_spare', nvme_available_spare), + ('available_spare_threshold', nvme_available_spare_threshold), + ('percentage_used', nvme_percentage_used), + ('media_errors', nvme_media_errors), + ('num_err_log_entries', nvme_num_err_log_entries), + ('data_units_read', nvme_data_units_read), + ('data_units_written', nvme_data_units_written), + ] + for key, metric in fields: + val = n.get(key) + if val is not None: + metric.labels(device).set(val) + +def export_identity(device, data): + model = data.get("model_name") or data.get("model_family") or "" + serial = data.get("serial_number") or "" + firmware = data.get("firmware_version") or "" + transport = data.get("device", {}).get("transport") or data.get("interface_name") or "" + # Set gauge value 1 (info pattern) + disk_info.labels(device, model, serial, firmware, transport).set(1) + +def export_health(device, data): + # ATA + passed = data.get("smart_status", {}).get("passed") + if passed is None: + # NVMe path: smart_status might appear too + passed = data.get("smart_status", {}).get("passed") + if passed is not None: + disk_healthy.labels(device).set(1 if passed else 0) + else: + # Unknown -> do not overwrite previous; or set -1 + disk_healthy.labels(device).set(-1) + +def process_device(device): + data = run_smartctl(device) + if not data: + return + try: + export_identity(device, data) + export_health(device, data) + if "ata_smart_attributes" in data: + export_ata_attributes(device, data) + if "nvme_smart_health_information_log" in data: + export_nvme(device, data) + # Last poll timestamp + disk_last_smart_poll_timestamp.labels(device).set(time.time()) + except Exception as e: + print(f"Processing error {device}: {e}") + exporter_errors_total.inc() + +@smart_poll_duration_seconds.time() +def poll_cycle(): + refresh_devices() + with LOCK: + devs = list(DEVICES) + if not devs: + return + # Parallel collection + threads = min(len(devs), 8) + with ThreadPoolExecutor(max_workers=threads) as exe: + futs = {exe.submit(process_device, d): d for d in devs} + for fut in as_completed(futs): + _ = fut.result() + collect_mdraid() + +def main(): + sanity_checks() + start_http_server(EXPORTER_PORT) + print(f"Started disk exporter on :{EXPORTER_PORT}, interval={CHECK_FREQUENCY}s") + + def _handle_signal(signum, frame): + print(f"Received signal {signum}, initiating shutdown...") + STOP_EVENT.set() + + # Register handlers (Kubernetes / Docker send SIGTERM) + signal.signal(signal.SIGTERM, _handle_signal) + signal.signal(signal.SIGINT, _handle_signal) + + try: + while not STOP_EVENT.is_set(): + start = time.time() + poll_cycle() + if STOP_EVENT.is_set(): + break + elapsed = time.time() - start + sleep_for = max(1, CHECK_FREQUENCY - elapsed) + # Sleep but wake early if signal received + STOP_EVENT.wait(timeout=sleep_for) + finally: + print("Exporter shutdown complete.") + # If any future cleanup is needed, add here. + +if __name__ == "__main__": + main() + + diff --git a/terraform/modules/utils/disk-exporter/resources/requirements.txt b/terraform/modules/utils/disk-exporter/resources/requirements.txt new file mode 100644 index 0000000..35f2419 --- /dev/null +++ b/terraform/modules/utils/disk-exporter/resources/requirements.txt @@ -0,0 +1 @@ +prometheus_client==0.23.1 \ No newline at end of file diff --git a/terraform/modules/utils/disk-exporter/variables.tf b/terraform/modules/utils/disk-exporter/variables.tf new file mode 100644 index 0000000..d72ac27 --- /dev/null +++ b/terraform/modules/utils/disk-exporter/variables.tf @@ -0,0 +1,40 @@ +variable "tag" { + description = "Python image tag" + type = string + default = "3.12-slim" +} + +variable "namespace" { + description = "Kubernetes namespace to deploy the prometheus exporter" + type = string + default = "prometheus" +} + +variable "persistent_folder" { + description = "The path to the persistent folder" + type = string +} + +variable "smartctl_static_version" { + description = "The smartctl static version to use" + type = string + default = "8.0-198-g34e7e4975211" +} + +variable "smartctl_sha256" { + description = "The smartctl static binary sha256 checksum (optional, leave empty if not used)" + type = string + default = "d7e779664248942bb519b254dcb49cddaec2593e192cc468254c58e64da20840" +} + +variable "check_frequency" { + description = "The frequency (in seconds) to check the disk health" + type = number + default = 600 +} + +variable "standby_mode" { + description = "Enable standby mode" + type = bool + default = false +} diff --git a/terraform/modules/utils/prometheus/prometheus.tf b/terraform/modules/utils/prometheus/prometheus.tf index 2cf3484..fae2b39 100644 --- a/terraform/modules/utils/prometheus/prometheus.tf +++ b/terraform/modules/utils/prometheus/prometheus.tf @@ -95,8 +95,9 @@ resource "kubernetes_deployment" "prometheus" { app = "prometheus" } annotations = { - "checksum/config" = local.prometheus_config_checksum - "checksum/rules" = local.prometheus_rules_checksum + "checksum/config" = local.prometheus_config_checksum + "checksum/rules" = local.prometheus_rules_checksum + "checksum/alertmanager" = local.alertmanager_config_checksum } }