#!/usr/bin/env python3 import os import re import json import time import threading import subprocess from concurrent.futures import ThreadPoolExecutor, as_completed from prometheus_client import start_http_server, Gauge, Counter, Summary import stat import tarfile import hashlib import urllib.request from io import BytesIO import signal # <-- added SMARTCTL = os.environ.get("SMARTCTL_PATH", "/usr/sbin/smartctl") CHECK_FREQUENCY = int(os.environ.get("CHECK_FREQUENCY", "600")) EXPORTER_PORT = int(os.environ.get("EXPORTER_PORT", "9100")) STANDBY_MODE = os.environ.get("STANDBY_MODE", "0") == "1" SMARTCTL_STATIC_VERSION = os.environ.get("SMARTCTL_STATIC_VERSION", "") SMARTCTL_SHA256 = os.environ.get("SMARTCTL_SHA256", "") SMARTCTL_CACHE_BASE = os.environ.get("SMARTCTL_CACHE_BASE", "/tmp/smartctl-cache") SMARTCTL_FORCE_DOWNLOAD = os.environ.get("SMARTCTL_FORCE_DOWNLOAD", "false") == "true" DEBUG = os.environ.get("DEBUG", "0") == "1" SMARTCTL_STATUS_BITS = { 0: "Command line did not parse", 1: "Device open failed", 2: "SMART status check returned 'DISK FAILING'", 3: "Prefail Attributes <= threshold", 4: "Usage Attributes <= threshold", 5: "Self-test log contains errors", 6: "Self-test in progress", 7: "Device has read SMART data (some internal error if also bit 1?)" } # -------- Prometheus Metrics -------- disk_healthy = Gauge('disk_healthy', 'SMART overall health status (1=passed,0=fail)', ['device']) disk_temperature = Gauge('disk_temperature', 'Drive temperature (C)', ['device']) disk_reallocated_sector_count = Gauge('disk_reallocated_sector_count', 'Reallocated Sector Count', ['device']) disk_reallocated_event_count = Gauge('disk_reallocated_event_count', 'Reallocated Event Count', ['device']) disk_offline_uncorrectable = Gauge('disk_offline_uncorrectable', 'Offline Uncorrectable Count', ['device']) disk_smart_attr_raw_value = Gauge('disk_smart_attr_raw_value', 'Raw value of SMART attribute', ['device','id','name']) disk_smart_attr_normalized_value = Gauge('disk_smart_attr_normalized_value', 'Normalized value of SMART attribute', ['device','id','name']) disk_info = Gauge('disk_info', 'Static disk identity info', ['device','model','serial','firmware','transport']) disk_last_smart_poll_timestamp = Gauge('disk_last_smart_poll_timestamp', 'Unix timestamp of last successful SMART poll', ['device']) # NVMe specific nvme_critical_warning = Gauge('nvme_critical_warning', 'NVMe critical warning bitmap', ['device']) nvme_available_spare = Gauge('nvme_available_spare', 'NVMe available spare (%)', ['device']) nvme_available_spare_threshold = Gauge('nvme_available_spare_threshold', 'NVMe available spare threshold (%)', ['device']) nvme_percentage_used = Gauge('nvme_percentage_used', 'NVMe percentage used (wear level)', ['device']) nvme_media_errors = Gauge('nvme_media_errors', 'NVMe media errors', ['device']) nvme_num_err_log_entries = Gauge('nvme_num_err_log_entries', 'NVMe number of error log entries', ['device']) nvme_data_units_read = Gauge('nvme_data_units_read', 'NVMe data units read (1000 * 512KiB)', ['device']) nvme_data_units_written = Gauge('nvme_data_units_written', 'NVMe data units written (1000 * 512KiB)', ['device']) nvme_temperature = Gauge('nvme_temperature', 'NVMe temperature (C)', ['device']) # mdadm (if any arrays) mdraid_degraded = Gauge('mdraid_degraded', 'MD RAID degraded flag (1=degraded)', ['array']) mdraid_sync_action = Gauge('mdraid_sync_action', 'MD RAID sync action indicator (idle=0,resync=1,recover=2,check=3,repair=4,reshape=5,unknown=99)', ['array']) # Exporter operational metrics exporter_errors_total = Counter('disk_exporter_errors_total', 'Total errors during SMART collection') smart_poll_duration_seconds = Summary('disk_smart_poll_duration_seconds', 'Duration of a full SMART poll cycle') smart_commands_total = Counter('disk_smart_commands_total', 'Total smartctl command invocations', ['result']) # Add exit code metric smartctl_exit_code = Gauge('smartctl_exit_code', 'Last smartctl raw exit code', ['device']) DEVICE_REGEX = re.compile(r'^(sd[a-z]+|nvme\d+n\d+)$') LOCK = threading.Lock() DEVICES = set() DEVICE_ROOT = os.environ.get("DEVICE_ROOT", "/dev") # NEW: root path for block devices (allows mounting host /dev elsewhere) # Graceful shutdown event STOP_EVENT = threading.Event() # <-- added # -------- Utility Functions -------- def _build_smartctl_url(version: str) -> str: return f"https://github.com/smartmontools/smartmontools-builds/releases/download/smartmontools-{version}/smartmontools-linux-x86_64-static-{version}.tar.gz" def _version_dir(version: str) -> str: return os.path.join(SMARTCTL_CACHE_BASE, version) def _cached_binary_path(version: str) -> str: return os.path.join(_version_dir(version), "smartctl") def _cached_sha_file(version: str) -> str: return os.path.join(_version_dir(version), ".archive_sha256") def _url_exists(url: str) -> bool: try: req = urllib.request.Request(url, method="HEAD") with urllib.request.urlopen(req, timeout=15): return True except Exception: # Some servers may not allow HEAD; fallback to optimistic True return True def _validate_cached(version: str) -> bool: bin_path = _cached_binary_path(version) if not (os.path.isfile(bin_path) and os.access(bin_path, os.X_OK)): return False if SMARTCTL_SHA256: sha_file = _cached_sha_file(version) if not os.path.isfile(sha_file): return False recorded = open(sha_file).read().strip() if recorded.lower() != SMARTCTL_SHA256.lower(): print(f"Cached smartctl for {version} sha mismatch (have {recorded}, expected {SMARTCTL_SHA256}); will re-download.") return False return True def _download_and_extract_smartctl(version: str): url = _build_smartctl_url(version) if not _url_exists(url): print(f"URL not reachable (HEAD failed): {url}") return None print(f"Fetching static smartctl archive: {url}") try: with urllib.request.urlopen(url, timeout=120) as r: archive = r.read() except Exception as e: print(f"Download failed: {e}") return None if SMARTCTL_SHA256: digest = hashlib.sha256(archive).hexdigest() if digest.lower() != SMARTCTL_SHA256.lower(): print(f"Archive SHA256 mismatch (expected {SMARTCTL_SHA256} got {digest})") return None vdir = _version_dir(version) os.makedirs(vdir, exist_ok=True) try: with tarfile.open(fileobj=BytesIO(archive), mode="r:gz") as tf: member = next((m for m in tf.getmembers() if m.name.endswith("/smartctl") or m.name == "smartctl"), None) if not member: print("smartctl binary not found inside archive") return None tf.extract(member, vdir) extracted = os.path.join(vdir, member.name) final_path = _cached_binary_path(version) if extracted != final_path: os.replace(extracted, final_path) os.chmod(final_path, os.stat(final_path).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) if SMARTCTL_SHA256: with open(_cached_sha_file(version), "w") as f: f.write(SMARTCTL_SHA256.lower()) print(f"Static smartctl ready: {final_path}") return final_path except Exception as e: print(f"Extraction error: {e}") return None def ensure_smartctl(): global SMARTCTL # If user-specified path works, keep it if os.path.isfile(SMARTCTL) and os.access(SMARTCTL, os.X_OK): return if SMARTCTL_STATIC_VERSION: if not SMARTCTL_FORCE_DOWNLOAD and _validate_cached(SMARTCTL_STATIC_VERSION): SMARTCTL = _cached_binary_path(SMARTCTL_STATIC_VERSION) print(f"Using cached smartctl: {SMARTCTL}") return # Download fresh alt = _download_and_extract_smartctl(SMARTCTL_STATIC_VERSION) if alt: SMARTCTL = alt return # Fallback search for p in ("/usr/bin/smartctl", "/usr/sbin/smartctl"): if os.path.isfile(p) and os.access(p, os.X_OK): SMARTCTL = p return def sanity_checks(): ensure_smartctl() if not os.path.isfile(SMARTCTL): print("smartctl not found. Install smartmontools or set SMARTCTL_STATIC_VERSION. Exiting.") raise SystemExit(1) def list_block_devices(): # lsblk returns both disks & partitions. Filter to TYPE=disk to avoid partitions. try: res = subprocess.run( ["lsblk", "-dn", "-o", "NAME,TYPE"], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) except subprocess.CalledProcessError as e: print(f"lsblk failed: {e.stderr}") exporter_errors_total.inc() return set() devs = set() for line in res.stdout.strip().splitlines(): parts = line.split() if len(parts) != 2: continue name, typ = parts if typ != "disk": continue if DEVICE_REGEX.match(name): devs.add(name) return devs def refresh_devices(): global DEVICES current = list_block_devices() with LOCK: added = current - DEVICES removed = DEVICES - current if added: print(f"New devices detected: {', '.join(sorted(added))}") if removed: print(f"Devices removed: {', '.join(sorted(removed))}") DEVICES = current def _decode_smartctl_exit(code: int) -> str: if code == 0: return "OK" bits = [] for bit, desc in SMARTCTL_STATUS_BITS.items(): if code & (1 << bit): bits.append(f"{bit}:{desc}") return "; ".join(bits) if bits else f"raw:{code}" def run_smartctl(device): dev_path = f"{DEVICE_ROOT}/{device}" cmd = [SMARTCTL, "-a", "-j", dev_path] if STANDBY_MODE: cmd.insert(1, "-n") cmd.insert(2, "standby") start = time.time() try: proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) out = proc.stdout.strip() exit_code = proc.returncode smartctl_exit_code.labels(device).set(exit_code) if not out: if DEBUG: print(f"[DEBUG] {device} smartctl produced no stdout. stderr='{proc.stderr.strip()}' rc={exit_code} bits={_decode_smartctl_exit(exit_code)}") smart_commands_total.labels('empty_output').inc() return None try: data = json.loads(out) except json.JSONDecodeError: if DEBUG: print(f"[DEBUG] {device} JSON parse error. stderr='{proc.stderr.strip()}'") smart_commands_total.labels('json_error').inc() return None if exit_code == 0: smart_commands_total.labels('ok').inc() else: smart_commands_total.labels('nonzero_exit').inc() if DEBUG: print(f"[DEBUG] {device} smartctl non-zero rc={exit_code} bits={_decode_smartctl_exit(exit_code)} stderr='{proc.stderr.strip()}'") # If no model & no SMART sections, log once if DEBUG and not any(k in data for k in ("ata_smart_attributes","nvme_smart_health_information_log")): print(f"[DEBUG] {device} SMART sections missing. Keys: {list(data.keys())}") return data except Exception as e: print(f"smartctl failure {device}: {e}") exporter_errors_total.inc() smart_commands_total.labels('exception').inc() return None finally: _ = time.time() - start # duration per device if needed def map_md_sync_action(val): # Map string -> numeric for gauge mapping = { 'idle':0,'resync':1,'recover':2,'check':3,'repair':4,'reshape':5 } return mapping.get(val, 99) def collect_mdraid(): # Parse /proc/mdstat and sysfs for degraded + sync_action mdstat = "/proc/mdstat" if not os.path.isfile(mdstat): return try: with open(mdstat,'r') as f: text = f.read() except Exception: return arrays = re.findall(r'^(md\d+)\s*:\s*(\w+).*$', text, re.MULTILINE) for name,_ in arrays: base = f"/sys/block/{name}/md" degraded_file = os.path.join(base, "degraded") sync_action_file = os.path.join(base, "sync_action") try: if os.path.isfile(degraded_file): with open(degraded_file) as f: d = int(f.read().strip()) mdraid_degraded.labels(name).set(1 if d > 0 else 0) if os.path.isfile(sync_action_file): with open(sync_action_file) as f: act = f.read().strip() mdraid_sync_action.labels(name).set(map_md_sync_action(act)) except Exception: exporter_errors_total.inc() def export_ata_attributes(device, data): attrs = data.get("ata_smart_attributes", {}).get("table", []) for a in attrs: attr_id = str(a.get("id")) name = a.get("name") or attr_id raw_val = a.get("raw", {}).get("value") normalized = a.get("value") if raw_val is not None: disk_smart_attr_raw_value.labels(device, attr_id, name).set(raw_val) if normalized is not None: disk_smart_attr_normalized_value.labels(device, attr_id, name).set(normalized) # Legacy named metrics if attr_id == "5": disk_reallocated_sector_count.labels(device).set(raw_val or 0) elif attr_id in ("190","194") and raw_val is not None: disk_temperature.labels(device).set(raw_val) elif attr_id == "196": disk_reallocated_event_count.labels(device).set(raw_val or 0) elif attr_id == "198": disk_offline_uncorrectable.labels(device).set(raw_val or 0) def export_nvme(device, data): n = data.get("nvme_smart_health_information_log", {}) if not n: return # Temps in Kelvin in smartctl JSON? smartctl usually returns Celsius already for 'temperature' temp = n.get("temperature") if temp is not None: nvme_temperature.labels(device).set(temp) disk_temperature.labels(device).set(temp) fields = [ ('critical_warning', nvme_critical_warning), ('available_spare', nvme_available_spare), ('available_spare_threshold', nvme_available_spare_threshold), ('percentage_used', nvme_percentage_used), ('media_errors', nvme_media_errors), ('num_err_log_entries', nvme_num_err_log_entries), ('data_units_read', nvme_data_units_read), ('data_units_written', nvme_data_units_written), ] for key, metric in fields: val = n.get(key) if val is not None: metric.labels(device).set(val) def export_identity(device, data): model = data.get("model_name") or data.get("model_family") or "" serial = data.get("serial_number") or "" firmware = data.get("firmware_version") or "" transport = data.get("device", {}).get("transport") or data.get("interface_name") or "" # Set gauge value 1 (info pattern) disk_info.labels(device, model, serial, firmware, transport).set(1) def export_health(device, data): # ATA passed = data.get("smart_status", {}).get("passed") if passed is None: # NVMe path: smart_status might appear too passed = data.get("smart_status", {}).get("passed") if passed is not None: disk_healthy.labels(device).set(1 if passed else 0) else: # Unknown -> do not overwrite previous; or set -1 disk_healthy.labels(device).set(-1) def process_device(device): data = run_smartctl(device) if not data: return try: export_identity(device, data) export_health(device, data) if "ata_smart_attributes" in data: export_ata_attributes(device, data) if "nvme_smart_health_information_log" in data: export_nvme(device, data) # Last poll timestamp disk_last_smart_poll_timestamp.labels(device).set(time.time()) except Exception as e: print(f"Processing error {device}: {e}") exporter_errors_total.inc() @smart_poll_duration_seconds.time() def poll_cycle(): refresh_devices() with LOCK: devs = list(DEVICES) if not devs: return # Parallel collection threads = min(len(devs), 8) with ThreadPoolExecutor(max_workers=threads) as exe: futs = {exe.submit(process_device, d): d for d in devs} for fut in as_completed(futs): _ = fut.result() collect_mdraid() def main(): sanity_checks() start_http_server(EXPORTER_PORT) print(f"Started disk exporter on :{EXPORTER_PORT}, interval={CHECK_FREQUENCY}s") def _handle_signal(signum, frame): print(f"Received signal {signum}, initiating shutdown...") STOP_EVENT.set() # Register handlers (Kubernetes / Docker send SIGTERM) signal.signal(signal.SIGTERM, _handle_signal) signal.signal(signal.SIGINT, _handle_signal) try: while not STOP_EVENT.is_set(): start = time.time() poll_cycle() if STOP_EVENT.is_set(): break elapsed = time.time() - start sleep_for = max(1, CHECK_FREQUENCY - elapsed) # Sleep but wake early if signal received STOP_EVENT.wait(timeout=sleep_for) finally: print("Exporter shutdown complete.") # If any future cleanup is needed, add here. if __name__ == "__main__": main()