Files

433 lines
17 KiB
Python
Raw Permalink Normal View History

#!/usr/bin/env python3
import os
import re
import json
import time
import threading
import subprocess
from concurrent.futures import ThreadPoolExecutor, as_completed
from prometheus_client import start_http_server, Gauge, Counter, Summary
import stat
import tarfile
import hashlib
import urllib.request
from io import BytesIO
import signal # <-- added
SMARTCTL = os.environ.get("SMARTCTL_PATH", "/usr/sbin/smartctl")
CHECK_FREQUENCY = int(os.environ.get("CHECK_FREQUENCY", "600"))
EXPORTER_PORT = int(os.environ.get("EXPORTER_PORT", "9100"))
STANDBY_MODE = os.environ.get("STANDBY_MODE", "0") == "1"
SMARTCTL_STATIC_VERSION = os.environ.get("SMARTCTL_STATIC_VERSION", "")
SMARTCTL_SHA256 = os.environ.get("SMARTCTL_SHA256", "")
SMARTCTL_CACHE_BASE = os.environ.get("SMARTCTL_CACHE_BASE", "/tmp/smartctl-cache")
SMARTCTL_FORCE_DOWNLOAD = os.environ.get("SMARTCTL_FORCE_DOWNLOAD", "false") == "true"
DEBUG = os.environ.get("DEBUG", "0") == "1"
SMARTCTL_STATUS_BITS = {
0: "Command line did not parse",
1: "Device open failed",
2: "SMART status check returned 'DISK FAILING'",
3: "Prefail Attributes <= threshold",
4: "Usage Attributes <= threshold",
5: "Self-test log contains errors",
6: "Self-test in progress",
7: "Device has read SMART data (some internal error if also bit 1?)"
}
# -------- Prometheus Metrics --------
disk_healthy = Gauge('disk_healthy', 'SMART overall health status (1=passed,0=fail)', ['device'])
disk_temperature = Gauge('disk_temperature', 'Drive temperature (C)', ['device'])
disk_reallocated_sector_count = Gauge('disk_reallocated_sector_count', 'Reallocated Sector Count', ['device'])
disk_reallocated_event_count = Gauge('disk_reallocated_event_count', 'Reallocated Event Count', ['device'])
disk_offline_uncorrectable = Gauge('disk_offline_uncorrectable', 'Offline Uncorrectable Count', ['device'])
disk_smart_attr_raw_value = Gauge('disk_smart_attr_raw_value', 'Raw value of SMART attribute', ['device','id','name'])
disk_smart_attr_normalized_value = Gauge('disk_smart_attr_normalized_value', 'Normalized value of SMART attribute', ['device','id','name'])
disk_info = Gauge('disk_info', 'Static disk identity info', ['device','model','serial','firmware','transport'])
disk_last_smart_poll_timestamp = Gauge('disk_last_smart_poll_timestamp', 'Unix timestamp of last successful SMART poll', ['device'])
# NVMe specific
nvme_critical_warning = Gauge('nvme_critical_warning', 'NVMe critical warning bitmap', ['device'])
nvme_available_spare = Gauge('nvme_available_spare', 'NVMe available spare (%)', ['device'])
nvme_available_spare_threshold = Gauge('nvme_available_spare_threshold', 'NVMe available spare threshold (%)', ['device'])
nvme_percentage_used = Gauge('nvme_percentage_used', 'NVMe percentage used (wear level)', ['device'])
nvme_media_errors = Gauge('nvme_media_errors', 'NVMe media errors', ['device'])
nvme_num_err_log_entries = Gauge('nvme_num_err_log_entries', 'NVMe number of error log entries', ['device'])
nvme_data_units_read = Gauge('nvme_data_units_read', 'NVMe data units read (1000 * 512KiB)', ['device'])
nvme_data_units_written = Gauge('nvme_data_units_written', 'NVMe data units written (1000 * 512KiB)', ['device'])
nvme_temperature = Gauge('nvme_temperature', 'NVMe temperature (C)', ['device'])
# mdadm (if any arrays)
mdraid_degraded = Gauge('mdraid_degraded', 'MD RAID degraded flag (1=degraded)', ['array'])
mdraid_sync_action = Gauge('mdraid_sync_action', 'MD RAID sync action indicator (idle=0,resync=1,recover=2,check=3,repair=4,reshape=5,unknown=99)', ['array'])
# Exporter operational metrics
exporter_errors_total = Counter('disk_exporter_errors_total', 'Total errors during SMART collection')
smart_poll_duration_seconds = Summary('disk_smart_poll_duration_seconds', 'Duration of a full SMART poll cycle')
smart_commands_total = Counter('disk_smart_commands_total', 'Total smartctl command invocations', ['result'])
# Add exit code metric
smartctl_exit_code = Gauge('smartctl_exit_code', 'Last smartctl raw exit code', ['device'])
DEVICE_REGEX = re.compile(r'^(sd[a-z]+|nvme\d+n\d+)$')
LOCK = threading.Lock()
DEVICES = set()
DEVICE_ROOT = os.environ.get("DEVICE_ROOT", "/dev") # NEW: root path for block devices (allows mounting host /dev elsewhere)
# Graceful shutdown event
STOP_EVENT = threading.Event() # <-- added
# -------- Utility Functions --------
def _build_smartctl_url(version: str) -> str:
return f"https://github.com/smartmontools/smartmontools-builds/releases/download/smartmontools-{version}/smartmontools-linux-x86_64-static-{version}.tar.gz"
def _version_dir(version: str) -> str:
return os.path.join(SMARTCTL_CACHE_BASE, version)
def _cached_binary_path(version: str) -> str:
return os.path.join(_version_dir(version), "smartctl")
def _cached_sha_file(version: str) -> str:
return os.path.join(_version_dir(version), ".archive_sha256")
def _url_exists(url: str) -> bool:
try:
req = urllib.request.Request(url, method="HEAD")
with urllib.request.urlopen(req, timeout=15):
return True
except Exception:
# Some servers may not allow HEAD; fallback to optimistic True
return True
def _validate_cached(version: str) -> bool:
bin_path = _cached_binary_path(version)
if not (os.path.isfile(bin_path) and os.access(bin_path, os.X_OK)):
return False
if SMARTCTL_SHA256:
sha_file = _cached_sha_file(version)
if not os.path.isfile(sha_file):
return False
recorded = open(sha_file).read().strip()
if recorded.lower() != SMARTCTL_SHA256.lower():
print(f"Cached smartctl for {version} sha mismatch (have {recorded}, expected {SMARTCTL_SHA256}); will re-download.")
return False
return True
def _download_and_extract_smartctl(version: str):
url = _build_smartctl_url(version)
if not _url_exists(url):
print(f"URL not reachable (HEAD failed): {url}")
return None
print(f"Fetching static smartctl archive: {url}")
try:
with urllib.request.urlopen(url, timeout=120) as r:
archive = r.read()
except Exception as e:
print(f"Download failed: {e}")
return None
if SMARTCTL_SHA256:
digest = hashlib.sha256(archive).hexdigest()
if digest.lower() != SMARTCTL_SHA256.lower():
print(f"Archive SHA256 mismatch (expected {SMARTCTL_SHA256} got {digest})")
return None
vdir = _version_dir(version)
os.makedirs(vdir, exist_ok=True)
try:
with tarfile.open(fileobj=BytesIO(archive), mode="r:gz") as tf:
member = next((m for m in tf.getmembers() if m.name.endswith("/smartctl") or m.name == "smartctl"), None)
if not member:
print("smartctl binary not found inside archive")
return None
tf.extract(member, vdir)
extracted = os.path.join(vdir, member.name)
final_path = _cached_binary_path(version)
if extracted != final_path:
os.replace(extracted, final_path)
os.chmod(final_path, os.stat(final_path).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
if SMARTCTL_SHA256:
with open(_cached_sha_file(version), "w") as f:
f.write(SMARTCTL_SHA256.lower())
print(f"Static smartctl ready: {final_path}")
return final_path
except Exception as e:
print(f"Extraction error: {e}")
return None
def ensure_smartctl():
global SMARTCTL
# If user-specified path works, keep it
if os.path.isfile(SMARTCTL) and os.access(SMARTCTL, os.X_OK):
return
if SMARTCTL_STATIC_VERSION:
if not SMARTCTL_FORCE_DOWNLOAD and _validate_cached(SMARTCTL_STATIC_VERSION):
SMARTCTL = _cached_binary_path(SMARTCTL_STATIC_VERSION)
print(f"Using cached smartctl: {SMARTCTL}")
return
# Download fresh
alt = _download_and_extract_smartctl(SMARTCTL_STATIC_VERSION)
if alt:
SMARTCTL = alt
return
# Fallback search
for p in ("/usr/bin/smartctl", "/usr/sbin/smartctl"):
if os.path.isfile(p) and os.access(p, os.X_OK):
SMARTCTL = p
return
def sanity_checks():
ensure_smartctl()
if not os.path.isfile(SMARTCTL):
print("smartctl not found. Install smartmontools or set SMARTCTL_STATIC_VERSION. Exiting.")
raise SystemExit(1)
def list_block_devices():
# lsblk returns both disks & partitions. Filter to TYPE=disk to avoid partitions.
try:
res = subprocess.run(
["lsblk", "-dn", "-o", "NAME,TYPE"],
check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
)
except subprocess.CalledProcessError as e:
print(f"lsblk failed: {e.stderr}")
exporter_errors_total.inc()
return set()
devs = set()
for line in res.stdout.strip().splitlines():
parts = line.split()
if len(parts) != 2:
continue
name, typ = parts
if typ != "disk":
continue
if DEVICE_REGEX.match(name):
devs.add(name)
return devs
def refresh_devices():
global DEVICES
current = list_block_devices()
with LOCK:
added = current - DEVICES
removed = DEVICES - current
if added:
print(f"New devices detected: {', '.join(sorted(added))}")
if removed:
print(f"Devices removed: {', '.join(sorted(removed))}")
DEVICES = current
def _decode_smartctl_exit(code: int) -> str:
if code == 0:
return "OK"
bits = []
for bit, desc in SMARTCTL_STATUS_BITS.items():
if code & (1 << bit):
bits.append(f"{bit}:{desc}")
return "; ".join(bits) if bits else f"raw:{code}"
def run_smartctl(device):
dev_path = f"{DEVICE_ROOT}/{device}"
cmd = [SMARTCTL, "-a", "-j", dev_path]
if STANDBY_MODE:
cmd.insert(1, "-n")
cmd.insert(2, "standby")
start = time.time()
try:
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
out = proc.stdout.strip()
exit_code = proc.returncode
smartctl_exit_code.labels(device).set(exit_code)
if not out:
if DEBUG:
print(f"[DEBUG] {device} smartctl produced no stdout. stderr='{proc.stderr.strip()}' rc={exit_code} bits={_decode_smartctl_exit(exit_code)}")
smart_commands_total.labels('empty_output').inc()
return None
try:
data = json.loads(out)
except json.JSONDecodeError:
if DEBUG:
print(f"[DEBUG] {device} JSON parse error. stderr='{proc.stderr.strip()}'")
smart_commands_total.labels('json_error').inc()
return None
if exit_code == 0:
smart_commands_total.labels('ok').inc()
else:
smart_commands_total.labels('nonzero_exit').inc()
if DEBUG:
print(f"[DEBUG] {device} smartctl non-zero rc={exit_code} bits={_decode_smartctl_exit(exit_code)} stderr='{proc.stderr.strip()}'")
# If no model & no SMART sections, log once
if DEBUG and not any(k in data for k in ("ata_smart_attributes","nvme_smart_health_information_log")):
print(f"[DEBUG] {device} SMART sections missing. Keys: {list(data.keys())}")
return data
except Exception as e:
print(f"smartctl failure {device}: {e}")
exporter_errors_total.inc()
smart_commands_total.labels('exception').inc()
return None
finally:
_ = time.time() - start # duration per device if needed
def map_md_sync_action(val):
# Map string -> numeric for gauge
mapping = {
'idle':0,'resync':1,'recover':2,'check':3,'repair':4,'reshape':5
}
return mapping.get(val, 99)
def collect_mdraid():
# Parse /proc/mdstat and sysfs for degraded + sync_action
mdstat = "/proc/mdstat"
if not os.path.isfile(mdstat):
return
try:
with open(mdstat,'r') as f:
text = f.read()
except Exception:
return
arrays = re.findall(r'^(md\d+)\s*:\s*(\w+).*$', text, re.MULTILINE)
for name,_ in arrays:
base = f"/sys/block/{name}/md"
degraded_file = os.path.join(base, "degraded")
sync_action_file = os.path.join(base, "sync_action")
try:
if os.path.isfile(degraded_file):
with open(degraded_file) as f:
d = int(f.read().strip())
mdraid_degraded.labels(name).set(1 if d > 0 else 0)
if os.path.isfile(sync_action_file):
with open(sync_action_file) as f:
act = f.read().strip()
mdraid_sync_action.labels(name).set(map_md_sync_action(act))
except Exception:
exporter_errors_total.inc()
def export_ata_attributes(device, data):
attrs = data.get("ata_smart_attributes", {}).get("table", [])
for a in attrs:
attr_id = str(a.get("id"))
name = a.get("name") or attr_id
raw_val = a.get("raw", {}).get("value")
normalized = a.get("value")
if raw_val is not None:
disk_smart_attr_raw_value.labels(device, attr_id, name).set(raw_val)
if normalized is not None:
disk_smart_attr_normalized_value.labels(device, attr_id, name).set(normalized)
# Legacy named metrics
if attr_id == "5":
disk_reallocated_sector_count.labels(device).set(raw_val or 0)
elif attr_id in ("190","194") and raw_val is not None:
disk_temperature.labels(device).set(raw_val)
elif attr_id == "196":
disk_reallocated_event_count.labels(device).set(raw_val or 0)
elif attr_id == "198":
disk_offline_uncorrectable.labels(device).set(raw_val or 0)
def export_nvme(device, data):
n = data.get("nvme_smart_health_information_log", {})
if not n:
return
# Temps in Kelvin in smartctl JSON? smartctl usually returns Celsius already for 'temperature'
temp = n.get("temperature")
if temp is not None:
nvme_temperature.labels(device).set(temp)
disk_temperature.labels(device).set(temp)
fields = [
('critical_warning', nvme_critical_warning),
('available_spare', nvme_available_spare),
('available_spare_threshold', nvme_available_spare_threshold),
('percentage_used', nvme_percentage_used),
('media_errors', nvme_media_errors),
('num_err_log_entries', nvme_num_err_log_entries),
('data_units_read', nvme_data_units_read),
('data_units_written', nvme_data_units_written),
]
for key, metric in fields:
val = n.get(key)
if val is not None:
metric.labels(device).set(val)
def export_identity(device, data):
model = data.get("model_name") or data.get("model_family") or ""
serial = data.get("serial_number") or ""
firmware = data.get("firmware_version") or ""
transport = data.get("device", {}).get("transport") or data.get("interface_name") or ""
# Set gauge value 1 (info pattern)
disk_info.labels(device, model, serial, firmware, transport).set(1)
def export_health(device, data):
# ATA
passed = data.get("smart_status", {}).get("passed")
if passed is None:
# NVMe path: smart_status might appear too
passed = data.get("smart_status", {}).get("passed")
if passed is not None:
disk_healthy.labels(device).set(1 if passed else 0)
else:
# Unknown -> do not overwrite previous; or set -1
disk_healthy.labels(device).set(-1)
def process_device(device):
data = run_smartctl(device)
if not data:
return
try:
export_identity(device, data)
export_health(device, data)
if "ata_smart_attributes" in data:
export_ata_attributes(device, data)
if "nvme_smart_health_information_log" in data:
export_nvme(device, data)
# Last poll timestamp
disk_last_smart_poll_timestamp.labels(device).set(time.time())
except Exception as e:
print(f"Processing error {device}: {e}")
exporter_errors_total.inc()
@smart_poll_duration_seconds.time()
def poll_cycle():
refresh_devices()
with LOCK:
devs = list(DEVICES)
if not devs:
return
# Parallel collection
threads = min(len(devs), 8)
with ThreadPoolExecutor(max_workers=threads) as exe:
futs = {exe.submit(process_device, d): d for d in devs}
for fut in as_completed(futs):
_ = fut.result()
collect_mdraid()
def main():
sanity_checks()
start_http_server(EXPORTER_PORT)
print(f"Started disk exporter on :{EXPORTER_PORT}, interval={CHECK_FREQUENCY}s")
def _handle_signal(signum, frame):
print(f"Received signal {signum}, initiating shutdown...")
STOP_EVENT.set()
# Register handlers (Kubernetes / Docker send SIGTERM)
signal.signal(signal.SIGTERM, _handle_signal)
signal.signal(signal.SIGINT, _handle_signal)
try:
while not STOP_EVENT.is_set():
start = time.time()
poll_cycle()
if STOP_EVENT.is_set():
break
elapsed = time.time() - start
sleep_for = max(1, CHECK_FREQUENCY - elapsed)
# Sleep but wake early if signal received
STOP_EVENT.wait(timeout=sleep_for)
finally:
print("Exporter shutdown complete.")
# If any future cleanup is needed, add here.
if __name__ == "__main__":
main()