alexandre.pires
429a94332e
- Introduced a new disk exporter module to monitor disk health using SMART data. - Updated `apps.tf` to include the disk exporter module with necessary parameters. - Added local variables for smartctl version and checksum in `config.tf`. - Enhanced `alertmanager.yml` to handle DiskUnhealthy alerts with specific grouping and dispatch settings. - Created new Prometheus rules for monitoring disk health, including alerts for unhealthy disks. - Updated Prometheus configuration to scrape metrics from the new disk exporter. - Enhanced Telegram notification template to include resolved alerts and improved formatting. - Updated the auth-exporter module to specify the Python image tag correctly. - Created the disk-exporter module with Kubernetes resources, including deployment, service account, and config map. - Added necessary Python scripts and requirements for the disk exporter functionality.
433 lines
17 KiB
Python
433 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
import os
|
|
import re
|
|
import json
|
|
import time
|
|
import threading
|
|
import subprocess
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from prometheus_client import start_http_server, Gauge, Counter, Summary
|
|
import stat
|
|
import tarfile
|
|
import hashlib
|
|
import urllib.request
|
|
from io import BytesIO
|
|
import signal # <-- added
|
|
|
|
SMARTCTL = os.environ.get("SMARTCTL_PATH", "/usr/sbin/smartctl")
|
|
CHECK_FREQUENCY = int(os.environ.get("CHECK_FREQUENCY", "600"))
|
|
EXPORTER_PORT = int(os.environ.get("EXPORTER_PORT", "9100"))
|
|
STANDBY_MODE = os.environ.get("STANDBY_MODE", "0") == "1"
|
|
|
|
SMARTCTL_STATIC_VERSION = os.environ.get("SMARTCTL_STATIC_VERSION", "")
|
|
SMARTCTL_SHA256 = os.environ.get("SMARTCTL_SHA256", "")
|
|
SMARTCTL_CACHE_BASE = os.environ.get("SMARTCTL_CACHE_BASE", "/tmp/smartctl-cache")
|
|
SMARTCTL_FORCE_DOWNLOAD = os.environ.get("SMARTCTL_FORCE_DOWNLOAD", "false") == "true"
|
|
|
|
DEBUG = os.environ.get("DEBUG", "0") == "1"
|
|
SMARTCTL_STATUS_BITS = {
|
|
0: "Command line did not parse",
|
|
1: "Device open failed",
|
|
2: "SMART status check returned 'DISK FAILING'",
|
|
3: "Prefail Attributes <= threshold",
|
|
4: "Usage Attributes <= threshold",
|
|
5: "Self-test log contains errors",
|
|
6: "Self-test in progress",
|
|
7: "Device has read SMART data (some internal error if also bit 1?)"
|
|
}
|
|
# -------- Prometheus Metrics --------
|
|
disk_healthy = Gauge('disk_healthy', 'SMART overall health status (1=passed,0=fail)', ['device'])
|
|
disk_temperature = Gauge('disk_temperature', 'Drive temperature (C)', ['device'])
|
|
disk_reallocated_sector_count = Gauge('disk_reallocated_sector_count', 'Reallocated Sector Count', ['device'])
|
|
disk_reallocated_event_count = Gauge('disk_reallocated_event_count', 'Reallocated Event Count', ['device'])
|
|
disk_offline_uncorrectable = Gauge('disk_offline_uncorrectable', 'Offline Uncorrectable Count', ['device'])
|
|
|
|
disk_smart_attr_raw_value = Gauge('disk_smart_attr_raw_value', 'Raw value of SMART attribute', ['device','id','name'])
|
|
disk_smart_attr_normalized_value = Gauge('disk_smart_attr_normalized_value', 'Normalized value of SMART attribute', ['device','id','name'])
|
|
|
|
disk_info = Gauge('disk_info', 'Static disk identity info', ['device','model','serial','firmware','transport'])
|
|
disk_last_smart_poll_timestamp = Gauge('disk_last_smart_poll_timestamp', 'Unix timestamp of last successful SMART poll', ['device'])
|
|
|
|
# NVMe specific
|
|
nvme_critical_warning = Gauge('nvme_critical_warning', 'NVMe critical warning bitmap', ['device'])
|
|
nvme_available_spare = Gauge('nvme_available_spare', 'NVMe available spare (%)', ['device'])
|
|
nvme_available_spare_threshold = Gauge('nvme_available_spare_threshold', 'NVMe available spare threshold (%)', ['device'])
|
|
nvme_percentage_used = Gauge('nvme_percentage_used', 'NVMe percentage used (wear level)', ['device'])
|
|
nvme_media_errors = Gauge('nvme_media_errors', 'NVMe media errors', ['device'])
|
|
nvme_num_err_log_entries = Gauge('nvme_num_err_log_entries', 'NVMe number of error log entries', ['device'])
|
|
nvme_data_units_read = Gauge('nvme_data_units_read', 'NVMe data units read (1000 * 512KiB)', ['device'])
|
|
nvme_data_units_written = Gauge('nvme_data_units_written', 'NVMe data units written (1000 * 512KiB)', ['device'])
|
|
nvme_temperature = Gauge('nvme_temperature', 'NVMe temperature (C)', ['device'])
|
|
|
|
# mdadm (if any arrays)
|
|
mdraid_degraded = Gauge('mdraid_degraded', 'MD RAID degraded flag (1=degraded)', ['array'])
|
|
mdraid_sync_action = Gauge('mdraid_sync_action', 'MD RAID sync action indicator (idle=0,resync=1,recover=2,check=3,repair=4,reshape=5,unknown=99)', ['array'])
|
|
|
|
# Exporter operational metrics
|
|
exporter_errors_total = Counter('disk_exporter_errors_total', 'Total errors during SMART collection')
|
|
smart_poll_duration_seconds = Summary('disk_smart_poll_duration_seconds', 'Duration of a full SMART poll cycle')
|
|
smart_commands_total = Counter('disk_smart_commands_total', 'Total smartctl command invocations', ['result'])
|
|
|
|
# Add exit code metric
|
|
smartctl_exit_code = Gauge('smartctl_exit_code', 'Last smartctl raw exit code', ['device'])
|
|
|
|
DEVICE_REGEX = re.compile(r'^(sd[a-z]+|nvme\d+n\d+)$')
|
|
LOCK = threading.Lock()
|
|
DEVICES = set()
|
|
DEVICE_ROOT = os.environ.get("DEVICE_ROOT", "/dev") # NEW: root path for block devices (allows mounting host /dev elsewhere)
|
|
|
|
# Graceful shutdown event
|
|
STOP_EVENT = threading.Event() # <-- added
|
|
|
|
# -------- Utility Functions --------
|
|
def _build_smartctl_url(version: str) -> str:
|
|
return f"https://github.com/smartmontools/smartmontools-builds/releases/download/smartmontools-{version}/smartmontools-linux-x86_64-static-{version}.tar.gz"
|
|
|
|
def _version_dir(version: str) -> str:
|
|
return os.path.join(SMARTCTL_CACHE_BASE, version)
|
|
|
|
def _cached_binary_path(version: str) -> str:
|
|
return os.path.join(_version_dir(version), "smartctl")
|
|
|
|
def _cached_sha_file(version: str) -> str:
|
|
return os.path.join(_version_dir(version), ".archive_sha256")
|
|
|
|
def _url_exists(url: str) -> bool:
|
|
try:
|
|
req = urllib.request.Request(url, method="HEAD")
|
|
with urllib.request.urlopen(req, timeout=15):
|
|
return True
|
|
except Exception:
|
|
# Some servers may not allow HEAD; fallback to optimistic True
|
|
return True
|
|
|
|
def _validate_cached(version: str) -> bool:
|
|
bin_path = _cached_binary_path(version)
|
|
if not (os.path.isfile(bin_path) and os.access(bin_path, os.X_OK)):
|
|
return False
|
|
if SMARTCTL_SHA256:
|
|
sha_file = _cached_sha_file(version)
|
|
if not os.path.isfile(sha_file):
|
|
return False
|
|
recorded = open(sha_file).read().strip()
|
|
if recorded.lower() != SMARTCTL_SHA256.lower():
|
|
print(f"Cached smartctl for {version} sha mismatch (have {recorded}, expected {SMARTCTL_SHA256}); will re-download.")
|
|
return False
|
|
return True
|
|
|
|
def _download_and_extract_smartctl(version: str):
|
|
url = _build_smartctl_url(version)
|
|
if not _url_exists(url):
|
|
print(f"URL not reachable (HEAD failed): {url}")
|
|
return None
|
|
print(f"Fetching static smartctl archive: {url}")
|
|
try:
|
|
with urllib.request.urlopen(url, timeout=120) as r:
|
|
archive = r.read()
|
|
except Exception as e:
|
|
print(f"Download failed: {e}")
|
|
return None
|
|
if SMARTCTL_SHA256:
|
|
digest = hashlib.sha256(archive).hexdigest()
|
|
if digest.lower() != SMARTCTL_SHA256.lower():
|
|
print(f"Archive SHA256 mismatch (expected {SMARTCTL_SHA256} got {digest})")
|
|
return None
|
|
vdir = _version_dir(version)
|
|
os.makedirs(vdir, exist_ok=True)
|
|
try:
|
|
with tarfile.open(fileobj=BytesIO(archive), mode="r:gz") as tf:
|
|
member = next((m for m in tf.getmembers() if m.name.endswith("/smartctl") or m.name == "smartctl"), None)
|
|
if not member:
|
|
print("smartctl binary not found inside archive")
|
|
return None
|
|
tf.extract(member, vdir)
|
|
extracted = os.path.join(vdir, member.name)
|
|
final_path = _cached_binary_path(version)
|
|
if extracted != final_path:
|
|
os.replace(extracted, final_path)
|
|
os.chmod(final_path, os.stat(final_path).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
|
if SMARTCTL_SHA256:
|
|
with open(_cached_sha_file(version), "w") as f:
|
|
f.write(SMARTCTL_SHA256.lower())
|
|
print(f"Static smartctl ready: {final_path}")
|
|
return final_path
|
|
except Exception as e:
|
|
print(f"Extraction error: {e}")
|
|
return None
|
|
|
|
def ensure_smartctl():
|
|
global SMARTCTL
|
|
# If user-specified path works, keep it
|
|
if os.path.isfile(SMARTCTL) and os.access(SMARTCTL, os.X_OK):
|
|
return
|
|
if SMARTCTL_STATIC_VERSION:
|
|
if not SMARTCTL_FORCE_DOWNLOAD and _validate_cached(SMARTCTL_STATIC_VERSION):
|
|
SMARTCTL = _cached_binary_path(SMARTCTL_STATIC_VERSION)
|
|
print(f"Using cached smartctl: {SMARTCTL}")
|
|
return
|
|
# Download fresh
|
|
alt = _download_and_extract_smartctl(SMARTCTL_STATIC_VERSION)
|
|
if alt:
|
|
SMARTCTL = alt
|
|
return
|
|
# Fallback search
|
|
for p in ("/usr/bin/smartctl", "/usr/sbin/smartctl"):
|
|
if os.path.isfile(p) and os.access(p, os.X_OK):
|
|
SMARTCTL = p
|
|
return
|
|
|
|
def sanity_checks():
|
|
ensure_smartctl()
|
|
if not os.path.isfile(SMARTCTL):
|
|
print("smartctl not found. Install smartmontools or set SMARTCTL_STATIC_VERSION. Exiting.")
|
|
raise SystemExit(1)
|
|
|
|
def list_block_devices():
|
|
# lsblk returns both disks & partitions. Filter to TYPE=disk to avoid partitions.
|
|
try:
|
|
res = subprocess.run(
|
|
["lsblk", "-dn", "-o", "NAME,TYPE"],
|
|
check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
|
|
)
|
|
except subprocess.CalledProcessError as e:
|
|
print(f"lsblk failed: {e.stderr}")
|
|
exporter_errors_total.inc()
|
|
return set()
|
|
devs = set()
|
|
for line in res.stdout.strip().splitlines():
|
|
parts = line.split()
|
|
if len(parts) != 2:
|
|
continue
|
|
name, typ = parts
|
|
if typ != "disk":
|
|
continue
|
|
if DEVICE_REGEX.match(name):
|
|
devs.add(name)
|
|
return devs
|
|
|
|
def refresh_devices():
|
|
global DEVICES
|
|
current = list_block_devices()
|
|
with LOCK:
|
|
added = current - DEVICES
|
|
removed = DEVICES - current
|
|
if added:
|
|
print(f"New devices detected: {', '.join(sorted(added))}")
|
|
if removed:
|
|
print(f"Devices removed: {', '.join(sorted(removed))}")
|
|
DEVICES = current
|
|
|
|
def _decode_smartctl_exit(code: int) -> str:
|
|
if code == 0:
|
|
return "OK"
|
|
bits = []
|
|
for bit, desc in SMARTCTL_STATUS_BITS.items():
|
|
if code & (1 << bit):
|
|
bits.append(f"{bit}:{desc}")
|
|
return "; ".join(bits) if bits else f"raw:{code}"
|
|
|
|
def run_smartctl(device):
|
|
dev_path = f"{DEVICE_ROOT}/{device}"
|
|
cmd = [SMARTCTL, "-a", "-j", dev_path]
|
|
if STANDBY_MODE:
|
|
cmd.insert(1, "-n")
|
|
cmd.insert(2, "standby")
|
|
start = time.time()
|
|
try:
|
|
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
out = proc.stdout.strip()
|
|
exit_code = proc.returncode
|
|
smartctl_exit_code.labels(device).set(exit_code)
|
|
if not out:
|
|
if DEBUG:
|
|
print(f"[DEBUG] {device} smartctl produced no stdout. stderr='{proc.stderr.strip()}' rc={exit_code} bits={_decode_smartctl_exit(exit_code)}")
|
|
smart_commands_total.labels('empty_output').inc()
|
|
return None
|
|
try:
|
|
data = json.loads(out)
|
|
except json.JSONDecodeError:
|
|
if DEBUG:
|
|
print(f"[DEBUG] {device} JSON parse error. stderr='{proc.stderr.strip()}'")
|
|
smart_commands_total.labels('json_error').inc()
|
|
return None
|
|
if exit_code == 0:
|
|
smart_commands_total.labels('ok').inc()
|
|
else:
|
|
smart_commands_total.labels('nonzero_exit').inc()
|
|
if DEBUG:
|
|
print(f"[DEBUG] {device} smartctl non-zero rc={exit_code} bits={_decode_smartctl_exit(exit_code)} stderr='{proc.stderr.strip()}'")
|
|
# If no model & no SMART sections, log once
|
|
if DEBUG and not any(k in data for k in ("ata_smart_attributes","nvme_smart_health_information_log")):
|
|
print(f"[DEBUG] {device} SMART sections missing. Keys: {list(data.keys())}")
|
|
return data
|
|
except Exception as e:
|
|
print(f"smartctl failure {device}: {e}")
|
|
exporter_errors_total.inc()
|
|
smart_commands_total.labels('exception').inc()
|
|
return None
|
|
finally:
|
|
_ = time.time() - start # duration per device if needed
|
|
|
|
def map_md_sync_action(val):
|
|
# Map string -> numeric for gauge
|
|
mapping = {
|
|
'idle':0,'resync':1,'recover':2,'check':3,'repair':4,'reshape':5
|
|
}
|
|
return mapping.get(val, 99)
|
|
|
|
def collect_mdraid():
|
|
# Parse /proc/mdstat and sysfs for degraded + sync_action
|
|
mdstat = "/proc/mdstat"
|
|
if not os.path.isfile(mdstat):
|
|
return
|
|
try:
|
|
with open(mdstat,'r') as f:
|
|
text = f.read()
|
|
except Exception:
|
|
return
|
|
arrays = re.findall(r'^(md\d+)\s*:\s*(\w+).*$', text, re.MULTILINE)
|
|
for name,_ in arrays:
|
|
base = f"/sys/block/{name}/md"
|
|
degraded_file = os.path.join(base, "degraded")
|
|
sync_action_file = os.path.join(base, "sync_action")
|
|
try:
|
|
if os.path.isfile(degraded_file):
|
|
with open(degraded_file) as f:
|
|
d = int(f.read().strip())
|
|
mdraid_degraded.labels(name).set(1 if d > 0 else 0)
|
|
if os.path.isfile(sync_action_file):
|
|
with open(sync_action_file) as f:
|
|
act = f.read().strip()
|
|
mdraid_sync_action.labels(name).set(map_md_sync_action(act))
|
|
except Exception:
|
|
exporter_errors_total.inc()
|
|
|
|
def export_ata_attributes(device, data):
|
|
attrs = data.get("ata_smart_attributes", {}).get("table", [])
|
|
for a in attrs:
|
|
attr_id = str(a.get("id"))
|
|
name = a.get("name") or attr_id
|
|
raw_val = a.get("raw", {}).get("value")
|
|
normalized = a.get("value")
|
|
if raw_val is not None:
|
|
disk_smart_attr_raw_value.labels(device, attr_id, name).set(raw_val)
|
|
if normalized is not None:
|
|
disk_smart_attr_normalized_value.labels(device, attr_id, name).set(normalized)
|
|
# Legacy named metrics
|
|
if attr_id == "5":
|
|
disk_reallocated_sector_count.labels(device).set(raw_val or 0)
|
|
elif attr_id in ("190","194") and raw_val is not None:
|
|
disk_temperature.labels(device).set(raw_val)
|
|
elif attr_id == "196":
|
|
disk_reallocated_event_count.labels(device).set(raw_val or 0)
|
|
elif attr_id == "198":
|
|
disk_offline_uncorrectable.labels(device).set(raw_val or 0)
|
|
|
|
def export_nvme(device, data):
|
|
n = data.get("nvme_smart_health_information_log", {})
|
|
if not n:
|
|
return
|
|
# Temps in Kelvin in smartctl JSON? smartctl usually returns Celsius already for 'temperature'
|
|
temp = n.get("temperature")
|
|
if temp is not None:
|
|
nvme_temperature.labels(device).set(temp)
|
|
disk_temperature.labels(device).set(temp)
|
|
fields = [
|
|
('critical_warning', nvme_critical_warning),
|
|
('available_spare', nvme_available_spare),
|
|
('available_spare_threshold', nvme_available_spare_threshold),
|
|
('percentage_used', nvme_percentage_used),
|
|
('media_errors', nvme_media_errors),
|
|
('num_err_log_entries', nvme_num_err_log_entries),
|
|
('data_units_read', nvme_data_units_read),
|
|
('data_units_written', nvme_data_units_written),
|
|
]
|
|
for key, metric in fields:
|
|
val = n.get(key)
|
|
if val is not None:
|
|
metric.labels(device).set(val)
|
|
|
|
def export_identity(device, data):
|
|
model = data.get("model_name") or data.get("model_family") or ""
|
|
serial = data.get("serial_number") or ""
|
|
firmware = data.get("firmware_version") or ""
|
|
transport = data.get("device", {}).get("transport") or data.get("interface_name") or ""
|
|
# Set gauge value 1 (info pattern)
|
|
disk_info.labels(device, model, serial, firmware, transport).set(1)
|
|
|
|
def export_health(device, data):
|
|
# ATA
|
|
passed = data.get("smart_status", {}).get("passed")
|
|
if passed is None:
|
|
# NVMe path: smart_status might appear too
|
|
passed = data.get("smart_status", {}).get("passed")
|
|
if passed is not None:
|
|
disk_healthy.labels(device).set(1 if passed else 0)
|
|
else:
|
|
# Unknown -> do not overwrite previous; or set -1
|
|
disk_healthy.labels(device).set(-1)
|
|
|
|
def process_device(device):
|
|
data = run_smartctl(device)
|
|
if not data:
|
|
return
|
|
try:
|
|
export_identity(device, data)
|
|
export_health(device, data)
|
|
if "ata_smart_attributes" in data:
|
|
export_ata_attributes(device, data)
|
|
if "nvme_smart_health_information_log" in data:
|
|
export_nvme(device, data)
|
|
# Last poll timestamp
|
|
disk_last_smart_poll_timestamp.labels(device).set(time.time())
|
|
except Exception as e:
|
|
print(f"Processing error {device}: {e}")
|
|
exporter_errors_total.inc()
|
|
|
|
@smart_poll_duration_seconds.time()
|
|
def poll_cycle():
|
|
refresh_devices()
|
|
with LOCK:
|
|
devs = list(DEVICES)
|
|
if not devs:
|
|
return
|
|
# Parallel collection
|
|
threads = min(len(devs), 8)
|
|
with ThreadPoolExecutor(max_workers=threads) as exe:
|
|
futs = {exe.submit(process_device, d): d for d in devs}
|
|
for fut in as_completed(futs):
|
|
_ = fut.result()
|
|
collect_mdraid()
|
|
|
|
def main():
|
|
sanity_checks()
|
|
start_http_server(EXPORTER_PORT)
|
|
print(f"Started disk exporter on :{EXPORTER_PORT}, interval={CHECK_FREQUENCY}s")
|
|
|
|
def _handle_signal(signum, frame):
|
|
print(f"Received signal {signum}, initiating shutdown...")
|
|
STOP_EVENT.set()
|
|
|
|
# Register handlers (Kubernetes / Docker send SIGTERM)
|
|
signal.signal(signal.SIGTERM, _handle_signal)
|
|
signal.signal(signal.SIGINT, _handle_signal)
|
|
|
|
try:
|
|
while not STOP_EVENT.is_set():
|
|
start = time.time()
|
|
poll_cycle()
|
|
if STOP_EVENT.is_set():
|
|
break
|
|
elapsed = time.time() - start
|
|
sleep_for = max(1, CHECK_FREQUENCY - elapsed)
|
|
# Sleep but wake early if signal received
|
|
STOP_EVENT.wait(timeout=sleep_for)
|
|
finally:
|
|
print("Exporter shutdown complete.")
|
|
# If any future cleanup is needed, add here.
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|
|
|