feat: add disk exporter module and related configurations
- Introduced a new disk exporter module to monitor disk health using SMART data. - Updated `apps.tf` to include the disk exporter module with necessary parameters. - Added local variables for smartctl version and checksum in `config.tf`. - Enhanced `alertmanager.yml` to handle DiskUnhealthy alerts with specific grouping and dispatch settings. - Created new Prometheus rules for monitoring disk health, including alerts for unhealthy disks. - Updated Prometheus configuration to scrape metrics from the new disk exporter. - Enhanced Telegram notification template to include resolved alerts and improved formatting. - Updated the auth-exporter module to specify the Python image tag correctly. - Created the disk-exporter module with Kubernetes resources, including deployment, service account, and config map. - Added necessary Python scripts and requirements for the disk exporter functionality.
This commit is contained in:
@@ -0,0 +1,432 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import time
|
||||
import threading
|
||||
import subprocess
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from prometheus_client import start_http_server, Gauge, Counter, Summary
|
||||
import stat
|
||||
import tarfile
|
||||
import hashlib
|
||||
import urllib.request
|
||||
from io import BytesIO
|
||||
import signal # <-- added
|
||||
|
||||
SMARTCTL = os.environ.get("SMARTCTL_PATH", "/usr/sbin/smartctl")
|
||||
CHECK_FREQUENCY = int(os.environ.get("CHECK_FREQUENCY", "600"))
|
||||
EXPORTER_PORT = int(os.environ.get("EXPORTER_PORT", "9100"))
|
||||
STANDBY_MODE = os.environ.get("STANDBY_MODE", "0") == "1"
|
||||
|
||||
SMARTCTL_STATIC_VERSION = os.environ.get("SMARTCTL_STATIC_VERSION", "")
|
||||
SMARTCTL_SHA256 = os.environ.get("SMARTCTL_SHA256", "")
|
||||
SMARTCTL_CACHE_BASE = os.environ.get("SMARTCTL_CACHE_BASE", "/tmp/smartctl-cache")
|
||||
SMARTCTL_FORCE_DOWNLOAD = os.environ.get("SMARTCTL_FORCE_DOWNLOAD", "false") == "true"
|
||||
|
||||
DEBUG = os.environ.get("DEBUG", "0") == "1"
|
||||
SMARTCTL_STATUS_BITS = {
|
||||
0: "Command line did not parse",
|
||||
1: "Device open failed",
|
||||
2: "SMART status check returned 'DISK FAILING'",
|
||||
3: "Prefail Attributes <= threshold",
|
||||
4: "Usage Attributes <= threshold",
|
||||
5: "Self-test log contains errors",
|
||||
6: "Self-test in progress",
|
||||
7: "Device has read SMART data (some internal error if also bit 1?)"
|
||||
}
|
||||
# -------- Prometheus Metrics --------
|
||||
disk_healthy = Gauge('disk_healthy', 'SMART overall health status (1=passed,0=fail)', ['device'])
|
||||
disk_temperature = Gauge('disk_temperature', 'Drive temperature (C)', ['device'])
|
||||
disk_reallocated_sector_count = Gauge('disk_reallocated_sector_count', 'Reallocated Sector Count', ['device'])
|
||||
disk_reallocated_event_count = Gauge('disk_reallocated_event_count', 'Reallocated Event Count', ['device'])
|
||||
disk_offline_uncorrectable = Gauge('disk_offline_uncorrectable', 'Offline Uncorrectable Count', ['device'])
|
||||
|
||||
disk_smart_attr_raw_value = Gauge('disk_smart_attr_raw_value', 'Raw value of SMART attribute', ['device','id','name'])
|
||||
disk_smart_attr_normalized_value = Gauge('disk_smart_attr_normalized_value', 'Normalized value of SMART attribute', ['device','id','name'])
|
||||
|
||||
disk_info = Gauge('disk_info', 'Static disk identity info', ['device','model','serial','firmware','transport'])
|
||||
disk_last_smart_poll_timestamp = Gauge('disk_last_smart_poll_timestamp', 'Unix timestamp of last successful SMART poll', ['device'])
|
||||
|
||||
# NVMe specific
|
||||
nvme_critical_warning = Gauge('nvme_critical_warning', 'NVMe critical warning bitmap', ['device'])
|
||||
nvme_available_spare = Gauge('nvme_available_spare', 'NVMe available spare (%)', ['device'])
|
||||
nvme_available_spare_threshold = Gauge('nvme_available_spare_threshold', 'NVMe available spare threshold (%)', ['device'])
|
||||
nvme_percentage_used = Gauge('nvme_percentage_used', 'NVMe percentage used (wear level)', ['device'])
|
||||
nvme_media_errors = Gauge('nvme_media_errors', 'NVMe media errors', ['device'])
|
||||
nvme_num_err_log_entries = Gauge('nvme_num_err_log_entries', 'NVMe number of error log entries', ['device'])
|
||||
nvme_data_units_read = Gauge('nvme_data_units_read', 'NVMe data units read (1000 * 512KiB)', ['device'])
|
||||
nvme_data_units_written = Gauge('nvme_data_units_written', 'NVMe data units written (1000 * 512KiB)', ['device'])
|
||||
nvme_temperature = Gauge('nvme_temperature', 'NVMe temperature (C)', ['device'])
|
||||
|
||||
# mdadm (if any arrays)
|
||||
mdraid_degraded = Gauge('mdraid_degraded', 'MD RAID degraded flag (1=degraded)', ['array'])
|
||||
mdraid_sync_action = Gauge('mdraid_sync_action', 'MD RAID sync action indicator (idle=0,resync=1,recover=2,check=3,repair=4,reshape=5,unknown=99)', ['array'])
|
||||
|
||||
# Exporter operational metrics
|
||||
exporter_errors_total = Counter('disk_exporter_errors_total', 'Total errors during SMART collection')
|
||||
smart_poll_duration_seconds = Summary('disk_smart_poll_duration_seconds', 'Duration of a full SMART poll cycle')
|
||||
smart_commands_total = Counter('disk_smart_commands_total', 'Total smartctl command invocations', ['result'])
|
||||
|
||||
# Add exit code metric
|
||||
smartctl_exit_code = Gauge('smartctl_exit_code', 'Last smartctl raw exit code', ['device'])
|
||||
|
||||
DEVICE_REGEX = re.compile(r'^(sd[a-z]+|nvme\d+n\d+)$')
|
||||
LOCK = threading.Lock()
|
||||
DEVICES = set()
|
||||
DEVICE_ROOT = os.environ.get("DEVICE_ROOT", "/dev") # NEW: root path for block devices (allows mounting host /dev elsewhere)
|
||||
|
||||
# Graceful shutdown event
|
||||
STOP_EVENT = threading.Event() # <-- added
|
||||
|
||||
# -------- Utility Functions --------
|
||||
def _build_smartctl_url(version: str) -> str:
|
||||
return f"https://github.com/smartmontools/smartmontools-builds/releases/download/smartmontools-{version}/smartmontools-linux-x86_64-static-{version}.tar.gz"
|
||||
|
||||
def _version_dir(version: str) -> str:
|
||||
return os.path.join(SMARTCTL_CACHE_BASE, version)
|
||||
|
||||
def _cached_binary_path(version: str) -> str:
|
||||
return os.path.join(_version_dir(version), "smartctl")
|
||||
|
||||
def _cached_sha_file(version: str) -> str:
|
||||
return os.path.join(_version_dir(version), ".archive_sha256")
|
||||
|
||||
def _url_exists(url: str) -> bool:
|
||||
try:
|
||||
req = urllib.request.Request(url, method="HEAD")
|
||||
with urllib.request.urlopen(req, timeout=15):
|
||||
return True
|
||||
except Exception:
|
||||
# Some servers may not allow HEAD; fallback to optimistic True
|
||||
return True
|
||||
|
||||
def _validate_cached(version: str) -> bool:
|
||||
bin_path = _cached_binary_path(version)
|
||||
if not (os.path.isfile(bin_path) and os.access(bin_path, os.X_OK)):
|
||||
return False
|
||||
if SMARTCTL_SHA256:
|
||||
sha_file = _cached_sha_file(version)
|
||||
if not os.path.isfile(sha_file):
|
||||
return False
|
||||
recorded = open(sha_file).read().strip()
|
||||
if recorded.lower() != SMARTCTL_SHA256.lower():
|
||||
print(f"Cached smartctl for {version} sha mismatch (have {recorded}, expected {SMARTCTL_SHA256}); will re-download.")
|
||||
return False
|
||||
return True
|
||||
|
||||
def _download_and_extract_smartctl(version: str):
|
||||
url = _build_smartctl_url(version)
|
||||
if not _url_exists(url):
|
||||
print(f"URL not reachable (HEAD failed): {url}")
|
||||
return None
|
||||
print(f"Fetching static smartctl archive: {url}")
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=120) as r:
|
||||
archive = r.read()
|
||||
except Exception as e:
|
||||
print(f"Download failed: {e}")
|
||||
return None
|
||||
if SMARTCTL_SHA256:
|
||||
digest = hashlib.sha256(archive).hexdigest()
|
||||
if digest.lower() != SMARTCTL_SHA256.lower():
|
||||
print(f"Archive SHA256 mismatch (expected {SMARTCTL_SHA256} got {digest})")
|
||||
return None
|
||||
vdir = _version_dir(version)
|
||||
os.makedirs(vdir, exist_ok=True)
|
||||
try:
|
||||
with tarfile.open(fileobj=BytesIO(archive), mode="r:gz") as tf:
|
||||
member = next((m for m in tf.getmembers() if m.name.endswith("/smartctl") or m.name == "smartctl"), None)
|
||||
if not member:
|
||||
print("smartctl binary not found inside archive")
|
||||
return None
|
||||
tf.extract(member, vdir)
|
||||
extracted = os.path.join(vdir, member.name)
|
||||
final_path = _cached_binary_path(version)
|
||||
if extracted != final_path:
|
||||
os.replace(extracted, final_path)
|
||||
os.chmod(final_path, os.stat(final_path).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
if SMARTCTL_SHA256:
|
||||
with open(_cached_sha_file(version), "w") as f:
|
||||
f.write(SMARTCTL_SHA256.lower())
|
||||
print(f"Static smartctl ready: {final_path}")
|
||||
return final_path
|
||||
except Exception as e:
|
||||
print(f"Extraction error: {e}")
|
||||
return None
|
||||
|
||||
def ensure_smartctl():
|
||||
global SMARTCTL
|
||||
# If user-specified path works, keep it
|
||||
if os.path.isfile(SMARTCTL) and os.access(SMARTCTL, os.X_OK):
|
||||
return
|
||||
if SMARTCTL_STATIC_VERSION:
|
||||
if not SMARTCTL_FORCE_DOWNLOAD and _validate_cached(SMARTCTL_STATIC_VERSION):
|
||||
SMARTCTL = _cached_binary_path(SMARTCTL_STATIC_VERSION)
|
||||
print(f"Using cached smartctl: {SMARTCTL}")
|
||||
return
|
||||
# Download fresh
|
||||
alt = _download_and_extract_smartctl(SMARTCTL_STATIC_VERSION)
|
||||
if alt:
|
||||
SMARTCTL = alt
|
||||
return
|
||||
# Fallback search
|
||||
for p in ("/usr/bin/smartctl", "/usr/sbin/smartctl"):
|
||||
if os.path.isfile(p) and os.access(p, os.X_OK):
|
||||
SMARTCTL = p
|
||||
return
|
||||
|
||||
def sanity_checks():
|
||||
ensure_smartctl()
|
||||
if not os.path.isfile(SMARTCTL):
|
||||
print("smartctl not found. Install smartmontools or set SMARTCTL_STATIC_VERSION. Exiting.")
|
||||
raise SystemExit(1)
|
||||
|
||||
def list_block_devices():
|
||||
# lsblk returns both disks & partitions. Filter to TYPE=disk to avoid partitions.
|
||||
try:
|
||||
res = subprocess.run(
|
||||
["lsblk", "-dn", "-o", "NAME,TYPE"],
|
||||
check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"lsblk failed: {e.stderr}")
|
||||
exporter_errors_total.inc()
|
||||
return set()
|
||||
devs = set()
|
||||
for line in res.stdout.strip().splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) != 2:
|
||||
continue
|
||||
name, typ = parts
|
||||
if typ != "disk":
|
||||
continue
|
||||
if DEVICE_REGEX.match(name):
|
||||
devs.add(name)
|
||||
return devs
|
||||
|
||||
def refresh_devices():
|
||||
global DEVICES
|
||||
current = list_block_devices()
|
||||
with LOCK:
|
||||
added = current - DEVICES
|
||||
removed = DEVICES - current
|
||||
if added:
|
||||
print(f"New devices detected: {', '.join(sorted(added))}")
|
||||
if removed:
|
||||
print(f"Devices removed: {', '.join(sorted(removed))}")
|
||||
DEVICES = current
|
||||
|
||||
def _decode_smartctl_exit(code: int) -> str:
|
||||
if code == 0:
|
||||
return "OK"
|
||||
bits = []
|
||||
for bit, desc in SMARTCTL_STATUS_BITS.items():
|
||||
if code & (1 << bit):
|
||||
bits.append(f"{bit}:{desc}")
|
||||
return "; ".join(bits) if bits else f"raw:{code}"
|
||||
|
||||
def run_smartctl(device):
|
||||
dev_path = f"{DEVICE_ROOT}/{device}"
|
||||
cmd = [SMARTCTL, "-a", "-j", dev_path]
|
||||
if STANDBY_MODE:
|
||||
cmd.insert(1, "-n")
|
||||
cmd.insert(2, "standby")
|
||||
start = time.time()
|
||||
try:
|
||||
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
out = proc.stdout.strip()
|
||||
exit_code = proc.returncode
|
||||
smartctl_exit_code.labels(device).set(exit_code)
|
||||
if not out:
|
||||
if DEBUG:
|
||||
print(f"[DEBUG] {device} smartctl produced no stdout. stderr='{proc.stderr.strip()}' rc={exit_code} bits={_decode_smartctl_exit(exit_code)}")
|
||||
smart_commands_total.labels('empty_output').inc()
|
||||
return None
|
||||
try:
|
||||
data = json.loads(out)
|
||||
except json.JSONDecodeError:
|
||||
if DEBUG:
|
||||
print(f"[DEBUG] {device} JSON parse error. stderr='{proc.stderr.strip()}'")
|
||||
smart_commands_total.labels('json_error').inc()
|
||||
return None
|
||||
if exit_code == 0:
|
||||
smart_commands_total.labels('ok').inc()
|
||||
else:
|
||||
smart_commands_total.labels('nonzero_exit').inc()
|
||||
if DEBUG:
|
||||
print(f"[DEBUG] {device} smartctl non-zero rc={exit_code} bits={_decode_smartctl_exit(exit_code)} stderr='{proc.stderr.strip()}'")
|
||||
# If no model & no SMART sections, log once
|
||||
if DEBUG and not any(k in data for k in ("ata_smart_attributes","nvme_smart_health_information_log")):
|
||||
print(f"[DEBUG] {device} SMART sections missing. Keys: {list(data.keys())}")
|
||||
return data
|
||||
except Exception as e:
|
||||
print(f"smartctl failure {device}: {e}")
|
||||
exporter_errors_total.inc()
|
||||
smart_commands_total.labels('exception').inc()
|
||||
return None
|
||||
finally:
|
||||
_ = time.time() - start # duration per device if needed
|
||||
|
||||
def map_md_sync_action(val):
|
||||
# Map string -> numeric for gauge
|
||||
mapping = {
|
||||
'idle':0,'resync':1,'recover':2,'check':3,'repair':4,'reshape':5
|
||||
}
|
||||
return mapping.get(val, 99)
|
||||
|
||||
def collect_mdraid():
|
||||
# Parse /proc/mdstat and sysfs for degraded + sync_action
|
||||
mdstat = "/proc/mdstat"
|
||||
if not os.path.isfile(mdstat):
|
||||
return
|
||||
try:
|
||||
with open(mdstat,'r') as f:
|
||||
text = f.read()
|
||||
except Exception:
|
||||
return
|
||||
arrays = re.findall(r'^(md\d+)\s*:\s*(\w+).*$', text, re.MULTILINE)
|
||||
for name,_ in arrays:
|
||||
base = f"/sys/block/{name}/md"
|
||||
degraded_file = os.path.join(base, "degraded")
|
||||
sync_action_file = os.path.join(base, "sync_action")
|
||||
try:
|
||||
if os.path.isfile(degraded_file):
|
||||
with open(degraded_file) as f:
|
||||
d = int(f.read().strip())
|
||||
mdraid_degraded.labels(name).set(1 if d > 0 else 0)
|
||||
if os.path.isfile(sync_action_file):
|
||||
with open(sync_action_file) as f:
|
||||
act = f.read().strip()
|
||||
mdraid_sync_action.labels(name).set(map_md_sync_action(act))
|
||||
except Exception:
|
||||
exporter_errors_total.inc()
|
||||
|
||||
def export_ata_attributes(device, data):
|
||||
attrs = data.get("ata_smart_attributes", {}).get("table", [])
|
||||
for a in attrs:
|
||||
attr_id = str(a.get("id"))
|
||||
name = a.get("name") or attr_id
|
||||
raw_val = a.get("raw", {}).get("value")
|
||||
normalized = a.get("value")
|
||||
if raw_val is not None:
|
||||
disk_smart_attr_raw_value.labels(device, attr_id, name).set(raw_val)
|
||||
if normalized is not None:
|
||||
disk_smart_attr_normalized_value.labels(device, attr_id, name).set(normalized)
|
||||
# Legacy named metrics
|
||||
if attr_id == "5":
|
||||
disk_reallocated_sector_count.labels(device).set(raw_val or 0)
|
||||
elif attr_id in ("190","194") and raw_val is not None:
|
||||
disk_temperature.labels(device).set(raw_val)
|
||||
elif attr_id == "196":
|
||||
disk_reallocated_event_count.labels(device).set(raw_val or 0)
|
||||
elif attr_id == "198":
|
||||
disk_offline_uncorrectable.labels(device).set(raw_val or 0)
|
||||
|
||||
def export_nvme(device, data):
|
||||
n = data.get("nvme_smart_health_information_log", {})
|
||||
if not n:
|
||||
return
|
||||
# Temps in Kelvin in smartctl JSON? smartctl usually returns Celsius already for 'temperature'
|
||||
temp = n.get("temperature")
|
||||
if temp is not None:
|
||||
nvme_temperature.labels(device).set(temp)
|
||||
disk_temperature.labels(device).set(temp)
|
||||
fields = [
|
||||
('critical_warning', nvme_critical_warning),
|
||||
('available_spare', nvme_available_spare),
|
||||
('available_spare_threshold', nvme_available_spare_threshold),
|
||||
('percentage_used', nvme_percentage_used),
|
||||
('media_errors', nvme_media_errors),
|
||||
('num_err_log_entries', nvme_num_err_log_entries),
|
||||
('data_units_read', nvme_data_units_read),
|
||||
('data_units_written', nvme_data_units_written),
|
||||
]
|
||||
for key, metric in fields:
|
||||
val = n.get(key)
|
||||
if val is not None:
|
||||
metric.labels(device).set(val)
|
||||
|
||||
def export_identity(device, data):
|
||||
model = data.get("model_name") or data.get("model_family") or ""
|
||||
serial = data.get("serial_number") or ""
|
||||
firmware = data.get("firmware_version") or ""
|
||||
transport = data.get("device", {}).get("transport") or data.get("interface_name") or ""
|
||||
# Set gauge value 1 (info pattern)
|
||||
disk_info.labels(device, model, serial, firmware, transport).set(1)
|
||||
|
||||
def export_health(device, data):
|
||||
# ATA
|
||||
passed = data.get("smart_status", {}).get("passed")
|
||||
if passed is None:
|
||||
# NVMe path: smart_status might appear too
|
||||
passed = data.get("smart_status", {}).get("passed")
|
||||
if passed is not None:
|
||||
disk_healthy.labels(device).set(1 if passed else 0)
|
||||
else:
|
||||
# Unknown -> do not overwrite previous; or set -1
|
||||
disk_healthy.labels(device).set(-1)
|
||||
|
||||
def process_device(device):
|
||||
data = run_smartctl(device)
|
||||
if not data:
|
||||
return
|
||||
try:
|
||||
export_identity(device, data)
|
||||
export_health(device, data)
|
||||
if "ata_smart_attributes" in data:
|
||||
export_ata_attributes(device, data)
|
||||
if "nvme_smart_health_information_log" in data:
|
||||
export_nvme(device, data)
|
||||
# Last poll timestamp
|
||||
disk_last_smart_poll_timestamp.labels(device).set(time.time())
|
||||
except Exception as e:
|
||||
print(f"Processing error {device}: {e}")
|
||||
exporter_errors_total.inc()
|
||||
|
||||
@smart_poll_duration_seconds.time()
|
||||
def poll_cycle():
|
||||
refresh_devices()
|
||||
with LOCK:
|
||||
devs = list(DEVICES)
|
||||
if not devs:
|
||||
return
|
||||
# Parallel collection
|
||||
threads = min(len(devs), 8)
|
||||
with ThreadPoolExecutor(max_workers=threads) as exe:
|
||||
futs = {exe.submit(process_device, d): d for d in devs}
|
||||
for fut in as_completed(futs):
|
||||
_ = fut.result()
|
||||
collect_mdraid()
|
||||
|
||||
def main():
|
||||
sanity_checks()
|
||||
start_http_server(EXPORTER_PORT)
|
||||
print(f"Started disk exporter on :{EXPORTER_PORT}, interval={CHECK_FREQUENCY}s")
|
||||
|
||||
def _handle_signal(signum, frame):
|
||||
print(f"Received signal {signum}, initiating shutdown...")
|
||||
STOP_EVENT.set()
|
||||
|
||||
# Register handlers (Kubernetes / Docker send SIGTERM)
|
||||
signal.signal(signal.SIGTERM, _handle_signal)
|
||||
signal.signal(signal.SIGINT, _handle_signal)
|
||||
|
||||
try:
|
||||
while not STOP_EVENT.is_set():
|
||||
start = time.time()
|
||||
poll_cycle()
|
||||
if STOP_EVENT.is_set():
|
||||
break
|
||||
elapsed = time.time() - start
|
||||
sleep_for = max(1, CHECK_FREQUENCY - elapsed)
|
||||
# Sleep but wake early if signal received
|
||||
STOP_EVENT.wait(timeout=sleep_for)
|
||||
finally:
|
||||
print("Exporter shutdown complete.")
|
||||
# If any future cleanup is needed, add here.
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user