Add GPU monitoring services and configurations

- Create systemd service templates for llama_exporter, nvidia_exporter, and podman.
- Add Prometheus configuration template for GPU metrics scraping.
- Introduce variables for GPU monitoring role in main.yml.
- Implement tasks for syncing llama models, including user setup and package installation.
- Update podman tasks to ensure proper sudo access and lingering for the podman user.
- Modify inventory to include gpu_monitoring group.
- Add Terraform modules for deploying Vaultwarden, including ingress and service configurations.
- Create Grafana dashboard for real-time GPU monitoring metrics.
- Update secrets and environment files to include Vaultwarden admin token.

ansible/roles/gpu_monitoring/defaults/main.yml

@@ -0,0 +1,68 @@
+---
+# gpu_monitoring defaults
+
+# Prometheus configuration
+prometheus_image: "docker.io/prom/prometheus:latest"
+prometheus_image_tag: "latest"
+prometheus_version: "latest"
+prometheus_port: 9091
+prometheus_container_port: 9090
+prometheus_data_dir: "/mnt/data/prometheus"
+prometheus_retention_days: "30"
+prometheus_retention_size: "10GB"
+prometheus_scrape_interval: "15s"
+prometheus_evaluation_interval: "15s"
+prometheus_scrape_timeout: "10s"
+
+# Prometheus container
+prometheus_container_memory: "1g"
+prometheus_user: prometheus
+prometheus_group: prometheus
+prometheus_user_home: "/mnt/data/prometheus"
+prometheus_network_name: prometheus-net
+prometheus_host_port: "{{ prometheus_port }}"
+prometheus_host_gateway: "host.containers.internal"
+prometheus_extra_groups: []
+prometheus_user_folders:
+  - .prometheus
+  - .cache
+
+# NVIDIA GPU Exporter configuration (NVML-based, works with consumer GPUs)
+nvidia_exporter_enabled: true
+nvidia_exporter_port: 9400
+nvidia_exporter_user: nvidia_exporter
+nvidia_exporter_user_home: "/home/nvidia_exporter"
+nvidia_exporter_install_dir: "/opt/nvidia_exporter"
+nvidia_exporter_venv_path: "/opt/nvidia_exporter/.venv"
+
+# AMD GPU Exporter configuration
+amd_exporter_enabled: true
+amd_exporter_port: 9500
+amd_exporter_user: amd_exporter
+amd_exporter_user_home: "/home/amd_exporter"
+amd_exporter_install_dir: "/opt/amdgpu_exporter"
+amd_exporter_venv_path: "/opt/amdgpu_exporter/.venv"
+amd_exporter_python_version: "python3"
+amd_exporter_sysfs_gpu_pattern: "/sys/class/drm/card?/device/"
+
+# Llama Exporter configuration
+llama_exporter_enabled: true
+llama_exporter_port: 9550
+llama_exporter_user: llama_exporter
+llama_exporter_user_home: "/home/llama_exporter"
+llama_exporter_install_dir: "/opt/llama_exporter"
+llama_exporter_venv_path: "/opt/llama_exporter/.venv"
+llama_exporter_scrape_interval: 15
+llama_exporter_scrape_timeout: 5
+
+# Llama.cpp scrape targets
+llama_exporter_targets: []
+
+# Firewall configuration
+prometheus_fw_port: "{{ prometheus_port }}"
+prometheus_fw_proto: "tcp"
+prometheus_fw_allowed_networks:
+  - cidr: "10.19.4.0/24"
+    comment: "local network"
+  - cidr: "10.5.5.5/32"
+    comment: "Laptop Wireguard"

ansible/roles/gpu_monitoring/files/amd_gpu_exporter.py

@@ -0,0 +1,290 @@
+#!/usr/bin/env python3
+"""AMD GPU sysfs metrics exporter for Prometheus."""
+
+import glob
+import os
+import re
+import time
+import logging
+import json
+import subprocess
+from http.server import HTTPServer, BaseHTTPRequestHandler
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
+logger = logging.getLogger(__name__)
+
+AMD_VENDOR_ID = 0x1002
+
+POLL_INTERVAL = 10
+
+
+def _read_sysfs(path):
+    try:
+        with open(path, "r") as f:
+            raw = f.read().strip()
+            return int(raw, 0)
+    except (OSError, ValueError):
+        return None
+
+
+def _read_first_int(paths):
+    for path in paths:
+        val = _read_sysfs(path)
+        if val is not None:
+            return val
+    return None
+
+
+def _metric_value(entry):
+    if isinstance(entry, dict):
+        value = entry.get("value")
+        if isinstance(value, (int, float)):
+            return float(value)
+    return None
+
+
+def _collect_amd_smi_metrics(results, set_metric):
+    """Populate metrics from amd-smi JSON output. Returns True when data is collected."""
+    cmd = ["amd-smi", "metric", "-m", "-u", "-p", "-t", "--json", "--gpu", "all"]
+    try:
+        proc = subprocess.run(cmd, capture_output=True, text=True, timeout=8, check=True)
+    except (OSError, subprocess.SubprocessError):
+        return False
+
+    try:
+        payload = json.loads(proc.stdout)
+    except (json.JSONDecodeError, TypeError):
+        return False
+
+    gpu_rows = payload.get("gpu_data", []) if isinstance(payload, dict) else []
+    if not gpu_rows:
+        return False
+
+    got_any = False
+    for row in gpu_rows:
+        if not isinstance(row, dict):
+            continue
+        gpu_id = row.get("gpu")
+        if gpu_id is None:
+            continue
+
+        labels = {"gpu": f"amd{gpu_id}", "gpu_name": f"AMD_GPU_{gpu_id}"}
+        set_metric("gpu_exists", labels, 1.0)
+        got_any = True
+
+        usage = row.get("usage", {}) if isinstance(row.get("usage"), dict) else {}
+        gfx_activity = _metric_value(usage.get("gfx_activity"))
+        if gfx_activity is not None:
+            set_metric("gpu_util_percent", labels, gfx_activity)
+
+        umc_activity = _metric_value(usage.get("umc_activity"))
+        if umc_activity is not None:
+            set_metric("memory_used_percent", labels, umc_activity)
+
+        power = row.get("power", {}) if isinstance(row.get("power"), dict) else {}
+        socket_power = _metric_value(power.get("socket_power"))
+        if socket_power is not None:
+            set_metric("power_watts", labels, socket_power)
+
+        temp = row.get("temperature", {}) if isinstance(row.get("temperature"), dict) else {}
+        edge_temp = _metric_value(temp.get("edge"))
+        if edge_temp is not None:
+            set_metric("temperature_celsius", labels, edge_temp)
+
+        mem_usage = row.get("mem_usage", {}) if isinstance(row.get("mem_usage"), dict) else {}
+        used_vram = _metric_value(mem_usage.get("used_vram"))
+        free_vram = _metric_value(mem_usage.get("free_vram"))
+        total_vram = _metric_value(mem_usage.get("total_vram"))
+
+        if used_vram is not None:
+            set_metric("mem_used_bytes", labels, used_vram * 1024 * 1024)
+        if free_vram is not None:
+            set_metric("mem_avail_bytes", labels, free_vram * 1024 * 1024)
+        if total_vram is not None and used_vram is not None and total_vram > 0:
+            set_metric("memory_used_percent", labels, (used_vram / total_vram) * 100.0)
+
+    return got_any
+
+
+def _get_card_ids():
+    paths = sorted(glob.glob("/sys/class/drm/card[0-9]*"))
+    seen = set()
+    card_ids = []
+    for p in paths:
+        name = os.path.basename(p)
+        dev_match = re.match(r"card(\d+)", name)
+        if dev_match:
+            idx = int(dev_match.group(1))
+            if idx not in seen:
+                seen.add(idx)
+                card_ids.append(idx)
+    return card_ids
+
+
+def _is_amd_gpu(idx):
+    vendor_path = f"/sys/class/drm/card{idx}/device/vendor"
+    vendor = _read_sysfs(vendor_path)
+    return vendor == AMD_VENDOR_ID
+
+
+def _get_gpu_name(idx):
+    if _is_amd_gpu(idx):
+        device_id = _read_sysfs(f"/sys/class/drm/card{idx}/device/device")
+        if device_id is not None:
+            return f"AMD_GPU_{device_id:#06x}"
+        return f"AMD_GPU_card{idx}"
+    return f"UNKNOWN_card{idx}"
+
+
+def _collect_metrics():
+    results = {}
+
+    def _set_metric(metric_name, labels, value):
+        label_key = tuple(sorted(labels.items()))
+        results[(metric_name, label_key)] = value
+
+    # Prefer amd-smi for robust telemetry on kernels where some sysfs nodes are busy.
+    if _collect_amd_smi_metrics(results, _set_metric):
+        return results
+
+    card_ids = _get_card_ids()
+
+    if not card_ids:
+        logger.warning("No AMD GPU cards found in sysfs")
+        return results
+
+    for idx in card_ids:
+        if not _is_amd_gpu(idx):
+            continue
+
+        labels = {"gpu": f"card{idx}", "gpu_name": _get_gpu_name(idx)}
+
+        # Power (microwatts in hwmon on many kernels, milliwatts on older path)
+        power = _read_first_int([
+            f"/sys/class/drm/card{idx}/device/power_average",
+            f"/sys/class/drm/card{idx}/device/hwmon/hwmon0/power1_average",
+            f"/sys/class/drm/card{idx}/device/hwmon/hwmon1/power1_average",
+            f"/sys/class/drm/card{idx}/device/hwmon/hwmon2/power1_average",
+            f"/sys/class/drm/card{idx}/device/hwmon/hwmon3/power1_average",
+        ])
+        if power is not None:
+            # Heuristic: hwmon power1_average is usually microwatts.
+            if power > 1_000_000:
+                _set_metric("power_watts", labels, power / 1_000_000.0)
+            else:
+                _set_metric("power_watts", labels, power / 1000.0)
+
+        # Memory used (bytes) - support both gpu_mem_* and mem_info_vram_* layouts
+        mem_used = _read_first_int([
+            f"/sys/class/drm/card{idx}/device/gpu_mem_used_bytes",
+            f"/sys/class/drm/card{idx}/device/mem_info_vram_used",
+        ])
+        if mem_used is not None:
+            _set_metric("mem_used_bytes", labels, mem_used)
+
+        # Memory available (bytes)
+        mem_avail = _read_first_int([
+            f"/sys/class/drm/card{idx}/device/gpu_mem_avail_bytes",
+        ])
+
+        # If total VRAM is available, derive avail = total - used.
+        if mem_avail is None and mem_used is not None:
+            mem_total = _read_first_int([
+                f"/sys/class/drm/card{idx}/device/mem_info_vram_total",
+            ])
+            if mem_total is not None and mem_total >= mem_used:
+                mem_avail = mem_total - mem_used
+
+        if mem_avail is not None:
+            _set_metric("mem_avail_bytes", labels, mem_avail)
+
+        # Some kernels expose direct busy percentage.
+        mem_busy = _read_first_int([
+            f"/sys/class/drm/card{idx}/device/mem_busy_percent",
+        ])
+        if mem_busy is not None:
+            _set_metric("memory_used_percent", labels, float(mem_busy))
+
+        if mem_used is not None and mem_avail is not None and (mem_used + mem_avail) > 0:
+            _set_metric("memory_used_percent", labels, (mem_used / (mem_used + mem_avail)) * 100.0)
+
+        # Temperature (milliCelsius)
+        temp = _read_first_int([
+            f"/sys/class/drm/card{idx}/device/temp_edge_input",
+            f"/sys/class/drm/card{idx}/device/hwmon/hwmon0/temp1_input",
+            f"/sys/class/drm/card{idx}/device/hwmon/hwmon1/temp1_input",
+            f"/sys/class/drm/card{idx}/device/hwmon/hwmon2/temp1_input",
+            f"/sys/class/drm/card{idx}/device/hwmon/hwmon3/temp1_input",
+        ])
+        if temp is not None:
+            _set_metric("temperature_celsius", labels, temp / 1000.0)
+
+        # Clock speed (MHz) - clock[1] = performance state
+        clock_path = f"/sys/class/drm/card{idx}/device/pp_cur_clk"
+        try:
+            with open(clock_path, "r") as f:
+                clocks_raw = f.read().strip()
+                clocks = [int(c) for c in clocks_raw.split()]
+                if len(clocks) >= 2:
+                    _set_metric("gpu_clock_mhz", labels, clocks[1])
+        except (OSError, ValueError):
+            pass
+
+        # Existence info
+        _set_metric("gpu_exists", labels, 1.0)
+
+    return results
+
+
+class MetricsHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path != "/metrics":
+            self.send_response(404)
+            self.end_headers()
+            return
+
+        metrics = _collect_metrics()
+        lines = []
+
+        for (metric_type, label_pairs), value in sorted(metrics.items()):
+            labels = dict(label_pairs)
+            label_str = ",".join(f'{k}="{v}"' for k, v in sorted(labels.items()))
+            if label_str:
+                line = f'{metric_type}{{{label_str}}} {value}'
+            else:
+                line = f"{metric_type} {value}"
+            lines.append(line)
+
+        body = "\n".join(lines) + "\n" if lines else "# no metrics available\n"
+
+        self.send_response(200)
+        self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+        self.end_headers()
+        self.wfile.write(body.encode("utf-8"))
+
+    def log_message(self, format, *args):
+        logger.debug("%s - - %s", self.address_string(), format % args)
+
+
+def main():
+    port = int(os.environ.get("AMD_EXPORTER_PORT", "9500"))
+    host = os.environ.get("AMD_EXPORTER_BIND", "0.0.0.0")
+
+    server = HTTPServer((host, port), MetricsHandler)
+    logger.info("Starting AMD GPU Exporter on %s:%d", host, port)
+    logger.info("Polling sysfs every %d seconds", POLL_INTERVAL)
+
+    def _update_loop():
+        while True:
+            _collect_metrics()
+            time.sleep(POLL_INTERVAL)
+
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        logger.info("Shutting down")
+        server.shutdown()
+
+
+if __name__ == "__main__":
+    main()

ansible/roles/gpu_monitoring/files/llama_exporter.py

@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+"""LLM inference metrics exporter for Prometheus.
+
+Scrape targets:
+    - llama.cpp HTTP API servers (standard /metrics or custom health)
+"""
+
+import json
+import os
+import time
+import logging
+import urllib.request
+import urllib.error
+from http.server import HTTPServer, BaseHTTPRequestHandler
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
+logger = logging.getLogger(__name__)
+
+POLL_INTERVAL = int(os.environ.get("LLAMA_EXPORTER_INTERVAL", "15"))
+SCRAPE_TIMEOUT = int(os.environ.get("LLAMA_EXPORTER_TIMEOUT", "5"))
+
+LLAMA_CPP_DEFAULTS = [
+    {"name": "llama-cpp-8012", "url": "http://localhost:8012"},
+]
+
+
+def _fetch_json(url, timeout=SCRAPE_TIMEOUT):
+    try:
+        req = urllib.request.Request(url, headers={"Accept": "application/json"})
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return json.loads(resp.read().decode("utf-8"))
+    except Exception as e:
+        logger.debug("Failed to fetch %s: %s", url, e)
+        return None
+
+
+def _collect_llama_cpp(target):
+    metrics = {}
+
+    def _set_metric(metric_name, labels, value):
+        label_key = tuple(sorted(labels.items()))
+        metrics[(metric_name, label_key)] = value
+
+    base_url = target["url"].rstrip("/")
+
+    # Check health endpoint
+    health_url = f"{base_url}/health"
+    health = _fetch_json(health_url)
+    if health and isinstance(health, dict):
+        status = health.get("status", "unknown")
+        _set_metric("llama_server_health", {"server": target["name"], "model": health.get("model", "unknown")}, 1.0 if status == "ok" else 0.0)
+    else:
+        _set_metric("llama_server_health", {"server": target["name"], "model": "unknown"}, 0.0)
+
+    # Check /metrics endpoint for llama.cpp built-in metrics
+    metrics_url = f"{base_url}/metrics"
+    try:
+        req = urllib.request.Request(metrics_url, headers={"Accept": "text/plain"})
+        with urllib.request.urlopen(req, timeout=SCRAPE_TIMEOUT) as resp:
+            body = resp.read().decode("utf-8")
+            for line in body.splitlines():
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+                # Parse simple key value pairs
+                if "{" in line:
+                    key_part = line.split("{")[0]
+                    labels_part = line.split("{")[1].rstrip("}").split("}")[0] if "}" in line else ""
+                    value = line.split()[-1] if line.split() else "1"
+                    label_str = ""
+                    if labels_part:
+                        label_str = ",".join(f'{{{k}="{v}"}}' for k, v in (
+                            pair.split("=") for pair in labels_part.split(",") if "=" in pair
+                        ))
+                    _set_metric(key_part.strip(), {"server": target["name"], "raw": labels_part}, float(value) if value.replace(".", "").replace("-", "").isdigit() else value)
+                else:
+                    parts = line.split()
+                    if len(parts) >= 2:
+                        try:
+                            float(parts[1])
+                            _set_metric(parts[0], {"server": target["name"]}, parts[1])
+                        except ValueError:
+                            pass
+    except Exception as e:
+        logger.debug("Failed to scrape %s/metrics: %s", metrics_url, e)
+
+    # Check /model endpoint for model info
+    model_url = f"{base_url}/models"
+    models = _fetch_json(model_url)
+    if models and isinstance(models, dict):
+        model_list = models.get("data", [models]) if "data" in models else [models]
+        for m in model_list:
+            if not isinstance(m, dict):
+                continue
+            model_id = m.get("id", "unknown")
+            _set_metric("llama_models_loaded", {"server": target["name"], "model": model_id}, 1.0)
+            ctx_max = m.get("context_size", m.get("n_ctx", 0))
+            if ctx_max:
+                _set_metric("llama_model_context_max", {"server": target["name"], "model": model_id}, int(ctx_max))
+            trained = m.get("train_tokens", m.get("n_train", 0))
+            if trained:
+                _set_metric("llama_model_tokens_trained", {"server": target["name"], "model": model_id}, int(trained))
+
+    return metrics
+
+
+
+class MetricsHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path != "/metrics":
+            self.send_response(404)
+            self.end_headers()
+            return
+
+        all_metrics = {}
+
+        # Scrape llama.cpp targets
+        targets = []
+        env_targets = os.environ.get("LLAMA_TARGETS", "")
+        if env_targets:
+            try:
+                targets = json.loads(env_targets)
+            except json.JSONDecodeError:
+                logger.warning("Invalid LLAMA_TARGETS JSON, using defaults")
+                targets = []
+
+        if not targets:
+            targets = LLAMA_CPP_DEFAULTS
+
+        for target in targets:
+            if "url" not in target or "name" not in target:
+                target["name"] = target.get("name", "default")
+                target["url"] = target.get("url", "http://localhost:8012")
+            try:
+                all_metrics.update(_collect_llama_cpp(target))
+            except Exception as e:
+                logger.error("Error scraping llama.cpp target %s: %s", target.get("name", "unknown"), e)
+
+        lines = []
+        for (metric_type, label_pairs), value in sorted(all_metrics.items()):
+            labels = dict(label_pairs)
+            if not labels:
+                line = f"{metric_type} {value}"
+            else:
+                label_str = ",".join(f'{k}="{v}"' for k, v in sorted(labels.items()))
+                line = f"{metric_type}{{{label_str}}} {value}"
+            lines.append(line)
+
+        body = "\n".join(lines) + "\n" if lines else "# no metrics available\n"
+
+        self.send_response(200)
+        self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+        self.end_headers()
+        self.wfile.write(body.encode("utf-8"))
+
+    def log_message(self, format, *args):
+        logger.debug("%s - - %s", self.address_string(), format % args)
+
+
+def main():
+    port = int(os.environ.get("LLAMA_EXPORTER_PORT", "9550"))
+    host = os.environ.get("LLAMA_EXPORTER_BIND", "0.0.0.0")
+
+    server = HTTPServer((host, port), MetricsHandler)
+    logger.info("Starting Llama Exporter on %s:%d", host, port)
+    logger.info("Polling every %d seconds, timeout %ds", POLL_INTERVAL, SCRAPE_TIMEOUT)
+
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        logger.info("Shutting down")
+        server.shutdown()
+
+
+if __name__ == "__main__":
+    main()

ansible/roles/gpu_monitoring/files/nvidia_gpu_exporter.py

@@ -0,0 +1,436 @@
+# nvidia_gpu_exporter - NVIDIA GPU Prometheus exporter using NVML (nvidia-ml-py)
+# Works with consumer GPUs (RTX series, GTX series) and datacenter GPUs
+#
+# Exposes metrics via HTTP at /metrics
+# Required package: nvidia-ml-py (pip)
+
+import http.server
+import json
+import threading
+import time
+import sys
+import signal
+import traceback
+import subprocess
+import csv
+
+try:
+    import pynvml
+except ImportError:
+    print("ERROR: pynvml not installed. Run: pip install nvidia-ml-py", file=sys.stderr)
+    sys.exit(1)
+
+# Globals
+port = 9400
+stop_event = threading.Event()
+nvidia_initialized = False
+gpus_detected = 0
+gpu_handles = {}
+gpu_names = {}
+last_metrics = {}
+last_metrics_time = 0
+metrics_lock = threading.Lock()
+
+
+def signal_handler(sig, frame):
+    stop_event.set()
+
+
+def format_value(value, dtype):
+    """Format a value based on its NVML type."""
+    if value is None:
+        return "0"
+    try:
+        if dtype == pynvml.NVML_FEATURE_COUNTER_DATA_TYPE_UINT32:
+            return str(int(value))
+        elif dtype == pynvml.NVML_FEATURE_COUNTER_DATA_TYPE_UINT64:
+            return str(int(value))
+        elif dtype == pynvml.NVML_FEATURE_COUNTER_DATA_TYPE_FLOAT:
+            return f"{float(value):.6f}"
+        else:
+            return str(value)
+    except (ValueError, TypeError):
+        return "0"
+
+
+def extract_nvml_stats_pair(stats):
+    """Return (session_count, avg_fps) from NVML stats object or tuple."""
+    if stats is None:
+        return 0.0, 0.0
+
+    # Newer bindings may return an object with named fields.
+    if hasattr(stats, "sessionCount") and hasattr(stats, "avgFps"):
+        return float(stats.sessionCount), float(stats.avgFps)
+
+    # Older bindings may return a tuple/list.
+    if isinstance(stats, (tuple, list)) and len(stats) >= 2:
+        return float(stats[0]), float(stats[1])
+
+    return 0.0, 0.0
+
+
+def _to_float_or_none(value):
+    if value is None:
+        return None
+    raw = str(value).strip()
+    if raw == "" or raw.upper() == "N/A":
+        return None
+    try:
+        return float(raw)
+    except ValueError:
+        return None
+
+
+def _to_bool_float_or_none(value):
+    if value is None:
+        return None
+    raw = str(value).strip().lower()
+    if raw in {"1", "true", "yes", "active", "enabled"}:
+        return 1.0
+    if raw in {"0", "false", "no", "not active", "disabled", "n/a"}:
+        return 0.0
+    return None
+
+
+def collect_nvidia_smi_metrics():
+    """Collect supplemental metrics via nvidia-smi keyed by GPU index."""
+    fields = [
+        "index",
+        "pstate",
+        "pcie.link.gen.current",
+        "pcie.link.width.current",
+        "clocks_event_reasons.sw_power_cap",
+        "clocks_event_reasons.hw_thermal_slowdown",
+        "clocks_event_reasons.hw_power_brake_slowdown",
+    ]
+
+    cmd = [
+        "nvidia-smi",
+        f"--query-gpu={','.join(fields)}",
+        "--format=csv,noheader,nounits",
+    ]
+
+    try:
+        proc = subprocess.run(cmd, capture_output=True, text=True, timeout=5, check=True)
+    except (OSError, subprocess.SubprocessError):
+        return {}
+
+    metrics_by_index = {}
+    reader = csv.reader(proc.stdout.splitlines())
+    for row in reader:
+        if len(row) < len(fields):
+            continue
+
+        idx_raw = row[0].strip()
+        if not idx_raw.isdigit():
+            continue
+        idx = int(idx_raw)
+
+        smi_metrics = {}
+
+        pstate_raw = row[1].strip()
+        if pstate_raw.startswith("P") and pstate_raw[1:].isdigit():
+            smi_metrics["pstate"] = float(int(pstate_raw[1:]))
+
+        pcie_gen = _to_float_or_none(row[2])
+        if pcie_gen is not None:
+            smi_metrics["pcie_link_gen_current"] = pcie_gen
+
+        pcie_width = _to_float_or_none(row[3])
+        if pcie_width is not None:
+            smi_metrics["pcie_link_width_current"] = pcie_width
+
+        sw_power_cap = _to_bool_float_or_none(row[4])
+        if sw_power_cap is not None:
+            smi_metrics["throttle_sw_power_cap"] = sw_power_cap
+
+        hw_thermal = _to_bool_float_or_none(row[5])
+        if hw_thermal is not None:
+            smi_metrics["throttle_hw_thermal_slowdown"] = hw_thermal
+
+        hw_power_brake = _to_bool_float_or_none(row[6])
+        if hw_power_brake is not None:
+            smi_metrics["throttle_hw_power_brake_slowdown"] = hw_power_brake
+
+        if smi_metrics:
+            metrics_by_index[idx] = smi_metrics
+
+    return metrics_by_index
+
+
+def get_gpu_metrics():
+    """Collect all GPU metrics from NVML."""
+    global gpus_detected, gpu_handles, gpu_names, last_metrics, last_metrics_time, nvidia_initialized
+
+    if not nvidia_initialized:
+        return {}
+
+    metrics = {}
+    now = time.time()
+    current_time = int(now)
+
+    # Global metrics
+    metrics["gpu_count"] = float(gpus_detected)
+    metrics["instance"] = "0"
+
+    smi_metrics_by_index = collect_nvidia_smi_metrics()
+
+    for i in range(gpus_detected):
+        try:
+            handle = gpu_handles.get(i)
+            name = gpu_names.get(i, "Unknown")
+            gpu_prefix = f"gpu_{i}"
+
+            # Basic info (set once)
+            if f"{gpu_prefix}_info" not in last_metrics or current_time == 0:
+                metrics[f"{gpu_prefix}_info"] = 1.0
+                metrics[f"{gpu_prefix}_name"] = float(name_to_float(name))
+
+            # Power
+            try:
+                power_usage = pynvml.nvmlDeviceGetPowerUsage(handle)
+                metrics[f"{gpu_prefix}_power_watts"] = power_usage / 1000.0
+            except pynvml.NVMLError:
+                metrics[f"{gpu_prefix}_power_watts"] = 0.0
+
+            try:
+                power_limit = pynvml.nvmlDeviceGetPowerManagementLimit(handle)
+                if power_limit:
+                    metrics[f"{gpu_prefix}_power_limit_watts"] = power_limit / 1000.0
+            except pynvml.NVMLError:
+                pass
+
+            # Temperature
+            try:
+                temp = pynvml.nvmlDeviceGetTemperature(handle, pynvml.NVML_TEMPERATURE_GPU)
+                metrics[f"{gpu_prefix}_temperature_celsius"] = float(temp)
+            except pynvml.NVMLError:
+                metrics[f"{gpu_prefix}_temperature_celsius"] = 0.0
+
+            # Memory
+            try:
+                mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle)
+                metrics[f"{gpu_prefix}_memory_used_bytes"] = mem_info.used
+                metrics[f"{gpu_prefix}_memory_total_bytes"] = mem_info.total
+                if mem_info.total > 0:
+                    metrics[f"{gpu_prefix}_memory_used_percent"] = (mem_info.used / mem_info.total) * 100.0
+            except pynvml.NVMLError:
+                metrics[f"{gpu_prefix}_memory_used_bytes"] = 0.0
+                metrics[f"{gpu_prefix}_memory_total_bytes"] = 0.0
+                metrics[f"{gpu_prefix}_memory_used_percent"] = 0.0
+
+            # Utilization
+            try:
+                util = pynvml.nvmlDeviceGetUtilizationRates(handle)
+                metrics[f"{gpu_prefix}_gpu_util_percent"] = float(util.gpu)
+                metrics[f"{gpu_prefix}_memory_util_percent"] = float(util.memory)
+            except pynvml.NVMLError:
+                metrics[f"{gpu_prefix}_gpu_util_percent"] = 0.0
+                metrics[f"{gpu_prefix}_memory_util_percent"] = 0.0
+
+            # Clock speeds
+            try:
+                clocks = pynvml.nvmlDeviceGetClockInfo(handle, pynvml.NVML_CLOCK_GRAPHICS)
+                metrics[f"{gpu_prefix}_clock_graphics_hz"] = float(clocks) * 1000000
+            except pynvml.NVMLError:
+                metrics[f"{gpu_prefix}_clock_graphics_hz"] = 0.0
+
+            try:
+                # pynvml exposes memory clock as NVML_CLOCK_MEM in some versions.
+                mem_clock_type = getattr(pynvml, "NVML_CLOCK_MEMORY", None)
+                if mem_clock_type is None:
+                    mem_clock_type = getattr(pynvml, "NVML_CLOCK_MEM", None)
+                if mem_clock_type is None:
+                    raise AttributeError("No NVML memory clock constant available")
+                mem_clock = pynvml.nvmlDeviceGetClockInfo(handle, mem_clock_type)
+                metrics[f"{gpu_prefix}_clock_memory_hz"] = float(mem_clock) * 1000000
+            except (pynvml.NVMLError, AttributeError):
+                metrics[f"{gpu_prefix}_clock_memory_hz"] = 0.0
+
+            try:
+                vol_clock = pynvml.nvmlDeviceGetClockInfo(handle, pynvml.NVML_CLOCK_VIDEO)
+                metrics[f"{gpu_prefix}_clock_video_hz"] = float(vol_clock) * 1000000
+            except pynvml.NVMLError:
+                metrics[f"{gpu_prefix}_clock_video_hz"] = 0.0
+
+            # Fan speed
+            try:
+                fan = pynvml.nvmlDeviceGetFanSpeed(handle)
+                metrics[f"{gpu_prefix}_fan_speed_percent"] = float(fan)
+            except pynvml.NVMLError:
+                metrics[f"{gpu_prefix}_fan_speed_percent"] = 0.0
+
+            # PCIe stats
+            try:
+                tx = pynvml.nvmlDeviceGetPcieThroughputStats(handle)
+                if hasattr(tx, 'tx_bytes'):
+                    metrics[f"{gpu_prefix}_pcie_tx_bytes_total"] = float(tx.tx_bytes)
+                    metrics[f"{gpu_prefix}_pcie_rx_bytes_total"] = float(tx.rx_bytes)
+            except (pynvml.NVMLError, AttributeError):
+                pass
+
+            # Compute/pending processes
+            try:
+                processes = pynvml.nvmlDeviceGetComputeRunningProcesses(handle)
+                metrics[f"{gpu_prefix}_processes"] = float(len(processes))
+            except pynvml.NVMLError:
+                metrics[f"{gpu_prefix}_processes"] = 0.0
+
+            # Encoder/Decoder stats
+            try:
+                enc_stats = pynvml.nvmlDeviceGetEncoderStats(handle)
+                enc_sessions, enc_avg_fps = extract_nvml_stats_pair(enc_stats)
+                metrics[f"{gpu_prefix}_encoder_sessions"] = enc_sessions
+                metrics[f"{gpu_prefix}_encoder_avg_fps"] = enc_avg_fps
+            except (pynvml.NVMLError, AttributeError, TypeError, ValueError):
+                metrics[f"{gpu_prefix}_encoder_sessions"] = 0.0
+                metrics[f"{gpu_prefix}_encoder_avg_fps"] = 0.0
+
+            try:
+                dec_stats = pynvml.nvmlDeviceGetDecoderStats(handle)
+                dec_sessions, dec_avg_fps = extract_nvml_stats_pair(dec_stats)
+                metrics[f"{gpu_prefix}_decoder_sessions"] = dec_sessions
+                metrics[f"{gpu_prefix}_decoder_avg_fps"] = dec_avg_fps
+            except (pynvml.NVMLError, AttributeError, TypeError, ValueError):
+                metrics[f"{gpu_prefix}_decoder_sessions"] = 0.0
+                metrics[f"{gpu_prefix}_decoder_avg_fps"] = 0.0
+
+            # Reboot/Pwr events
+            try:
+                inforom_fn = getattr(pynvml, "nvmlDeviceGetInforomConfigurationVersion", None)
+                if inforom_fn is None:
+                    raise AttributeError("Inforom API not available in this pynvml version")
+                events = inforom_fn(handle)
+                if events:
+                    metrics[f"{gpu_prefix}_firmware_version"] = 1.0
+            except (pynvml.NVMLError, AttributeError):
+                pass
+
+            # Supplemental nvidia-smi metrics (when available).
+            smi_metrics = smi_metrics_by_index.get(i, {})
+            for name, value in smi_metrics.items():
+                metrics[f"{gpu_prefix}_{name}"] = value
+
+        except pynvml.NVMLError:
+            traceback.print_exc()
+            continue
+
+    last_metrics = metrics
+    last_metrics_time = current_time
+    return metrics
+
+
+def name_to_float(name):
+    """Convert GPU name to a numeric value for prometheus."""
+    # Prometheus metrics don't support string values, so we use hash
+    return float(hash(name)) % 1000000
+
+
+def generate_prometheus_output(metrics):
+    """Format metrics as Prometheus text exposition format."""
+    lines = []
+    lines.append("# HELP gpu_info NVIDIA GPU information")
+    lines.append("# TYPE gpu_info gauge")
+    for i in range(gpus_detected):
+        lines.append(f'gpu_info{{gpu="{gpu_names.get(i, "Unknown")}",gpu_id="{i}"}} 1')
+
+    for key, value in sorted(metrics.items()):
+        # Skip info metric (handled above)
+        if key == "gpu_count":
+            lines.append(f"# HELP gpu_total Total number of NVIDIA GPUs")
+            lines.append("# TYPE gpu_total gauge")
+            lines.append(f"gpu_total {value}")
+            continue
+
+        if key == "instance":
+            continue
+
+        if key.endswith("_name"):
+            continue
+
+        if key.endswith("_info"):
+            continue
+
+        lines.append(f"# HELP {key} {key} metric")
+        lines.append("# TYPE " + key + " gauge")
+        if isinstance(value, float):
+            lines.append(f"{key} {value:.6f}")
+        else:
+            lines.append(f"{key} {value}")
+
+    return "\n".join(lines) + "\n"
+
+
+class MetricsHandler(http.server.BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path == "/metrics":
+            with metrics_lock:
+                metrics = get_gpu_metrics()
+            output = generate_prometheus_output(metrics)
+            self.send_response(200)
+            self.send_header("Content-Type", "text/plain; charset=utf-8")
+            self.end_headers()
+            self.wfile.write(output.encode())
+        elif self.path == "/health":
+            self.send_response(200)
+            self.send_header("Content-Type", "text/plain")
+            self.end_headers()
+            self.wfile.write("OK\n".encode())
+        else:
+            self.send_response(404)
+            self.end_headers()
+
+    def log_message(self, format, *args):
+        # Suppress default logging
+        pass
+
+
+def run_server():
+    global port, stop_event, nvidia_initialized, gpus_detected, gpu_handles, gpu_names
+
+    try:
+        pynvml.nvmlInit()
+        nvidia_initialized = True
+    except pynvml.NVMLError as e:
+        print(f"ERROR: Failed to initialize NVML: {e}", file=sys.stderr)
+        print("Make sure NVIDIA drivers are properly installed.", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        gpus_detected = pynvml.nvmlDeviceGetCount()
+        print(f"Found {gpus_detected} NVIDIA GPU(s)")
+
+        for i in range(gpus_detected):
+            try:
+                handle = pynvml.nvmlDeviceGetHandleByIndex(i)
+                gpu_handles[i] = handle
+                name = pynvml.nvmlDeviceGetName(handle)
+                if isinstance(name, bytes):
+                    name = name.decode("utf-8")
+                gpu_names[i] = name
+                print(f"  GPU {i}: {name}")
+            except pynvml.NVMLError:
+                pass
+    except pynvml.NVMLError as e:
+        print(f"ERROR: Failed to enumerate devices: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    server = http.server.HTTPServer(("0.0.0.0", port), MetricsHandler)
+    print(f"NVIDIA GPU Exporter listening on port {port}")
+
+    signal.signal(signal.SIGTERM, signal_handler)
+    signal.signal(signal.SIGINT, signal_handler)
+
+    # Poll so SIGTERM can stop the loop quickly instead of waiting on long timeouts.
+    server.timeout = 1
+
+    try:
+        while not stop_event.is_set():
+            server.handle_request()
+    finally:
+        server.server_close()
+        pynvml.nvmlShutdown()
+
+
+if __name__ == "__main__":
+    run_server()

ansible/roles/gpu_monitoring/handlers/main.yml

@@ -0,0 +1,27 @@
+---
+- name: Reload systemd daemon
+  become: true
+  ansible.builtin.systemd:
+    daemon_reload: true
+  listen: Reload systemd daemon
+
+- name: Restart nvidia_exporter
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ nvidia_exporter_service_name }}"
+    state: restarted
+  listen: Restart nvidia_exporter
+
+- name: Restart amd_exporter
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ amd_exporter_service_name }}"
+    state: restarted
+  listen: Restart amd_exporter
+
+- name: Restart llama_exporter
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ llama_exporter_service_name }}"
+    state: restarted
+  listen: Restart llama_exporter

ansible/roles/gpu_monitoring/meta/main.yml

@@ -0,0 +1,15 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Deploy Prometheus monitoring stack on gpu-01 for GPU and LLM metrics
+  company: A13Labs
+  role_name: gpu_monitoring
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: Fedora
+      versions:
+        - "39"
+        - "40"
+        - "41"

ansible/roles/gpu_monitoring/molecule/default/converge.yml

@@ -0,0 +1,11 @@
+---
+- name: Converge
+  hosts: all
+  gather_facts: true
+  tasks:
+    - name: Include gpu_monitoring role
+      ansible.builtin.include_role:
+        name: "gpu_monitoring"
+      vars:
+        amd_exporter_enabled: false
+        llama_exporter_enabled: false

ansible/roles/gpu_monitoring/molecule/default/molecule.yml

@@ -0,0 +1,17 @@
+---
+driver:
+  name: podman
+platforms:
+  - name: instance
+    image: fedora:latest
+    pre_build_image: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    privileged: true
+provisioner:
+  name: ansible
+  playbooks:
+    converge: converge.yml
+    prepare: prepare.yml
+verifier:
+  name: ansible

ansible/roles/gpu_monitoring/molecule/default/prepare.yml

@@ -0,0 +1,10 @@
+---
+- name: Prepare
+  hosts: all
+  gather_facts: false
+  tasks:
+    - name: Update cache
+      ansible.builtin.raw: dnf update -y
+
+    - name: Install required packages
+      ansible.builtin.raw: dnf install -y python3

ansible/roles/gpu_monitoring/tasks/amd_exporter.yml

@@ -0,0 +1,82 @@
+---
+- name: Create amd_exporter group
+  become: true
+  ansible.builtin.group:
+    name: "{{ amd_exporter_user }}"
+    state: present
+    system: true
+
+- name: Create amd_exporter user
+  become: true
+  ansible.builtin.user:
+    name: "{{ amd_exporter_user }}"
+    group: "{{ amd_exporter_user }}"
+    home: "{{ amd_exporter_user_home }}"
+    create_home: true
+    shell: /usr/sbin/nologin
+    system: true
+
+- name: Create amd_exporter install directory
+  become: true
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ amd_exporter_user }}"
+    group: "{{ amd_exporter_user }}"
+    mode: "0755"
+  loop:
+    - "{{ amd_exporter_install_dir }}"
+    - "{{ amd_exporter_install_dir }}/logs"
+
+- name: Copy amd_gpu_exporter script
+  become: true
+  ansible.builtin.copy:
+    src: amd_gpu_exporter.py
+    dest: "{{ amd_exporter_script_path }}"
+    owner: "{{ amd_exporter_user }}"
+    group: "{{ amd_exporter_user }}"
+    mode: "0755"
+  notify:
+    - Restart amd_exporter
+
+- name: Create Python virtual environment
+  become: true
+  ansible.builtin.command:
+    cmd: "{{ amd_exporter_python_version }} -m venv {{ amd_exporter_venv_path }}"
+    creates: "{{ amd_exporter_venv_path }}/bin/activate"
+
+- name: Install Python dependencies in venv
+  become: true
+  ansible.builtin.pip:
+    name:
+      - prometheus-client
+      - requests
+    virtualenv: "{{ amd_exporter_venv_path }}"
+    state: present
+
+- name: Fix ownership of amd_exporter directory
+  become: true
+  ansible.builtin.file:
+    path: "{{ amd_exporter_install_dir }}"
+    owner: "{{ amd_exporter_user }}"
+    group: "{{ amd_exporter_user }}"
+    recurse: true
+
+- name: Generate amd_exporter systemd service
+  become: true
+  ansible.builtin.template:
+    src: amdgpu_exporter.service.j2
+    dest: "/etc/systemd/system/{{ amd_exporter_service_name }}.service"
+    owner: root
+    group: root
+    mode: "0644"
+  notify:
+    - Reload systemd daemon
+    - Restart amd_exporter
+
+- name: Enable amd_exporter service
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ amd_exporter_service_name }}"
+    enabled: true
+    state: started

ansible/roles/gpu_monitoring/tasks/firewall.yml

@@ -0,0 +1,13 @@
+---
+- name: Add Prometheus firewall rules
+  become: true
+  community.general.ufw:
+    rule: allow
+    src: "{{ item.cidr }}"
+    port: "{{ prometheus_fw_port | string }}"
+    proto: "{{ prometheus_fw_proto }}"
+    comment: "{{ item.comment | default(omit) }}"
+  loop: "{{ prometheus_fw_allowed_networks }}"
+  loop_control:
+    label: "{{ item.cidr }} -> port {{ prometheus_fw_port }}"
+  when: item.cidr is defined and item.cidr | length > 0

ansible/roles/gpu_monitoring/tasks/llama_exporter.yml

@@ -0,0 +1,82 @@
+---
+- name: Create llama_exporter group
+  become: true
+  ansible.builtin.group:
+    name: "{{ llama_exporter_user }}"
+    state: present
+    system: true
+
+- name: Create llama_exporter user
+  become: true
+  ansible.builtin.user:
+    name: "{{ llama_exporter_user }}"
+    group: "{{ llama_exporter_user }}"
+    home: "{{ llama_exporter_user_home }}"
+    create_home: true
+    shell: /usr/sbin/nologin
+    system: true
+
+- name: Create llama_exporter install directory
+  become: true
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ llama_exporter_user }}"
+    group: "{{ llama_exporter_user }}"
+    mode: "0755"
+  loop:
+    - "{{ llama_exporter_install_dir }}"
+    - "{{ llama_exporter_install_dir }}/logs"
+
+- name: Copy llama_exporter script
+  become: true
+  ansible.builtin.copy:
+    src: llama_exporter.py
+    dest: "{{ llama_exporter_script_path }}"
+    owner: "{{ llama_exporter_user }}"
+    group: "{{ llama_exporter_user }}"
+    mode: "0755"
+  notify:
+    - Restart llama_exporter
+
+- name: Create Python virtual environment
+  become: true
+  ansible.builtin.command:
+    cmd: "{{ ansible_facts['python']['executable'] }} -m venv {{ llama_exporter_venv_path }}"
+    creates: "{{ llama_exporter_venv_path }}/bin/activate"
+
+- name: Install Python dependencies in llama_exporter venv
+  become: true
+  ansible.builtin.pip:
+    name:
+      - prometheus-client
+      - requests
+    virtualenv: "{{ llama_exporter_venv_path }}"
+    state: present
+
+- name: Fix ownership of llama_exporter directory
+  become: true
+  ansible.builtin.file:
+    path: "{{ llama_exporter_install_dir }}"
+    owner: "{{ llama_exporter_user }}"
+    group: "{{ llama_exporter_user }}"
+    recurse: true
+
+- name: Generate llama_exporter systemd service
+  become: true
+  ansible.builtin.template:
+    src: llama_exporter.service.j2
+    dest: "/etc/systemd/system/{{ llama_exporter_service_name }}.service"
+    owner: root
+    group: root
+    mode: "0644"
+  notify:
+    - Reload systemd daemon
+    - Restart llama_exporter
+
+- name: Enable llama_exporter service
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ llama_exporter_service_name }}"
+    enabled: true
+    state: started

ansible/roles/gpu_monitoring/tasks/main.yml

@@ -0,0 +1,28 @@
+---
+- name: Include NVIDIA GPU Exporter setup
+  when: nvidia_exporter_enabled | default(true) | bool
+  ansible.builtin.include_tasks: nvidia_exporter.yml
+  tags:
+    - roles::gpu_monitoring::nvidia_exporter
+
+- name: Include AMD GPU Exporter setup
+  when: amd_exporter_enabled | default(true) | bool
+  ansible.builtin.include_tasks: amd_exporter.yml
+  tags:
+    - roles::gpu_monitoring::amd_exporter
+
+- name: Include Llama Exporter setup
+  when: llama_exporter_enabled | default(true) | bool
+  ansible.builtin.include_tasks: llama_exporter.yml
+  tags:
+    - roles::gpu_monitoring::llama_exporter
+
+- name: Include Prometheus setup
+  ansible.builtin.include_tasks: prometheus.yml
+  tags:
+    - roles::gpu_monitoring::prometheus
+
+- name: Include firewall rules
+  ansible.builtin.include_tasks: firewall.yml
+  tags:
+    - roles::gpu_monitoring::firewall

ansible/roles/gpu_monitoring/tasks/nvidia_exporter.yml

@@ -0,0 +1,84 @@
+---
+# NVIDIA GPU Exporter using NVML (pynvml)
+# Works with consumer GPUs (RTX/GTX) and datacenter GPUs
+# No DCGM required - uses pynvml Python library
+
+- name: Create NVIDIA exporter group
+  ansible.builtin.group:
+    name: "{{ nvidia_exporter_user }}"
+    state: present
+    system: true
+  become: true
+
+- name: Create NVIDIA exporter user
+  ansible.builtin.user:
+    name: "{{ nvidia_exporter_user }}"
+    group: "{{ nvidia_exporter_user }}"
+    home: "{{ nvidia_exporter_user_home }}"
+    system: true
+    create_home: true
+    shell: /usr/sbin/nologin
+  become: true
+
+- name: Create NVIDIA exporter install directory
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ nvidia_exporter_user }}"
+    group: "{{ nvidia_exporter_user }}"
+    mode: "0755"
+  loop:
+    - "{{ nvidia_exporter_install_dir }}"
+    - "{{ nvidia_exporter_install_dir }}/logs"
+  become: true
+
+- name: Copy NVIDIA exporter script
+  ansible.builtin.copy:
+    src: nvidia_gpu_exporter.py
+    dest: "{{ nvidia_exporter_script_path }}"
+    owner: "{{ nvidia_exporter_user }}"
+    group: "{{ nvidia_exporter_user }}"
+    mode: "0755"
+  become: true
+  notify:
+    - Restart nvidia_exporter
+
+- name: Create Python virtual environment for NVIDIA exporter
+  ansible.builtin.command:
+    cmd: "{{ ansible_facts['python']['executable'] }} -m venv {{ nvidia_exporter_venv_path }}"
+    creates: "{{ nvidia_exporter_venv_path }}/bin/activate"
+  become: true
+
+- name: Install nvidia-ml-py in NVIDIA exporter venv
+  ansible.builtin.pip:
+    name: "nvidia-ml-py"
+    virtualenv: "{{ nvidia_exporter_venv_path }}"
+    state: present
+  become: true
+
+- name: Fix ownership of NVIDIA exporter directory
+  ansible.builtin.file:
+    path: "{{ nvidia_exporter_install_dir }}"
+    owner: "{{ nvidia_exporter_user }}"
+    group: "{{ nvidia_exporter_user }}"
+    recurse: true
+  become: true
+
+- name: Copy NVIDIA exporter systemd service
+  ansible.builtin.template:
+    src: nvidia_exporter.service.j2
+    dest: "/etc/systemd/system/{{ nvidia_exporter_service_name }}.service"
+    owner: root
+    group: root
+    mode: "0644"
+  become: true
+  notify:
+    - Reload systemd daemon
+    - Restart nvidia_exporter
+
+- name: Ensure NVIDIA exporter is enabled and started
+  ansible.builtin.systemd:
+    name: "{{ nvidia_exporter_service_name }}"
+    enabled: true
+    state: started
+  become: true

ansible/roles/gpu_monitoring/tasks/prometheus.yml

@@ -0,0 +1,206 @@
+---
+# Prometheus - Podman container with standard podman.yml pattern
+# Follows the pattern in playbook_podman_llama_vulkan.yml
+
+- name: Prometheus | Check if required vars are set
+  ansible.builtin.fail:
+    msg: "Required variables are not set for Podman setup. Please check your playbook variables.)"
+  when: prometheus_user is not defined or prometheus_group is not defined or prometheus_user_home is not defined
+  tags:
+    - prometheus
+
+- name: Prometheus | Create prometheus group
+  become: true
+  ansible.builtin.group:
+    name: "{{ prometheus_group }}"
+    system: true
+  tags:
+    - prometheus
+
+- name: Prometheus | Create prometheus user
+  become: true
+  ansible.builtin.user:
+    name: "{{ prometheus_user }}"
+    groups: "{{ prometheus_extra_groups | default([]) }}"
+    shell: /bin/bash
+    home: "{{ prometheus_user_home }}"
+    group: "{{ prometheus_group }}"
+  tags:
+    - prometheus
+
+- name: Prometheus | Disable password login for prometheus user
+  ansible.builtin.command: passwd -d "{{ prometheus_user }}"
+  become: true
+  changed_when: false
+  tags:
+    - prometheus
+
+- name: Prometheus | Create prometheus user home
+  become: true
+  ansible.builtin.file:
+    path: "{{ prometheus_user_home }}"
+    state: directory
+    owner: "{{ prometheus_user }}"
+    group: "{{ prometheus_group }}"
+    mode: "0750"
+  tags:
+    - prometheus
+
+- name: Prometheus | Create build folders
+  become: true
+  ansible.builtin.file:
+    path: "{{ prometheus_user_home }}/build"
+    state: directory
+    owner: "{{ prometheus_user }}"
+    group: "{{ prometheus_group }}"
+    mode: "0750"
+  tags:
+    - prometheus
+
+- name: Prometheus | Create folders
+  become: true
+  ansible.builtin.file:
+    path: "{{ prometheus_user_home }}/{{ item }}"
+    state: directory
+    owner: "{{ prometheus_user }}"
+    group: "{{ prometheus_group }}"
+    mode: "0750"
+  loop: "{{ prometheus_user_folders | default([]) }}"
+  tags:
+    - prometheus
+
+- name: Prometheus | Add SSH Authorized key
+  become: true
+  ansible.posix.authorized_key:
+    user: "{{ prometheus_user }}"
+    key: "{{ prometheus_user_pubkey }}"
+    state: "{{ 'present' if prometheus_user_pubkey is defined else 'absent' }}"
+  when: prometheus_user_pubkey is defined and prometheus_user_pubkey != ""
+  tags:
+    - prometheus
+
+- name: Prometheus | SELinux tasks (RedHat only)
+  become: true
+  when: ansible_os_family == 'RedHat'
+  tags:
+    - prometheus
+  block:
+    - name: Prometheus | Persistently set SELinux context
+      community.general.sefcontext:
+        target: "{{ prometheus_user_home }}(/.*)?"
+        setype: user_home_dir_t
+        state: present
+
+    - name: Prometheus | Persistently set SELinux context (ssh authorized keys)
+      community.general.sefcontext:
+        target: "{{ prometheus_user_home }}/.ssh/authorized_keys"
+        setype: ssh_home_t
+        state: present
+      when: prometheus_user_pubkey is defined and prometheus_user_pubkey != ""
+
+    - name: Prometheus | Apply SELinux context
+      ansible.builtin.command: restorecon -Rv {{ prometheus_user_home }}
+      changed_when: false
+
+- name: Prometheus | Enable lingering for prometheus user
+  become: true
+  ansible.builtin.command: loginctl enable-linger {{ prometheus_user }}
+  register: linger_status
+  changed_when: >-
+    'created' in linger_status.stdout or
+    (linger_status.rc == 0 and not
+      ('linger file already exists' in linger_status.stderr or
+      'linger file does not exist' in linger_status.stderr))
+  failed_when:
+    - linger_status.rc != 0 and not
+      ('linger file already exists' in linger_status.stderr)
+  tags:
+    - prometheus
+
+- name: Prometheus | Get prometheus user UID
+  ansible.builtin.command: id -u {{ prometheus_user }}
+  changed_when: false
+  register: prometheus_uid
+  tags:
+    - prometheus
+
+- name: Prometheus | Deploy Prometheus configuration
+  ansible.builtin.template:
+    src: prometheus.yml.j2
+    dest: "{{ prometheus_user_home }}/prometheus.yml"
+    owner: "{{ prometheus_user }}"
+    group: "{{ prometheus_group }}"
+    mode: "0644"
+  tags:
+    - prometheus
+
+- name: Prometheus | Create Prometheus data directory
+  ansible.builtin.file:
+    path: "{{ prometheus_data_dir }}"
+    state: directory
+    owner: "{{ prometheus_user }}"
+    group: "{{ prometheus_group }}"
+    mode: "0700"
+  tags:
+    - prometheus
+
+- name: Prometheus | Run Podman tasks as prometheus user
+  become: true
+  become_user: "{{ prometheus_user }}"
+  environment:
+    XDG_RUNTIME_DIR: "/run/user/{{ prometheus_uid.stdout }}"
+  tags:
+    - prometheus
+  block:
+    - name: Prometheus | Pull Prometheus image
+      containers.podman.podman_image:
+        name: "docker.io/prom/prometheus:{{ prometheus_image_tag }}"
+        force: false
+
+    - name: Prometheus | Create podman network
+      containers.podman.podman_network:
+        name: "{{ prometheus_network_name }}"
+
+    - name: Prometheus | Create Podman container
+      containers.podman.podman_container:
+        name: prometheus
+        image: "docker.io/prom/prometheus:{{ prometheus_image_tag }}"
+        state: started
+        recreate: true
+        user: "{{ prometheus_uid.stdout }}"
+        network: "{{ prometheus_network_name }}"
+        network_alias:
+          - prometheus
+        ports:
+          - "0.0.0.0:{{ prometheus_host_port }}:{{ prometheus_container_port }}/tcp"
+        volumes:
+          - "{{ prometheus_data_dir }}:/prometheus"
+          - "{{ prometheus_user_home }}/prometheus.yml:/etc/prometheus/prometheus.yml"
+        memory: "{{ prometheus_container_memory }}"
+        memory_swap: "0"
+        env:
+          TZ: "{{ timezone | default('UTC') }}"
+          PROMETHEUS_PORT: "{{ prometheus_port | string }}"
+
+        systemd: "true"
+        restart: false
+        detach: true
+        label:
+          a13labs.prometheus: "true"
+          a13labs.service: "prometheus"
+          a13labs.team: "infra"
+        cmd_args:
+          - "--userns=keep-id"
+          - "--security-opt=label=disable"
+        restart_policy: unless-stopped
+        stop_signal: 1
+- name: Prometheus | Ensure systemd is reloaded after container creation
+  become: true
+  become_user: "{{ prometheus_user }}"
+  environment:
+    XDG_RUNTIME_DIR: "/run/user/{{ prometheus_uid.stdout }}"
+  ansible.builtin.systemd:
+    daemon_reload: true
+    scope: user
+  tags:
+    - prometheus

ansible/roles/gpu_monitoring/templates/amdgpu_exporter.service.j2

@@ -0,0 +1,30 @@
+[Unit]
+Description=AMD GPU Sysfs Metrics Exporter
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User={{ amd_exporter_user }}
+Group={{ amd_exporter_user }}
+WorkingDirectory={{ amd_exporter_install_dir }}
+Environment="AMD_EXPORTER_PORT={{ amd_exporter_port }}"
+Environment="AMD_EXPORTER_BIND=0.0.0.0"
+Environment="PATH={{ amd_exporter_venv_bin }}:/usr/bin:/bin"
+ExecStart={{ amd_exporter_venv_bin }}/python3 {{ amd_exporter_script_path }}
+Restart=always
+RestartSec=10
+TimeoutStopSec=10
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=amdgpu_exporter
+
+# Security hardening
+ProtectSystem=strict
+ProtectHome=true
+ReadWritePaths={{ amd_exporter_install_dir }} /var/log
+NoNewPrivileges=true
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/gpu_monitoring/templates/llama_exporter.service.j2

@@ -0,0 +1,35 @@
+[Unit]
+Description=LLM Inference Metrics Exporter (llama.cpp + Ollama)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User={{ llama_exporter_user }}
+Group={{ llama_exporter_user }}
+WorkingDirectory={{ llama_exporter_install_dir }}
+Environment="LLAMA_EXPORTER_PORT={{ llama_exporter_port }}"
+Environment="LLAMA_EXPORTER_BIND=0.0.0.0"
+Environment="LLAMA_EXPORTER_INTERVAL={{ llama_exporter_scrape_interval }}"
+Environment="LLAMA_EXPORTER_TIMEOUT={{ llama_exporter_scrape_timeout }}"
+{% if llama_exporter_targets and llama_exporter_targets != "" %}
+Environment="LLAMA_TARGETS={{ llama_exporter_targets | to_json }}"
+{% endif %}
+Environment="PATH={{ llama_exporter_venv_bin }}:/usr/bin:/bin"
+ExecStart={{ llama_exporter_venv_bin }}/python3 {{ llama_exporter_script_path }}
+Restart=always
+RestartSec=10
+TimeoutStopSec=10
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=llama_exporter
+
+# Security hardening
+ProtectSystem=strict
+ProtectHome=true
+ReadWritePaths={{ llama_exporter_install_dir }} /var/log
+NoNewPrivileges=true
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/gpu_monitoring/templates/nvidia_exporter.service.j2

@@ -0,0 +1,34 @@
+# nvidia_gpu_exporter - Prometheus exporter for NVIDIA GPUs via NVML
+#
+# Exposes GPU metrics at /metrics port {{ nvidia_exporter_port }}
+
+[Unit]
+Description=NVIDIA GPU Prometheus Exporter
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User={{ nvidia_exporter_user }}
+Group={{ nvidia_exporter_user }}
+WorkingDirectory={{ nvidia_exporter_install_dir }}
+Environment="PYTHONUNBUFFERED=1"
+ExecStart={{ nvidia_exporter_venv_path }}/bin/python3 {{ nvidia_exporter_script_path }}
+Restart=on-failure
+RestartSec=5
+TimeoutStopSec=10
+StandardOutput=journal
+StandardError=journal
+
+# Security hardening
+NoNewPrivileges=true
+ProtectSystem=strict
+ProtectHome=true
+ReadOnlyPaths=/sys
+PrivateTmp=true
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectControlGroups=true
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/gpu_monitoring/templates/podman.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=Podman service for {{ item.name }}
+After=local-fs.target network-online.target
+
+[Service]
+Type=oneshot
+User={{ prometheus_user }}
+Group={{ prometheus_group }}
+RemainAfterExit=true
+ExecStart={{ podman_binary | default('/usr/bin/podman') }} start {{ item.name }}
+ExecStop={{ podman_binary | default('/usr/bin/podman') }} stop {{ item.name }}
+
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/gpu_monitoring/templates/prometheus.yml.j2

@@ -0,0 +1,30 @@
+global:
+  scrape_interval: {{ prometheus_scrape_interval }}
+  evaluation_interval: {{ prometheus_evaluation_interval }}
+  scrape_timeout: {{ prometheus_scrape_timeout }}
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ["localhost:{{ prometheus_container_port }}"]
+
+  - job_name: nvidia-gpu
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["{{ prometheus_host_gateway }}:{{ nvidia_exporter_port }}"]
+        labels:
+          datacenter: "{{ inventory_hostname }}"
+
+  - job_name: amdgpu_exporter
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["{{ prometheus_host_gateway }}:{{ amd_exporter_port }}"]
+        labels:
+          datacenter: "{{ inventory_hostname }}"
+
+  - job_name: llama_exporter
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["{{ prometheus_host_gateway }}:{{ llama_exporter_port }}"]
+        labels:
+          datacenter: "{{ inventory_hostname }}"

ansible/roles/gpu_monitoring/vars/main.yml

@@ -0,0 +1,23 @@
+---
+# Internal role variables (non-overridable)
+
+# Container names
+gpu_monitoring_prometheus_container_name: "prometheus"
+
+# Derived paths
+nvidia_exporter_script_path: "{{ nvidia_exporter_install_dir }}/nvidia_gpu_exporter.py"
+nvidia_exporter_venv_bin: "{{ nvidia_exporter_venv_path }}/bin"
+nvidia_exporter_service_name: "nvidia_exporter"
+
+amd_exporter_script_path: "{{ amd_exporter_install_dir }}/amd_gpu_exporter.py"
+amd_exporter_venv_bin: "{{ amd_exporter_venv_path }}/bin"
+amd_exporter_service_name: "amdgpu_exporter"
+
+llama_exporter_script_path: "{{ llama_exporter_install_dir }}/llama_exporter.py"
+llama_exporter_venv_bin: "{{ llama_exporter_venv_path }}/bin"
+llama_exporter_service_name: "llama_exporter"
+
+# Default llama.cpp targets if none provided
+llama_exporter_targets_default:
+  - name: "llama-cpp-8012"
+    url: "http://localhost:8012"