Add GPU monitoring services and configurations

- Create systemd service templates for llama_exporter, nvidia_exporter, and podman.
- Add Prometheus configuration template for GPU metrics scraping.
- Introduce variables for GPU monitoring role in main.yml.
- Implement tasks for syncing llama models, including user setup and package installation.
- Update podman tasks to ensure proper sudo access and lingering for the podman user.
- Modify inventory to include gpu_monitoring group.
- Add Terraform modules for deploying Vaultwarden, including ingress and service configurations.
- Create Grafana dashboard for real-time GPU monitoring metrics.
- Update secrets and environment files to include Vaultwarden admin token.

ansible/files/scripts/llama_hf_sync.py

@@ -0,0 +1,284 @@
+#!/usr/bin/env python3
+"""Synchronize llama.cpp models from Hugging Face into a managed local directory."""
+
+from __future__ import annotations
+
+import argparse
+import configparser
+import json
+import os
+import re
+import shutil
+import sys
+from pathlib import Path
+from typing import Any
+
+from huggingface_hub import HfApi, snapshot_download
+
+
+def _sanitize_slug(value: str) -> str:
+    slug = re.sub(r"[^a-zA-Z0-9._-]+", "-", value.strip())
+    slug = slug.strip("-._")
+    return slug.lower() or "model"
+
+
+def _resolve_repo_file(api: HfApi, model: dict[str, Any]) -> str:
+    repo_files = api.list_repo_files(repo_id=model["model_id"], revision=model["revision"], repo_type="model")
+
+    hf_file = model.get("hf_file")
+    if hf_file:
+        if hf_file in repo_files:
+            return hf_file
+        raise RuntimeError(
+            f"Requested hf_file '{hf_file}' was not found in repo "
+            f"{model['model_id']}@{model['revision']}"
+        )
+
+    quant_upper = (model.get("quant") or "").upper()
+    ggufs = [item for item in repo_files if item.lower().endswith(".gguf")]
+    matches = [item for item in ggufs if quant_upper in Path(item).name.upper()]
+    if matches:
+        return sorted(matches, key=lambda name: (len(Path(name).name), name))[0]
+
+    raise RuntimeError(
+        f"No GGUF file containing quant '{model.get('quant')}' found in "
+        f"repo {model['model_id']}@{model['revision']}. Set hf_file explicitly if needed."
+    )
+
+
+def _load_models(models_file: Path) -> list[dict[str, Any]]:
+    data = json.loads(models_file.read_text(encoding="utf-8"))
+    if not isinstance(data, list):
+        raise RuntimeError("models file must contain a JSON array")
+
+    normalized: list[dict[str, Any]] = []
+    for item in data:
+        if not isinstance(item, dict):
+            raise RuntimeError("each model entry must be an object")
+        if item.get("enable", True) is False:
+            continue
+
+        model_id = str(item.get("model_id", "")).strip()
+        quant = str(item.get("quant", "")).strip()
+        revision = str(item.get("revision", "")).strip()
+        hf_file = str(item.get("hf_file", "")).strip() or None
+        name = str(item.get("name", "")).strip()
+        preset = item.get("preset", {})
+
+        if preset is None:
+            preset = {}
+        if not isinstance(preset, dict):
+            raise RuntimeError(f"preset must be an object for model '{name or model_id or 'unknown'}'")
+
+        if not name:
+            raise RuntimeError(f"name is required for model '{model_id or 'unknown'}'")
+        if not model_id:
+            raise RuntimeError("model_id is required for all models")
+        if not quant and not hf_file:
+            raise RuntimeError(f"quant or hf_file is required for model '{model_id}'")
+        if not revision:
+            raise RuntimeError(f"revision is required for model '{model_id}'")
+
+        normalized.append(
+            {
+                "model_id": model_id,
+                "quant": quant,
+                "revision": revision,
+                "hf_file": hf_file,
+                "name": name,
+                "preset": preset,
+            }
+        )
+
+    return normalized
+
+
+def _load_preset_global(preset_global_file: Path | None) -> dict[str, Any]:
+    if preset_global_file is None:
+        return {}
+
+    if not preset_global_file.exists():
+        return {}
+
+    data = json.loads(preset_global_file.read_text(encoding="utf-8"))
+    if data is None:
+        return {}
+    if not isinstance(data, dict):
+        raise RuntimeError("global preset options must be a JSON object")
+    return data
+
+
+def _to_preset_value(value: Any) -> str:
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, (int, float, str)):
+        return str(value)
+    return json.dumps(value, separators=(",", ":"))
+
+
+def _normalize_preset_options(options: dict[str, Any], scope: str) -> dict[str, str]:
+    normalized: dict[str, str] = {}
+    for key, value in options.items():
+        key_str = str(key).strip()
+        if not key_str:
+            raise RuntimeError(f"{scope} preset option keys must be non-empty")
+        if value is None:
+            continue
+        normalized[key_str] = _to_preset_value(value)
+    return normalized
+
+
+def _download_model(model: dict[str, Any], target_dir: Path, repo_file: str, dry_run: bool) -> None:
+    if dry_run:
+        return
+
+    snapshot_download(
+        repo_id=model["model_id"],
+        revision=model["revision"],
+        local_dir=str(target_dir),
+        allow_patterns=[repo_file],
+        local_dir_use_symlinks=False,
+    )
+
+
+def _write_preset(preset_file: Path, entries: list[dict[str, Any]], global_options: dict[str, Any]) -> None:
+    parser = configparser.ConfigParser(interpolation=None)
+    parser.optionxform = str
+    parser["*"] = _normalize_preset_options(global_options, "global")
+
+    for entry in entries:
+        model_options = _normalize_preset_options(entry.get("preset", {}), f"model '{entry['name']}'")
+        model_options["model"] = entry["container_model_path"]
+        parser[entry["name"]] = model_options
+
+    preset_file.parent.mkdir(parents=True, exist_ok=True)
+    with preset_file.open("w", encoding="utf-8") as fh:
+        # Router preset format supports top-level version key.
+        fh.write("version = 1\n\n")
+        parser.write(fh)
+
+
+def _prune_unmanaged(managed_dir: Path, links_dir: Path, keep_slugs: set[str], dry_run: bool) -> tuple[list[str], list[str]]:
+    pruned_dirs: list[str] = []
+    pruned_links: list[str] = []
+
+    if managed_dir.exists():
+        for child in managed_dir.iterdir():
+            if not child.is_dir():
+                continue
+            if child.name == ".router":
+                continue
+            if child.name not in keep_slugs:
+                pruned_dirs.append(child.name)
+                if not dry_run:
+                    shutil.rmtree(child)
+
+    if links_dir.exists():
+        for child in links_dir.iterdir():
+            if child.suffix != ".gguf":
+                continue
+            slug = child.stem
+            if slug not in keep_slugs:
+                pruned_links.append(child.name)
+                if not dry_run:
+                    child.unlink(missing_ok=True)
+
+    return pruned_dirs, pruned_links
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Sync Hugging Face GGUF models for llama.cpp router mode")
+    parser.add_argument("--models-file", required=True)
+    parser.add_argument("--managed-dir", required=True)
+    parser.add_argument("--links-dir", required=True)
+    parser.add_argument("--manifest-file", required=True)
+    parser.add_argument("--preset-file", required=True)
+    parser.add_argument("--preset-global-file")
+    parser.add_argument("--container-links-dir", required=True)
+    parser.add_argument("--prune", action="store_true")
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+
+    models_file = Path(args.models_file)
+    managed_dir = Path(args.managed_dir)
+    links_dir = Path(args.links_dir)
+    manifest_file = Path(args.manifest_file)
+    preset_file = Path(args.preset_file)
+    preset_global_file = Path(args.preset_global_file) if args.preset_global_file else None
+
+    managed_dir.mkdir(parents=True, exist_ok=True)
+    links_dir.mkdir(parents=True, exist_ok=True)
+    manifest_file.parent.mkdir(parents=True, exist_ok=True)
+    preset_file.parent.mkdir(parents=True, exist_ok=True)
+
+    models = _load_models(models_file)
+    global_options = _load_preset_global(preset_global_file)
+    api = HfApi()
+    entries: list[dict[str, Any]] = []
+
+    for model in models:
+        slug = _sanitize_slug(model["name"])
+        target_dir = managed_dir / slug
+        target_dir.mkdir(parents=True, exist_ok=True)
+        repo_file = _resolve_repo_file(api, model)
+
+        _download_model(model, target_dir, repo_file, args.dry_run)
+        selected_file = target_dir / repo_file
+        if not args.dry_run and not selected_file.exists():
+            raise RuntimeError(f"Expected downloaded file not found: {selected_file}")
+
+        link_path = links_dir / f"{slug}.gguf"
+        container_model_path = f"{args.container_links_dir.rstrip('/')}/{slug}.gguf"
+
+        if not args.dry_run:
+            if link_path.exists() or link_path.is_symlink():
+                link_path.unlink()
+            # Keep symlink targets relative so they remain valid inside the
+            # container-mounted /models tree.
+            relative_target = os.path.relpath(selected_file, start=link_path.parent)
+            link_path.symlink_to(relative_target)
+
+        entries.append(
+            {
+                "name": model["name"],
+                "slug": slug,
+                "model_id": model["model_id"],
+                "revision": model["revision"],
+                "quant": model["quant"],
+                "hf_file": model.get("hf_file"),
+                "preset": model.get("preset", {}),
+                "repo_file": repo_file,
+                "selected_file": str(selected_file),
+                "link_path": str(link_path),
+                "container_model_path": container_model_path,
+            }
+        )
+
+    pruned_dirs: list[str] = []
+    pruned_links: list[str] = []
+    if args.prune:
+        keep_slugs = {entry["slug"] for entry in entries}
+        pruned_dirs, pruned_links = _prune_unmanaged(managed_dir, links_dir, keep_slugs, args.dry_run)
+
+    result = {
+        "models": entries,
+        "dry_run": args.dry_run,
+        "prune": args.prune,
+        "pruned_dirs": pruned_dirs,
+        "pruned_links": pruned_links,
+    }
+
+    if not args.dry_run:
+        manifest_file.write_text(json.dumps(result, indent=2), encoding="utf-8")
+        _write_preset(preset_file, entries, global_options)
+
+    print(json.dumps(result))
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except Exception as exc:
+        print(f"ERROR: {exc}", file=sys.stderr)
+        raise SystemExit(1)

ansible/host_vars/gpu-01.lab.alexpires.me.yml

@@ -47,7 +47,8 @@ fw_allowed_ports:
   - { rule: "allow", port: "8999", proto: "tcp", from: "10.19.4.0/24" } # ComfyUI - SimpleHttpServer (local network)
   - { rule: "allow", port: "8012", proto: "tcp", from: "10.19.4.0/24" } # Qwen2 (local network)
   - { rule: "allow", port: "8012", proto: "tcp", from: "10.5.5.5/32" } # Qwen2 (Laptop)
-  - { rule: "allow", port: "9090", proto: "tcp", from: "10.19.4.0/24" } # local network
+  - { rule: "allow", port: "9091", proto: "tcp", from: "10.19.4.0/24" } # Prometheus (local network)
+  - { rule: "allow", port: "9091", proto: "tcp", from: "10.5.5.5/32" } # Prometheus (Laptop)
 ufw_outgoing_traffic:
   - 22 # SSH
   - 53 # DNS
@@ -139,3 +140,55 @@ ollama_model_files:
     name: "qwen3.5-4b-32k:latest"
   - file: "ollama/qwen3.5-4b-64k.modelfile"
     name: "qwen3.5-4b-64k:latest"
+llama_models:
+  # TODO: replace revision values with pinned commit SHAs for strict reproducibility.
+  - name: "gpt-oss-20b-q4-0"
+    model_id: "unsloth/gpt-oss-20b-GGUF"
+    quant: "Q4_0"
+    revision: "main"
+  - name: "gemma-4-26b-a4b-ud-iq4-xs"
+    model_id: "unsloth/gemma-4-26B-A4B-it-GGUF"
+    quant: "UD-IQ4_XS"
+    revision: "main"
+  - name: "gemma-4-31b-iq4-xs"
+    model_id: "unsloth/gemma-4-31B-it-GGUF"
+    quant: "IQ4_XS"
+    revision: "main"
+  - name: "qwen3-6-35b-a3b-mtp-ud-iq4-xs"
+    model_id: "unsloth/Qwen3.6-35B-A3B-MTP-GGUF"
+    quant: "UD-IQ4_XS"
+    revision: "main"
+  - name: "qwen3-5-27b-mtp-iq4-xs"
+    model_id: "unsloth/Qwen3.5-27B-MTP-GGUF"
+    quant: "IQ4_XS"
+    revision: "main"
+  - name: "qwen3-5-35b-a3b-ud-iq4-xs"
+    model_id: "unsloth/Qwen3.5-35B-A3B-GGUF"
+    quant: "UD-IQ4_XS"
+    revision: "main"
+  - name: "ministral-3-14b-instruct-iq4-xs"
+    model_id: "unsloth/Ministral-3-14B-Instruct-2512-GGUF"
+    quant: "IQ4_XS"
+    revision: "main"
+  - name: "granite-4-1-30b-iq4-xs"
+    model_id: "unsloth/granite-4.1-30b-GGUF"
+    quant: "IQ4_XS"
+    revision: "main"
+  - name: "qwen3-5-9b-iq4-xs"
+    model_id: "unsloth/Qwen3.5-9B-GGUF"
+    quant: "IQ4_XS"
+    revision: "main"
+  - name: "qwen3-6-35b-a3b-claude47-distilled-iq4-xs"
+    model_id: "lordx64/Qwen3.6-35B-A3B-Claude-4.7-Opus-Reasoning-Distilled-IQ4_XS-GGUF"
+    quant: "IQ4_XS"
+    revision: "main"
+  - name: "devstral-small-2-24b-instruct-iq4-xs"
+    model_id: "unsloth/Devstral-Small-2-24B-Instruct-2512-GGUF"
+    quant: "IQ4_XS"
+    revision: "main"
+
+# Optional global model preset options applied under [*] in models.ini.
+# Keys map to llama.cpp CLI args without leading dashes, short args, or LLAMA_ARG_* env names.
+llama_preset_global:
+  c: 131072
+  n-gpu-layers: all

ansible/playbook_gpu_monitoring.yml

@@ -0,0 +1,18 @@
+---
+- name: GPU Monitoring Stack Setup
+  hosts: gpu_monitoring
+  gather_facts: true
+  become: true
+
+  pre_tasks:
+    - name: Check that host is Fedora
+      ansible.builtin.assert:
+        that:
+          - ansible_os_family == "RedHat"
+        fail_msg: "This playbook only supports RedHat-family OS (Fedora)."
+
+  roles:
+    - role: "gpu_monitoring"
+      tags:
+        - roles
+        - roles::gpu_monitoring

ansible/playbook_podman_llama_rocm.yml

@@ -1,30 +0,0 @@
-- name: Setup llama.cpp - ROCm
-  hosts: llamacpp
-  vars:
-    podman_user: "{{ llama_user | default('llama') }}"
-    podman_group: "{{ llama_group | default('llama') }}"
-    podman_extra_groups: "users,video"
-    podman_user_home: "{{ llama_home | default('/home/llama') }}"
-    podman_user_folders:
-      - data
-    pods:
-      - name: llamacpp-rocm
-        build:
-          dockerfile: "{{ lookup('file', 'dockerfiles/Dockerfile.llamacpp.rocm') }}"
-        env:
-          LLAMA_ARG_MODELS_DIR: "/home/worker/.llamacpp"
-          LLAMA_ARG_THREADS: 6
-          LLAMA_ARG_PARALLEL: 1
-          LLAMA_ARG_CACHE: "true"
-        ports:
-          - "0.0.0.0:{{ llama_port | default(11435) }}:11434/tcp"
-        volumes:
-          - "{{ llama_home }}/data:/home/worker/.llamacpp:Z"
-        device:
-          - "/dev/kfd:/dev/kfd:rw"
-          - "/dev/dri:/dev/dri:rw"
-
-  tasks:
-    - name: Setup Pods
-      ansible.builtin.include_tasks:
-        file: tasks/podman.yml

ansible/playbook_podman_llama_vulkan.yml

@@ -0,0 +1,101 @@
+- name: Setup llama.cpp - Vulkan
+  hosts: llamacpp
+  vars:
+    podman_user: "{{ llama_user | default('llama') }}"
+    podman_group: "{{ llama_group | default('llama') }}"
+    podman_extra_groups: "users,video"
+    podman_user_home: "{{ llama_home | default('/home/llama') }}"
+    __llama_models__: "{{ llama_models | default([]) }}"
+    __llama_router_dir__: "{{ podman_user_home }}/models/.router"
+    __llama_managed_dir__: "{{ podman_user_home }}/models/managed"
+    __llama_links_dir__: "{{ podman_user_home }}/models/managed-links"
+    __llama_manifest_file__: "{{ __llama_router_dir__ }}/manifest.json"
+    __llama_models_input_file__: "{{ __llama_router_dir__ }}/desired-models.json"
+    __llama_preset_global__: "{{ llama_preset_global | default({}) }}"
+    __llama_preset_global_input_file__: "{{ __llama_router_dir__ }}/preset-global.json"
+    __llama_preset_file__: "{{ __llama_router_dir__ }}/models.ini"
+    __llama_sync_script__: "{{ __llama_router_dir__ }}/llama_hf_sync.py"
+    __llama_sync_venv__: "{{ llama_sync_venv | default(__llama_router_dir__ ~ '/.venv') }}"
+    __llama_sync_python__: "{{ __llama_sync_venv__ }}/bin/python"
+    __llama_models_max__: "{{ llama_models_max_loaded | default(1) }}"
+    podman_user_folders:
+      - models
+      - .cache
+      - .cache/llama.cpp
+    pods:
+      - name: llama.cpp
+        repo:
+          image: ghcr.io/ggml-org/llama.cpp:server-vulkan
+        ports:
+          - "0.0.0.0:{{ llama_port | default(8012) }}:8080/tcp"
+        command:
+          - "-t"
+          - "{{ llama_cmd_threads | default('12') }}"
+          - "-np"
+          - "{{ llama_cmd_parallel | default('1') }}"
+          - "-b"
+          - "{{ llama_cmd_batch_size | default('1024') }}"
+          - "-ub"
+          - "{{ llama_cmd_ubatch_size | default('512') }}"
+          - "-fa"
+          - "{{ llama_cmd_flash_attn | default('on') }}"
+          - "-ctk"
+          - "{{ llama_cmd_cache_type_k | default('q4_0') }}"
+          - "-ctv"
+          - "{{ llama_cmd_cache_type_v | default('q4_0') }}"
+          - "-cram"
+          - "{{ llama_cmd_cache_reuse | default('-1') }}"
+          - "-sm"
+          - "{{ llama_cmd_split_mode | default('layer') }}"
+          - "-dev"
+          - "{{ llama_cmd_devices | default('Vulkan1,Vulkan2') }}"
+          - "-ts"
+          - "{{ llama_cmd_tensor_split | default('8,12') }}"
+          - "-fit"
+          - "{{ llama_cmd_fit | default('off') }}"
+          - "-mg"
+          - "{{ llama_cmd_main_gpu | default('1') }}"
+          - "--mmap"
+          - "--models-preset"
+          - "{{ llama_cmd_models_preset_path | default('/models/.router/models.ini') }}"
+          - "--models-max"
+          - "{{ llama_cmd_models_max | default(__llama_models_max__) }}"
+        volumes:
+          - "{{ podman_user_home }}/models:/models:Z"
+          - "{{ podman_user_home }}/.cache:/app/.cache:Z"
+        device:
+          - "nvidia.com/gpu=all"
+          - "/dev/kfd:/dev/kfd:rw"
+          - "/dev/dri:/dev/dri:rw"
+  tasks:
+    - name: Sync llama models from Hugging Face
+      ansible.builtin.include_tasks:
+        file: tasks/llama_models_sync.yml
+      when: llama_models_sync_enabled | default(true)
+
+    - name: Setup Pods
+      ansible.builtin.include_tasks:
+        file: tasks/podman.yml
+
+    - name: Query llama router model list
+      ansible.builtin.uri:
+        url: "http://127.0.0.1:{{ llama_port | default(8012) }}/models?reload=1"
+        method: GET
+        status_code: 200
+      register: __llama_router_models__
+      changed_when: false
+      when:
+        - llama_models_sync_enabled | default(true)
+        - not (llama_models_dry_run | default(false))
+
+    - name: Validate expected model aliases are available in router
+      ansible.builtin.assert:
+        that:
+          - item.name in (__llama_router_models__.json.data | map(attribute='id') | list)
+        fail_msg: "Model name '{{ item.name }}' not present in llama router /models output"
+      loop: "{{ __llama_sync_result__.models | default([]) }}"
+      loop_control:
+        label: "{{ item.name }}"
+      when:
+        - llama_models_sync_enabled | default(true)
+        - not (llama_models_dry_run | default(false))

ansible/playbook_podman_qwen3.6.yml

@@ -1,4 +1,4 @@
-- name: Setup Qwen3.6 - Coder (FIM - LLAMA.cpp)
+- name: Setup Qwen3.6 - Coder
   hosts: qwen3_6_host
   vars:
     podman_user: "{{ qwen3_Z6_user | default('qwen36') }}"
@@ -6,7 +6,7 @@
     podman_extra_groups: "users,video"
     podman_user_home: "{{ qwen3_6_home | default('/home/qwen36') }}"
     llama_model: "{{ qwen3_6_model | default('bartowski/Qwen_Qwen3.6-35B-A3B-GGUF:IQ4_XS') }}"
-    __qwen3_6_context_length__: "{{ qwen3_6_context_length | default(65537) }}"
+    __qwen3_6_context_length__: "{{ qwen3_6_context_length | default(262144) }}"
     podman_user_folders:
       - models
       - .cache
@@ -14,38 +14,50 @@
     pods:
       - name: qwen3.6
         repo:
-          image: ghcr.io/ggml-org/llama.cpp:server-cuda
+          image: ghcr.io/ggml-org/llama.cpp:server-vulkan
         ports:
           - "0.0.0.0:{{ qwen3_6_port | default(8012) }}:8080/tcp"
         command:
+          - "-t"
+          - "12"
           - "-hf"
           - "{{ llama_model }}"
           - "-ngl"
-          - "40"
-          - "-ncmoe"
-          - "32"
+          - "999"
           - "-c"
           - "{{ __qwen3_6_context_length__ }}"
           - "-np"
           - "1"
-          - "-ub"
-          - "512"
           - "-b"
           - "1024"
+          - "-ub"
+          - "512"
           - "-fa"
           - "on"
-          - "--no-mmap"
           - "-ctk"
-          - "iq4_nl"
+          - "q4_0"
           - "-ctv"
-          - "iq4_nl"
+          - "q8_0"
           - "-cram"
           - "{{ __qwen3_6_context_length__ }}"
+          - "-sm"
+          - "layer"
+          - "-dev"
+          - "Vulkan1,Vulkan2"
+          - "-ts"
+          - "8,12"
+          - "-fit"
+          - "off"
+          - "-mg"
+          - 1
+          - "--mmap"
         volumes:
           - "{{ podman_user_home }}/models:/models:Z"
           - "{{ podman_user_home }}/.cache:/app/.cache:Z"
         device:
           - "nvidia.com/gpu=all"
+          - "/dev/kfd:/dev/kfd:rw"
+          - "/dev/dri:/dev/dri:rw"
   tasks:
     - name: Setup Pods
       ansible.builtin.include_tasks:

ansible/roles/gpu_monitoring/defaults/main.yml

@@ -0,0 +1,68 @@
+---
+# gpu_monitoring defaults
+
+# Prometheus configuration
+prometheus_image: "docker.io/prom/prometheus:latest"
+prometheus_image_tag: "latest"
+prometheus_version: "latest"
+prometheus_port: 9091
+prometheus_container_port: 9090
+prometheus_data_dir: "/mnt/data/prometheus"
+prometheus_retention_days: "30"
+prometheus_retention_size: "10GB"
+prometheus_scrape_interval: "15s"
+prometheus_evaluation_interval: "15s"
+prometheus_scrape_timeout: "10s"
+
+# Prometheus container
+prometheus_container_memory: "1g"
+prometheus_user: prometheus
+prometheus_group: prometheus
+prometheus_user_home: "/mnt/data/prometheus"
+prometheus_network_name: prometheus-net
+prometheus_host_port: "{{ prometheus_port }}"
+prometheus_host_gateway: "host.containers.internal"
+prometheus_extra_groups: []
+prometheus_user_folders:
+  - .prometheus
+  - .cache
+
+# NVIDIA GPU Exporter configuration (NVML-based, works with consumer GPUs)
+nvidia_exporter_enabled: true
+nvidia_exporter_port: 9400
+nvidia_exporter_user: nvidia_exporter
+nvidia_exporter_user_home: "/home/nvidia_exporter"
+nvidia_exporter_install_dir: "/opt/nvidia_exporter"
+nvidia_exporter_venv_path: "/opt/nvidia_exporter/.venv"
+
+# AMD GPU Exporter configuration
+amd_exporter_enabled: true
+amd_exporter_port: 9500
+amd_exporter_user: amd_exporter
+amd_exporter_user_home: "/home/amd_exporter"
+amd_exporter_install_dir: "/opt/amdgpu_exporter"
+amd_exporter_venv_path: "/opt/amdgpu_exporter/.venv"
+amd_exporter_python_version: "python3"
+amd_exporter_sysfs_gpu_pattern: "/sys/class/drm/card?/device/"
+
+# Llama Exporter configuration
+llama_exporter_enabled: true
+llama_exporter_port: 9550
+llama_exporter_user: llama_exporter
+llama_exporter_user_home: "/home/llama_exporter"
+llama_exporter_install_dir: "/opt/llama_exporter"
+llama_exporter_venv_path: "/opt/llama_exporter/.venv"
+llama_exporter_scrape_interval: 15
+llama_exporter_scrape_timeout: 5
+
+# Llama.cpp scrape targets
+llama_exporter_targets: []
+
+# Firewall configuration
+prometheus_fw_port: "{{ prometheus_port }}"
+prometheus_fw_proto: "tcp"
+prometheus_fw_allowed_networks:
+  - cidr: "10.19.4.0/24"
+    comment: "local network"
+  - cidr: "10.5.5.5/32"
+    comment: "Laptop Wireguard"

ansible/roles/gpu_monitoring/files/amd_gpu_exporter.py

@@ -0,0 +1,290 @@
+#!/usr/bin/env python3
+"""AMD GPU sysfs metrics exporter for Prometheus."""
+
+import glob
+import os
+import re
+import time
+import logging
+import json
+import subprocess
+from http.server import HTTPServer, BaseHTTPRequestHandler
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
+logger = logging.getLogger(__name__)
+
+AMD_VENDOR_ID = 0x1002
+
+POLL_INTERVAL = 10
+
+
+def _read_sysfs(path):
+    try:
+        with open(path, "r") as f:
+            raw = f.read().strip()
+            return int(raw, 0)
+    except (OSError, ValueError):
+        return None
+
+
+def _read_first_int(paths):
+    for path in paths:
+        val = _read_sysfs(path)
+        if val is not None:
+            return val
+    return None
+
+
+def _metric_value(entry):
+    if isinstance(entry, dict):
+        value = entry.get("value")
+        if isinstance(value, (int, float)):
+            return float(value)
+    return None
+
+
+def _collect_amd_smi_metrics(results, set_metric):
+    """Populate metrics from amd-smi JSON output. Returns True when data is collected."""
+    cmd = ["amd-smi", "metric", "-m", "-u", "-p", "-t", "--json", "--gpu", "all"]
+    try:
+        proc = subprocess.run(cmd, capture_output=True, text=True, timeout=8, check=True)
+    except (OSError, subprocess.SubprocessError):
+        return False
+
+    try:
+        payload = json.loads(proc.stdout)
+    except (json.JSONDecodeError, TypeError):
+        return False
+
+    gpu_rows = payload.get("gpu_data", []) if isinstance(payload, dict) else []
+    if not gpu_rows:
+        return False
+
+    got_any = False
+    for row in gpu_rows:
+        if not isinstance(row, dict):
+            continue
+        gpu_id = row.get("gpu")
+        if gpu_id is None:
+            continue
+
+        labels = {"gpu": f"amd{gpu_id}", "gpu_name": f"AMD_GPU_{gpu_id}"}
+        set_metric("gpu_exists", labels, 1.0)
+        got_any = True
+
+        usage = row.get("usage", {}) if isinstance(row.get("usage"), dict) else {}
+        gfx_activity = _metric_value(usage.get("gfx_activity"))
+        if gfx_activity is not None:
+            set_metric("gpu_util_percent", labels, gfx_activity)
+
+        umc_activity = _metric_value(usage.get("umc_activity"))
+        if umc_activity is not None:
+            set_metric("memory_used_percent", labels, umc_activity)
+
+        power = row.get("power", {}) if isinstance(row.get("power"), dict) else {}
+        socket_power = _metric_value(power.get("socket_power"))
+        if socket_power is not None:
+            set_metric("power_watts", labels, socket_power)
+
+        temp = row.get("temperature", {}) if isinstance(row.get("temperature"), dict) else {}
+        edge_temp = _metric_value(temp.get("edge"))
+        if edge_temp is not None:
+            set_metric("temperature_celsius", labels, edge_temp)
+
+        mem_usage = row.get("mem_usage", {}) if isinstance(row.get("mem_usage"), dict) else {}
+        used_vram = _metric_value(mem_usage.get("used_vram"))
+        free_vram = _metric_value(mem_usage.get("free_vram"))
+        total_vram = _metric_value(mem_usage.get("total_vram"))
+
+        if used_vram is not None:
+            set_metric("mem_used_bytes", labels, used_vram * 1024 * 1024)
+        if free_vram is not None:
+            set_metric("mem_avail_bytes", labels, free_vram * 1024 * 1024)
+        if total_vram is not None and used_vram is not None and total_vram > 0:
+            set_metric("memory_used_percent", labels, (used_vram / total_vram) * 100.0)
+
+    return got_any
+
+
+def _get_card_ids():
+    paths = sorted(glob.glob("/sys/class/drm/card[0-9]*"))
+    seen = set()
+    card_ids = []
+    for p in paths:
+        name = os.path.basename(p)
+        dev_match = re.match(r"card(\d+)", name)
+        if dev_match:
+            idx = int(dev_match.group(1))
+            if idx not in seen:
+                seen.add(idx)
+                card_ids.append(idx)
+    return card_ids
+
+
+def _is_amd_gpu(idx):
+    vendor_path = f"/sys/class/drm/card{idx}/device/vendor"
+    vendor = _read_sysfs(vendor_path)
+    return vendor == AMD_VENDOR_ID
+
+
+def _get_gpu_name(idx):
+    if _is_amd_gpu(idx):
+        device_id = _read_sysfs(f"/sys/class/drm/card{idx}/device/device")
+        if device_id is not None:
+            return f"AMD_GPU_{device_id:#06x}"
+        return f"AMD_GPU_card{idx}"
+    return f"UNKNOWN_card{idx}"
+
+
+def _collect_metrics():
+    results = {}
+
+    def _set_metric(metric_name, labels, value):
+        label_key = tuple(sorted(labels.items()))
+        results[(metric_name, label_key)] = value
+
+    # Prefer amd-smi for robust telemetry on kernels where some sysfs nodes are busy.
+    if _collect_amd_smi_metrics(results, _set_metric):
+        return results
+
+    card_ids = _get_card_ids()
+
+    if not card_ids:
+        logger.warning("No AMD GPU cards found in sysfs")
+        return results
+
+    for idx in card_ids:
+        if not _is_amd_gpu(idx):
+            continue
+
+        labels = {"gpu": f"card{idx}", "gpu_name": _get_gpu_name(idx)}
+
+        # Power (microwatts in hwmon on many kernels, milliwatts on older path)
+        power = _read_first_int([
+            f"/sys/class/drm/card{idx}/device/power_average",
+            f"/sys/class/drm/card{idx}/device/hwmon/hwmon0/power1_average",
+            f"/sys/class/drm/card{idx}/device/hwmon/hwmon1/power1_average",
+            f"/sys/class/drm/card{idx}/device/hwmon/hwmon2/power1_average",
+            f"/sys/class/drm/card{idx}/device/hwmon/hwmon3/power1_average",
+        ])
+        if power is not None:
+            # Heuristic: hwmon power1_average is usually microwatts.
+            if power > 1_000_000:
+                _set_metric("power_watts", labels, power / 1_000_000.0)
+            else:
+                _set_metric("power_watts", labels, power / 1000.0)
+
+        # Memory used (bytes) - support both gpu_mem_* and mem_info_vram_* layouts
+        mem_used = _read_first_int([
+            f"/sys/class/drm/card{idx}/device/gpu_mem_used_bytes",
+            f"/sys/class/drm/card{idx}/device/mem_info_vram_used",
+        ])
+        if mem_used is not None:
+            _set_metric("mem_used_bytes", labels, mem_used)
+
+        # Memory available (bytes)
+        mem_avail = _read_first_int([
+            f"/sys/class/drm/card{idx}/device/gpu_mem_avail_bytes",
+        ])
+
+        # If total VRAM is available, derive avail = total - used.
+        if mem_avail is None and mem_used is not None:
+            mem_total = _read_first_int([
+                f"/sys/class/drm/card{idx}/device/mem_info_vram_total",
+            ])
+            if mem_total is not None and mem_total >= mem_used:
+                mem_avail = mem_total - mem_used
+
+        if mem_avail is not None:
+            _set_metric("mem_avail_bytes", labels, mem_avail)
+
+        # Some kernels expose direct busy percentage.
+        mem_busy = _read_first_int([
+            f"/sys/class/drm/card{idx}/device/mem_busy_percent",
+        ])
+        if mem_busy is not None:
+            _set_metric("memory_used_percent", labels, float(mem_busy))
+
+        if mem_used is not None and mem_avail is not None and (mem_used + mem_avail) > 0:
+            _set_metric("memory_used_percent", labels, (mem_used / (mem_used + mem_avail)) * 100.0)
+
+        # Temperature (milliCelsius)
+        temp = _read_first_int([
+            f"/sys/class/drm/card{idx}/device/temp_edge_input",
+            f"/sys/class/drm/card{idx}/device/hwmon/hwmon0/temp1_input",
+            f"/sys/class/drm/card{idx}/device/hwmon/hwmon1/temp1_input",
+            f"/sys/class/drm/card{idx}/device/hwmon/hwmon2/temp1_input",
+            f"/sys/class/drm/card{idx}/device/hwmon/hwmon3/temp1_input",
+        ])
+        if temp is not None:
+            _set_metric("temperature_celsius", labels, temp / 1000.0)
+
+        # Clock speed (MHz) - clock[1] = performance state
+        clock_path = f"/sys/class/drm/card{idx}/device/pp_cur_clk"
+        try:
+            with open(clock_path, "r") as f:
+                clocks_raw = f.read().strip()
+                clocks = [int(c) for c in clocks_raw.split()]
+                if len(clocks) >= 2:
+                    _set_metric("gpu_clock_mhz", labels, clocks[1])
+        except (OSError, ValueError):
+            pass
+
+        # Existence info
+        _set_metric("gpu_exists", labels, 1.0)
+
+    return results
+
+
+class MetricsHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path != "/metrics":
+            self.send_response(404)
+            self.end_headers()
+            return
+
+        metrics = _collect_metrics()
+        lines = []
+
+        for (metric_type, label_pairs), value in sorted(metrics.items()):
+            labels = dict(label_pairs)
+            label_str = ",".join(f'{k}="{v}"' for k, v in sorted(labels.items()))
+            if label_str:
+                line = f'{metric_type}{{{label_str}}} {value}'
+            else:
+                line = f"{metric_type} {value}"
+            lines.append(line)
+
+        body = "\n".join(lines) + "\n" if lines else "# no metrics available\n"
+
+        self.send_response(200)
+        self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+        self.end_headers()
+        self.wfile.write(body.encode("utf-8"))
+
+    def log_message(self, format, *args):
+        logger.debug("%s - - %s", self.address_string(), format % args)
+
+
+def main():
+    port = int(os.environ.get("AMD_EXPORTER_PORT", "9500"))
+    host = os.environ.get("AMD_EXPORTER_BIND", "0.0.0.0")
+
+    server = HTTPServer((host, port), MetricsHandler)
+    logger.info("Starting AMD GPU Exporter on %s:%d", host, port)
+    logger.info("Polling sysfs every %d seconds", POLL_INTERVAL)
+
+    def _update_loop():
+        while True:
+            _collect_metrics()
+            time.sleep(POLL_INTERVAL)
+
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        logger.info("Shutting down")
+        server.shutdown()
+
+
+if __name__ == "__main__":
+    main()

ansible/roles/gpu_monitoring/files/llama_exporter.py

@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+"""LLM inference metrics exporter for Prometheus.
+
+Scrape targets:
+    - llama.cpp HTTP API servers (standard /metrics or custom health)
+"""
+
+import json
+import os
+import time
+import logging
+import urllib.request
+import urllib.error
+from http.server import HTTPServer, BaseHTTPRequestHandler
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
+logger = logging.getLogger(__name__)
+
+POLL_INTERVAL = int(os.environ.get("LLAMA_EXPORTER_INTERVAL", "15"))
+SCRAPE_TIMEOUT = int(os.environ.get("LLAMA_EXPORTER_TIMEOUT", "5"))
+
+LLAMA_CPP_DEFAULTS = [
+    {"name": "llama-cpp-8012", "url": "http://localhost:8012"},
+]
+
+
+def _fetch_json(url, timeout=SCRAPE_TIMEOUT):
+    try:
+        req = urllib.request.Request(url, headers={"Accept": "application/json"})
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return json.loads(resp.read().decode("utf-8"))
+    except Exception as e:
+        logger.debug("Failed to fetch %s: %s", url, e)
+        return None
+
+
+def _collect_llama_cpp(target):
+    metrics = {}
+
+    def _set_metric(metric_name, labels, value):
+        label_key = tuple(sorted(labels.items()))
+        metrics[(metric_name, label_key)] = value
+
+    base_url = target["url"].rstrip("/")
+
+    # Check health endpoint
+    health_url = f"{base_url}/health"
+    health = _fetch_json(health_url)
+    if health and isinstance(health, dict):
+        status = health.get("status", "unknown")
+        _set_metric("llama_server_health", {"server": target["name"], "model": health.get("model", "unknown")}, 1.0 if status == "ok" else 0.0)
+    else:
+        _set_metric("llama_server_health", {"server": target["name"], "model": "unknown"}, 0.0)
+
+    # Check /metrics endpoint for llama.cpp built-in metrics
+    metrics_url = f"{base_url}/metrics"
+    try:
+        req = urllib.request.Request(metrics_url, headers={"Accept": "text/plain"})
+        with urllib.request.urlopen(req, timeout=SCRAPE_TIMEOUT) as resp:
+            body = resp.read().decode("utf-8")
+            for line in body.splitlines():
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+                # Parse simple key value pairs
+                if "{" in line:
+                    key_part = line.split("{")[0]
+                    labels_part = line.split("{")[1].rstrip("}").split("}")[0] if "}" in line else ""
+                    value = line.split()[-1] if line.split() else "1"
+                    label_str = ""
+                    if labels_part:
+                        label_str = ",".join(f'{{{k}="{v}"}}' for k, v in (
+                            pair.split("=") for pair in labels_part.split(",") if "=" in pair
+                        ))
+                    _set_metric(key_part.strip(), {"server": target["name"], "raw": labels_part}, float(value) if value.replace(".", "").replace("-", "").isdigit() else value)
+                else:
+                    parts = line.split()
+                    if len(parts) >= 2:
+                        try:
+                            float(parts[1])
+                            _set_metric(parts[0], {"server": target["name"]}, parts[1])
+                        except ValueError:
+                            pass
+    except Exception as e:
+        logger.debug("Failed to scrape %s/metrics: %s", metrics_url, e)
+
+    # Check /model endpoint for model info
+    model_url = f"{base_url}/models"
+    models = _fetch_json(model_url)
+    if models and isinstance(models, dict):
+        model_list = models.get("data", [models]) if "data" in models else [models]
+        for m in model_list:
+            if not isinstance(m, dict):
+                continue
+            model_id = m.get("id", "unknown")
+            _set_metric("llama_models_loaded", {"server": target["name"], "model": model_id}, 1.0)
+            ctx_max = m.get("context_size", m.get("n_ctx", 0))
+            if ctx_max:
+                _set_metric("llama_model_context_max", {"server": target["name"], "model": model_id}, int(ctx_max))
+            trained = m.get("train_tokens", m.get("n_train", 0))
+            if trained:
+                _set_metric("llama_model_tokens_trained", {"server": target["name"], "model": model_id}, int(trained))
+
+    return metrics
+
+
+
+class MetricsHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path != "/metrics":
+            self.send_response(404)
+            self.end_headers()
+            return
+
+        all_metrics = {}
+
+        # Scrape llama.cpp targets
+        targets = []
+        env_targets = os.environ.get("LLAMA_TARGETS", "")
+        if env_targets:
+            try:
+                targets = json.loads(env_targets)
+            except json.JSONDecodeError:
+                logger.warning("Invalid LLAMA_TARGETS JSON, using defaults")
+                targets = []
+
+        if not targets:
+            targets = LLAMA_CPP_DEFAULTS
+
+        for target in targets:
+            if "url" not in target or "name" not in target:
+                target["name"] = target.get("name", "default")
+                target["url"] = target.get("url", "http://localhost:8012")
+            try:
+                all_metrics.update(_collect_llama_cpp(target))
+            except Exception as e:
+                logger.error("Error scraping llama.cpp target %s: %s", target.get("name", "unknown"), e)
+
+        lines = []
+        for (metric_type, label_pairs), value in sorted(all_metrics.items()):
+            labels = dict(label_pairs)
+            if not labels:
+                line = f"{metric_type} {value}"
+            else:
+                label_str = ",".join(f'{k}="{v}"' for k, v in sorted(labels.items()))
+                line = f"{metric_type}{{{label_str}}} {value}"
+            lines.append(line)
+
+        body = "\n".join(lines) + "\n" if lines else "# no metrics available\n"
+
+        self.send_response(200)
+        self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+        self.end_headers()
+        self.wfile.write(body.encode("utf-8"))
+
+    def log_message(self, format, *args):
+        logger.debug("%s - - %s", self.address_string(), format % args)
+
+
+def main():
+    port = int(os.environ.get("LLAMA_EXPORTER_PORT", "9550"))
+    host = os.environ.get("LLAMA_EXPORTER_BIND", "0.0.0.0")
+
+    server = HTTPServer((host, port), MetricsHandler)
+    logger.info("Starting Llama Exporter on %s:%d", host, port)
+    logger.info("Polling every %d seconds, timeout %ds", POLL_INTERVAL, SCRAPE_TIMEOUT)
+
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        logger.info("Shutting down")
+        server.shutdown()
+
+
+if __name__ == "__main__":
+    main()

ansible/roles/gpu_monitoring/files/nvidia_gpu_exporter.py

@@ -0,0 +1,436 @@
+# nvidia_gpu_exporter - NVIDIA GPU Prometheus exporter using NVML (nvidia-ml-py)
+# Works with consumer GPUs (RTX series, GTX series) and datacenter GPUs
+#
+# Exposes metrics via HTTP at /metrics
+# Required package: nvidia-ml-py (pip)
+
+import http.server
+import json
+import threading
+import time
+import sys
+import signal
+import traceback
+import subprocess
+import csv
+
+try:
+    import pynvml
+except ImportError:
+    print("ERROR: pynvml not installed. Run: pip install nvidia-ml-py", file=sys.stderr)
+    sys.exit(1)
+
+# Globals
+port = 9400
+stop_event = threading.Event()
+nvidia_initialized = False
+gpus_detected = 0
+gpu_handles = {}
+gpu_names = {}
+last_metrics = {}
+last_metrics_time = 0
+metrics_lock = threading.Lock()
+
+
+def signal_handler(sig, frame):
+    stop_event.set()
+
+
+def format_value(value, dtype):
+    """Format a value based on its NVML type."""
+    if value is None:
+        return "0"
+    try:
+        if dtype == pynvml.NVML_FEATURE_COUNTER_DATA_TYPE_UINT32:
+            return str(int(value))
+        elif dtype == pynvml.NVML_FEATURE_COUNTER_DATA_TYPE_UINT64:
+            return str(int(value))
+        elif dtype == pynvml.NVML_FEATURE_COUNTER_DATA_TYPE_FLOAT:
+            return f"{float(value):.6f}"
+        else:
+            return str(value)
+    except (ValueError, TypeError):
+        return "0"
+
+
+def extract_nvml_stats_pair(stats):
+    """Return (session_count, avg_fps) from NVML stats object or tuple."""
+    if stats is None:
+        return 0.0, 0.0
+
+    # Newer bindings may return an object with named fields.
+    if hasattr(stats, "sessionCount") and hasattr(stats, "avgFps"):
+        return float(stats.sessionCount), float(stats.avgFps)
+
+    # Older bindings may return a tuple/list.
+    if isinstance(stats, (tuple, list)) and len(stats) >= 2:
+        return float(stats[0]), float(stats[1])
+
+    return 0.0, 0.0
+
+
+def _to_float_or_none(value):
+    if value is None:
+        return None
+    raw = str(value).strip()
+    if raw == "" or raw.upper() == "N/A":
+        return None
+    try:
+        return float(raw)
+    except ValueError:
+        return None
+
+
+def _to_bool_float_or_none(value):
+    if value is None:
+        return None
+    raw = str(value).strip().lower()
+    if raw in {"1", "true", "yes", "active", "enabled"}:
+        return 1.0
+    if raw in {"0", "false", "no", "not active", "disabled", "n/a"}:
+        return 0.0
+    return None
+
+
+def collect_nvidia_smi_metrics():
+    """Collect supplemental metrics via nvidia-smi keyed by GPU index."""
+    fields = [
+        "index",
+        "pstate",
+        "pcie.link.gen.current",
+        "pcie.link.width.current",
+        "clocks_event_reasons.sw_power_cap",
+        "clocks_event_reasons.hw_thermal_slowdown",
+        "clocks_event_reasons.hw_power_brake_slowdown",
+    ]
+
+    cmd = [
+        "nvidia-smi",
+        f"--query-gpu={','.join(fields)}",
+        "--format=csv,noheader,nounits",
+    ]
+
+    try:
+        proc = subprocess.run(cmd, capture_output=True, text=True, timeout=5, check=True)
+    except (OSError, subprocess.SubprocessError):
+        return {}
+
+    metrics_by_index = {}
+    reader = csv.reader(proc.stdout.splitlines())
+    for row in reader:
+        if len(row) < len(fields):
+            continue
+
+        idx_raw = row[0].strip()
+        if not idx_raw.isdigit():
+            continue
+        idx = int(idx_raw)
+
+        smi_metrics = {}
+
+        pstate_raw = row[1].strip()
+        if pstate_raw.startswith("P") and pstate_raw[1:].isdigit():
+            smi_metrics["pstate"] = float(int(pstate_raw[1:]))
+
+        pcie_gen = _to_float_or_none(row[2])
+        if pcie_gen is not None:
+            smi_metrics["pcie_link_gen_current"] = pcie_gen
+
+        pcie_width = _to_float_or_none(row[3])
+        if pcie_width is not None:
+            smi_metrics["pcie_link_width_current"] = pcie_width
+
+        sw_power_cap = _to_bool_float_or_none(row[4])
+        if sw_power_cap is not None:
+            smi_metrics["throttle_sw_power_cap"] = sw_power_cap
+
+        hw_thermal = _to_bool_float_or_none(row[5])
+        if hw_thermal is not None:
+            smi_metrics["throttle_hw_thermal_slowdown"] = hw_thermal
+
+        hw_power_brake = _to_bool_float_or_none(row[6])
+        if hw_power_brake is not None:
+            smi_metrics["throttle_hw_power_brake_slowdown"] = hw_power_brake
+
+        if smi_metrics:
+            metrics_by_index[idx] = smi_metrics
+
+    return metrics_by_index
+
+
+def get_gpu_metrics():
+    """Collect all GPU metrics from NVML."""
+    global gpus_detected, gpu_handles, gpu_names, last_metrics, last_metrics_time, nvidia_initialized
+
+    if not nvidia_initialized:
+        return {}
+
+    metrics = {}
+    now = time.time()
+    current_time = int(now)
+
+    # Global metrics
+    metrics["gpu_count"] = float(gpus_detected)
+    metrics["instance"] = "0"
+
+    smi_metrics_by_index = collect_nvidia_smi_metrics()
+
+    for i in range(gpus_detected):
+        try:
+            handle = gpu_handles.get(i)
+            name = gpu_names.get(i, "Unknown")
+            gpu_prefix = f"gpu_{i}"
+
+            # Basic info (set once)
+            if f"{gpu_prefix}_info" not in last_metrics or current_time == 0:
+                metrics[f"{gpu_prefix}_info"] = 1.0
+                metrics[f"{gpu_prefix}_name"] = float(name_to_float(name))
+
+            # Power
+            try:
+                power_usage = pynvml.nvmlDeviceGetPowerUsage(handle)
+                metrics[f"{gpu_prefix}_power_watts"] = power_usage / 1000.0
+            except pynvml.NVMLError:
+                metrics[f"{gpu_prefix}_power_watts"] = 0.0
+
+            try:
+                power_limit = pynvml.nvmlDeviceGetPowerManagementLimit(handle)
+                if power_limit:
+                    metrics[f"{gpu_prefix}_power_limit_watts"] = power_limit / 1000.0
+            except pynvml.NVMLError:
+                pass
+
+            # Temperature
+            try:
+                temp = pynvml.nvmlDeviceGetTemperature(handle, pynvml.NVML_TEMPERATURE_GPU)
+                metrics[f"{gpu_prefix}_temperature_celsius"] = float(temp)
+            except pynvml.NVMLError:
+                metrics[f"{gpu_prefix}_temperature_celsius"] = 0.0
+
+            # Memory
+            try:
+                mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle)
+                metrics[f"{gpu_prefix}_memory_used_bytes"] = mem_info.used
+                metrics[f"{gpu_prefix}_memory_total_bytes"] = mem_info.total
+                if mem_info.total > 0:
+                    metrics[f"{gpu_prefix}_memory_used_percent"] = (mem_info.used / mem_info.total) * 100.0
+            except pynvml.NVMLError:
+                metrics[f"{gpu_prefix}_memory_used_bytes"] = 0.0
+                metrics[f"{gpu_prefix}_memory_total_bytes"] = 0.0
+                metrics[f"{gpu_prefix}_memory_used_percent"] = 0.0
+
+            # Utilization
+            try:
+                util = pynvml.nvmlDeviceGetUtilizationRates(handle)
+                metrics[f"{gpu_prefix}_gpu_util_percent"] = float(util.gpu)
+                metrics[f"{gpu_prefix}_memory_util_percent"] = float(util.memory)
+            except pynvml.NVMLError:
+                metrics[f"{gpu_prefix}_gpu_util_percent"] = 0.0
+                metrics[f"{gpu_prefix}_memory_util_percent"] = 0.0
+
+            # Clock speeds
+            try:
+                clocks = pynvml.nvmlDeviceGetClockInfo(handle, pynvml.NVML_CLOCK_GRAPHICS)
+                metrics[f"{gpu_prefix}_clock_graphics_hz"] = float(clocks) * 1000000
+            except pynvml.NVMLError:
+                metrics[f"{gpu_prefix}_clock_graphics_hz"] = 0.0
+
+            try:
+                # pynvml exposes memory clock as NVML_CLOCK_MEM in some versions.
+                mem_clock_type = getattr(pynvml, "NVML_CLOCK_MEMORY", None)
+                if mem_clock_type is None:
+                    mem_clock_type = getattr(pynvml, "NVML_CLOCK_MEM", None)
+                if mem_clock_type is None:
+                    raise AttributeError("No NVML memory clock constant available")
+                mem_clock = pynvml.nvmlDeviceGetClockInfo(handle, mem_clock_type)
+                metrics[f"{gpu_prefix}_clock_memory_hz"] = float(mem_clock) * 1000000
+            except (pynvml.NVMLError, AttributeError):
+                metrics[f"{gpu_prefix}_clock_memory_hz"] = 0.0
+
+            try:
+                vol_clock = pynvml.nvmlDeviceGetClockInfo(handle, pynvml.NVML_CLOCK_VIDEO)
+                metrics[f"{gpu_prefix}_clock_video_hz"] = float(vol_clock) * 1000000
+            except pynvml.NVMLError:
+                metrics[f"{gpu_prefix}_clock_video_hz"] = 0.0
+
+            # Fan speed
+            try:
+                fan = pynvml.nvmlDeviceGetFanSpeed(handle)
+                metrics[f"{gpu_prefix}_fan_speed_percent"] = float(fan)
+            except pynvml.NVMLError:
+                metrics[f"{gpu_prefix}_fan_speed_percent"] = 0.0
+
+            # PCIe stats
+            try:
+                tx = pynvml.nvmlDeviceGetPcieThroughputStats(handle)
+                if hasattr(tx, 'tx_bytes'):
+                    metrics[f"{gpu_prefix}_pcie_tx_bytes_total"] = float(tx.tx_bytes)
+                    metrics[f"{gpu_prefix}_pcie_rx_bytes_total"] = float(tx.rx_bytes)
+            except (pynvml.NVMLError, AttributeError):
+                pass
+
+            # Compute/pending processes
+            try:
+                processes = pynvml.nvmlDeviceGetComputeRunningProcesses(handle)
+                metrics[f"{gpu_prefix}_processes"] = float(len(processes))
+            except pynvml.NVMLError:
+                metrics[f"{gpu_prefix}_processes"] = 0.0
+
+            # Encoder/Decoder stats
+            try:
+                enc_stats = pynvml.nvmlDeviceGetEncoderStats(handle)
+                enc_sessions, enc_avg_fps = extract_nvml_stats_pair(enc_stats)
+                metrics[f"{gpu_prefix}_encoder_sessions"] = enc_sessions
+                metrics[f"{gpu_prefix}_encoder_avg_fps"] = enc_avg_fps
+            except (pynvml.NVMLError, AttributeError, TypeError, ValueError):
+                metrics[f"{gpu_prefix}_encoder_sessions"] = 0.0
+                metrics[f"{gpu_prefix}_encoder_avg_fps"] = 0.0
+
+            try:
+                dec_stats = pynvml.nvmlDeviceGetDecoderStats(handle)
+                dec_sessions, dec_avg_fps = extract_nvml_stats_pair(dec_stats)
+                metrics[f"{gpu_prefix}_decoder_sessions"] = dec_sessions
+                metrics[f"{gpu_prefix}_decoder_avg_fps"] = dec_avg_fps
+            except (pynvml.NVMLError, AttributeError, TypeError, ValueError):
+                metrics[f"{gpu_prefix}_decoder_sessions"] = 0.0
+                metrics[f"{gpu_prefix}_decoder_avg_fps"] = 0.0
+
+            # Reboot/Pwr events
+            try:
+                inforom_fn = getattr(pynvml, "nvmlDeviceGetInforomConfigurationVersion", None)
+                if inforom_fn is None:
+                    raise AttributeError("Inforom API not available in this pynvml version")
+                events = inforom_fn(handle)
+                if events:
+                    metrics[f"{gpu_prefix}_firmware_version"] = 1.0
+            except (pynvml.NVMLError, AttributeError):
+                pass
+
+            # Supplemental nvidia-smi metrics (when available).
+            smi_metrics = smi_metrics_by_index.get(i, {})
+            for name, value in smi_metrics.items():
+                metrics[f"{gpu_prefix}_{name}"] = value
+
+        except pynvml.NVMLError:
+            traceback.print_exc()
+            continue
+
+    last_metrics = metrics
+    last_metrics_time = current_time
+    return metrics
+
+
+def name_to_float(name):
+    """Convert GPU name to a numeric value for prometheus."""
+    # Prometheus metrics don't support string values, so we use hash
+    return float(hash(name)) % 1000000
+
+
+def generate_prometheus_output(metrics):
+    """Format metrics as Prometheus text exposition format."""
+    lines = []
+    lines.append("# HELP gpu_info NVIDIA GPU information")
+    lines.append("# TYPE gpu_info gauge")
+    for i in range(gpus_detected):
+        lines.append(f'gpu_info{{gpu="{gpu_names.get(i, "Unknown")}",gpu_id="{i}"}} 1')
+
+    for key, value in sorted(metrics.items()):
+        # Skip info metric (handled above)
+        if key == "gpu_count":
+            lines.append(f"# HELP gpu_total Total number of NVIDIA GPUs")
+            lines.append("# TYPE gpu_total gauge")
+            lines.append(f"gpu_total {value}")
+            continue
+
+        if key == "instance":
+            continue
+
+        if key.endswith("_name"):
+            continue
+
+        if key.endswith("_info"):
+            continue
+
+        lines.append(f"# HELP {key} {key} metric")
+        lines.append("# TYPE " + key + " gauge")
+        if isinstance(value, float):
+            lines.append(f"{key} {value:.6f}")
+        else:
+            lines.append(f"{key} {value}")
+
+    return "\n".join(lines) + "\n"
+
+
+class MetricsHandler(http.server.BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path == "/metrics":
+            with metrics_lock:
+                metrics = get_gpu_metrics()
+            output = generate_prometheus_output(metrics)
+            self.send_response(200)
+            self.send_header("Content-Type", "text/plain; charset=utf-8")
+            self.end_headers()
+            self.wfile.write(output.encode())
+        elif self.path == "/health":
+            self.send_response(200)
+            self.send_header("Content-Type", "text/plain")
+            self.end_headers()
+            self.wfile.write("OK\n".encode())
+        else:
+            self.send_response(404)
+            self.end_headers()
+
+    def log_message(self, format, *args):
+        # Suppress default logging
+        pass
+
+
+def run_server():
+    global port, stop_event, nvidia_initialized, gpus_detected, gpu_handles, gpu_names
+
+    try:
+        pynvml.nvmlInit()
+        nvidia_initialized = True
+    except pynvml.NVMLError as e:
+        print(f"ERROR: Failed to initialize NVML: {e}", file=sys.stderr)
+        print("Make sure NVIDIA drivers are properly installed.", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        gpus_detected = pynvml.nvmlDeviceGetCount()
+        print(f"Found {gpus_detected} NVIDIA GPU(s)")
+
+        for i in range(gpus_detected):
+            try:
+                handle = pynvml.nvmlDeviceGetHandleByIndex(i)
+                gpu_handles[i] = handle
+                name = pynvml.nvmlDeviceGetName(handle)
+                if isinstance(name, bytes):
+                    name = name.decode("utf-8")
+                gpu_names[i] = name
+                print(f"  GPU {i}: {name}")
+            except pynvml.NVMLError:
+                pass
+    except pynvml.NVMLError as e:
+        print(f"ERROR: Failed to enumerate devices: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    server = http.server.HTTPServer(("0.0.0.0", port), MetricsHandler)
+    print(f"NVIDIA GPU Exporter listening on port {port}")
+
+    signal.signal(signal.SIGTERM, signal_handler)
+    signal.signal(signal.SIGINT, signal_handler)
+
+    # Poll so SIGTERM can stop the loop quickly instead of waiting on long timeouts.
+    server.timeout = 1
+
+    try:
+        while not stop_event.is_set():
+            server.handle_request()
+    finally:
+        server.server_close()
+        pynvml.nvmlShutdown()
+
+
+if __name__ == "__main__":
+    run_server()

ansible/roles/gpu_monitoring/handlers/main.yml

@@ -0,0 +1,27 @@
+---
+- name: Reload systemd daemon
+  become: true
+  ansible.builtin.systemd:
+    daemon_reload: true
+  listen: Reload systemd daemon
+
+- name: Restart nvidia_exporter
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ nvidia_exporter_service_name }}"
+    state: restarted
+  listen: Restart nvidia_exporter
+
+- name: Restart amd_exporter
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ amd_exporter_service_name }}"
+    state: restarted
+  listen: Restart amd_exporter
+
+- name: Restart llama_exporter
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ llama_exporter_service_name }}"
+    state: restarted
+  listen: Restart llama_exporter

ansible/roles/gpu_monitoring/meta/main.yml

@@ -0,0 +1,15 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Deploy Prometheus monitoring stack on gpu-01 for GPU and LLM metrics
+  company: A13Labs
+  role_name: gpu_monitoring
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: Fedora
+      versions:
+        - "39"
+        - "40"
+        - "41"

ansible/roles/gpu_monitoring/molecule/default/converge.yml

@@ -0,0 +1,11 @@
+---
+- name: Converge
+  hosts: all
+  gather_facts: true
+  tasks:
+    - name: Include gpu_monitoring role
+      ansible.builtin.include_role:
+        name: "gpu_monitoring"
+      vars:
+        amd_exporter_enabled: false
+        llama_exporter_enabled: false

ansible/roles/gpu_monitoring/molecule/default/molecule.yml

@@ -0,0 +1,17 @@
+---
+driver:
+  name: podman
+platforms:
+  - name: instance
+    image: fedora:latest
+    pre_build_image: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    privileged: true
+provisioner:
+  name: ansible
+  playbooks:
+    converge: converge.yml
+    prepare: prepare.yml
+verifier:
+  name: ansible

ansible/roles/gpu_monitoring/molecule/default/prepare.yml

@@ -0,0 +1,10 @@
+---
+- name: Prepare
+  hosts: all
+  gather_facts: false
+  tasks:
+    - name: Update cache
+      ansible.builtin.raw: dnf update -y
+
+    - name: Install required packages
+      ansible.builtin.raw: dnf install -y python3

ansible/roles/gpu_monitoring/tasks/amd_exporter.yml

@@ -0,0 +1,82 @@
+---
+- name: Create amd_exporter group
+  become: true
+  ansible.builtin.group:
+    name: "{{ amd_exporter_user }}"
+    state: present
+    system: true
+
+- name: Create amd_exporter user
+  become: true
+  ansible.builtin.user:
+    name: "{{ amd_exporter_user }}"
+    group: "{{ amd_exporter_user }}"
+    home: "{{ amd_exporter_user_home }}"
+    create_home: true
+    shell: /usr/sbin/nologin
+    system: true
+
+- name: Create amd_exporter install directory
+  become: true
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ amd_exporter_user }}"
+    group: "{{ amd_exporter_user }}"
+    mode: "0755"
+  loop:
+    - "{{ amd_exporter_install_dir }}"
+    - "{{ amd_exporter_install_dir }}/logs"
+
+- name: Copy amd_gpu_exporter script
+  become: true
+  ansible.builtin.copy:
+    src: amd_gpu_exporter.py
+    dest: "{{ amd_exporter_script_path }}"
+    owner: "{{ amd_exporter_user }}"
+    group: "{{ amd_exporter_user }}"
+    mode: "0755"
+  notify:
+    - Restart amd_exporter
+
+- name: Create Python virtual environment
+  become: true
+  ansible.builtin.command:
+    cmd: "{{ amd_exporter_python_version }} -m venv {{ amd_exporter_venv_path }}"
+    creates: "{{ amd_exporter_venv_path }}/bin/activate"
+
+- name: Install Python dependencies in venv
+  become: true
+  ansible.builtin.pip:
+    name:
+      - prometheus-client
+      - requests
+    virtualenv: "{{ amd_exporter_venv_path }}"
+    state: present
+
+- name: Fix ownership of amd_exporter directory
+  become: true
+  ansible.builtin.file:
+    path: "{{ amd_exporter_install_dir }}"
+    owner: "{{ amd_exporter_user }}"
+    group: "{{ amd_exporter_user }}"
+    recurse: true
+
+- name: Generate amd_exporter systemd service
+  become: true
+  ansible.builtin.template:
+    src: amdgpu_exporter.service.j2
+    dest: "/etc/systemd/system/{{ amd_exporter_service_name }}.service"
+    owner: root
+    group: root
+    mode: "0644"
+  notify:
+    - Reload systemd daemon
+    - Restart amd_exporter
+
+- name: Enable amd_exporter service
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ amd_exporter_service_name }}"
+    enabled: true
+    state: started

ansible/roles/gpu_monitoring/tasks/firewall.yml

@@ -0,0 +1,13 @@
+---
+- name: Add Prometheus firewall rules
+  become: true
+  community.general.ufw:
+    rule: allow
+    src: "{{ item.cidr }}"
+    port: "{{ prometheus_fw_port | string }}"
+    proto: "{{ prometheus_fw_proto }}"
+    comment: "{{ item.comment | default(omit) }}"
+  loop: "{{ prometheus_fw_allowed_networks }}"
+  loop_control:
+    label: "{{ item.cidr }} -> port {{ prometheus_fw_port }}"
+  when: item.cidr is defined and item.cidr | length > 0

ansible/roles/gpu_monitoring/tasks/llama_exporter.yml

@@ -0,0 +1,82 @@
+---
+- name: Create llama_exporter group
+  become: true
+  ansible.builtin.group:
+    name: "{{ llama_exporter_user }}"
+    state: present
+    system: true
+
+- name: Create llama_exporter user
+  become: true
+  ansible.builtin.user:
+    name: "{{ llama_exporter_user }}"
+    group: "{{ llama_exporter_user }}"
+    home: "{{ llama_exporter_user_home }}"
+    create_home: true
+    shell: /usr/sbin/nologin
+    system: true
+
+- name: Create llama_exporter install directory
+  become: true
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ llama_exporter_user }}"
+    group: "{{ llama_exporter_user }}"
+    mode: "0755"
+  loop:
+    - "{{ llama_exporter_install_dir }}"
+    - "{{ llama_exporter_install_dir }}/logs"
+
+- name: Copy llama_exporter script
+  become: true
+  ansible.builtin.copy:
+    src: llama_exporter.py
+    dest: "{{ llama_exporter_script_path }}"
+    owner: "{{ llama_exporter_user }}"
+    group: "{{ llama_exporter_user }}"
+    mode: "0755"
+  notify:
+    - Restart llama_exporter
+
+- name: Create Python virtual environment
+  become: true
+  ansible.builtin.command:
+    cmd: "{{ ansible_facts['python']['executable'] }} -m venv {{ llama_exporter_venv_path }}"
+    creates: "{{ llama_exporter_venv_path }}/bin/activate"
+
+- name: Install Python dependencies in llama_exporter venv
+  become: true
+  ansible.builtin.pip:
+    name:
+      - prometheus-client
+      - requests
+    virtualenv: "{{ llama_exporter_venv_path }}"
+    state: present
+
+- name: Fix ownership of llama_exporter directory
+  become: true
+  ansible.builtin.file:
+    path: "{{ llama_exporter_install_dir }}"
+    owner: "{{ llama_exporter_user }}"
+    group: "{{ llama_exporter_user }}"
+    recurse: true
+
+- name: Generate llama_exporter systemd service
+  become: true
+  ansible.builtin.template:
+    src: llama_exporter.service.j2
+    dest: "/etc/systemd/system/{{ llama_exporter_service_name }}.service"
+    owner: root
+    group: root
+    mode: "0644"
+  notify:
+    - Reload systemd daemon
+    - Restart llama_exporter
+
+- name: Enable llama_exporter service
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ llama_exporter_service_name }}"
+    enabled: true
+    state: started

ansible/roles/gpu_monitoring/tasks/main.yml

@@ -0,0 +1,28 @@
+---
+- name: Include NVIDIA GPU Exporter setup
+  when: nvidia_exporter_enabled | default(true) | bool
+  ansible.builtin.include_tasks: nvidia_exporter.yml
+  tags:
+    - roles::gpu_monitoring::nvidia_exporter
+
+- name: Include AMD GPU Exporter setup
+  when: amd_exporter_enabled | default(true) | bool
+  ansible.builtin.include_tasks: amd_exporter.yml
+  tags:
+    - roles::gpu_monitoring::amd_exporter
+
+- name: Include Llama Exporter setup
+  when: llama_exporter_enabled | default(true) | bool
+  ansible.builtin.include_tasks: llama_exporter.yml
+  tags:
+    - roles::gpu_monitoring::llama_exporter
+
+- name: Include Prometheus setup
+  ansible.builtin.include_tasks: prometheus.yml
+  tags:
+    - roles::gpu_monitoring::prometheus
+
+- name: Include firewall rules
+  ansible.builtin.include_tasks: firewall.yml
+  tags:
+    - roles::gpu_monitoring::firewall

ansible/roles/gpu_monitoring/tasks/nvidia_exporter.yml

@@ -0,0 +1,84 @@
+---
+# NVIDIA GPU Exporter using NVML (pynvml)
+# Works with consumer GPUs (RTX/GTX) and datacenter GPUs
+# No DCGM required - uses pynvml Python library
+
+- name: Create NVIDIA exporter group
+  ansible.builtin.group:
+    name: "{{ nvidia_exporter_user }}"
+    state: present
+    system: true
+  become: true
+
+- name: Create NVIDIA exporter user
+  ansible.builtin.user:
+    name: "{{ nvidia_exporter_user }}"
+    group: "{{ nvidia_exporter_user }}"
+    home: "{{ nvidia_exporter_user_home }}"
+    system: true
+    create_home: true
+    shell: /usr/sbin/nologin
+  become: true
+
+- name: Create NVIDIA exporter install directory
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ nvidia_exporter_user }}"
+    group: "{{ nvidia_exporter_user }}"
+    mode: "0755"
+  loop:
+    - "{{ nvidia_exporter_install_dir }}"
+    - "{{ nvidia_exporter_install_dir }}/logs"
+  become: true
+
+- name: Copy NVIDIA exporter script
+  ansible.builtin.copy:
+    src: nvidia_gpu_exporter.py
+    dest: "{{ nvidia_exporter_script_path }}"
+    owner: "{{ nvidia_exporter_user }}"
+    group: "{{ nvidia_exporter_user }}"
+    mode: "0755"
+  become: true
+  notify:
+    - Restart nvidia_exporter
+
+- name: Create Python virtual environment for NVIDIA exporter
+  ansible.builtin.command:
+    cmd: "{{ ansible_facts['python']['executable'] }} -m venv {{ nvidia_exporter_venv_path }}"
+    creates: "{{ nvidia_exporter_venv_path }}/bin/activate"
+  become: true
+
+- name: Install nvidia-ml-py in NVIDIA exporter venv
+  ansible.builtin.pip:
+    name: "nvidia-ml-py"
+    virtualenv: "{{ nvidia_exporter_venv_path }}"
+    state: present
+  become: true
+
+- name: Fix ownership of NVIDIA exporter directory
+  ansible.builtin.file:
+    path: "{{ nvidia_exporter_install_dir }}"
+    owner: "{{ nvidia_exporter_user }}"
+    group: "{{ nvidia_exporter_user }}"
+    recurse: true
+  become: true
+
+- name: Copy NVIDIA exporter systemd service
+  ansible.builtin.template:
+    src: nvidia_exporter.service.j2
+    dest: "/etc/systemd/system/{{ nvidia_exporter_service_name }}.service"
+    owner: root
+    group: root
+    mode: "0644"
+  become: true
+  notify:
+    - Reload systemd daemon
+    - Restart nvidia_exporter
+
+- name: Ensure NVIDIA exporter is enabled and started
+  ansible.builtin.systemd:
+    name: "{{ nvidia_exporter_service_name }}"
+    enabled: true
+    state: started
+  become: true

ansible/roles/gpu_monitoring/tasks/prometheus.yml

@@ -0,0 +1,206 @@
+---
+# Prometheus - Podman container with standard podman.yml pattern
+# Follows the pattern in playbook_podman_llama_vulkan.yml
+
+- name: Prometheus | Check if required vars are set
+  ansible.builtin.fail:
+    msg: "Required variables are not set for Podman setup. Please check your playbook variables.)"
+  when: prometheus_user is not defined or prometheus_group is not defined or prometheus_user_home is not defined
+  tags:
+    - prometheus
+
+- name: Prometheus | Create prometheus group
+  become: true
+  ansible.builtin.group:
+    name: "{{ prometheus_group }}"
+    system: true
+  tags:
+    - prometheus
+
+- name: Prometheus | Create prometheus user
+  become: true
+  ansible.builtin.user:
+    name: "{{ prometheus_user }}"
+    groups: "{{ prometheus_extra_groups | default([]) }}"
+    shell: /bin/bash
+    home: "{{ prometheus_user_home }}"
+    group: "{{ prometheus_group }}"
+  tags:
+    - prometheus
+
+- name: Prometheus | Disable password login for prometheus user
+  ansible.builtin.command: passwd -d "{{ prometheus_user }}"
+  become: true
+  changed_when: false
+  tags:
+    - prometheus
+
+- name: Prometheus | Create prometheus user home
+  become: true
+  ansible.builtin.file:
+    path: "{{ prometheus_user_home }}"
+    state: directory
+    owner: "{{ prometheus_user }}"
+    group: "{{ prometheus_group }}"
+    mode: "0750"
+  tags:
+    - prometheus
+
+- name: Prometheus | Create build folders
+  become: true
+  ansible.builtin.file:
+    path: "{{ prometheus_user_home }}/build"
+    state: directory
+    owner: "{{ prometheus_user }}"
+    group: "{{ prometheus_group }}"
+    mode: "0750"
+  tags:
+    - prometheus
+
+- name: Prometheus | Create folders
+  become: true
+  ansible.builtin.file:
+    path: "{{ prometheus_user_home }}/{{ item }}"
+    state: directory
+    owner: "{{ prometheus_user }}"
+    group: "{{ prometheus_group }}"
+    mode: "0750"
+  loop: "{{ prometheus_user_folders | default([]) }}"
+  tags:
+    - prometheus
+
+- name: Prometheus | Add SSH Authorized key
+  become: true
+  ansible.posix.authorized_key:
+    user: "{{ prometheus_user }}"
+    key: "{{ prometheus_user_pubkey }}"
+    state: "{{ 'present' if prometheus_user_pubkey is defined else 'absent' }}"
+  when: prometheus_user_pubkey is defined and prometheus_user_pubkey != ""
+  tags:
+    - prometheus
+
+- name: Prometheus | SELinux tasks (RedHat only)
+  become: true
+  when: ansible_os_family == 'RedHat'
+  tags:
+    - prometheus
+  block:
+    - name: Prometheus | Persistently set SELinux context
+      community.general.sefcontext:
+        target: "{{ prometheus_user_home }}(/.*)?"
+        setype: user_home_dir_t
+        state: present
+
+    - name: Prometheus | Persistently set SELinux context (ssh authorized keys)
+      community.general.sefcontext:
+        target: "{{ prometheus_user_home }}/.ssh/authorized_keys"
+        setype: ssh_home_t
+        state: present
+      when: prometheus_user_pubkey is defined and prometheus_user_pubkey != ""
+
+    - name: Prometheus | Apply SELinux context
+      ansible.builtin.command: restorecon -Rv {{ prometheus_user_home }}
+      changed_when: false
+
+- name: Prometheus | Enable lingering for prometheus user
+  become: true
+  ansible.builtin.command: loginctl enable-linger {{ prometheus_user }}
+  register: linger_status
+  changed_when: >-
+    'created' in linger_status.stdout or
+    (linger_status.rc == 0 and not
+      ('linger file already exists' in linger_status.stderr or
+      'linger file does not exist' in linger_status.stderr))
+  failed_when:
+    - linger_status.rc != 0 and not
+      ('linger file already exists' in linger_status.stderr)
+  tags:
+    - prometheus
+
+- name: Prometheus | Get prometheus user UID
+  ansible.builtin.command: id -u {{ prometheus_user }}
+  changed_when: false
+  register: prometheus_uid
+  tags:
+    - prometheus
+
+- name: Prometheus | Deploy Prometheus configuration
+  ansible.builtin.template:
+    src: prometheus.yml.j2
+    dest: "{{ prometheus_user_home }}/prometheus.yml"
+    owner: "{{ prometheus_user }}"
+    group: "{{ prometheus_group }}"
+    mode: "0644"
+  tags:
+    - prometheus
+
+- name: Prometheus | Create Prometheus data directory
+  ansible.builtin.file:
+    path: "{{ prometheus_data_dir }}"
+    state: directory
+    owner: "{{ prometheus_user }}"
+    group: "{{ prometheus_group }}"
+    mode: "0700"
+  tags:
+    - prometheus
+
+- name: Prometheus | Run Podman tasks as prometheus user
+  become: true
+  become_user: "{{ prometheus_user }}"
+  environment:
+    XDG_RUNTIME_DIR: "/run/user/{{ prometheus_uid.stdout }}"
+  tags:
+    - prometheus
+  block:
+    - name: Prometheus | Pull Prometheus image
+      containers.podman.podman_image:
+        name: "docker.io/prom/prometheus:{{ prometheus_image_tag }}"
+        force: false
+
+    - name: Prometheus | Create podman network
+      containers.podman.podman_network:
+        name: "{{ prometheus_network_name }}"
+
+    - name: Prometheus | Create Podman container
+      containers.podman.podman_container:
+        name: prometheus
+        image: "docker.io/prom/prometheus:{{ prometheus_image_tag }}"
+        state: started
+        recreate: true
+        user: "{{ prometheus_uid.stdout }}"
+        network: "{{ prometheus_network_name }}"
+        network_alias:
+          - prometheus
+        ports:
+          - "0.0.0.0:{{ prometheus_host_port }}:{{ prometheus_container_port }}/tcp"
+        volumes:
+          - "{{ prometheus_data_dir }}:/prometheus"
+          - "{{ prometheus_user_home }}/prometheus.yml:/etc/prometheus/prometheus.yml"
+        memory: "{{ prometheus_container_memory }}"
+        memory_swap: "0"
+        env:
+          TZ: "{{ timezone | default('UTC') }}"
+          PROMETHEUS_PORT: "{{ prometheus_port | string }}"
+
+        systemd: "true"
+        restart: false
+        detach: true
+        label:
+          a13labs.prometheus: "true"
+          a13labs.service: "prometheus"
+          a13labs.team: "infra"
+        cmd_args:
+          - "--userns=keep-id"
+          - "--security-opt=label=disable"
+        restart_policy: unless-stopped
+        stop_signal: 1
+- name: Prometheus | Ensure systemd is reloaded after container creation
+  become: true
+  become_user: "{{ prometheus_user }}"
+  environment:
+    XDG_RUNTIME_DIR: "/run/user/{{ prometheus_uid.stdout }}"
+  ansible.builtin.systemd:
+    daemon_reload: true
+    scope: user
+  tags:
+    - prometheus

ansible/roles/gpu_monitoring/templates/amdgpu_exporter.service.j2

@@ -0,0 +1,30 @@
+[Unit]
+Description=AMD GPU Sysfs Metrics Exporter
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User={{ amd_exporter_user }}
+Group={{ amd_exporter_user }}
+WorkingDirectory={{ amd_exporter_install_dir }}
+Environment="AMD_EXPORTER_PORT={{ amd_exporter_port }}"
+Environment="AMD_EXPORTER_BIND=0.0.0.0"
+Environment="PATH={{ amd_exporter_venv_bin }}:/usr/bin:/bin"
+ExecStart={{ amd_exporter_venv_bin }}/python3 {{ amd_exporter_script_path }}
+Restart=always
+RestartSec=10
+TimeoutStopSec=10
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=amdgpu_exporter
+
+# Security hardening
+ProtectSystem=strict
+ProtectHome=true
+ReadWritePaths={{ amd_exporter_install_dir }} /var/log
+NoNewPrivileges=true
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/gpu_monitoring/templates/llama_exporter.service.j2

@@ -0,0 +1,35 @@
+[Unit]
+Description=LLM Inference Metrics Exporter (llama.cpp + Ollama)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User={{ llama_exporter_user }}
+Group={{ llama_exporter_user }}
+WorkingDirectory={{ llama_exporter_install_dir }}
+Environment="LLAMA_EXPORTER_PORT={{ llama_exporter_port }}"
+Environment="LLAMA_EXPORTER_BIND=0.0.0.0"
+Environment="LLAMA_EXPORTER_INTERVAL={{ llama_exporter_scrape_interval }}"
+Environment="LLAMA_EXPORTER_TIMEOUT={{ llama_exporter_scrape_timeout }}"
+{% if llama_exporter_targets and llama_exporter_targets != "" %}
+Environment="LLAMA_TARGETS={{ llama_exporter_targets | to_json }}"
+{% endif %}
+Environment="PATH={{ llama_exporter_venv_bin }}:/usr/bin:/bin"
+ExecStart={{ llama_exporter_venv_bin }}/python3 {{ llama_exporter_script_path }}
+Restart=always
+RestartSec=10
+TimeoutStopSec=10
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=llama_exporter
+
+# Security hardening
+ProtectSystem=strict
+ProtectHome=true
+ReadWritePaths={{ llama_exporter_install_dir }} /var/log
+NoNewPrivileges=true
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/gpu_monitoring/templates/nvidia_exporter.service.j2

@@ -0,0 +1,34 @@
+# nvidia_gpu_exporter - Prometheus exporter for NVIDIA GPUs via NVML
+#
+# Exposes GPU metrics at /metrics port {{ nvidia_exporter_port }}
+
+[Unit]
+Description=NVIDIA GPU Prometheus Exporter
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User={{ nvidia_exporter_user }}
+Group={{ nvidia_exporter_user }}
+WorkingDirectory={{ nvidia_exporter_install_dir }}
+Environment="PYTHONUNBUFFERED=1"
+ExecStart={{ nvidia_exporter_venv_path }}/bin/python3 {{ nvidia_exporter_script_path }}
+Restart=on-failure
+RestartSec=5
+TimeoutStopSec=10
+StandardOutput=journal
+StandardError=journal
+
+# Security hardening
+NoNewPrivileges=true
+ProtectSystem=strict
+ProtectHome=true
+ReadOnlyPaths=/sys
+PrivateTmp=true
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectControlGroups=true
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/gpu_monitoring/templates/podman.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=Podman service for {{ item.name }}
+After=local-fs.target network-online.target
+
+[Service]
+Type=oneshot
+User={{ prometheus_user }}
+Group={{ prometheus_group }}
+RemainAfterExit=true
+ExecStart={{ podman_binary | default('/usr/bin/podman') }} start {{ item.name }}
+ExecStop={{ podman_binary | default('/usr/bin/podman') }} stop {{ item.name }}
+
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/gpu_monitoring/templates/prometheus.yml.j2

@@ -0,0 +1,30 @@
+global:
+  scrape_interval: {{ prometheus_scrape_interval }}
+  evaluation_interval: {{ prometheus_evaluation_interval }}
+  scrape_timeout: {{ prometheus_scrape_timeout }}
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ["localhost:{{ prometheus_container_port }}"]
+
+  - job_name: nvidia-gpu
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["{{ prometheus_host_gateway }}:{{ nvidia_exporter_port }}"]
+        labels:
+          datacenter: "{{ inventory_hostname }}"
+
+  - job_name: amdgpu_exporter
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["{{ prometheus_host_gateway }}:{{ amd_exporter_port }}"]
+        labels:
+          datacenter: "{{ inventory_hostname }}"
+
+  - job_name: llama_exporter
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["{{ prometheus_host_gateway }}:{{ llama_exporter_port }}"]
+        labels:
+          datacenter: "{{ inventory_hostname }}"

ansible/roles/gpu_monitoring/vars/main.yml

@@ -0,0 +1,23 @@
+---
+# Internal role variables (non-overridable)
+
+# Container names
+gpu_monitoring_prometheus_container_name: "prometheus"
+
+# Derived paths
+nvidia_exporter_script_path: "{{ nvidia_exporter_install_dir }}/nvidia_gpu_exporter.py"
+nvidia_exporter_venv_bin: "{{ nvidia_exporter_venv_path }}/bin"
+nvidia_exporter_service_name: "nvidia_exporter"
+
+amd_exporter_script_path: "{{ amd_exporter_install_dir }}/amd_gpu_exporter.py"
+amd_exporter_venv_bin: "{{ amd_exporter_venv_path }}/bin"
+amd_exporter_service_name: "amdgpu_exporter"
+
+llama_exporter_script_path: "{{ llama_exporter_install_dir }}/llama_exporter.py"
+llama_exporter_venv_bin: "{{ llama_exporter_venv_path }}/bin"
+llama_exporter_service_name: "llama_exporter"
+
+# Default llama.cpp targets if none provided
+llama_exporter_targets_default:
+  - name: "llama-cpp-8012"
+    url: "http://localhost:8012"

ansible/tasks/llama_models_sync.yml

@@ -0,0 +1,146 @@
+- name: Ensure podman user exists before model sync
+  ansible.builtin.include_tasks:
+    file: podman/user.yml
+
+- name: Enable sudo access to podman user (temporary)
+  become: true
+  ansible.builtin.lineinfile:
+    path: "/etc/sudoers.d/ansible_{{ podman_user }}_tmp"
+    create: true
+    mode: "0440"
+    line: "{{ ansible_user_id }} ALL=({{ podman_user }}) NOPASSWD: ALL"
+    validate: "visudo -cf %s"
+
+- name: Validate llama_models schema
+  ansible.builtin.assert:
+    that:
+      - __llama_models__ is iterable
+    fail_msg: "llama_models must be a list"
+
+- name: Install Python packages required for llama model sync
+  become: true
+  ansible.builtin.package:
+    name: "{{ __llama_sync_system_packages__ }}"
+    state: present
+  vars:
+    __llama_sync_system_packages__: >-
+      {{
+        {
+          'Debian': ['python3-pip', 'python3-packaging', 'python3-venv'],
+          'RedHat': ['python3-pip', 'python3-packaging'],
+          'Archlinux': ['python-pip', 'python-packaging']
+        }[ansible_os_family]
+      }}
+
+- name: Install Hugging Face hub client in llama model sync virtualenv
+  become: true
+  become_user: "{{ podman_user }}"
+  ansible.builtin.pip:
+    name:
+      - pip
+      - setuptools
+      - packaging
+      - huggingface_hub>=0.32.0
+    state: present
+    virtualenv: "{{ __llama_sync_venv__ }}"
+    virtualenv_command: "python3 -m venv"
+
+- name: Create llama router model directories
+  become: true
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: directory
+    owner: "{{ podman_user }}"
+    group: "{{ podman_group }}"
+    mode: "{{ item.mode }}"
+  loop:
+    - { path: "{{ podman_user_home }}/models", mode: "0750" }
+    - { path: "{{ __llama_managed_dir__ }}", mode: "0750" }
+    - { path: "{{ __llama_links_dir__ }}", mode: "0750" }
+    - { path: "{{ __llama_router_dir__ }}", mode: "0750" }
+
+- name: Copy llama Hugging Face sync helper
+  become: true
+  ansible.builtin.copy:
+    src: scripts/llama_hf_sync.py
+    dest: "{{ __llama_sync_script__ }}"
+    owner: "{{ podman_user }}"
+    group: "{{ podman_group }}"
+    mode: "0750"
+
+- name: Write desired llama model manifest input
+  become: true
+  ansible.builtin.copy:
+    content: "{{ __llama_models__ | to_nice_json }}"
+    dest: "{{ __llama_models_input_file__ }}"
+    owner: "{{ podman_user }}"
+    group: "{{ podman_group }}"
+    mode: "0640"
+
+- name: Write global llama preset options
+  become: true
+  ansible.builtin.copy:
+    content: "{{ __llama_preset_global__ | to_nice_json }}"
+    dest: "{{ __llama_preset_global_input_file__ }}"
+    owner: "{{ podman_user }}"
+    group: "{{ podman_group }}"
+    mode: "0640"
+
+- name: Run llama model sync from Hugging Face
+  vars:
+    __llama_sync_argv__: >-
+      {{
+        [
+          __llama_sync_python__,
+          __llama_sync_script__,
+          '--models-file',
+          __llama_models_input_file__,
+          '--managed-dir',
+          __llama_managed_dir__,
+          '--links-dir',
+          __llama_links_dir__,
+          '--manifest-file',
+          __llama_manifest_file__,
+          '--preset-file',
+          __llama_preset_file__,
+          '--preset-global-file',
+          __llama_preset_global_input_file__,
+          '--container-links-dir',
+          '/models/managed-links'
+        ]
+        + (['--prune'] if llama_models_prune_enabled | default(true) else [])
+        + (['--dry-run'] if llama_models_dry_run | default(false) else [])
+      }}
+  become: true
+  become_user: "{{ podman_user }}"
+  ansible.builtin.command:
+    argv: "{{ __llama_sync_argv__ }}"
+  environment:
+    HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') | default('', true) }}"
+  register: __llama_sync_output__
+  changed_when: false
+
+- name: Parse llama sync output
+  ansible.builtin.set_fact:
+    __llama_sync_result__: "{{ __llama_sync_output__.stdout | from_json }}"
+
+- name: Ensure router preset file exists when not in dry-run mode
+  become: true
+  become_user: "{{ podman_user }}"
+  ansible.builtin.stat:
+    path: "{{ __llama_preset_file__ }}"
+  register: __llama_preset_stat__
+  when: not (llama_models_dry_run | default(false))
+
+- name: Ensure router preset file exists when not in dry-run mode
+  ansible.builtin.assert:
+    that:
+      - __llama_preset_stat__.stat.exists
+    fail_msg: "llama router preset file was not created"
+  when: not (llama_models_dry_run | default(false))
+
+- name: Remove temporary sudo access
+  become: true
+  ansible.builtin.file:
+    path: "/etc/sudoers.d/ansible_{{ podman_user }}_tmp"
+    state: absent

ansible/tasks/podman.yml

@@ -34,7 +34,7 @@
     path: "/etc/sudoers.d/ansible_{{ podman_user }}_tmp"
     create: true
     mode: "0440"
-    line: "{{ ansible_user_id }} ALL=({{ podman_user }}) NOPASSWD: ALL"
+    line: "{{ ansible_facts['user_id'] }} ALL=({{ podman_user }}) NOPASSWD: ALL"
     validate: "visudo -cf %s"
 
 - name: Enable lingering for podman user
@@ -42,10 +42,10 @@
   ansible.builtin.command: loginctl enable-linger {{ podman_user }}
   register: linger_status
   changed_when: >-
-    "'created' in linger_status.stdout or
+    'created' in linger_status.stdout or
     (linger_status.rc == 0 and not
       ('linger file already exists' in linger_status.stderr or
-      'linger file does not exist' in linger_status.stderr))"
+      'linger file does not exist' in linger_status.stderr))
   failed_when:
     - linger_status.rc != 0 and not
       ('linger file already exists' in linger_status.stderr)