feat(vaultwarden): add SMTP configuration options and enhance signup settings

- Introduced SMTP settings for Vaultwarden including host, port, security, and authentication details.
- Added variables for signup verification, 2FA settings, password hints, and logging options.
- Updated Vaultwarden deployment to utilize new SMTP configurations.
- Enhanced Grafana module to support dynamic dashboard and datasource provisioning.
- Added LLM proxy configuration for Open Web UI with necessary environment variables.

ansible/amd-rocm-container.yml

@@ -1,31 +0,0 @@
-- name: Add ROCm repository
-  become: true
-  ansible.builtin.dnf_repository:
-    name: rocm
-    description: ROCm Repository
-    baseurl: https://repo.radeon.com/rocm/el9/latest/main/
-    gpgcheck: yes
-    gpgkey: https://repo.radeon.com/rocm/rocm.gpg.key
-    enabled: yes
-
-- name: Add AMD Graphics repository
-  become: true
-  ansible.builtin.dnf_repository:
-    name: rocmgraphics
-    description: AMD Graphics Repository
-    baseurl: https://repo.radeon.com/graphics/latest/el/9.6/main/x86_64/
-    gpgcheck: yes
-    gpgkey: https://repo.radeon.com/rocm/rocm.gpg.key
-    enabled: yes
-
-- name: Install rocm-dkms and dependencies
-  become: true
-  ansible.builtin.package:
-    name:
-      - rocm
-      - rocm-developer-tools
-      - hipblas-devel
-      - hip-devel
-      - rocwmma-devel
-      - rocm-opencl-devel
-    state: present

ansible/host_vars/gpu-01.lab.alexpires.me.yml

@@ -88,8 +88,6 @@ duo_users: ["*", "!provision", "!comfyui", "!kokoro", "!ollama"]
 duo_api_hostname: api-7cda1654.duosecurity.com
 
 # podman services settings
-ollama_home: "/mnt/data/ollama"
-llama_home: "/mnt/data/llamacpp"
 podman_user_pubkey: "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAHHOPBR9p9kq5Cqzpe4cr3jHnweaYrHPXv5sXNzt+sCXP54uc5rWUBhxW2OQVvQzJ47dEVhEKi4WSC7LcuKS2G5AQDzWXNiasHvYIYQU3F/EknVCZnsiXYqXphYkJA6rJCNRnISZCIC1mocq6PB7J08ONdRFCvjfUBuVRT8BNGXNmQ/zQ=="
 
 # ComfyUI settings
@@ -108,39 +106,18 @@ certbot_domains:
   - "gpu-01.lab.alexpires.me"
 
 certbot_cockpit_domain: "gpu-01.lab.alexpires.me"
-ollama_models:
-  # Thinking models
-  - gpt-oss:latest
-  - gemma4:e4b
-  - qwen3.5:9b
-  - qwen3.5:4b
-  # Non-thinking models
-  - ministral-3:3b
-  - ministral-3:8b
-  - ministral-3:14b
-  - granite4.1:3b
-  - granite4.1:8b
-  - deepseek-r1:latest
-  - qwen2.5-coder:3b
-  # Embedding models
-  - embeddinggemma:latest
-  - nomic-embed-text:latest
-  # Whisper models
-  - dimavz/whisper-tiny:latest
-ollama_model_files:
-  - file: "ollama/gpt-oss-latest-32k.modelfile"
-    name: "gpt-oss-32k:latest"
-  - file: "ollama/gemma4-e4b-32k.modelfile"
-    name: "gemma4-e4b-32k:latest"
-  - file: "ollama/qwen3.5-9b-32k.modelfile"
-    name: "qwen3.5-9b-32k:latest"
-  - file: "ollama/qwen3.5-9b-64k.modelfile"
-    name: "qwen3.5-9b-64k:latest"
-  - file: "ollama/qwen3.5-4b-32k.modelfile"
-    name: "qwen3.5-4b-32k:latest"
-  - file: "ollama/qwen3.5-4b-64k.modelfile"
-    name: "qwen3.5-4b-64k:latest"
-llama_models:
+
+# Llama.cpp server settings
+llama_server_home: "/mnt/data/llamacpp"
+llama_server_user_pubkey: "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAHHOPBR9p9kq5Cqzpe4cr3jHnweaYrHPXv5sXNzt+sCXP54uc5rWUBhxW2OQVvQzJ47dEVhEKi4WSC7LcuKS2G5AQDzWXNiasHvYIYQU3F/EknVCZnsiXYqXphYkJA6rJCNRnISZCIC1mocq6PB7J08ONdRFCvjfUBuVRT8BNGXNmQ/zQ=="
+
+llama_server_port: 11434
+llama_server_devices:
+  - "nvidia.com/gpu=all"
+  - "/dev/kfd:/dev/kfd:rw"
+  - "/dev/dri:/dev/dri:rw"
+
+llama_server_models:
   # TODO: replace revision values with pinned commit SHAs for strict reproducibility.
   - name: "gpt-oss-20b-q4-0"
     model_id: "unsloth/gpt-oss-20b-GGUF"
@@ -186,9 +163,13 @@ llama_models:
     model_id: "unsloth/Devstral-Small-2-24B-Instruct-2512-GGUF"
     quant: "IQ4_XS"
     revision: "main"
+  - name: "qwen3-6-27b-mtp-iq4-xs"
+    model_id: "unsloth/Qwen3.6-27B-MTP-GGUF"
+    quant: "IQ4_XS"
+    revision: "main"
 
-# Optional global model preset options applied under [*] in models.ini.
-# Keys map to llama.cpp CLI args without leading dashes, short args, or LLAMA_ARG_* env names.
-llama_preset_global:
-  c: 131072
-  n-gpu-layers: all
+# # Optional global model preset options applied under [*] in models.ini.
+# # Keys map to llama.cpp CLI args without leading dashes, short args, or LLAMA_ARG_* env names.
+# llama_preset_global:
+#   c: 131072
+#   n-gpu-layers: all

ansible/playbook_amd_rocm.yml

@@ -1,7 +1,5 @@
 - name: Setup AMD ROCm Container Environment
   hosts: amd
 
-  tasks:
-    - name: Setup AMD ROCm Container Environment
-      ansible.builtin.include_tasks:
-        file: tasks/amd-rocm-container.yml
+  roles:
+    - amd_rocm

ansible/playbook_auto_suspend.yml

@@ -1,7 +1,5 @@
 - name: Setup Auto Suspend
   hosts: auto_suspend
 
-  tasks:
-    - name: Setup Auto Suspend
-      ansible.builtin.include_tasks:
-        file: tasks/auto_suspend.yml
+  roles:
+    - auto_suspend

ansible/playbook_llama_server.yml

@@ -0,0 +1,4 @@
+- name: Setup llama.cpp - Vulkan
+  hosts: llamacpp
+  roles:
+    - llama_server

ansible/playbook_nvidia_cdi_generator.yml

@@ -1,7 +1,5 @@
 - name: Setup NVidia CDI Generator
   hosts: nvidia
 
-  tasks:
-    - name: Setup nvidia-cdi-generator
-      ansible.builtin.include_tasks:
-        file: tasks/nvidia-cdi-generator.yml
+  roles:
+    - nvidia_cdi_generator

ansible/playbook_podman_comfyui.yml

@@ -45,7 +45,5 @@
         device:
           - "nvidia.com/gpu=all"
 
-  tasks:
-    - name: Setup Pods
-      ansible.builtin.include_tasks:
-        file: tasks/podman.yml
+  roles:
+    - podman

ansible/playbook_podman_kokoro.yml

@@ -26,7 +26,5 @@
         # device:
         #   - "nvidia.com/gpu=all"
 
-  tasks:
-    - name: Setup Pods
-      ansible.builtin.include_tasks:
-        file: tasks/podman.yml
+  roles:
+    - podman

ansible/playbook_podman_llama_vulkan.yml

@@ -5,24 +5,11 @@
     podman_group: "{{ llama_group | default('llama') }}"
     podman_extra_groups: "users,video"
     podman_user_home: "{{ llama_home | default('/home/llama') }}"
-    __llama_models__: "{{ llama_models | default([]) }}"
-    __llama_router_dir__: "{{ podman_user_home }}/models/.router"
-    __llama_managed_dir__: "{{ podman_user_home }}/models/managed"
-    __llama_links_dir__: "{{ podman_user_home }}/models/managed-links"
-    __llama_manifest_file__: "{{ __llama_router_dir__ }}/manifest.json"
-    __llama_models_input_file__: "{{ __llama_router_dir__ }}/desired-models.json"
-    __llama_preset_global__: "{{ llama_preset_global | default({}) }}"
-    __llama_preset_global_input_file__: "{{ __llama_router_dir__ }}/preset-global.json"
-    __llama_preset_file__: "{{ __llama_router_dir__ }}/models.ini"
-    __llama_sync_script__: "{{ __llama_router_dir__ }}/llama_hf_sync.py"
-    __llama_sync_venv__: "{{ llama_sync_venv | default(__llama_router_dir__ ~ '/.venv') }}"
-    __llama_sync_python__: "{{ __llama_sync_venv__ }}/bin/python"
-    __llama_models_max__: "{{ llama_models_max_loaded | default(1) }}"
     podman_user_folders:
       - models
       - .cache
       - .cache/llama.cpp
-    pods:
+    podman_pods:
       - name: llama.cpp
         repo:
           image: ghcr.io/ggml-org/llama.cpp:server-vulkan
@@ -67,16 +54,11 @@
           - "nvidia.com/gpu=all"
           - "/dev/kfd:/dev/kfd:rw"
           - "/dev/dri:/dev/dri:rw"
+  roles:
+    - llama_models_sync
+    - podman
+
   tasks:
-    - name: Sync llama models from Hugging Face
-      ansible.builtin.include_tasks:
-        file: tasks/llama_models_sync.yml
-      when: llama_models_sync_enabled | default(true)
-
-    - name: Setup Pods
-      ansible.builtin.include_tasks:
-        file: tasks/podman.yml
-
     - name: Query llama router model list
       ansible.builtin.uri:
         url: "http://127.0.0.1:{{ llama_port | default(8012) }}/models?reload=1"

ansible/playbook_podman_ollama.yml

@@ -27,7 +27,5 @@
           - "nvidia.com/gpu=all"
           - "/dev/kfd:/dev/kfd:rw"
           - "/dev/dri:/dev/dri:rw"
-  tasks:
-    - name: Setup Pods
-      ansible.builtin.include_tasks:
-        file: tasks/podman.yml
+  roles:
+    - podman

ansible/playbook_podman_ollama_rocm.yml

@@ -28,11 +28,10 @@
           - "/dev/kfd:/dev/kfd:rw"
           - "/dev/dri:/dev/dri:rw"
 
-  tasks:
-    - name: Setup Pods
-      ansible.builtin.include_tasks:
-        file: tasks/podman.yml
+  roles:
+    - podman
 
+  tasks:
     - name: Enable sudo access to podman user (temporary)
       become: true
       ansible.builtin.lineinfile:

ansible/playbook_podman_qwen2.yml

@@ -31,7 +31,5 @@
         device:
           - "/dev/kfd:/dev/kfd:rw"
           - "/dev/dri:/dev/dri:rw"
-  tasks:
-    - name: Setup Pods
-      ansible.builtin.include_tasks:
-        file: tasks/podman.yml
+  roles:
+    - podman

ansible/playbook_podman_qwen3.6.yml

@@ -58,7 +58,5 @@
           - "nvidia.com/gpu=all"
           - "/dev/kfd:/dev/kfd:rw"
           - "/dev/dri:/dev/dri:rw"
-  tasks:
-    - name: Setup Pods
-      ansible.builtin.include_tasks:
-        file: tasks/podman.yml
+  roles:
+    - podman

ansible/playbook_windows_game.yml

@@ -1,7 +1,5 @@
 - name: Setup Windows Game Optimization
   hosts: windows_game
 
-  tasks:
-    - name: Setup Optimizations
-      ansible.builtin.include_tasks:
-        file: tasks/windows_game.yml
+  roles:
+    - windows_game

ansible/playbook_windows_prometheus.yml

@@ -1,7 +1,5 @@
 - name: Setup Prometheus on Windows
   hosts: windows_prometheus
 
-  tasks:
-    - name: Setup Prometheus
-      ansible.builtin.include_tasks:
-        file: tasks/windows_prometheus.yml
+  roles:
+    - windows_prometheus

ansible/playbook_windows_ssh.yml

@@ -1,7 +1,5 @@
 - name: Setup ssh on Windows
   hosts: windows
 
-  tasks:
-    - name: Setup ssh
-      ansible.builtin.include_tasks:
-        file: tasks/windows_ssh.yml
+  roles:
+    - windows_ssh

ansible/playbook_wol.yml

@@ -1,7 +1,5 @@
 - name: Setup WakeOnLan
   hosts: wol
 
-  tasks:
-    - name: Setup WakeOnLan
-      ansible.builtin.include_tasks:
-        file: tasks/wol.yml
+  roles:
+    - wol

ansible/roles/amd_rocm/defaults/main.yml

@@ -0,0 +1,13 @@
+---
+# amd_rocm defaults
+
+rocm_repo_baseurl: "https://repo.radeon.com/rocm/el9/latest/main/"
+rocm_repo_gpgkey: "https://repo.radeon.com/rocm/rocm.gpg.key"
+rocmgraphics_repo_baseurl: "https://repo.radeon.com/graphics/latest/el/9.6/main/x86_64/"
+rocm_packages:
+  - rocm
+  - rocm-developer-tools
+  - hipblas-devel
+  - hip-devel
+  - rocwmma-devel
+  - rocm-opencl-devel

ansible/roles/amd_rocm/meta/main.yml

@@ -0,0 +1,18 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Install AMD ROCm repository and packages for GPU computing
+  company: A13Labs
+  role_name: amd_rocm
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: Fedora
+      versions:
+        - "39"
+        - "40"
+        - "41"
+    - name: EL
+      versions:
+        - "9"

ansible/roles/amd_rocm/tasks/main.yml

@@ -0,0 +1,52 @@
+---
+- name: Add ROCm repository (yum)
+  when: ansible_pkg_mgr == 'yum'
+  become: true
+  ansible.builtin.yum_repository:
+    name: rocm
+    description: ROCm Repository
+    baseurl: "{{ rocm_repo_baseurl }}"
+    gpgcheck: true
+    gpgkey: "{{ rocm_repo_gpgkey }}"
+    enabled: true
+    state: present
+
+- name: Add ROCm repository (dnf)
+  when: ansible_pkg_mgr == 'dnf'
+  become: true
+  ansible.builtin.dnf_repository:
+    name: rocm
+    description: ROCm Repository
+    baseurl: "{{ rocm_repo_baseurl }}"
+    gpgcheck: yes
+    gpgkey: "{{ rocm_repo_gpgkey }}"
+    enabled: yes
+
+- name: Add AMD Graphics repository (yum)
+  when: ansible_pkg_mgr == 'yum'
+  become: true
+  ansible.builtin.yum_repository:
+    name: rocmgraphics
+    description: AMD Graphics Repository
+    baseurl: "{{ rocmgraphics_repo_baseurl }}"
+    gpgkey: "{{ rocm_repo_gpgkey }}"
+    gpgcheck: true
+    enabled: true
+    state: present
+
+- name: Add AMD Graphics repository (dnf)
+  when: ansible_pkg_mgr == 'dnf'
+  become: true
+  ansible.builtin.dnf_repository:
+    name: rocmgraphics
+    description: AMD Graphics Repository
+    baseurl: "{{ rocmgraphics_repo_baseurl }}"
+    gpgkey: "{{ rocm_repo_gpgkey }}"
+    gpgcheck: yes
+    enabled: yes
+
+- name: Install rocm-dkms and dependencies
+  become: true
+  ansible.builtin.package:
+    name: "{{ rocm_packages }}"
+    state: present

ansible/roles/auto_suspend/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+auto_suspend_grub_config_path_bios: /boot/grub2/grub.cfg

ansible/roles/auto_suspend/meta/main.yml

@@ -0,0 +1,13 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Configure automatic system suspend based on idle time
+  company: A13Labs
+  role_name: auto_suspend
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: EL
+      versions:
+        - "9"

ansible/roles/auto_suspend/tasks/main.yml

@@ -1,3 +1,4 @@
+---
 - name: Install dependencies
   become: true
   ansible.builtin.package:

ansible/roles/gpu_monitoring/defaults/main.yml

@@ -46,18 +46,6 @@ amd_exporter_python_version: "python3"
 amd_exporter_sysfs_gpu_pattern: "/sys/class/drm/card?/device/"
 
 # Llama Exporter configuration
-llama_exporter_enabled: true
-llama_exporter_port: 9550
-llama_exporter_user: llama_exporter
-llama_exporter_user_home: "/home/llama_exporter"
-llama_exporter_install_dir: "/opt/llama_exporter"
-llama_exporter_venv_path: "/opt/llama_exporter/.venv"
-llama_exporter_scrape_interval: 15
-llama_exporter_scrape_timeout: 5
-
-# Llama.cpp scrape targets
-llama_exporter_targets: []
-
 # Firewall configuration
 prometheus_fw_port: "{{ prometheus_port }}"
 prometheus_fw_proto: "tcp"

ansible/roles/gpu_monitoring/files/llama_exporter.py

@@ -1,176 +0,0 @@
-#!/usr/bin/env python3
-"""LLM inference metrics exporter for Prometheus.
-
-Scrape targets:
-    - llama.cpp HTTP API servers (standard /metrics or custom health)
-"""
-
-import json
-import os
-import time
-import logging
-import urllib.request
-import urllib.error
-from http.server import HTTPServer, BaseHTTPRequestHandler
-
-logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
-logger = logging.getLogger(__name__)
-
-POLL_INTERVAL = int(os.environ.get("LLAMA_EXPORTER_INTERVAL", "15"))
-SCRAPE_TIMEOUT = int(os.environ.get("LLAMA_EXPORTER_TIMEOUT", "5"))
-
-LLAMA_CPP_DEFAULTS = [
-    {"name": "llama-cpp-8012", "url": "http://localhost:8012"},
-]
-
-
-def _fetch_json(url, timeout=SCRAPE_TIMEOUT):
-    try:
-        req = urllib.request.Request(url, headers={"Accept": "application/json"})
-        with urllib.request.urlopen(req, timeout=timeout) as resp:
-            return json.loads(resp.read().decode("utf-8"))
-    except Exception as e:
-        logger.debug("Failed to fetch %s: %s", url, e)
-        return None
-
-
-def _collect_llama_cpp(target):
-    metrics = {}
-
-    def _set_metric(metric_name, labels, value):
-        label_key = tuple(sorted(labels.items()))
-        metrics[(metric_name, label_key)] = value
-
-    base_url = target["url"].rstrip("/")
-
-    # Check health endpoint
-    health_url = f"{base_url}/health"
-    health = _fetch_json(health_url)
-    if health and isinstance(health, dict):
-        status = health.get("status", "unknown")
-        _set_metric("llama_server_health", {"server": target["name"], "model": health.get("model", "unknown")}, 1.0 if status == "ok" else 0.0)
-    else:
-        _set_metric("llama_server_health", {"server": target["name"], "model": "unknown"}, 0.0)
-
-    # Check /metrics endpoint for llama.cpp built-in metrics
-    metrics_url = f"{base_url}/metrics"
-    try:
-        req = urllib.request.Request(metrics_url, headers={"Accept": "text/plain"})
-        with urllib.request.urlopen(req, timeout=SCRAPE_TIMEOUT) as resp:
-            body = resp.read().decode("utf-8")
-            for line in body.splitlines():
-                line = line.strip()
-                if not line or line.startswith("#"):
-                    continue
-                # Parse simple key value pairs
-                if "{" in line:
-                    key_part = line.split("{")[0]
-                    labels_part = line.split("{")[1].rstrip("}").split("}")[0] if "}" in line else ""
-                    value = line.split()[-1] if line.split() else "1"
-                    label_str = ""
-                    if labels_part:
-                        label_str = ",".join(f'{{{k}="{v}"}}' for k, v in (
-                            pair.split("=") for pair in labels_part.split(",") if "=" in pair
-                        ))
-                    _set_metric(key_part.strip(), {"server": target["name"], "raw": labels_part}, float(value) if value.replace(".", "").replace("-", "").isdigit() else value)
-                else:
-                    parts = line.split()
-                    if len(parts) >= 2:
-                        try:
-                            float(parts[1])
-                            _set_metric(parts[0], {"server": target["name"]}, parts[1])
-                        except ValueError:
-                            pass
-    except Exception as e:
-        logger.debug("Failed to scrape %s/metrics: %s", metrics_url, e)
-
-    # Check /model endpoint for model info
-    model_url = f"{base_url}/models"
-    models = _fetch_json(model_url)
-    if models and isinstance(models, dict):
-        model_list = models.get("data", [models]) if "data" in models else [models]
-        for m in model_list:
-            if not isinstance(m, dict):
-                continue
-            model_id = m.get("id", "unknown")
-            _set_metric("llama_models_loaded", {"server": target["name"], "model": model_id}, 1.0)
-            ctx_max = m.get("context_size", m.get("n_ctx", 0))
-            if ctx_max:
-                _set_metric("llama_model_context_max", {"server": target["name"], "model": model_id}, int(ctx_max))
-            trained = m.get("train_tokens", m.get("n_train", 0))
-            if trained:
-                _set_metric("llama_model_tokens_trained", {"server": target["name"], "model": model_id}, int(trained))
-
-    return metrics
-
-
-
-class MetricsHandler(BaseHTTPRequestHandler):
-    def do_GET(self):
-        if self.path != "/metrics":
-            self.send_response(404)
-            self.end_headers()
-            return
-
-        all_metrics = {}
-
-        # Scrape llama.cpp targets
-        targets = []
-        env_targets = os.environ.get("LLAMA_TARGETS", "")
-        if env_targets:
-            try:
-                targets = json.loads(env_targets)
-            except json.JSONDecodeError:
-                logger.warning("Invalid LLAMA_TARGETS JSON, using defaults")
-                targets = []
-
-        if not targets:
-            targets = LLAMA_CPP_DEFAULTS
-
-        for target in targets:
-            if "url" not in target or "name" not in target:
-                target["name"] = target.get("name", "default")
-                target["url"] = target.get("url", "http://localhost:8012")
-            try:
-                all_metrics.update(_collect_llama_cpp(target))
-            except Exception as e:
-                logger.error("Error scraping llama.cpp target %s: %s", target.get("name", "unknown"), e)
-
-        lines = []
-        for (metric_type, label_pairs), value in sorted(all_metrics.items()):
-            labels = dict(label_pairs)
-            if not labels:
-                line = f"{metric_type} {value}"
-            else:
-                label_str = ",".join(f'{k}="{v}"' for k, v in sorted(labels.items()))
-                line = f"{metric_type}{{{label_str}}} {value}"
-            lines.append(line)
-
-        body = "\n".join(lines) + "\n" if lines else "# no metrics available\n"
-
-        self.send_response(200)
-        self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
-        self.end_headers()
-        self.wfile.write(body.encode("utf-8"))
-
-    def log_message(self, format, *args):
-        logger.debug("%s - - %s", self.address_string(), format % args)
-
-
-def main():
-    port = int(os.environ.get("LLAMA_EXPORTER_PORT", "9550"))
-    host = os.environ.get("LLAMA_EXPORTER_BIND", "0.0.0.0")
-
-    server = HTTPServer((host, port), MetricsHandler)
-    logger.info("Starting Llama Exporter on %s:%d", host, port)
-    logger.info("Polling every %d seconds, timeout %ds", POLL_INTERVAL, SCRAPE_TIMEOUT)
-
-    try:
-        server.serve_forever()
-    except KeyboardInterrupt:
-        logger.info("Shutting down")
-        server.shutdown()
-
-
-if __name__ == "__main__":
-    main()

ansible/roles/gpu_monitoring/handlers/main.yml

@@ -18,10 +18,3 @@
     name: "{{ amd_exporter_service_name }}"
     state: restarted
   listen: Restart amd_exporter
-
-- name: Restart llama_exporter
-  become: true
-  ansible.builtin.systemd:
-    name: "{{ llama_exporter_service_name }}"
-    state: restarted
-  listen: Restart llama_exporter

ansible/roles/gpu_monitoring/molecule/default/converge.yml

@@ -8,4 +8,3 @@
         name: "gpu_monitoring"
       vars:
         amd_exporter_enabled: false
-        llama_exporter_enabled: false

ansible/roles/gpu_monitoring/tasks/main.yml

@@ -11,12 +11,6 @@
   tags:
     - roles::gpu_monitoring::amd_exporter
 
-- name: Include Llama Exporter setup
-  when: llama_exporter_enabled | default(true) | bool
-  ansible.builtin.include_tasks: llama_exporter.yml
-  tags:
-    - roles::gpu_monitoring::llama_exporter
-
 - name: Include Prometheus setup
   ansible.builtin.include_tasks: prometheus.yml
   tags:

ansible/roles/gpu_monitoring/templates/llama_exporter.service.j2

@@ -1,35 +0,0 @@
-[Unit]
-Description=LLM Inference Metrics Exporter (llama.cpp + Ollama)
-After=network-online.target
-Wants=network-online.target
-
-[Service]
-Type=simple
-User={{ llama_exporter_user }}
-Group={{ llama_exporter_user }}
-WorkingDirectory={{ llama_exporter_install_dir }}
-Environment="LLAMA_EXPORTER_PORT={{ llama_exporter_port }}"
-Environment="LLAMA_EXPORTER_BIND=0.0.0.0"
-Environment="LLAMA_EXPORTER_INTERVAL={{ llama_exporter_scrape_interval }}"
-Environment="LLAMA_EXPORTER_TIMEOUT={{ llama_exporter_scrape_timeout }}"
-{% if llama_exporter_targets and llama_exporter_targets != "" %}
-Environment="LLAMA_TARGETS={{ llama_exporter_targets | to_json }}"
-{% endif %}
-Environment="PATH={{ llama_exporter_venv_bin }}:/usr/bin:/bin"
-ExecStart={{ llama_exporter_venv_bin }}/python3 {{ llama_exporter_script_path }}
-Restart=always
-RestartSec=10
-TimeoutStopSec=10
-StandardOutput=journal
-StandardError=journal
-SyslogIdentifier=llama_exporter
-
-# Security hardening
-ProtectSystem=strict
-ProtectHome=true
-ReadWritePaths={{ llama_exporter_install_dir }} /var/log
-NoNewPrivileges=true
-PrivateTmp=true
-
-[Install]
-WantedBy=multi-user.target

ansible/roles/gpu_monitoring/templates/prometheus.yml.j2

@@ -21,10 +21,3 @@ scrape_configs:
       - targets: ["{{ prometheus_host_gateway }}:{{ amd_exporter_port }}"]
         labels:
           datacenter: "{{ inventory_hostname }}"
-
-  - job_name: llama_exporter
-    metrics_path: /metrics
-    static_configs:
-      - targets: ["{{ prometheus_host_gateway }}:{{ llama_exporter_port }}"]
-        labels:
-          datacenter: "{{ inventory_hostname }}"

ansible/roles/gpu_monitoring/vars/main.yml

@@ -12,12 +12,3 @@ nvidia_exporter_service_name: "nvidia_exporter"
 amd_exporter_script_path: "{{ amd_exporter_install_dir }}/amd_gpu_exporter.py"
 amd_exporter_venv_bin: "{{ amd_exporter_venv_path }}/bin"
 amd_exporter_service_name: "amdgpu_exporter"
-
-llama_exporter_script_path: "{{ llama_exporter_install_dir }}/llama_exporter.py"
-llama_exporter_venv_bin: "{{ llama_exporter_venv_path }}/bin"
-llama_exporter_service_name: "llama_exporter"
-
-# Default llama.cpp targets if none provided
-llama_exporter_targets_default:
-  - name: "llama-cpp-8012"
-    url: "http://localhost:8012"

ansible/roles/llama_server/defaults/main.yml

@@ -0,0 +1,26 @@
+llama_server_user: "llama"
+llama_server_group: "llama"
+llama_server_home: "/home/llama"
+llama_server_pubkey: ""
+llama_server_models: []
+llama_server_models_max: 1
+llama_server_image: "ghcr.io/ggml-org/llama.cpp"
+llama_server_tag: "server-vulkan"
+llama_server_extra_groups: "users,video"
+llama_server_devices: []
+llama_server_env: {}
+llama_server_port: 8080
+llama_server_preset_global: {}
+
+# Llama Exporter configuration
+llama_server_exporter_enabled: true
+llama_server_exporter_port: 9550
+llama_server_exporter_user: llama_exporter
+llama_server_exporter_user_home: "/opt/llama_exporter"
+llama_server_exporter_install_dir: "/opt/llama_exporter"
+llama_server_exporter_venv_path: "/opt/llama_exporter/.venv"
+llama_server_exporter_scrape_interval: 15
+llama_server_exporter_scrape_timeout: 5
+
+# Llama.cpp scrape targets
+llama_server_exporter_targets: []

ansible/roles/llama_server/files/scripts/llama_exporter.py

@@ -0,0 +1,389 @@
+#!/usr/bin/env python3
+"""LLM inference metrics exporter for Prometheus.
+
+Background thread scrapes only loaded models to avoid triggering model loads
+in llama-server, then the HTTP handler serves the cached data immediately.
+
+Exposes metrics for every known model from /models; unloaded models show
+zero values without ever requesting /metrics?model=<unloaded>.
+
+Uses persistent JSON cache on disk to survive restarts and compute counter
+deltas for Prometheus rate/irate queries.
+"""
+
+import json
+import os
+import time
+import threading
+import logging
+import urllib.request
+import urllib.error
+from http.server import HTTPServer, BaseHTTPRequestHandler
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
+logger = logging.getLogger(__name__)
+
+POLL_INTERVAL = int(os.environ.get("LLAMA_EXPORTER_INTERVAL", "15"))
+SCRAPE_TIMEOUT = int(os.environ.get("LLAMA_EXPORTER_TIMEOUT", "5"))
+CACHE_FILE = os.environ.get(
+    "LLAMA_EXPORTER_CACHE",
+    os.path.join(os.path.dirname(os.path.abspath(__file__)), "llama_exporter_cache.json"),
+)
+
+LLAMA_CPP_DEFAULTS = [
+    {"name": "llama.cpp", "url": "http://localhost:11434"},
+]
+
+# All llama.cpp metrics we expose per model (counters + gauges).
+ALL_METRICS = [
+    "llamacpp:prompt_tokens_total",
+    "llamacpp:prompt_seconds_total",
+    "llamacpp:tokens_predicted_total",
+    "llamacpp:tokens_predicted_seconds_total",
+    "llamacpp:n_decode_total",
+    "llamacpp:n_tokens_max",
+    "llamacpp:prompt_tokens_seconds",
+    "llamacpp:predicted_tokens_seconds",
+    "llamacpp:requests_processing",
+    "llamacpp:requests_deferred",
+    "llamacpp:n_busy_slots_per_decode",
+]
+
+# Counter metrics that accumulate over time — exposed as _delta for rate() queries.
+COUNTER_METRICS = {
+    "llamacpp:prompt_tokens_total",
+    "llamacpp:prompt_seconds_total",
+    "llamacpp:tokens_predicted_total",
+    "llamacpp:tokens_predicted_seconds_total",
+    "llamacpp:n_decode_total",
+}
+
+
+class Cache:
+    """Persistent, thread-safe metrics cache with delta computation."""
+
+    def __init__(self):
+        self._lock = threading.Lock()
+        self._data = {}  # { model_id: { metric_name: value } }
+        self._known_models = set()
+        self._health = None
+        self._loaded = {}
+
+        # Previous state for delta computation: { model_id: { metric_name: value } }
+        self._prev = {}
+        self._prev_timestamp = 0.0
+        self._deltas = {}
+
+        # Load persisted state
+        self._load_cache()
+
+    def _load_cache(self):
+        """Load previous scrape state from disk."""
+        try:
+            with open(CACHE_FILE, "r") as f:
+                state = json.load(f)
+            self._prev = state.get("models", {})
+            self._prev_timestamp = state.get("timestamp", 0.0)
+            logger.info("Loaded cache from %s (timestamp=%s, models=%d)",
+                        CACHE_FILE, self._prev_timestamp, len(self._prev))
+        except FileNotFoundError:
+            logger.info("No cache file found at %s", CACHE_FILE)
+        except Exception as e:
+            logger.warning("Failed to load cache: %s", e)
+
+    def _save_cache(self, current_data, known_models, loaded):
+        """Save current scrape state to disk."""
+        try:
+            state = {
+                "timestamp": time.time(),
+                "known": list(known_models),
+                "loaded": list(loaded),
+                "models": {},
+            }
+            for mid, metrics in current_data.items():
+                state["models"][mid] = dict(metrics)
+            tmp = CACHE_FILE + ".tmp"
+            with open(tmp, "w") as f:
+                json.dump(state, f)
+            os.replace(tmp, CACHE_FILE)
+        except Exception as e:
+            logger.error("Failed to save cache: %s", e)
+
+    def _compute_deltas_for_data(self, current_data):
+        """Compute deltas given current data and stored previous state.
+
+        Only counter metrics get deltas. On counter reset (value went backward),
+        delta is silently 0 (no entry added).
+        """
+        deltas = {}
+        if not self._prev:
+            return deltas
+
+        for mid, prev_metrics in self._prev.items():
+            mid_deltas = {}
+            cur_metrics = current_data.get(mid, {})
+            for mname, prev_val in prev_metrics.items():
+                # Only compute deltas for counter metrics
+                if mname not in COUNTER_METRICS:
+                    continue
+                cur_val = cur_metrics.get(mname)
+                if cur_val is None:
+                    continue
+                diff = cur_val - prev_val
+                # Counter reset: value went backward, delta is 0 (wrapped)
+                if diff < 0:
+                    continue
+                if diff > 0:
+                    mid_deltas[mname] = diff
+            if mid_deltas:
+                deltas[mid] = mid_deltas
+        return deltas
+
+    def update(self, models_data, metric_lines, known_models=None):
+        """Called by the background thread after a successful scrape cycle."""
+        with self._lock:
+            # Discover model IDs and loaded status from /models endpoint
+            known = set()
+            loaded = {}
+            if models_data and isinstance(models_data, dict):
+                model_list = models_data.get("data", [])
+                for m in model_list:
+                    if not isinstance(m, dict):
+                        continue
+                    mid = m.get("id", "unknown")
+                    known.add(mid)
+                    status_data = m.get("status", {})
+                    if isinstance(status_data, dict) and status_data.get("value") == "loaded":
+                        loaded[mid] = 1.0
+                    self._health = {"status": "ok", "model": mid}
+
+            if not known:
+                self._health = {"status": "error", "model": "unknown"}
+
+            # Build new metrics dict for all known models
+            new_data = {}
+            for mid in known:
+                new_data[mid] = {}
+
+            # Apply parsed metric values from /metrics?model=<id>
+            for metric_name, model_id, value in metric_lines:
+                if model_id in known:
+                    new_data[model_id][metric_name] = value
+
+            # Ensure every known model has all ALL_METRICS entries
+            for mid in known:
+                for mname in ALL_METRICS:
+                    if mname not in new_data[mid]:
+                        new_data[mid][mname] = 0.0
+
+            # Previously known models no longer in the list get zeroed out
+            for mid in self._known_models - known:
+                new_data[mid] = {m: 0.0 for m in ALL_METRICS}
+
+            # Compute deltas before updating previous state
+            deltas = self._compute_deltas_for_data(new_data)
+
+            # Save previous state for next cycle
+            self._save_cache(new_data, known, loaded)
+
+            # Update state
+            self._prev = {mid: dict(metrics) for mid, metrics in new_data.items()}
+            self._known_models = known
+            self._data = new_data
+            self._loaded = loaded
+            self._deltas = deltas
+
+    def snapshot(self):
+        """Return a frozen copy of the current cache state including deltas."""
+        with self._lock:
+            return {
+                "data": {k: dict(v) for k, v in self._data.items()},
+                "known": set(self._known_models),
+                "health": dict(self._health) if self._health else {"status": "error", "model": "unknown"},
+                "loaded": dict(self._loaded),
+                "deltas": {k: dict(v) for k, v in self._deltas.items()},
+            }
+
+
+cache = Cache()
+
+
+def _fetch_json(url, timeout=SCRAPE_TIMEOUT):
+    try:
+        req = urllib.request.Request(url, headers={"Accept": "application/json"})
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return json.loads(resp.read().decode("utf-8"))
+    except Exception as e:
+        logger.debug("Failed to fetch %s: %s", url, e)
+        return None
+
+
+def _fetch_text(url, timeout=SCRAPE_TIMEOUT):
+    try:
+        req = urllib.request.Request(url, headers={"Accept": "text/plain"})
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return resp.read().decode("utf-8")
+    except Exception as e:
+        logger.debug("Failed to fetch %s: %s", url, e)
+        return None
+
+
+def _scrape_cycle():
+    """One full scrape cycle: discover models, then scrape metrics per model."""
+    targets = []
+    env_targets = os.environ.get("LLAMA_TARGETS", "")
+    if env_targets:
+        try:
+            targets = json.loads(env_targets)
+        except json.JSONDecodeError:
+            targets = []
+    if not targets:
+        targets = LLAMA_CPP_DEFAULTS
+
+    all_metric_lines = []
+    models_data = None
+    known_models = {}  # { model_id: base_url, ... }
+
+    for target in targets:
+        base_url = target["url"].rstrip("/")
+
+        # Fetch /models to discover models and their status
+        models_data = _fetch_json(f"{base_url}/models")
+
+        if models_data and isinstance(models_data, dict):
+            model_list = models_data.get("data", [])
+            for m in model_list:
+                if not isinstance(m, dict) or "id" not in m:
+                    continue
+                model_id = m["id"]
+                known_models[model_id] = base_url
+                # Only scrape metrics from loaded models to avoid triggering loads
+                status_data = m.get("status", {})
+                if isinstance(status_data, dict) and status_data.get("value") != "loaded":
+                    continue
+                # Scrape /metrics for loaded models only
+                metrics_url = f"{base_url}/metrics?model={model_id}"
+                body = _fetch_text(metrics_url)
+                if body:
+                    for line in body.splitlines():
+                        parsed = _parse_metric_line(line)
+                        if parsed:
+                            metric_name, metric_value = parsed
+                            if metric_name in ALL_METRICS:
+                                all_metric_lines.append((metric_name, model_id, metric_value))
+                else:
+                    logger.debug("No metrics body for model %s", model_id)
+
+    # Update the shared cache
+    cache.update(models_data, all_metric_lines, known_models)
+
+
+def _parse_metric_line(line):
+    """Parse a single Prometheus metric line. Returns (name, value) or None."""
+    line = line.strip()
+    if not line or line.startswith("#"):
+        return None
+    try:
+        # Handle lines with labels: metric_name{label="val"} value
+        if "{" in line:
+            parts = line.split("{")
+            name = parts[0].strip()
+            rest = parts[1]
+            # value is the last token after the closing }
+            value = rest.rsplit("}", 1)[-1].strip().split()[-1] if "}" in rest else rest.strip()
+        else:
+            parts = line.split()
+            name = parts[0]
+            value = parts[1] if len(parts) >= 2 else "1"
+        # Try to parse as float
+        float(value)
+        return (name, float(value))
+    except (ValueError, IndexError):
+        return None
+
+
+def _background_scrape():
+    """Background thread: periodically scrape and update cache."""
+    logger.info("Background scraper started (interval=%ds)", POLL_INTERVAL)
+    # Do one immediate scrape on startup
+    _scrape_cycle()
+    while True:
+        try:
+            time.sleep(POLL_INTERVAL)
+            _scrape_cycle()
+        except Exception as e:
+            logger.error("Scrape cycle error: %s", e)
+
+
+class MetricsHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path != "/metrics":
+            self.send_response(404)
+            self.end_headers()
+            return
+
+        snap = cache.snapshot()
+        lines = []
+
+        def _fmt(metric_name, model_id, value):
+            return metric_name + '{' + 'server="llama-cpp-11434",model="' + model_id + '"} ' + str(value)
+
+        # Health metric
+        health = snap["health"]
+        status = health.get("status", "error")
+        health_model = health.get("model", "unknown")
+        health_val = 1.0 if status == "ok" else 0.0
+        lines.append(_fmt("llama_server_health", health_model, health_val))
+
+        # Loaded metrics
+        for mid in snap["loaded"]:
+            lines.append(_fmt("llama_models_loaded", mid, snap["loaded"][mid]))
+
+        # Per-model metrics from cache (absolute values)
+        for mid in sorted(snap["data"]):
+            metrics = snap["data"][mid]
+            for mname in ALL_METRICS:
+                value = metrics.get(mname, 0.0)
+                lines.append(_fmt(mname, mid, value))
+
+        # Per-model delta metrics (counters as change since last scrape)
+        deltas = snap.get("deltas", {})
+        for mid in sorted(deltas):
+            delta_metrics = deltas[mid]
+            for mname in sorted(delta_metrics):
+                value = delta_metrics[mname]
+                lines.append(_fmt(mname + "_delta", mid, value))
+
+        body = "\n".join(lines) + "\n" if lines else "# no metrics available\n"
+
+        self.send_response(200)
+        self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+        self.end_headers()
+        self.wfile.write(body.encode("utf-8"))
+
+    def log_message(self, format, *args):
+        logger.debug("%s - - %s", self.address_string(), format % args)
+
+
+def main():
+    port = int(os.environ.get("LLAMA_EXPORTER_PORT", "9550"))
+    host = os.environ.get("LLAMA_EXPORTER_BIND", "0.0.0.0")
+
+    # Start background scraper thread
+    scraper = threading.Thread(target=_background_scrape, daemon=True)
+    scraper.start()
+
+    # Start HTTP server
+    server = HTTPServer((host, port), MetricsHandler)
+    logger.info("Starting Llama Exporter on %s:%d (interval=%ds, timeout=%ds, cache=%s)",
+                host, port, POLL_INTERVAL, SCRAPE_TIMEOUT, CACHE_FILE)
+
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        logger.info("Shutting down")
+        server.shutdown()
+
+
+if __name__ == "__main__":
+    main()

ansible/roles/llama_server/files/scripts/test_llama_exporter.py

@@ -0,0 +1,231 @@
+#!/usr/bin/env python3
+"""Tests for llama_exporter cache persistence and delta computation."""
+
+import json
+import os
+import sys
+import tempfile
+import shutil
+import unittest
+
+# Setup temp cache file before importing the module
+TEMP_DIR = tempfile.mkdtemp()
+os.environ["LLAMA_EXPORTER_CACHE"] = os.path.join(TEMP_DIR, "test_cache.json")
+
+# Import from parent directory (scripts)
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from llama_exporter import Cache
+
+
+class TestCachePersistenceAndDeltas(unittest.TestCase):
+
+    def setUp(self):
+        # Ensure temp dir exists (pytest may clean it between test methods)
+        os.makedirs(TEMP_DIR, exist_ok=True)
+        self.cache = Cache()
+
+    def tearDown(self):
+        shutil.rmtree(TEMP_DIR, ignore_errors=True)
+
+    def test_cold_start_no_deltas(self):
+        """First scrape should produce no deltas since there's no previous state."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+                {"id": "model-v2", "status": {"value": "unloaded"}},
+            ]
+        }
+        metric_lines = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 100.0),
+            ("llamacpp:prompt_seconds_total", "model-v1", 2.5),
+            ("llamacpp:n_tokens_max", "model-v1", 4096.0),
+        ]
+        known_models = {"model-v1": "http://localhost:11434", "model-v2": "http://localhost:11434"}
+
+        self.cache.update(models_data, metric_lines, known_models)
+
+        snap = self.cache.snapshot()
+        # No deltas on first scrape
+        self.assertEqual(snap["deltas"], {})
+        # Absolute values are correct
+        self.assertEqual(snap["data"]["model-v1"]["llamacpp:prompt_tokens_total"], 100.0)
+        # Unloaded model has zeros
+        self.assertEqual(snap["data"]["model-v2"]["llamacpp:prompt_tokens_total"], 0.0)
+        # Only loaded model is in loaded set
+        self.assertIn("model-v1", snap["loaded"])
+        self.assertNotIn("model-v2", snap["loaded"])
+
+    def test_delta_computation(self):
+        """Second scrape should produce deltas from first scrape."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+            ]
+        }
+
+        # First scrape
+        metric_lines_1 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 100.0),
+            ("llamacpp:prompt_seconds_total", "model-v1", 2.5),
+        ]
+        known_models_1 = {"model-v1": "http://localhost:11434"}
+        self.cache.update(models_data, metric_lines_1, known_models_1)
+
+        # Second scrape - values increased
+        metric_lines_2 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 250.0),
+            ("llamacpp:prompt_seconds_total", "model-v1", 5.0),
+        ]
+        self.cache.update(models_data, metric_lines_2, known_models_1)
+
+        snap = self.cache.snapshot()
+        # Deltas should be computed
+        self.assertIn("model-v1", snap["deltas"])
+        self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_tokens_total"], 150.0)
+        self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_seconds_total"], 2.5)
+
+    def test_counter_reset(self):
+        """If a counter value goes backward, delta should be 0."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+            ]
+        }
+
+        # First scrape
+        metric_lines_1 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 100.0),
+        ]
+        known_models_1 = {"model-v1": "http://localhost:11434"}
+        self.cache.update(models_data, metric_lines_1, known_models_1)
+
+        # Second scrape - counter reset (model was reloaded)
+        metric_lines_2 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 10.0),
+        ]
+        self.cache.update(models_data, metric_lines_2, known_models_1)
+
+        snap = self.cache.snapshot()
+        # On counter reset, delta is implicitly 0 (no entry in deltas)
+        self.assertNotIn("model-v1", snap["deltas"])
+
+    def test_gauge_metrics_no_delta(self):
+        """Gauge metrics should not get delta entries (only counters do)."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+            ]
+        }
+
+        # First scrape
+        metric_lines_1 = [
+            ("llamacpp:n_tokens_max", "model-v1", 4096.0),
+            ("llamacpp:requests_processing", "model-v1", 2.0),
+        ]
+        known_models_1 = {"model-v1": "http://localhost:11434"}
+        self.cache.update(models_data, metric_lines_1, known_models_1)
+
+        # Second scrape
+        metric_lines_2 = [
+            ("llamacpp:n_tokens_max", "model-v1", 8192.0),
+            ("llamacpp:requests_processing", "model-v1", 5.0),
+        ]
+        self.cache.update(models_data, metric_lines_2, known_models_1)
+
+        snap = self.cache.snapshot()
+        # Gauge metrics should not have deltas
+        self.assertNotIn("model-v1", snap["deltas"])
+
+    def test_persistence_across_restarts(self):
+        """Delta computation should work across cache reloads from disk."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+            ]
+        }
+
+        # First scrape (first process instance)
+        metric_lines_1 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 100.0),
+            ("llamacpp:prompt_seconds_total", "model-v1", 2.5),
+        ]
+        known_models_1 = {"model-v1": "http://localhost:11434"}
+        self.cache.update(models_data, metric_lines_1, known_models_1)
+
+        # Simulate restart: create new Cache instance (loads from disk)
+        self.cache._save_cache(self.cache._data, self.cache._known_models, self.cache._loaded)
+
+        # Create a new cache instance (simulates restart)
+        new_cache = Cache()
+
+        # Second scrape after restart
+        metric_lines_2 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 300.0),
+            ("llamacpp:prompt_seconds_total", "model-v1", 8.0),
+        ]
+        known_models_2 = {"model-v1": "http://localhost:11434"}
+        new_cache.update(models_data, metric_lines_2, known_models_2)
+
+        snap = new_cache.snapshot()
+        # Deltas should be computed using persisted previous state
+        self.assertIn("model-v1", snap["deltas"])
+        self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_tokens_total"], 200.0)
+        self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_seconds_total"], 5.5)
+
+    def test_unloaded_model_zero_values(self):
+        """Unloaded models should appear with zero values but never trigger /metrics calls."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+                {"id": "model-v2", "status": {"value": "unloaded"}},
+                {"id": "model-v3", "status": {"value": "unloaded"}},
+            ]
+        }
+        metric_lines = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 50.0),
+        ]
+        known_models = {
+            "model-v1": "http://localhost:11434",
+            "model-v2": "http://localhost:11434",
+            "model-v3": "http://localhost:11434",
+        }
+
+        self.cache.update(models_data, metric_lines, known_models)
+
+        snap = self.cache.snapshot()
+        # Loaded model has its actual value
+        self.assertEqual(snap["data"]["model-v1"]["llamacpp:prompt_tokens_total"], 50.0)
+        # Unloaded models have zeros for all metrics
+        for m in ["model-v2", "model-v3"]:
+            self.assertEqual(snap["data"][m]["llamacpp:prompt_tokens_total"], 0.0)
+            self.assertEqual(snap["data"][m]["llamacpp:n_tokens_max"], 0.0)
+
+    def test_removed_model_zeroed(self):
+        """Models removed from llama-server should be zeroed out."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+            ]
+        }
+
+        # First scrape with two models
+        metric_lines_1 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 100.0),
+        ]
+        known_models_1 = {"model-v1": "http://localhost:11434"}
+        self.cache.update(models_data, metric_lines_1, known_models_1)
+
+        # Second scrape with only model-v1 (model-v2 was removed)
+        metric_lines_2 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 200.0),
+        ]
+        known_models_2 = {"model-v1": "http://localhost:11434"}
+        self.cache.update(models_data, metric_lines_2, known_models_2)
+
+        snap = self.cache.snapshot()
+        # model-v1 delta should be 100
+        self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_tokens_total"], 100.0)
+
+
+if __name__ == "__main__":
+    unittest.main()

ansible/roles/llama_server/handlers/main.yml

@@ -0,0 +1,13 @@
+---
+- name: Reload systemd daemon
+  become: true
+  ansible.builtin.systemd:
+    daemon_reload: true
+  listen: Reload systemd daemon
+
+- name: Restart llama_exporter
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ llama_server_exporter_service_name }}"
+    state: restarted
+  listen: Restart llama_exporter

ansible/roles/llama_server/meta/main.yml

@@ -0,0 +1,13 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Sync llama models from Hugging Face
+  company: A13Labs
+  role_name: llama_models_sync
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: EL
+      versions:
+        - "9"

ansible/roles/llama_server/tasks/llama_exporter.yml

@@ -2,16 +2,16 @@
 - name: Create llama_exporter group
   become: true
   ansible.builtin.group:
-    name: "{{ llama_exporter_user }}"
+    name: "{{ llama_server_exporter_user }}"
     state: present
     system: true
 
 - name: Create llama_exporter user
   become: true
   ansible.builtin.user:
-    name: "{{ llama_exporter_user }}"
-    group: "{{ llama_exporter_user }}"
-    home: "{{ llama_exporter_user_home }}"
+    name: "{{ llama_server_exporter_user }}"
+    group: "{{ llama_server_exporter_user }}"
+    home: "{{ llama_server_exporter_user_home }}"
     create_home: true
     shell: /usr/sbin/nologin
     system: true
@@ -21,20 +21,20 @@
   ansible.builtin.file:
     path: "{{ item }}"
     state: directory
-    owner: "{{ llama_exporter_user }}"
-    group: "{{ llama_exporter_user }}"
+    owner: "{{ llama_server_exporter_user }}"
+    group: "{{ llama_server_exporter_user }}"
     mode: "0755"
   loop:
-    - "{{ llama_exporter_install_dir }}"
-    - "{{ llama_exporter_install_dir }}/logs"
+    - "{{ llama_server_exporter_install_dir }}"
+    - "{{ llama_server_exporter_install_dir }}/logs"
 
 - name: Copy llama_exporter script
   become: true
   ansible.builtin.copy:
-    src: llama_exporter.py
-    dest: "{{ llama_exporter_script_path }}"
-    owner: "{{ llama_exporter_user }}"
-    group: "{{ llama_exporter_user }}"
+    src: scripts/llama_exporter.py
+    dest: "{{ llama_server_exporter_script_path }}"
+    owner: "{{ llama_server_exporter_user }}"
+    group: "{{ llama_server_exporter_user }}"
     mode: "0755"
   notify:
     - Restart llama_exporter
@@ -42,31 +42,22 @@
 - name: Create Python virtual environment
   become: true
   ansible.builtin.command:
-    cmd: "{{ ansible_facts['python']['executable'] }} -m venv {{ llama_exporter_venv_path }}"
-    creates: "{{ llama_exporter_venv_path }}/bin/activate"
-
-- name: Install Python dependencies in llama_exporter venv
-  become: true
-  ansible.builtin.pip:
-    name:
-      - prometheus-client
-      - requests
-    virtualenv: "{{ llama_exporter_venv_path }}"
-    state: present
+    cmd: "{{ ansible_facts['python']['executable'] }} -m venv {{ llama_server_exporter_venv_path }}"
+    creates: "{{ llama_server_exporter_venv_path }}/bin/activate"
 
 - name: Fix ownership of llama_exporter directory
   become: true
   ansible.builtin.file:
-    path: "{{ llama_exporter_install_dir }}"
-    owner: "{{ llama_exporter_user }}"
-    group: "{{ llama_exporter_user }}"
+    path: "{{ llama_server_exporter_install_dir }}"
+    owner: "{{ llama_server_exporter_user }}"
+    group: "{{ llama_server_exporter_user }}"
     recurse: true
 
 - name: Generate llama_exporter systemd service
   become: true
   ansible.builtin.template:
     src: llama_exporter.service.j2
-    dest: "/etc/systemd/system/{{ llama_exporter_service_name }}.service"
+    dest: "/etc/systemd/system/{{ llama_server_exporter_service_name }}.service"
     owner: root
     group: root
     mode: "0644"
@@ -77,6 +68,6 @@
 - name: Enable llama_exporter service
   become: true
   ansible.builtin.systemd:
-    name: "{{ llama_exporter_service_name }}"
+    name: "{{ llama_server_exporter_service_name }}"
     enabled: true
     state: started

ansible/roles/llama_server/tasks/main.yml

@@ -0,0 +1,358 @@
+---
+- name: Validate llama_server_models schema
+  ansible.builtin.assert:
+    that:
+      - llama_server_models is iterable
+    fail_msg: "llama_server_models must be a list"
+
+- name: Create group
+  become: true
+  ansible.builtin.group:
+    name: "{{ llama_server_group }}"
+    system: true
+
+- name: Create user
+  become: true
+  ansible.builtin.user:
+    name: "{{ llama_server_user }}"
+    groups: "{{ podman_extra_groups | default([]) }}"
+    shell: /bin/bash
+    home: "{{ llama_server_home }}"
+    group: "{{ llama_server_group }}"
+
+- name: Disable password login for user
+  ansible.builtin.command: passwd -d "{{ llama_server_user }}"
+  become: true
+  changed_when: false
+
+- name: Create user home
+  become: true
+  ansible.builtin.file:
+    path: "{{ llama_server_home }}"
+    state: directory
+    owner: "{{ llama_server_user }}"
+    group: "{{ llama_server_group }}"
+    mode: "0750"
+
+- name: Set fact podman_binary (Debian family)
+  ansible.builtin.set_fact:
+    podman_binary: /usr/bin/podman
+  when: ansible_os_family == 'Debian'
+
+- name: Set fact podman_binary (Red Hat family)
+  ansible.builtin.set_fact:
+    podman_binary: /usr/sbin/podman
+  when: ansible_os_family == 'RedHat'
+
+- name: Create required folders
+  become: true
+  ansible.builtin.file:
+    path: "{{ llama_server_home }}/{{ item }}"
+    state: directory
+    owner: "{{ llama_server_user }}"
+    group: "{{ llama_server_group }}"
+    mode: "0750"
+  loop:
+    - models
+    - .cache
+    - .cache/llama.cpp
+
+- name: "Add SSH Authorized key"
+  become: true
+  ansible.posix.authorized_key:
+    user: "{{ llama_server_user }}"
+    key: "{{ llama_server_pubkey }}"
+    state: "{{ 'present' if llama_server_pubkey is defined and llama_server_pubkey != '' else 'absent' }}"
+  when: llama_server_pubkey is defined and llama_server_pubkey != ""
+
+- name: SELinux tasks (RedHat only)
+  when: ansible_os_family == 'RedHat'
+  become: true
+  block:
+    - name: Persistently set SELinux context
+      community.general.sefcontext:
+        target: "{{ llama_server_home }}(/.*)?"
+        setype: user_home_dir_t
+        state: present
+
+    - name: Persistently set SELinux context (ssh authorized keys)
+      community.general.sefcontext:
+        target: "{{ llama_server_home }}/.ssh/authorized_keys"
+        setype: ssh_home_t
+        state: present
+      when: llama_server_pubkey is defined and llama_server_pubkey != ""
+
+    - name: Apply SELinux context
+      ansible.builtin.command: restorecon -Rv {{ llama_server_home }}
+      changed_when: false
+
+- name: Enable sudo access to llama user (temporary)
+  become: true
+  ansible.builtin.lineinfile:
+    path: "/etc/sudoers.d/ansible_{{ llama_server_user }}_tmp"
+    create: true
+    mode: "0440"
+    line: "{{ ansible_facts['user_id'] }} ALL=({{ llama_server_user }}) NOPASSWD: ALL"
+    validate: "visudo -cf %s"
+
+- name: Enable lingering for llama user
+  become: true
+  ansible.builtin.command: loginctl enable-linger {{ llama_server_user }}
+  register: linger_status
+  changed_when: >-
+    'created' in linger_status.stdout or
+    (linger_status.rc == 0 and not
+      ('linger file already exists' in linger_status.stderr or
+      'linger file does not exist' in linger_status.stderr))
+  failed_when:
+    - linger_status.rc != 0 and not
+      ('linger file already exists' in linger_status.stderr)
+
+- name: Get current user's UID
+  ansible.builtin.command: id -u {{ llama_server_user }}
+  changed_when: false
+  register: uid
+
+- name: Get current user's GID
+  ansible.builtin.command: id -g {{ llama_server_user }}
+  changed_when: false
+  register: gid
+
+- name: Run tasks as podman user
+  become: true
+  become_user: "{{ llama_server_user }}"
+  block:
+    - name: Pull the requested images
+      containers.podman.podman_image:
+        name: "{{ llama_server_image }}"
+        tag: "{{ llama_server_tag }}"
+        state: present
+
+    - name: Create pods
+      containers.podman.podman_container:
+        name: "llama.cpp"
+        image: "{{ llama_server_image }}:{{ llama_server_tag }}"
+        state: started
+        device: "{{ llama_server_devices | default(omit) }}"
+        env: "{{ llama_server_env | default(omit) }}"
+        ports: "0.0.0.0:{{ llama_server_port }}:8080/tcp"
+        volumes:
+          - "{{ llama_server_home }}/models:/models:Z"
+          - "{{ llama_server_home }}/.cache:/app/.cache:Z"
+        command:
+          - "-t"
+          - "{{ llama_server_argv_threads | default(10) }}"
+          - "-np"
+          - "{{ llama_server_argv_parallel | default(1) }}"
+          - "-b"
+          - "{{ llama_server_argv_batch_size | default(1024) }}"
+          - "-ub"
+          - "{{ llama_server_argv_ubatch_size | default(512) }}"
+          - "-fa"
+          - "{{ llama_server_argv_flash_attn | default('on') }}"
+          - "-ctk"
+          - "{{ llama_server_argv_cache_type_k | default('q4_0') }}"
+          - "-ctv"
+          - "{{ llama_server_argv_cache_type_v | default('q8_0') }}"
+          - "-cram"
+          - "{{ llama_server_argv_cache_reuse | default(-1) }}"
+          - "-sm"
+          - "{{ llama_server_argv_split_mode | default('layer') }}"
+          - "-dev"
+          - "{{ llama_server_argv_devices | default('Vulkan1,Vulkan2') }}"
+          - "-ts"
+          - "{{ llama_server_argv_tensor_split | default('8,12') }}"
+          - "-fit"
+          - "{{ llama_server_argv_fit | default('off') }}"
+          - "-mg"
+          - "{{ llama_server_argv_main_gpu | default(1) }}"
+          - "--mmap"
+          - "--metrics"
+          - "--models-preset"
+          - "/models/.router/models.ini"
+          - "--models-max"
+          - "{{ llama_server_models_max }}"
+
+        cmd_args:
+          - "--userns=keep-id"
+          - "--security-opt=label=disable"
+
+- name: Create systemd service file for pods
+  become: true
+  ansible.builtin.template:
+    src: podman.service.j2
+    dest: "/etc/systemd/system/podman-llama.cpp.service"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Reload systemd daemon
+  become: true
+  ansible.builtin.systemd:
+    daemon_reload: true
+
+- name: Enable containers at boot
+  become: true
+  ansible.builtin.systemd:
+    name: "podman-llama.cpp"
+    enabled: true
+    state: "started"
+
+- name: Install Python packages required for llama server model sync
+  become: true
+  ansible.builtin.package:
+    name: "{{ __llama_sync_system_packages__ }}"
+    state: present
+  vars:
+    __llama_sync_system_packages__: >-
+      {{
+        {
+          'Debian': ['python3-pip', 'python3-packaging', 'python3-venv'],
+          'RedHat': ['python3-pip', 'python3-packaging'],
+          'Archlinux': ['python-pip', 'python-packaging']
+        }[ansible_os_family]
+      }}
+
+- name: Install Hugging Face hub client in llama server sync virtualenv
+  become: true
+  become_user: "{{ llama_server_user }}"
+  ansible.builtin.pip:
+    name:
+      - pip
+      - setuptools
+      - packaging
+      - huggingface_hub>=0.32.0
+    state: present
+    virtualenv: "{{ llama_server_home }}/.router/.venv"
+    virtualenv_command: "python3 -m venv"
+
+- name: Create llama server router model directories
+  become: true
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: directory
+    owner: "{{ llama_server_user }}"
+    group: "{{ llama_server_group }}"
+    mode: "{{ item.mode }}"
+  loop:
+    - { path: "{{ llama_server_home }}/models", mode: "0750" }
+    - { path: "{{ llama_server_home }}/models/managed", mode: "0750" }
+    - { path: "{{ llama_server_home }}/models/managed-links", mode: "0750" }
+    - { path: "{{ llama_server_home }}/models/.router", mode: "0750" }
+
+- name: Copy llama server Hugging Face sync helper
+  become: true
+  ansible.builtin.copy:
+    src: scripts/llama_hf_sync.py
+    dest: "{{ llama_server_home }}/models/.router/llama_hf_sync.py"
+    owner: "{{ llama_server_user }}"
+    group: "{{ llama_server_group }}"
+    mode: "0750"
+
+- name: Write desired llama server model manifest input
+  become: true
+  ansible.builtin.copy:
+    content: "{{ llama_server_models | to_nice_json }}"
+    dest: "{{ llama_server_home }}/models/.router/desired-models.json"
+    owner: "{{ llama_server_user }}"
+    group: "{{ llama_server_group }}"
+    mode: "0640"
+
+- name: Write global llama server preset options
+  become: true
+  ansible.builtin.copy:
+    content: "{{ llama_server_preset_global | to_nice_json }}"
+    dest: "{{ llama_server_home }}/models/.router/preset-global.json"
+    owner: "{{ llama_server_user }}"
+    group: "{{ llama_server_group }}"
+    mode: "0640"
+
+- name: Set arguments for llama server model sync
+  ansible.builtin.set_fact:
+    __llama_preset_file__:
+      - "{{ llama_server_home }}/.router/.venv/bin/python"
+      - "{{ llama_server_home }}/models/.router/llama_hf_sync.py"
+      - "--models-file"
+      - "{{ llama_server_home }}/models/.router/desired-models.json"
+      - "--managed-dir"
+      - "{{ llama_server_home }}/models/managed"
+      - "--links-dir"
+      - "{{ llama_server_home }}/models/managed-links"
+      - "--manifest-file"
+      - "{{ llama_server_home }}/models/.router/manifest.json"
+      - "--preset-file"
+      - "{{ llama_server_home }}/models/.router/models.ini"
+      - "--preset-global-file"
+      - "{{ llama_server_home }}/models/.router/preset-global.json"
+      - "--container-links-dir"
+      - "/models/managed-links"
+
+- name: Run llama server model sync from Hugging Face
+  vars:
+    __llama_sync_argv__: >-
+      {{ __llama_preset_file__
+        + (['--prune'] if llama_server_models_prune_enabled | default(true) else [])
+        + (['--dry-run'] if llama_server_models_dry_run | default(false) else [])
+      }}
+  become: true
+  become_user: "{{ llama_server_user }}"
+  ansible.builtin.command:
+    argv: "{{ __llama_sync_argv__ }}"
+  environment:
+    HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') | default('', true) }}"
+  register: __llama_sync_output__
+  changed_when: false
+
+- name: Parse llama server sync output
+  ansible.builtin.set_fact:
+    __llama_sync_result__: "{{ __llama_sync_output__.stdout | from_json }}"
+
+- name: Stat llama router preset file (when not in dry-run mode)
+  become: true
+  become_user: "{{ llama_server_user }}"
+  ansible.builtin.stat:
+    path: "{{ llama_server_home }}/models/.router/models.ini"
+  register: __llama_preset_stat__
+  when: not (llama_server_models_dry_run | default(false))
+
+- name: Ensure router preset file exists when not in dry-run mode
+  ansible.builtin.assert:
+    that:
+      - __llama_preset_stat__.stat.exists
+    fail_msg: "llama router preset file was not created"
+  when: not (llama_server_models_dry_run | default(false))
+
+- name: Query llama router model list
+  ansible.builtin.uri:
+    url: "http://127.0.0.1:{{ llama_server_port | default(8080) }}/models?reload=1"
+    method: GET
+    status_code: 200
+  register: __llama_router_models__
+  changed_when: false
+  when:
+    - llama_server_models_sync_enabled | default(true)
+    - not (llama_server_models_dry_run | default(false))
+
+- name: Validate expected model aliases are available in router
+  ansible.builtin.assert:
+    that:
+      - item.name in (__llama_router_models__.json.data | map(attribute='id') | list)
+    fail_msg: "Model name '{{ item.name }}' not present in llama router /models output"
+  loop: "{{ __llama_sync_result__.models | default([]) }}"
+  loop_control:
+    label: "{{ item.name }}"
+  when:
+    - llama_server_models_sync_enabled | default(true)
+    - not (llama_server_models_dry_run | default(false))
+
+- name: Include Llama Exporter setup
+  when: llama_server_exporter_enabled | default(true) | bool
+  ansible.builtin.include_tasks: llama_exporter.yml
+  tags:
+    - roles::llama_server::llama_exporter
+
+- name: Remove temporary sudo access
+  become: true
+  ansible.builtin.file:
+    path: "/etc/sudoers.d/ansible_{{ llama_server_user }}_tmp"
+    state: absent

ansible/roles/llama_server/templates/llama_exporter.service.j2

@@ -0,0 +1,35 @@
+[Unit]
+Description=LLM Inference Metrics Exporter (llama.cpp + Ollama)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User={{ llama_server_exporter_user }}
+Group={{ llama_server_exporter_user }}
+WorkingDirectory={{ llama_server_exporter_install_dir }}
+Environment="LLAMA_EXPORTER_PORT={{ llama_server_exporter_port }}"
+Environment="LLAMA_EXPORTER_BIND=0.0.0.0"
+Environment="LLAMA_EXPORTER_INTERVAL={{ llama_server_exporter_scrape_interval }}"
+Environment="LLAMA_EXPORTER_TIMEOUT={{ llama_server_exporter_scrape_timeout }}"
+{% if llama_server_exporter_targets and llama_server_exporter_targets != "" %}
+Environment="LLAMA_TARGETS={{ llama_server_exporter_targets | to_json }}"
+{% endif %}
+Environment="PATH={{ llama_server_exporter_venv_bin }}:/usr/bin:/bin"
+ExecStart={{ llama_server_exporter_venv_bin }}/python3 {{ llama_server_exporter_script_path }}
+Restart=always
+RestartSec=10
+TimeoutStopSec=10
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=llama_exporter
+
+# Security hardening
+ProtectSystem=strict
+ProtectHome=true
+ReadWritePaths={{ llama_server_exporter_install_dir }} /var/log
+NoNewPrivileges=true
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/llama_server/templates/podman.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=Podman file for llama.cpp server
+After=local-fs.target network-online.target nvidia-cdi-generator.service
+
+[Service]
+Type=oneshot
+User={{ llama_server_user }}
+Group={{ llama_server_group }}
+RemainAfterExit=true
+ExecStart={{ podman_binary }} start podman-llama.cpp
+ExecStop={{ podman_binary }} stop podman-llama.cpp
+
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/llama_server/vars/main.yml

@@ -0,0 +1,11 @@
+---
+# Internal role variables (non-overridable)
+
+llama_server_exporter_script_path: "{{ llama_server_exporter_install_dir }}/llama_exporter.py"
+llama_server_exporter_venv_bin: "{{ llama_server_exporter_venv_path }}/bin"
+llama_server_exporter_service_name: "llama_exporter"
+
+# Default llama.cpp targets if none provided
+llama_server_exporter_targets_default:
+  - name: "llama-cpp-11434"
+    url: "http://localhost:11434"

ansible/roles/nvidia_cdi_generator/meta/main.yml

@@ -0,0 +1,13 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Install and configure NVIDIA CDI generator service
+  company: A13Labs
+  role_name: nvidia_cdi_generator
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: EL
+      versions:
+        - "9"

ansible/roles/nvidia_cdi_generator/tasks/main.yml

@@ -1,3 +1,4 @@
+---
 - name: Install nvidia-cdi-generator
   become: true
   ansible.builtin.package:

ansible/roles/podman/defaults/main.yml

@@ -0,0 +1,15 @@
+---
+# podman defaults
+
+podman_user: ""
+podman_group: ""
+podman_user_home: ""
+podman_extra_groups: []
+podman_user_folders: []
+podman_user_pubkey: ""
+podman_network_name: "none"
+podman_build_files: []
+podman_pods: []
+
+# Binary path - will be set by OS family in tasks
+# podman_binary is set as a fact in tasks/main.yml

ansible/roles/podman/meta/main.yml

@@ -0,0 +1,18 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Deploy and manage Podman users, pods, and containers
+  company: A13Labs
+  role_name: podman
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: Fedora
+      versions:
+        - "39"
+        - "40"
+        - "41"
+    - name: EL
+      versions:
+        - "9"

ansible/roles/podman/tasks/main.yml

@@ -1,32 +1,90 @@
-- name: Check if required vars are set
-  ansible.builtin.fail:
-    msg: "Required variables are not set for Podman setup. Please check your playbook variables.)"
-  when:
-    - podman_user is not defined
-    - podman_group is not defined
-    - podman_user_home is not defined
-
+---
 - name: Set fact podman_binary (Debian family)
-  when: ansible_os_family == 'Debian'
   ansible.builtin.set_fact:
     podman_binary: /usr/bin/podman
+  when: ansible_os_family == 'Debian'
 
 - name: Set fact podman_binary (Red Hat family)
-  when: ansible_os_family == 'RedHat'
   ansible.builtin.set_fact:
     podman_binary: /usr/sbin/podman
+  when: ansible_os_family == 'RedHat'
 
-- name: Install Podman and its dependencies
+- name: Create group
   become: true
-  ansible.builtin.package:
-    name:
-      - podman
-      - acl
-    state: present
+  ansible.builtin.group:
+    name: "{{ podman_group }}"
+    system: true
 
-- name: Setup user
-  ansible.builtin.include_tasks:
-    file: podman/user.yml
+- name: Create user
+  become: true
+  ansible.builtin.user:
+    name: "{{ podman_user }}"
+    groups: "{{ podman_extra_groups | default([]) }}"
+    shell: /bin/bash
+    home: "{{ podman_user_home }}"
+    group: "{{ podman_group }}"
+
+- name: Disable password login for user
+  ansible.builtin.command: passwd -d "{{ podman_user }}"
+  become: true
+  changed_when: false
+
+- name: Create user home
+  become: true
+  ansible.builtin.file:
+    path: "{{ podman_user_home }}"
+    state: directory
+    owner: "{{ podman_user }}"
+    group: "{{ podman_group }}"
+    mode: "0750"
+
+- name: Create build folders
+  become: true
+  ansible.builtin.file:
+    path: "{{ podman_user_home }}/build"
+    state: directory
+    owner: "{{ podman_user }}"
+    group: "{{ podman_group }}"
+    mode: "0750"
+
+- name: Create folders
+  become: true
+  ansible.builtin.file:
+    path: "{{ podman_user_home }}/{{ item }}"
+    state: directory
+    owner: "{{ podman_user }}"
+    group: "{{ podman_group }}"
+    mode: "0750"
+  loop: "{{ podman_user_folders }}"
+
+- name: "Add SSH Authorized key"
+  become: true
+  ansible.posix.authorized_key:
+    user: "{{ podman_user }}"
+    key: "{{ podman_user_pubkey }}"
+    state: "{{ 'present' if podman_user_pubkey is defined and podman_user_pubkey != '' else 'absent' }}"
+  when: podman_user_pubkey is defined and podman_user_pubkey != ""
+
+- name: SELinux tasks (RedHat only)
+  when: ansible_os_family == 'RedHat'
+  become: true
+  block:
+    - name: Persistently set SELinux context
+      community.general.sefcontext:
+        target: "{{ podman_user_home }}(/.*)?"
+        setype: user_home_dir_t
+        state: present
+
+    - name: Persistently set SELinux context (ssh authorized keys)
+      community.general.sefcontext:
+        target: "{{ podman_user_home }}/.ssh/authorized_keys"
+        setype: ssh_home_t
+        state: present
+      when: podman_user_pubkey is defined and podman_user_pubkey != ""
+
+    - name: Apply SELinux context
+      ansible.builtin.command: restorecon -Rv {{ podman_user_home }}
+      changed_when: false
 
 - name: Enable sudo access to podman user (temporary)
   become: true
@@ -91,7 +149,7 @@
           extra_args: "--build-arg UID={{ uid.stdout }} --build-arg GID={{ gid.stdout }} --security-opt=label=disable {{ item.build.args | default('') }}"
         tag: "latest"
         state: present
-      loop: "{{ pods }}"
+      loop: "{{ podman_pods }}"
       when: item.build is defined and item.build.dockerfile is defined
 
     - name: Pull the requested images
@@ -99,7 +157,7 @@
         name: "{{ item.repo.image }}"
         tag: "{{ item.repo.tag | default('latest') }}"
         state: present
-      loop: "{{ pods }}"
+      loop: "{{ podman_pods }}"
       when: item.repo is defined and item.repo.image is defined
 
     - name: Create pods
@@ -116,7 +174,7 @@
         cmd_args:
           - "--userns=keep-id"
           - "--security-opt=label=disable"
-      loop: "{{ pods }}"
+      loop: "{{ podman_pods }}"
 
 - name: Create systemd service file for pods
   become: true
@@ -126,7 +184,7 @@
     owner: root
     group: root
     mode: "0644"
-  loop: "{{ pods }}"
+  loop: "{{ podman_pods }}"
 
 - name: Reload systemd daemon
   become: true
@@ -139,7 +197,7 @@
     name: "podman-{{ item.name }}"
     enabled: "{{ item.systemd.enabled | default(true) }}"
     state: "{{ item.systemd.state | default('started') }}"
-  loop: "{{ pods }}"
+  loop: "{{ podman_pods }}"
 
 - name: Remove temporary sudo access
   become: true

ansible/roles/podman/templates/podman.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=Podman service for {{ item.name }}
+After=local-fs.target network-online.target
+
+[Service]
+Type=oneshot
+User={{ podman_user }}
+Group={{ podman_group }}
+RemainAfterExit=true
+ExecStart={{ podman_binary }} start {{ item.name }}
+ExecStop={{ podman_binary }} stop {{ item.name }}
+
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/windows_game/meta/main.yml

@@ -0,0 +1,11 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Windows game optimization (user creation, auto-login, power plan, service disable)
+  company: A13Labs
+  role_name: windows_game
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: windows

ansible/roles/windows_game/tasks/main.yml

@@ -1,3 +1,4 @@
+---
 - name: Create limited GameUser
   ansible.windows.win_user:
     name: "{{ game_user }}"
@@ -177,10 +178,3 @@
       - Power Management: Prefer Maximum Performance
       - Low Latency Mode: Ultra
       - Vertical Sync: Off or Fast Sync (depending on monitor)
-
-# - name: Set custom shell to launcher.bat
-#   ansible.windows.win_regedit:
-#     path: HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon
-#     name: Shell
-#     data: C:\GameShell\launcher.bat
-#     type: string

ansible/roles/windows_prometheus/meta/main.yml

@@ -0,0 +1,11 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Install Prometheus and Windows Exporter on Windows
+  company: A13Labs
+  role_name: windows_prometheus
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: windows

ansible/roles/windows_ssh/meta/main.yml

@@ -0,0 +1,11 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Install and configure OpenSSH Server on Windows
+  company: A13Labs
+  role_name: windows_ssh
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: windows

ansible/roles/windows_ssh/tasks/main.yml

@@ -4,9 +4,6 @@
 #     keys:
 #       - "ssh-ed25519 AAAA..."
 #       - "ssh-rsa AAAA..."
-#   - name: bob
-#     keys:
-#       - "ssh-rsa AAAA..."
 - name: Setting defaults for Windows OpenSSH Server installation
   ansible.builtin.set_fact:
     __ssh_default_ciphers__:
@@ -108,7 +105,7 @@
       Write-Output 'hostkeys-generated'
     } else {
       Write-Output 'ssh-keygen-not-found'
-      exit 2
+      exit(2)
     }
   register: ssh_keygen
   changed_when: "'hostkeys-generated' in ssh_keygen.stdout"
@@ -358,7 +355,7 @@
       if ($before -ne $after) {
         Write-Output 'admin-auth-keys-acl-changed'
       } else {
-        Write-Output 'admin-auth-keys-acl-unchanged'
+        Write-Output 'admin-auth-keys-unchanged'
       }
     } else {
       Write-Output 'no-admin-auth-keys'

ansible/roles/wol/meta/main.yml

@@ -0,0 +1,13 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Configure Wake-on-LAN via systemd-networkd
+  company: A13Labs
+  role_name: wol
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: EL
+      versions:
+        - "9"

ansible/roles/wol/tasks/main.yml

@@ -1,3 +1,4 @@
+---
 - name: Install ethtool
   become: true
   ansible.builtin.package:

ansible/tasks/amd-rocm-container.yml

@@ -1,33 +0,0 @@
-- name: Add ROCm repository
-  become: true
-  ansible.builtin.yum_repository:
-    name: rocm
-    description: ROCm Repository
-    baseurl: https://repo.radeon.com/rocm/el9/latest/main/
-    gpgcheck: true
-    gpgkey: https://repo.radeon.com/rocm/rocm.gpg.key
-    enabled: true
-    state: present
-
-- name: Add AMD Graphics repository
-  become: true
-  ansible.builtin.yum_repository:
-    name: rocmgraphics
-    description: AMD Graphics Repository
-    baseurl: https://repo.radeon.com/graphics/latest/el/9.6/main/x86_64/
-    gpgkey: https://repo.radeon.com/rocm/rocm.gpg.key
-    gpgcheck: true
-    enabled: true
-    state: present
-
-- name: Install rocm-dkms and dependencies
-  become: true
-  ansible.builtin.package:
-    name:
-      - rocm
-      - rocm-developer-tools
-      - hipblas-devel
-      - hip-devel
-      - rocwmma-devel
-      - rocm-opencl-devel
-    state: present

ansible/tasks/llama_models_sync.yml

@@ -1,146 +0,0 @@
-- name: Ensure podman user exists before model sync
-  ansible.builtin.include_tasks:
-    file: podman/user.yml
-
-- name: Enable sudo access to podman user (temporary)
-  become: true
-  ansible.builtin.lineinfile:
-    path: "/etc/sudoers.d/ansible_{{ podman_user }}_tmp"
-    create: true
-    mode: "0440"
-    line: "{{ ansible_user_id }} ALL=({{ podman_user }}) NOPASSWD: ALL"
-    validate: "visudo -cf %s"
-
-- name: Validate llama_models schema
-  ansible.builtin.assert:
-    that:
-      - __llama_models__ is iterable
-    fail_msg: "llama_models must be a list"
-
-- name: Install Python packages required for llama model sync
-  become: true
-  ansible.builtin.package:
-    name: "{{ __llama_sync_system_packages__ }}"
-    state: present
-  vars:
-    __llama_sync_system_packages__: >-
-      {{
-        {
-          'Debian': ['python3-pip', 'python3-packaging', 'python3-venv'],
-          'RedHat': ['python3-pip', 'python3-packaging'],
-          'Archlinux': ['python-pip', 'python-packaging']
-        }[ansible_os_family]
-      }}
-
-- name: Install Hugging Face hub client in llama model sync virtualenv
-  become: true
-  become_user: "{{ podman_user }}"
-  ansible.builtin.pip:
-    name:
-      - pip
-      - setuptools
-      - packaging
-      - huggingface_hub>=0.32.0
-    state: present
-    virtualenv: "{{ __llama_sync_venv__ }}"
-    virtualenv_command: "python3 -m venv"
-
-- name: Create llama router model directories
-  become: true
-  ansible.builtin.file:
-    path: "{{ item.path }}"
-    state: directory
-    owner: "{{ podman_user }}"
-    group: "{{ podman_group }}"
-    mode: "{{ item.mode }}"
-  loop:
-    - { path: "{{ podman_user_home }}/models", mode: "0750" }
-    - { path: "{{ __llama_managed_dir__ }}", mode: "0750" }
-    - { path: "{{ __llama_links_dir__ }}", mode: "0750" }
-    - { path: "{{ __llama_router_dir__ }}", mode: "0750" }
-
-- name: Copy llama Hugging Face sync helper
-  become: true
-  ansible.builtin.copy:
-    src: scripts/llama_hf_sync.py
-    dest: "{{ __llama_sync_script__ }}"
-    owner: "{{ podman_user }}"
-    group: "{{ podman_group }}"
-    mode: "0750"
-
-- name: Write desired llama model manifest input
-  become: true
-  ansible.builtin.copy:
-    content: "{{ __llama_models__ | to_nice_json }}"
-    dest: "{{ __llama_models_input_file__ }}"
-    owner: "{{ podman_user }}"
-    group: "{{ podman_group }}"
-    mode: "0640"
-
-- name: Write global llama preset options
-  become: true
-  ansible.builtin.copy:
-    content: "{{ __llama_preset_global__ | to_nice_json }}"
-    dest: "{{ __llama_preset_global_input_file__ }}"
-    owner: "{{ podman_user }}"
-    group: "{{ podman_group }}"
-    mode: "0640"
-
-- name: Run llama model sync from Hugging Face
-  vars:
-    __llama_sync_argv__: >-
-      {{
-        [
-          __llama_sync_python__,
-          __llama_sync_script__,
-          '--models-file',
-          __llama_models_input_file__,
-          '--managed-dir',
-          __llama_managed_dir__,
-          '--links-dir',
-          __llama_links_dir__,
-          '--manifest-file',
-          __llama_manifest_file__,
-          '--preset-file',
-          __llama_preset_file__,
-          '--preset-global-file',
-          __llama_preset_global_input_file__,
-          '--container-links-dir',
-          '/models/managed-links'
-        ]
-        + (['--prune'] if llama_models_prune_enabled | default(true) else [])
-        + (['--dry-run'] if llama_models_dry_run | default(false) else [])
-      }}
-  become: true
-  become_user: "{{ podman_user }}"
-  ansible.builtin.command:
-    argv: "{{ __llama_sync_argv__ }}"
-  environment:
-    HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') | default('', true) }}"
-  register: __llama_sync_output__
-  changed_when: false
-
-- name: Parse llama sync output
-  ansible.builtin.set_fact:
-    __llama_sync_result__: "{{ __llama_sync_output__.stdout | from_json }}"
-
-- name: Ensure router preset file exists when not in dry-run mode
-  become: true
-  become_user: "{{ podman_user }}"
-  ansible.builtin.stat:
-    path: "{{ __llama_preset_file__ }}"
-  register: __llama_preset_stat__
-  when: not (llama_models_dry_run | default(false))
-
-- name: Ensure router preset file exists when not in dry-run mode
-  ansible.builtin.assert:
-    that:
-      - __llama_preset_stat__.stat.exists
-    fail_msg: "llama router preset file was not created"
-  when: not (llama_models_dry_run | default(false))
-
-- name: Remove temporary sudo access
-  become: true
-  ansible.builtin.file:
-    path: "/etc/sudoers.d/ansible_{{ podman_user }}_tmp"
-    state: absent

ansible/tasks/podman/user.yml

@@ -1,84 +0,0 @@
-- name: Check if required vars are set
-  ansible.builtin.fail:
-    msg: "Required variables are not set for Podman setup. Please check your playbook variables.)"
-  when:
-    - podman_user is not defined
-    - podman_group is not defined
-    - podman_user_home is not defined
-
-- name: Create group
-  become: true
-  ansible.builtin.group:
-    name: "{{ podman_group }}"
-    system: true
-
-- name: Create user
-  become: true
-  ansible.builtin.user:
-    name: "{{ podman_user }}"
-    groups: "{{ podman_extra_groups | default([]) }}"
-    shell: /bin/bash
-    home: "{{ podman_user_home }}"
-    group: "{{ podman_group }}"
-
-- name: Disable password login for user
-  ansible.builtin.command: passwd -d "{{ podman_user }}"
-  become: true
-  changed_when: false
-
-- name: Create user home
-  become: true
-  ansible.builtin.file:
-    path: "{{ podman_user_home }}"
-    state: directory
-    owner: "{{ podman_user }}"
-    group: "{{ podman_group }}"
-    mode: "0750"
-
-- name: Create build folders
-  become: true
-  ansible.builtin.file:
-    path: "{{ podman_user_home }}/build"
-    state: directory
-    owner: "{{ podman_user }}"
-    group: "{{ podman_group }}"
-    mode: "0750"
-
-- name: Create folders
-  become: true
-  ansible.builtin.file:
-    path: "{{ podman_user_home }}/{{ item }}"
-    state: directory
-    owner: "{{ podman_user }}"
-    group: "{{ podman_group }}"
-    mode: "0750"
-  loop: "{{ podman_user_folders }}"
-
-- name: "Add SSH Authorized key"
-  become: true
-  ansible.posix.authorized_key:
-    user: "{{ podman_user }}"
-    key: "{{ podman_user_pubkey }}"
-    state: "{{ 'present' if podman_user_pubkey is defined else 'absent' }}"
-  when: podman_user_pubkey is defined and podman_user_pubkey != ""
-
-- name: SELinux tasks (RedHat only)
-  when: ansible_os_family == 'RedHat'
-  become: true
-  block:
-    - name: Persistently set SELinux context
-      community.general.sefcontext:
-        target: "{{ podman_user_home }}(/.*)?"
-        setype: user_home_dir_t
-        state: present
-
-    - name: Persistently set SELinux context (ssh authorized keys)
-      community.general.sefcontext:
-        target: "{{ podman_user_home }}/.ssh/authorized_keys"
-        setype: ssh_home_t
-        state: present
-      when: podman_user_pubkey is defined and podman_user_pubkey != ""
-
-    - name: Apply SELinux context
-      ansible.builtin.command: restorecon -Rv {{ podman_user_home }}
-      changed_when: false

ansible/tmp/gpu-realtime-dashboard.json

@@ -0,0 +1,810 @@
+{
+    "annotations": {
+        "list": [
+            {
+                "builtIn": 1,
+                "datasource": {
+                    "type": "grafana",
+                    "uid": "-- Grafana --"
+                },
+                "enable": true,
+                "hide": true,
+                "iconColor": "rgba(0, 211, 255, 1)",
+                "name": "Annotations & Alerts",
+                "type": "dashboard"
+            }
+        ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 0,
+    "id": null,
+    "links": [],
+    "liveNow": true,
+    "panels": [
+        {
+            "datasource": "$datasource",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "thresholds"
+                    },
+                    "mappings": [],
+                    "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                            {
+                                "color": "red",
+                                "value": null
+                            },
+                            {
+                                "color": "green",
+                                "value": 1
+                            }
+                        ]
+                    },
+                    "unit": "short"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 6,
+                "w": 24,
+                "x": 0,
+                "y": 0
+            },
+            "id": 1,
+            "options": {
+                "colorMode": "background",
+                "graphMode": "none",
+                "justifyMode": "auto",
+                "orientation": "auto",
+                "reduceOptions": {
+                    "calcs": [
+                        "lastNotNull"
+                    ],
+                    "fields": "",
+                    "values": false
+                },
+                "textMode": "auto"
+            },
+            "pluginVersion": "11.0.0",
+            "targets": [
+                {
+                    "editorMode": "code",
+                    "expr": "up{job=~\"prometheus|nvidia-gpu|amdgpu_exporter|llama_exporter\"}",
+                    "instant": true,
+                    "legendFormat": "{{job}}",
+                    "range": false,
+                    "refId": "A"
+                }
+            ],
+            "title": "Exporter Health",
+            "type": "stat"
+        },
+        {
+            "datasource": "$datasource",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "unit": "percent"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 6
+            },
+            "id": 2,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "table",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "pluginVersion": "11.0.0",
+            "targets": [
+                {
+                    "editorMode": "code",
+                    "expr": "{__name__=~\"gpu_.*_gpu_util_percent|gpu_util_percent\"}",
+                    "legendFormat": "{{__name__}} {{gpu}}",
+                    "range": true,
+                    "refId": "A"
+                }
+            ],
+            "title": "GPU Utilization",
+            "type": "timeseries"
+        },
+        {
+            "datasource": "$datasource",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "unit": "celsius"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 6
+            },
+            "id": 3,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "table",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "pluginVersion": "11.0.0",
+            "targets": [
+                {
+                    "editorMode": "code",
+                    "expr": "{__name__=~\"gpu_.*_temperature_celsius|temperature_celsius\"}",
+                    "legendFormat": "{{__name__}} {{gpu}}",
+                    "range": true,
+                    "refId": "A"
+                }
+            ],
+            "title": "GPU Temperature",
+            "type": "timeseries"
+        },
+        {
+            "datasource": "$datasource",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "unit": "watt"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 14
+            },
+            "id": 4,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "table",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "pluginVersion": "11.0.0",
+            "targets": [
+                {
+                    "editorMode": "code",
+                    "expr": "{__name__=~\"gpu_.*_power_watts|power_watts\"}",
+                    "legendFormat": "{{__name__}} {{gpu}}",
+                    "range": true,
+                    "refId": "A"
+                }
+            ],
+            "title": "GPU Power Draw",
+            "type": "timeseries"
+        },
+        {
+            "datasource": "$datasource",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "unit": "percent"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 14
+            },
+            "id": 5,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "table",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "pluginVersion": "11.0.0",
+            "targets": [
+                {
+                    "editorMode": "code",
+                    "expr": "{__name__=~\"gpu_.*_memory_used_percent\"}",
+                    "legendFormat": "nvidia",
+                    "range": true,
+                    "refId": "A"
+                },
+                {
+                    "editorMode": "code",
+                    "expr": "100 * mem_used_bytes / clamp_min(mem_used_bytes + mem_avail_bytes, 1)",
+                    "legendFormat": "amd {{gpu}}",
+                    "range": true,
+                    "refId": "B"
+                }
+            ],
+            "title": "GPU Memory Usage",
+            "type": "timeseries"
+        },
+        {
+            "datasource": "$datasource",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "unit": "s"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 22
+            },
+            "id": 6,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "table",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "pluginVersion": "11.0.0",
+            "targets": [
+                {
+                    "editorMode": "code",
+                    "expr": "scrape_duration_seconds{job=~\"nvidia-gpu|amdgpu_exporter|llama_exporter|prometheus\"}",
+                    "legendFormat": "{{job}}",
+                    "range": true,
+                    "refId": "A"
+                }
+            ],
+            "title": "Scrape Duration",
+            "type": "timeseries"
+        },
+        {
+            "datasource": "$datasource",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "unit": "short"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 22
+            },
+            "id": 7,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "table",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "pluginVersion": "11.0.0",
+            "targets": [
+                {
+                    "editorMode": "code",
+                    "expr": "llama_server_health{model=~\"$model\"}",
+                    "legendFormat": "health {{server}}",
+                    "range": true,
+                    "refId": "A"
+                },
+                {
+                    "editorMode": "code",
+                    "expr": "llama_models_loaded{model=~\"$model\"}",
+                    "legendFormat": "models {{server}} {{model}}",
+                    "range": true,
+                    "refId": "B"
+                }
+            ],
+            "title": "Llama Server Metrics",
+            "type": "timeseries"
+        },
+        {
+            "datasource": "$datasource",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "unit": "reqps"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 8,
+                "x": 0,
+                "y": 30
+            },
+            "id": 8,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "table",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "pluginVersion": "11.0.0",
+            "targets": [
+                {
+                    "editorMode": "code",
+                    "expr": "rate(llamacpp:tokens_predicted_total{model=~\"$model\"}[5m])",
+                    "legendFormat": "decode {{model}}",
+                    "range": true,
+                    "refId": "A"
+                },
+                {
+                    "editorMode": "code",
+                    "expr": "rate(llamacpp:prompt_tokens_total{model=~\"$model\"}[5m])",
+                    "legendFormat": "prompt {{model}}",
+                    "range": true,
+                    "refId": "B"
+                }
+            ],
+            "title": "Token Throughput",
+            "type": "timeseries"
+        },
+        {
+            "datasource": "$datasource",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "unit": "s"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 8,
+                "x": 8,
+                "y": 30
+            },
+            "id": 9,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "table",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "pluginVersion": "11.0.0",
+            "targets": [
+                {
+                    "editorMode": "code",
+                    "expr": "rate(llamacpp:tokens_predicted_seconds_total{model=~\"$model\"}[5m])",
+                    "legendFormat": "decode time {{model}}",
+                    "range": true,
+                    "refId": "A"
+                },
+                {
+                    "editorMode": "code",
+                    "expr": "rate(llamacpp:prompt_seconds_total{model=~\"$model\"}[5m])",
+                    "legendFormat": "prompt time {{model}}",
+                    "range": true,
+                    "refId": "B"
+                }
+            ],
+            "title": "Processing Time",
+            "type": "timeseries"
+        },
+        {
+            "datasource": "$datasource",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "unit": "short"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 8,
+                "x": 16,
+                "y": 30
+            },
+            "id": 10,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "table",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "pluginVersion": "11.0.0",
+            "targets": [
+                {
+                    "editorMode": "code",
+                    "expr": "llamacpp:requests_processing{model=~\"$model\"}",
+                    "legendFormat": "processing {{model}}",
+                    "range": true,
+                    "refId": "A"
+                },
+                {
+                    "editorMode": "code",
+                    "expr": "llamacpp:requests_deferred{model=~\"$model\"}",
+                    "legendFormat": "deferred {{model}}",
+                    "range": true,
+                    "refId": "B"
+                }
+            ],
+            "title": "Request Queue",
+            "type": "timeseries"
+        },
+        {
+            "datasource": "$datasource",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "unit": "short"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 38
+            },
+            "id": 11,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "table",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "pluginVersion": "11.0.0",
+            "targets": [
+                {
+                    "editorMode": "code",
+                    "expr": "llamacpp:n_busy_slots_per_decode{model=~\"$model\"}",
+                    "legendFormat": "busy slots {{model}}",
+                    "range": true,
+                    "refId": "A"
+                }
+            ],
+            "title": "Busy Slots Per Decode",
+            "type": "timeseries"
+        },
+        {
+            "datasource": "$datasource",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "unit": "short"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 38
+            },
+            "id": 12,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "table",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "pluginVersion": "11.0.0",
+            "targets": [
+                {
+                    "editorMode": "code",
+                    "expr": "llamacpp:n_tokens_max{model=~\"$model\"}",
+                    "legendFormat": "max tokens {{model}}",
+                    "range": true,
+                    "refId": "A"
+                }
+            ],
+            "title": "Max Tokens (Context Window)",
+            "type": "timeseries"
+        },
+        {
+            "datasource": "$datasource",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "unit": "short"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 46
+            },
+            "id": 13,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "table",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "pluginVersion": "11.0.0",
+            "targets": [
+                {
+                    "editorMode": "code",
+                    "expr": "{__name__=~\"gpu_.*_pstate\"}",
+                    "legendFormat": "{{__name__}}",
+                    "range": true,
+                    "refId": "A"
+                }
+            ],
+            "title": "NVIDIA P-State",
+            "type": "timeseries"
+        },
+        {
+            "datasource": "$datasource",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "unit": "short"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 30
+            },
+            "id": 9,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "table",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "pluginVersion": "11.0.0",
+            "targets": [
+                {
+                    "editorMode": "code",
+                    "expr": "{__name__=~\"gpu_.*_pcie_link_gen_current\"}",
+                    "legendFormat": "pcie gen",
+                    "range": true,
+                    "refId": "A"
+                },
+                {
+                    "editorMode": "code",
+                    "expr": "{__name__=~\"gpu_.*_pcie_link_width_current\"}",
+                    "legendFormat": "pcie width",
+                    "range": true,
+                    "refId": "B"
+                }
+            ],
+            "title": "NVIDIA PCIe Link",
+            "type": "timeseries"
+        },
+        {
+            "datasource": "$datasource",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "max": 1,
+                    "min": 0,
+                    "unit": "short"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 24,
+                "x": 0,
+                "y": 38
+            },
+            "id": 10,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "table",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "pluginVersion": "11.0.0",
+            "targets": [
+                {
+                    "editorMode": "code",
+                    "expr": "{__name__=~\"gpu_.*_throttle_sw_power_cap\"}",
+                    "legendFormat": "sw power cap",
+                    "range": true,
+                    "refId": "A"
+                },
+                {
+                    "editorMode": "code",
+                    "expr": "{__name__=~\"gpu_.*_throttle_hw_thermal_slowdown\"}",
+                    "legendFormat": "hw thermal slowdown",
+                    "range": true,
+                    "refId": "B"
+                },
+                {
+                    "editorMode": "code",
+                    "expr": "{__name__=~\"gpu_.*_throttle_hw_power_brake_slowdown\"}",
+                    "legendFormat": "hw power brake slowdown",
+                    "range": true,
+                    "refId": "C"
+                }
+            ],
+            "title": "NVIDIA Throttle Flags",
+            "type": "timeseries"
+        }
+    ],
+    "refresh": "5s",
+    "schemaVersion": 39,
+    "style": "dark",
+    "tags": [
+        "gpu",
+        "monitoring",
+        "realtime"
+    ],
+    "templating": {
+        "list": [
+            {
+                "current": {
+                    "selected": true,
+                    "text": "Prometheus",
+                    "value": "Prometheus"
+                },
+                "hide": 0,
+                "includeAll": false,
+                "label": "Datasource",
+                "multi": false,
+                "name": "datasource",
+                "options": [],
+                "query": "prometheus",
+                "refresh": 1,
+                "regex": "",
+                "skipUrlSync": false,
+                "type": "datasource"
+            },
+            {
+                "current": {
+                    "selected": false,
+                    "text": "All",
+                    "value": "$__all"
+                },
+                "datasource": {
+                    "type": "prometheus",
+                    "uid": "$datasource"
+                },
+                "definition": "label_values(llamacpp:tokens_predicted_total, model)",
+                "hide": 0,
+                "includeAll": true,
+                "label": "Model",
+                "multi": false,
+                "name": "model",
+                "options": [],
+                "query": "label_values(llamacpp:tokens_predicted_total, model)",
+                "refresh": 2,
+                "regex": "",
+                "skipUrlSync": false,
+                "sort": 1,
+                "type": "query"
+            }
+        ]
+    },
+    "time": {
+        "from": "now-30m",
+        "to": "now"
+    },
+    "timepicker": {},
+    "timezone": "",
+    "title": "GPU Monitoring Realtime",
+    "uid": "gpu-monitoring-realtime",
+    "version": 1,
+    "weekStart": ""
+}