feat(vaultwarden): add SMTP configuration options and enhance signup settings

- Introduced SMTP settings for Vaultwarden including host, port, security, and authentication details.
- Added variables for signup verification, 2FA settings, password hints, and logging options.
- Updated Vaultwarden deployment to utilize new SMTP configurations.
- Enhanced Grafana module to support dynamic dashboard and datasource provisioning.
- Added LLM proxy configuration for Open Web UI with necessary environment variables.

ansible/roles/amd_rocm/defaults/main.yml

@@ -0,0 +1,13 @@
+---
+# amd_rocm defaults
+
+rocm_repo_baseurl: "https://repo.radeon.com/rocm/el9/latest/main/"
+rocm_repo_gpgkey: "https://repo.radeon.com/rocm/rocm.gpg.key"
+rocmgraphics_repo_baseurl: "https://repo.radeon.com/graphics/latest/el/9.6/main/x86_64/"
+rocm_packages:
+  - rocm
+  - rocm-developer-tools
+  - hipblas-devel
+  - hip-devel
+  - rocwmma-devel
+  - rocm-opencl-devel

ansible/roles/amd_rocm/meta/main.yml

@@ -0,0 +1,18 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Install AMD ROCm repository and packages for GPU computing
+  company: A13Labs
+  role_name: amd_rocm
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: Fedora
+      versions:
+        - "39"
+        - "40"
+        - "41"
+    - name: EL
+      versions:
+        - "9"

ansible/roles/amd_rocm/tasks/main.yml

@@ -0,0 +1,52 @@
+---
+- name: Add ROCm repository (yum)
+  when: ansible_pkg_mgr == 'yum'
+  become: true
+  ansible.builtin.yum_repository:
+    name: rocm
+    description: ROCm Repository
+    baseurl: "{{ rocm_repo_baseurl }}"
+    gpgcheck: true
+    gpgkey: "{{ rocm_repo_gpgkey }}"
+    enabled: true
+    state: present
+
+- name: Add ROCm repository (dnf)
+  when: ansible_pkg_mgr == 'dnf'
+  become: true
+  ansible.builtin.dnf_repository:
+    name: rocm
+    description: ROCm Repository
+    baseurl: "{{ rocm_repo_baseurl }}"
+    gpgcheck: yes
+    gpgkey: "{{ rocm_repo_gpgkey }}"
+    enabled: yes
+
+- name: Add AMD Graphics repository (yum)
+  when: ansible_pkg_mgr == 'yum'
+  become: true
+  ansible.builtin.yum_repository:
+    name: rocmgraphics
+    description: AMD Graphics Repository
+    baseurl: "{{ rocmgraphics_repo_baseurl }}"
+    gpgkey: "{{ rocm_repo_gpgkey }}"
+    gpgcheck: true
+    enabled: true
+    state: present
+
+- name: Add AMD Graphics repository (dnf)
+  when: ansible_pkg_mgr == 'dnf'
+  become: true
+  ansible.builtin.dnf_repository:
+    name: rocmgraphics
+    description: AMD Graphics Repository
+    baseurl: "{{ rocmgraphics_repo_baseurl }}"
+    gpgkey: "{{ rocm_repo_gpgkey }}"
+    gpgcheck: yes
+    enabled: yes
+
+- name: Install rocm-dkms and dependencies
+  become: true
+  ansible.builtin.package:
+    name: "{{ rocm_packages }}"
+    state: present

ansible/roles/auto_suspend/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+auto_suspend_grub_config_path_bios: /boot/grub2/grub.cfg

ansible/roles/auto_suspend/meta/main.yml

@@ -0,0 +1,13 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Configure automatic system suspend based on idle time
+  company: A13Labs
+  role_name: auto_suspend
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: EL
+      versions:
+        - "9"

ansible/roles/auto_suspend/tasks/main.yml

@@ -0,0 +1,130 @@
+---
+- name: Install dependencies
+  become: true
+  ansible.builtin.package:
+    name: "{{ item }}"
+    state: present
+  loop:
+    - sysstat
+    - bc
+
+- name: Enable sysstat service
+  become: true
+  ansible.builtin.systemd:
+    name: sysstat
+    enabled: true
+    state: started
+
+- name: Read existing GRUB_CMDLINE_LINUX
+  ansible.builtin.shell: set -o pipefail && grep '^GRUB_CMDLINE_LINUX=' /etc/default/grub | cut -d= -f2- | tr -d '"'
+  register: grub_line
+  changed_when: false
+  failed_when: grub_line.stdout == ""
+
+- name: Set fact for existing_cmdline
+  ansible.builtin.set_fact:
+    existing_cmdline: "{{ grub_line.stdout }}"
+
+- name: Set fedora grub file locations
+  ansible.builtin.set_fact:
+    grub_config_path_bios: /boot/grub2/grub.cfg
+  when: ansible_distribution == "Fedora" and ansible_os_family == "RedHat"
+
+- name: Show current GRUB_CMDLINE_LINUX
+  ansible.builtin.debug:
+    msg: "Current GRUB_CMDLINE_LINUX: {{ existing_cmdline }}"
+
+- name: Updated GRUB_CMDLINE_LINUX
+  ansible.builtin.debug:
+    msg: "Current GRUB_CMDLINE_LINUX: {{ existing_cmdline | regex_replace(' *mem_sleep_default=\\S+', '') }} mem_sleep_default=deep"
+
+- name: Ensure mem_sleep_default=deep is set in GRUB_CMDLINE_LINUX
+  become: true
+  vars:
+    grub_cmdline_linux_line: >-
+      GRUB_CMDLINE_LINUX="{{ existing_cmdline | regex_replace(' *mem_sleep_default=\\S+', '') }} mem_sleep_default=deep"
+  ansible.builtin.lineinfile:
+    path: /etc/default/grub
+    regexp: '^GRUB_CMDLINE_LINUX='
+    line: "{{ grub_cmdline_linux_line }}"
+
+- name: Generate grub config for BIOS
+  become: true
+  changed_when: false
+  ansible.builtin.command: grub2-mkconfig -o {{ grub_config_path_bios }}
+
+- name: Create idle check and suspend script
+  become: true
+  ansible.builtin.copy:
+    dest: /usr/local/bin/auto_suspend_script.sh
+    mode: '0755'
+    owner: root
+    group: root
+    content: "{{ lookup('file', 'scripts/auto_suspend.sh') }}"
+
+- name: Create systemd service for auto suspend
+  become: true
+  ansible.builtin.copy:
+    dest: /etc/systemd/system/auto-suspend.service
+    content: |
+      [Unit]
+      Description=Check system idleness and suspend if idle
+      After=multi-user.target
+
+      [Service]
+      Type=oneshot
+      ExecStart=/usr/local/bin/auto_suspend_script.sh
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Create sleep.conf.d folder if it doesn't exist
+  become: true
+  ansible.builtin.file:
+    path: /etc/systemd/sleep.conf.d
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+
+- name: Create sleep.conf.d for deep sleep
+  become: true
+  ansible.builtin.copy:
+    dest: /etc/systemd/sleep.conf.d/mem-deep.conf
+    content: |
+      [Sleep]
+      MemorySleepMode=deep
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Create systemd timer for auto suspend
+  become: true
+  ansible.builtin.copy:
+    dest: /etc/systemd/system/auto-suspend.timer
+    content: |
+      [Unit]
+      Description=Run idle suspend check every 10 minutes
+
+      [Timer]
+      OnBootSec=10min
+      OnUnitActiveSec=10min
+      Unit=auto-suspend.service
+
+      [Install]
+      WantedBy=timers.target
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Reload systemd daemon
+  become: true
+  ansible.builtin.systemd:
+    daemon_reload: true
+
+- name: Enable and start auto-suspend timer
+  become: true
+  ansible.builtin.systemd:
+    name: auto-suspend.timer
+    enabled: true
+    state: started

ansible/roles/gpu_monitoring/defaults/main.yml

@@ -46,18 +46,6 @@ amd_exporter_python_version: "python3"
 amd_exporter_sysfs_gpu_pattern: "/sys/class/drm/card?/device/"
 
 # Llama Exporter configuration
-llama_exporter_enabled: true
-llama_exporter_port: 9550
-llama_exporter_user: llama_exporter
-llama_exporter_user_home: "/home/llama_exporter"
-llama_exporter_install_dir: "/opt/llama_exporter"
-llama_exporter_venv_path: "/opt/llama_exporter/.venv"
-llama_exporter_scrape_interval: 15
-llama_exporter_scrape_timeout: 5
-
-# Llama.cpp scrape targets
-llama_exporter_targets: []
-
 # Firewall configuration
 prometheus_fw_port: "{{ prometheus_port }}"
 prometheus_fw_proto: "tcp"

ansible/roles/gpu_monitoring/files/llama_exporter.py

@@ -1,176 +0,0 @@
-#!/usr/bin/env python3
-"""LLM inference metrics exporter for Prometheus.
-
-Scrape targets:
-    - llama.cpp HTTP API servers (standard /metrics or custom health)
-"""
-
-import json
-import os
-import time
-import logging
-import urllib.request
-import urllib.error
-from http.server import HTTPServer, BaseHTTPRequestHandler
-
-logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
-logger = logging.getLogger(__name__)
-
-POLL_INTERVAL = int(os.environ.get("LLAMA_EXPORTER_INTERVAL", "15"))
-SCRAPE_TIMEOUT = int(os.environ.get("LLAMA_EXPORTER_TIMEOUT", "5"))
-
-LLAMA_CPP_DEFAULTS = [
-    {"name": "llama-cpp-8012", "url": "http://localhost:8012"},
-]
-
-
-def _fetch_json(url, timeout=SCRAPE_TIMEOUT):
-    try:
-        req = urllib.request.Request(url, headers={"Accept": "application/json"})
-        with urllib.request.urlopen(req, timeout=timeout) as resp:
-            return json.loads(resp.read().decode("utf-8"))
-    except Exception as e:
-        logger.debug("Failed to fetch %s: %s", url, e)
-        return None
-
-
-def _collect_llama_cpp(target):
-    metrics = {}
-
-    def _set_metric(metric_name, labels, value):
-        label_key = tuple(sorted(labels.items()))
-        metrics[(metric_name, label_key)] = value
-
-    base_url = target["url"].rstrip("/")
-
-    # Check health endpoint
-    health_url = f"{base_url}/health"
-    health = _fetch_json(health_url)
-    if health and isinstance(health, dict):
-        status = health.get("status", "unknown")
-        _set_metric("llama_server_health", {"server": target["name"], "model": health.get("model", "unknown")}, 1.0 if status == "ok" else 0.0)
-    else:
-        _set_metric("llama_server_health", {"server": target["name"], "model": "unknown"}, 0.0)
-
-    # Check /metrics endpoint for llama.cpp built-in metrics
-    metrics_url = f"{base_url}/metrics"
-    try:
-        req = urllib.request.Request(metrics_url, headers={"Accept": "text/plain"})
-        with urllib.request.urlopen(req, timeout=SCRAPE_TIMEOUT) as resp:
-            body = resp.read().decode("utf-8")
-            for line in body.splitlines():
-                line = line.strip()
-                if not line or line.startswith("#"):
-                    continue
-                # Parse simple key value pairs
-                if "{" in line:
-                    key_part = line.split("{")[0]
-                    labels_part = line.split("{")[1].rstrip("}").split("}")[0] if "}" in line else ""
-                    value = line.split()[-1] if line.split() else "1"
-                    label_str = ""
-                    if labels_part:
-                        label_str = ",".join(f'{{{k}="{v}"}}' for k, v in (
-                            pair.split("=") for pair in labels_part.split(",") if "=" in pair
-                        ))
-                    _set_metric(key_part.strip(), {"server": target["name"], "raw": labels_part}, float(value) if value.replace(".", "").replace("-", "").isdigit() else value)
-                else:
-                    parts = line.split()
-                    if len(parts) >= 2:
-                        try:
-                            float(parts[1])
-                            _set_metric(parts[0], {"server": target["name"]}, parts[1])
-                        except ValueError:
-                            pass
-    except Exception as e:
-        logger.debug("Failed to scrape %s/metrics: %s", metrics_url, e)
-
-    # Check /model endpoint for model info
-    model_url = f"{base_url}/models"
-    models = _fetch_json(model_url)
-    if models and isinstance(models, dict):
-        model_list = models.get("data", [models]) if "data" in models else [models]
-        for m in model_list:
-            if not isinstance(m, dict):
-                continue
-            model_id = m.get("id", "unknown")
-            _set_metric("llama_models_loaded", {"server": target["name"], "model": model_id}, 1.0)
-            ctx_max = m.get("context_size", m.get("n_ctx", 0))
-            if ctx_max:
-                _set_metric("llama_model_context_max", {"server": target["name"], "model": model_id}, int(ctx_max))
-            trained = m.get("train_tokens", m.get("n_train", 0))
-            if trained:
-                _set_metric("llama_model_tokens_trained", {"server": target["name"], "model": model_id}, int(trained))
-
-    return metrics
-
-
-
-class MetricsHandler(BaseHTTPRequestHandler):
-    def do_GET(self):
-        if self.path != "/metrics":
-            self.send_response(404)
-            self.end_headers()
-            return
-
-        all_metrics = {}
-
-        # Scrape llama.cpp targets
-        targets = []
-        env_targets = os.environ.get("LLAMA_TARGETS", "")
-        if env_targets:
-            try:
-                targets = json.loads(env_targets)
-            except json.JSONDecodeError:
-                logger.warning("Invalid LLAMA_TARGETS JSON, using defaults")
-                targets = []
-
-        if not targets:
-            targets = LLAMA_CPP_DEFAULTS
-
-        for target in targets:
-            if "url" not in target or "name" not in target:
-                target["name"] = target.get("name", "default")
-                target["url"] = target.get("url", "http://localhost:8012")
-            try:
-                all_metrics.update(_collect_llama_cpp(target))
-            except Exception as e:
-                logger.error("Error scraping llama.cpp target %s: %s", target.get("name", "unknown"), e)
-
-        lines = []
-        for (metric_type, label_pairs), value in sorted(all_metrics.items()):
-            labels = dict(label_pairs)
-            if not labels:
-                line = f"{metric_type} {value}"
-            else:
-                label_str = ",".join(f'{k}="{v}"' for k, v in sorted(labels.items()))
-                line = f"{metric_type}{{{label_str}}} {value}"
-            lines.append(line)
-
-        body = "\n".join(lines) + "\n" if lines else "# no metrics available\n"
-
-        self.send_response(200)
-        self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
-        self.end_headers()
-        self.wfile.write(body.encode("utf-8"))
-
-    def log_message(self, format, *args):
-        logger.debug("%s - - %s", self.address_string(), format % args)
-
-
-def main():
-    port = int(os.environ.get("LLAMA_EXPORTER_PORT", "9550"))
-    host = os.environ.get("LLAMA_EXPORTER_BIND", "0.0.0.0")
-
-    server = HTTPServer((host, port), MetricsHandler)
-    logger.info("Starting Llama Exporter on %s:%d", host, port)
-    logger.info("Polling every %d seconds, timeout %ds", POLL_INTERVAL, SCRAPE_TIMEOUT)
-
-    try:
-        server.serve_forever()
-    except KeyboardInterrupt:
-        logger.info("Shutting down")
-        server.shutdown()
-
-
-if __name__ == "__main__":
-    main()

ansible/roles/gpu_monitoring/handlers/main.yml

@@ -18,10 +18,3 @@
     name: "{{ amd_exporter_service_name }}"
     state: restarted
   listen: Restart amd_exporter
-
-- name: Restart llama_exporter
-  become: true
-  ansible.builtin.systemd:
-    name: "{{ llama_exporter_service_name }}"
-    state: restarted
-  listen: Restart llama_exporter

ansible/roles/gpu_monitoring/molecule/default/converge.yml

@@ -8,4 +8,3 @@
         name: "gpu_monitoring"
       vars:
         amd_exporter_enabled: false
-        llama_exporter_enabled: false

ansible/roles/gpu_monitoring/tasks/main.yml

@@ -11,12 +11,6 @@
   tags:
     - roles::gpu_monitoring::amd_exporter
 
-- name: Include Llama Exporter setup
-  when: llama_exporter_enabled | default(true) | bool
-  ansible.builtin.include_tasks: llama_exporter.yml
-  tags:
-    - roles::gpu_monitoring::llama_exporter
-
 - name: Include Prometheus setup
   ansible.builtin.include_tasks: prometheus.yml
   tags:

ansible/roles/gpu_monitoring/templates/llama_exporter.service.j2

@@ -1,35 +0,0 @@
-[Unit]
-Description=LLM Inference Metrics Exporter (llama.cpp + Ollama)
-After=network-online.target
-Wants=network-online.target
-
-[Service]
-Type=simple
-User={{ llama_exporter_user }}
-Group={{ llama_exporter_user }}
-WorkingDirectory={{ llama_exporter_install_dir }}
-Environment="LLAMA_EXPORTER_PORT={{ llama_exporter_port }}"
-Environment="LLAMA_EXPORTER_BIND=0.0.0.0"
-Environment="LLAMA_EXPORTER_INTERVAL={{ llama_exporter_scrape_interval }}"
-Environment="LLAMA_EXPORTER_TIMEOUT={{ llama_exporter_scrape_timeout }}"
-{% if llama_exporter_targets and llama_exporter_targets != "" %}
-Environment="LLAMA_TARGETS={{ llama_exporter_targets | to_json }}"
-{% endif %}
-Environment="PATH={{ llama_exporter_venv_bin }}:/usr/bin:/bin"
-ExecStart={{ llama_exporter_venv_bin }}/python3 {{ llama_exporter_script_path }}
-Restart=always
-RestartSec=10
-TimeoutStopSec=10
-StandardOutput=journal
-StandardError=journal
-SyslogIdentifier=llama_exporter
-
-# Security hardening
-ProtectSystem=strict
-ProtectHome=true
-ReadWritePaths={{ llama_exporter_install_dir }} /var/log
-NoNewPrivileges=true
-PrivateTmp=true
-
-[Install]
-WantedBy=multi-user.target

ansible/roles/gpu_monitoring/templates/prometheus.yml.j2

@@ -21,10 +21,3 @@ scrape_configs:
       - targets: ["{{ prometheus_host_gateway }}:{{ amd_exporter_port }}"]
         labels:
           datacenter: "{{ inventory_hostname }}"
-
-  - job_name: llama_exporter
-    metrics_path: /metrics
-    static_configs:
-      - targets: ["{{ prometheus_host_gateway }}:{{ llama_exporter_port }}"]
-        labels:
-          datacenter: "{{ inventory_hostname }}"

ansible/roles/gpu_monitoring/vars/main.yml

@@ -12,12 +12,3 @@ nvidia_exporter_service_name: "nvidia_exporter"
 amd_exporter_script_path: "{{ amd_exporter_install_dir }}/amd_gpu_exporter.py"
 amd_exporter_venv_bin: "{{ amd_exporter_venv_path }}/bin"
 amd_exporter_service_name: "amdgpu_exporter"
-
-llama_exporter_script_path: "{{ llama_exporter_install_dir }}/llama_exporter.py"
-llama_exporter_venv_bin: "{{ llama_exporter_venv_path }}/bin"
-llama_exporter_service_name: "llama_exporter"
-
-# Default llama.cpp targets if none provided
-llama_exporter_targets_default:
-  - name: "llama-cpp-8012"
-    url: "http://localhost:8012"

ansible/roles/llama_server/defaults/main.yml

@@ -0,0 +1,26 @@
+llama_server_user: "llama"
+llama_server_group: "llama"
+llama_server_home: "/home/llama"
+llama_server_pubkey: ""
+llama_server_models: []
+llama_server_models_max: 1
+llama_server_image: "ghcr.io/ggml-org/llama.cpp"
+llama_server_tag: "server-vulkan"
+llama_server_extra_groups: "users,video"
+llama_server_devices: []
+llama_server_env: {}
+llama_server_port: 8080
+llama_server_preset_global: {}
+
+# Llama Exporter configuration
+llama_server_exporter_enabled: true
+llama_server_exporter_port: 9550
+llama_server_exporter_user: llama_exporter
+llama_server_exporter_user_home: "/opt/llama_exporter"
+llama_server_exporter_install_dir: "/opt/llama_exporter"
+llama_server_exporter_venv_path: "/opt/llama_exporter/.venv"
+llama_server_exporter_scrape_interval: 15
+llama_server_exporter_scrape_timeout: 5
+
+# Llama.cpp scrape targets
+llama_server_exporter_targets: []

ansible/roles/llama_server/files/scripts/llama_exporter.py

@@ -0,0 +1,389 @@
+#!/usr/bin/env python3
+"""LLM inference metrics exporter for Prometheus.
+
+Background thread scrapes only loaded models to avoid triggering model loads
+in llama-server, then the HTTP handler serves the cached data immediately.
+
+Exposes metrics for every known model from /models; unloaded models show
+zero values without ever requesting /metrics?model=<unloaded>.
+
+Uses persistent JSON cache on disk to survive restarts and compute counter
+deltas for Prometheus rate/irate queries.
+"""
+
+import json
+import os
+import time
+import threading
+import logging
+import urllib.request
+import urllib.error
+from http.server import HTTPServer, BaseHTTPRequestHandler
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
+logger = logging.getLogger(__name__)
+
+POLL_INTERVAL = int(os.environ.get("LLAMA_EXPORTER_INTERVAL", "15"))
+SCRAPE_TIMEOUT = int(os.environ.get("LLAMA_EXPORTER_TIMEOUT", "5"))
+CACHE_FILE = os.environ.get(
+    "LLAMA_EXPORTER_CACHE",
+    os.path.join(os.path.dirname(os.path.abspath(__file__)), "llama_exporter_cache.json"),
+)
+
+LLAMA_CPP_DEFAULTS = [
+    {"name": "llama.cpp", "url": "http://localhost:11434"},
+]
+
+# All llama.cpp metrics we expose per model (counters + gauges).
+ALL_METRICS = [
+    "llamacpp:prompt_tokens_total",
+    "llamacpp:prompt_seconds_total",
+    "llamacpp:tokens_predicted_total",
+    "llamacpp:tokens_predicted_seconds_total",
+    "llamacpp:n_decode_total",
+    "llamacpp:n_tokens_max",
+    "llamacpp:prompt_tokens_seconds",
+    "llamacpp:predicted_tokens_seconds",
+    "llamacpp:requests_processing",
+    "llamacpp:requests_deferred",
+    "llamacpp:n_busy_slots_per_decode",
+]
+
+# Counter metrics that accumulate over time — exposed as _delta for rate() queries.
+COUNTER_METRICS = {
+    "llamacpp:prompt_tokens_total",
+    "llamacpp:prompt_seconds_total",
+    "llamacpp:tokens_predicted_total",
+    "llamacpp:tokens_predicted_seconds_total",
+    "llamacpp:n_decode_total",
+}
+
+
+class Cache:
+    """Persistent, thread-safe metrics cache with delta computation."""
+
+    def __init__(self):
+        self._lock = threading.Lock()
+        self._data = {}  # { model_id: { metric_name: value } }
+        self._known_models = set()
+        self._health = None
+        self._loaded = {}
+
+        # Previous state for delta computation: { model_id: { metric_name: value } }
+        self._prev = {}
+        self._prev_timestamp = 0.0
+        self._deltas = {}
+
+        # Load persisted state
+        self._load_cache()
+
+    def _load_cache(self):
+        """Load previous scrape state from disk."""
+        try:
+            with open(CACHE_FILE, "r") as f:
+                state = json.load(f)
+            self._prev = state.get("models", {})
+            self._prev_timestamp = state.get("timestamp", 0.0)
+            logger.info("Loaded cache from %s (timestamp=%s, models=%d)",
+                        CACHE_FILE, self._prev_timestamp, len(self._prev))
+        except FileNotFoundError:
+            logger.info("No cache file found at %s", CACHE_FILE)
+        except Exception as e:
+            logger.warning("Failed to load cache: %s", e)
+
+    def _save_cache(self, current_data, known_models, loaded):
+        """Save current scrape state to disk."""
+        try:
+            state = {
+                "timestamp": time.time(),
+                "known": list(known_models),
+                "loaded": list(loaded),
+                "models": {},
+            }
+            for mid, metrics in current_data.items():
+                state["models"][mid] = dict(metrics)
+            tmp = CACHE_FILE + ".tmp"
+            with open(tmp, "w") as f:
+                json.dump(state, f)
+            os.replace(tmp, CACHE_FILE)
+        except Exception as e:
+            logger.error("Failed to save cache: %s", e)
+
+    def _compute_deltas_for_data(self, current_data):
+        """Compute deltas given current data and stored previous state.
+
+        Only counter metrics get deltas. On counter reset (value went backward),
+        delta is silently 0 (no entry added).
+        """
+        deltas = {}
+        if not self._prev:
+            return deltas
+
+        for mid, prev_metrics in self._prev.items():
+            mid_deltas = {}
+            cur_metrics = current_data.get(mid, {})
+            for mname, prev_val in prev_metrics.items():
+                # Only compute deltas for counter metrics
+                if mname not in COUNTER_METRICS:
+                    continue
+                cur_val = cur_metrics.get(mname)
+                if cur_val is None:
+                    continue
+                diff = cur_val - prev_val
+                # Counter reset: value went backward, delta is 0 (wrapped)
+                if diff < 0:
+                    continue
+                if diff > 0:
+                    mid_deltas[mname] = diff
+            if mid_deltas:
+                deltas[mid] = mid_deltas
+        return deltas
+
+    def update(self, models_data, metric_lines, known_models=None):
+        """Called by the background thread after a successful scrape cycle."""
+        with self._lock:
+            # Discover model IDs and loaded status from /models endpoint
+            known = set()
+            loaded = {}
+            if models_data and isinstance(models_data, dict):
+                model_list = models_data.get("data", [])
+                for m in model_list:
+                    if not isinstance(m, dict):
+                        continue
+                    mid = m.get("id", "unknown")
+                    known.add(mid)
+                    status_data = m.get("status", {})
+                    if isinstance(status_data, dict) and status_data.get("value") == "loaded":
+                        loaded[mid] = 1.0
+                    self._health = {"status": "ok", "model": mid}
+
+            if not known:
+                self._health = {"status": "error", "model": "unknown"}
+
+            # Build new metrics dict for all known models
+            new_data = {}
+            for mid in known:
+                new_data[mid] = {}
+
+            # Apply parsed metric values from /metrics?model=<id>
+            for metric_name, model_id, value in metric_lines:
+                if model_id in known:
+                    new_data[model_id][metric_name] = value
+
+            # Ensure every known model has all ALL_METRICS entries
+            for mid in known:
+                for mname in ALL_METRICS:
+                    if mname not in new_data[mid]:
+                        new_data[mid][mname] = 0.0
+
+            # Previously known models no longer in the list get zeroed out
+            for mid in self._known_models - known:
+                new_data[mid] = {m: 0.0 for m in ALL_METRICS}
+
+            # Compute deltas before updating previous state
+            deltas = self._compute_deltas_for_data(new_data)
+
+            # Save previous state for next cycle
+            self._save_cache(new_data, known, loaded)
+
+            # Update state
+            self._prev = {mid: dict(metrics) for mid, metrics in new_data.items()}
+            self._known_models = known
+            self._data = new_data
+            self._loaded = loaded
+            self._deltas = deltas
+
+    def snapshot(self):
+        """Return a frozen copy of the current cache state including deltas."""
+        with self._lock:
+            return {
+                "data": {k: dict(v) for k, v in self._data.items()},
+                "known": set(self._known_models),
+                "health": dict(self._health) if self._health else {"status": "error", "model": "unknown"},
+                "loaded": dict(self._loaded),
+                "deltas": {k: dict(v) for k, v in self._deltas.items()},
+            }
+
+
+cache = Cache()
+
+
+def _fetch_json(url, timeout=SCRAPE_TIMEOUT):
+    try:
+        req = urllib.request.Request(url, headers={"Accept": "application/json"})
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return json.loads(resp.read().decode("utf-8"))
+    except Exception as e:
+        logger.debug("Failed to fetch %s: %s", url, e)
+        return None
+
+
+def _fetch_text(url, timeout=SCRAPE_TIMEOUT):
+    try:
+        req = urllib.request.Request(url, headers={"Accept": "text/plain"})
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return resp.read().decode("utf-8")
+    except Exception as e:
+        logger.debug("Failed to fetch %s: %s", url, e)
+        return None
+
+
+def _scrape_cycle():
+    """One full scrape cycle: discover models, then scrape metrics per model."""
+    targets = []
+    env_targets = os.environ.get("LLAMA_TARGETS", "")
+    if env_targets:
+        try:
+            targets = json.loads(env_targets)
+        except json.JSONDecodeError:
+            targets = []
+    if not targets:
+        targets = LLAMA_CPP_DEFAULTS
+
+    all_metric_lines = []
+    models_data = None
+    known_models = {}  # { model_id: base_url, ... }
+
+    for target in targets:
+        base_url = target["url"].rstrip("/")
+
+        # Fetch /models to discover models and their status
+        models_data = _fetch_json(f"{base_url}/models")
+
+        if models_data and isinstance(models_data, dict):
+            model_list = models_data.get("data", [])
+            for m in model_list:
+                if not isinstance(m, dict) or "id" not in m:
+                    continue
+                model_id = m["id"]
+                known_models[model_id] = base_url
+                # Only scrape metrics from loaded models to avoid triggering loads
+                status_data = m.get("status", {})
+                if isinstance(status_data, dict) and status_data.get("value") != "loaded":
+                    continue
+                # Scrape /metrics for loaded models only
+                metrics_url = f"{base_url}/metrics?model={model_id}"
+                body = _fetch_text(metrics_url)
+                if body:
+                    for line in body.splitlines():
+                        parsed = _parse_metric_line(line)
+                        if parsed:
+                            metric_name, metric_value = parsed
+                            if metric_name in ALL_METRICS:
+                                all_metric_lines.append((metric_name, model_id, metric_value))
+                else:
+                    logger.debug("No metrics body for model %s", model_id)
+
+    # Update the shared cache
+    cache.update(models_data, all_metric_lines, known_models)
+
+
+def _parse_metric_line(line):
+    """Parse a single Prometheus metric line. Returns (name, value) or None."""
+    line = line.strip()
+    if not line or line.startswith("#"):
+        return None
+    try:
+        # Handle lines with labels: metric_name{label="val"} value
+        if "{" in line:
+            parts = line.split("{")
+            name = parts[0].strip()
+            rest = parts[1]
+            # value is the last token after the closing }
+            value = rest.rsplit("}", 1)[-1].strip().split()[-1] if "}" in rest else rest.strip()
+        else:
+            parts = line.split()
+            name = parts[0]
+            value = parts[1] if len(parts) >= 2 else "1"
+        # Try to parse as float
+        float(value)
+        return (name, float(value))
+    except (ValueError, IndexError):
+        return None
+
+
+def _background_scrape():
+    """Background thread: periodically scrape and update cache."""
+    logger.info("Background scraper started (interval=%ds)", POLL_INTERVAL)
+    # Do one immediate scrape on startup
+    _scrape_cycle()
+    while True:
+        try:
+            time.sleep(POLL_INTERVAL)
+            _scrape_cycle()
+        except Exception as e:
+            logger.error("Scrape cycle error: %s", e)
+
+
+class MetricsHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path != "/metrics":
+            self.send_response(404)
+            self.end_headers()
+            return
+
+        snap = cache.snapshot()
+        lines = []
+
+        def _fmt(metric_name, model_id, value):
+            return metric_name + '{' + 'server="llama-cpp-11434",model="' + model_id + '"} ' + str(value)
+
+        # Health metric
+        health = snap["health"]
+        status = health.get("status", "error")
+        health_model = health.get("model", "unknown")
+        health_val = 1.0 if status == "ok" else 0.0
+        lines.append(_fmt("llama_server_health", health_model, health_val))
+
+        # Loaded metrics
+        for mid in snap["loaded"]:
+            lines.append(_fmt("llama_models_loaded", mid, snap["loaded"][mid]))
+
+        # Per-model metrics from cache (absolute values)
+        for mid in sorted(snap["data"]):
+            metrics = snap["data"][mid]
+            for mname in ALL_METRICS:
+                value = metrics.get(mname, 0.0)
+                lines.append(_fmt(mname, mid, value))
+
+        # Per-model delta metrics (counters as change since last scrape)
+        deltas = snap.get("deltas", {})
+        for mid in sorted(deltas):
+            delta_metrics = deltas[mid]
+            for mname in sorted(delta_metrics):
+                value = delta_metrics[mname]
+                lines.append(_fmt(mname + "_delta", mid, value))
+
+        body = "\n".join(lines) + "\n" if lines else "# no metrics available\n"
+
+        self.send_response(200)
+        self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+        self.end_headers()
+        self.wfile.write(body.encode("utf-8"))
+
+    def log_message(self, format, *args):
+        logger.debug("%s - - %s", self.address_string(), format % args)
+
+
+def main():
+    port = int(os.environ.get("LLAMA_EXPORTER_PORT", "9550"))
+    host = os.environ.get("LLAMA_EXPORTER_BIND", "0.0.0.0")
+
+    # Start background scraper thread
+    scraper = threading.Thread(target=_background_scrape, daemon=True)
+    scraper.start()
+
+    # Start HTTP server
+    server = HTTPServer((host, port), MetricsHandler)
+    logger.info("Starting Llama Exporter on %s:%d (interval=%ds, timeout=%ds, cache=%s)",
+                host, port, POLL_INTERVAL, SCRAPE_TIMEOUT, CACHE_FILE)
+
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        logger.info("Shutting down")
+        server.shutdown()
+
+
+if __name__ == "__main__":
+    main()

ansible/roles/llama_server/files/scripts/llama_hf_sync.py

@@ -0,0 +1,284 @@
+#!/usr/bin/env python3
+"""Synchronize llama.cpp models from Hugging Face into a managed local directory."""
+
+from __future__ import annotations
+
+import argparse
+import configparser
+import json
+import os
+import re
+import shutil
+import sys
+from pathlib import Path
+from typing import Any
+
+from huggingface_hub import HfApi, snapshot_download
+
+
+def _sanitize_slug(value: str) -> str:
+    slug = re.sub(r"[^a-zA-Z0-9._-]+", "-", value.strip())
+    slug = slug.strip("-._")
+    return slug.lower() or "model"
+
+
+def _resolve_repo_file(api: HfApi, model: dict[str, Any]) -> str:
+    repo_files = api.list_repo_files(repo_id=model["model_id"], revision=model["revision"], repo_type="model")
+
+    hf_file = model.get("hf_file")
+    if hf_file:
+        if hf_file in repo_files:
+            return hf_file
+        raise RuntimeError(
+            f"Requested hf_file '{hf_file}' was not found in repo "
+            f"{model['model_id']}@{model['revision']}"
+        )
+
+    quant_upper = (model.get("quant") or "").upper()
+    ggufs = [item for item in repo_files if item.lower().endswith(".gguf")]
+    matches = [item for item in ggufs if quant_upper in Path(item).name.upper()]
+    if matches:
+        return sorted(matches, key=lambda name: (len(Path(name).name), name))[0]
+
+    raise RuntimeError(
+        f"No GGUF file containing quant '{model.get('quant')}' found in "
+        f"repo {model['model_id']}@{model['revision']}. Set hf_file explicitly if needed."
+    )
+
+
+def _load_models(models_file: Path) -> list[dict[str, Any]]:
+    data = json.loads(models_file.read_text(encoding="utf-8"))
+    if not isinstance(data, list):
+        raise RuntimeError("models file must contain a JSON array")
+
+    normalized: list[dict[str, Any]] = []
+    for item in data:
+        if not isinstance(item, dict):
+            raise RuntimeError("each model entry must be an object")
+        if item.get("enable", True) is False:
+            continue
+
+        model_id = str(item.get("model_id", "")).strip()
+        quant = str(item.get("quant", "")).strip()
+        revision = str(item.get("revision", "")).strip()
+        hf_file = str(item.get("hf_file", "")).strip() or None
+        name = str(item.get("name", "")).strip()
+        preset = item.get("preset", {})
+
+        if preset is None:
+            preset = {}
+        if not isinstance(preset, dict):
+            raise RuntimeError(f"preset must be an object for model '{name or model_id or 'unknown'}'")
+
+        if not name:
+            raise RuntimeError(f"name is required for model '{model_id or 'unknown'}'")
+        if not model_id:
+            raise RuntimeError("model_id is required for all models")
+        if not quant and not hf_file:
+            raise RuntimeError(f"quant or hf_file is required for model '{model_id}'")
+        if not revision:
+            raise RuntimeError(f"revision is required for model '{model_id}'")
+
+        normalized.append(
+            {
+                "model_id": model_id,
+                "quant": quant,
+                "revision": revision,
+                "hf_file": hf_file,
+                "name": name,
+                "preset": preset,
+            }
+        )
+
+    return normalized
+
+
+def _load_preset_global(preset_global_file: Path | None) -> dict[str, Any]:
+    if preset_global_file is None:
+        return {}
+
+    if not preset_global_file.exists():
+        return {}
+
+    data = json.loads(preset_global_file.read_text(encoding="utf-8"))
+    if data is None:
+        return {}
+    if not isinstance(data, dict):
+        raise RuntimeError("global preset options must be a JSON object")
+    return data
+
+
+def _to_preset_value(value: Any) -> str:
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, (int, float, str)):
+        return str(value)
+    return json.dumps(value, separators=(",", ":"))
+
+
+def _normalize_preset_options(options: dict[str, Any], scope: str) -> dict[str, str]:
+    normalized: dict[str, str] = {}
+    for key, value in options.items():
+        key_str = str(key).strip()
+        if not key_str:
+            raise RuntimeError(f"{scope} preset option keys must be non-empty")
+        if value is None:
+            continue
+        normalized[key_str] = _to_preset_value(value)
+    return normalized
+
+
+def _download_model(model: dict[str, Any], target_dir: Path, repo_file: str, dry_run: bool) -> None:
+    if dry_run:
+        return
+
+    snapshot_download(
+        repo_id=model["model_id"],
+        revision=model["revision"],
+        local_dir=str(target_dir),
+        allow_patterns=[repo_file],
+        local_dir_use_symlinks=False,
+    )
+
+
+def _write_preset(preset_file: Path, entries: list[dict[str, Any]], global_options: dict[str, Any]) -> None:
+    parser = configparser.ConfigParser(interpolation=None)
+    parser.optionxform = str
+    parser["*"] = _normalize_preset_options(global_options, "global")
+
+    for entry in entries:
+        model_options = _normalize_preset_options(entry.get("preset", {}), f"model '{entry['name']}'")
+        model_options["model"] = entry["container_model_path"]
+        parser[entry["name"]] = model_options
+
+    preset_file.parent.mkdir(parents=True, exist_ok=True)
+    with preset_file.open("w", encoding="utf-8") as fh:
+        # Router preset format supports top-level version key.
+        fh.write("version = 1\n\n")
+        parser.write(fh)
+
+
+def _prune_unmanaged(managed_dir: Path, links_dir: Path, keep_slugs: set[str], dry_run: bool) -> tuple[list[str], list[str]]:
+    pruned_dirs: list[str] = []
+    pruned_links: list[str] = []
+
+    if managed_dir.exists():
+        for child in managed_dir.iterdir():
+            if not child.is_dir():
+                continue
+            if child.name == ".router":
+                continue
+            if child.name not in keep_slugs:
+                pruned_dirs.append(child.name)
+                if not dry_run:
+                    shutil.rmtree(child)
+
+    if links_dir.exists():
+        for child in links_dir.iterdir():
+            if child.suffix != ".gguf":
+                continue
+            slug = child.stem
+            if slug not in keep_slugs:
+                pruned_links.append(child.name)
+                if not dry_run:
+                    child.unlink(missing_ok=True)
+
+    return pruned_dirs, pruned_links
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Sync Hugging Face GGUF models for llama.cpp router mode")
+    parser.add_argument("--models-file", required=True)
+    parser.add_argument("--managed-dir", required=True)
+    parser.add_argument("--links-dir", required=True)
+    parser.add_argument("--manifest-file", required=True)
+    parser.add_argument("--preset-file", required=True)
+    parser.add_argument("--preset-global-file")
+    parser.add_argument("--container-links-dir", required=True)
+    parser.add_argument("--prune", action="store_true")
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+
+    models_file = Path(args.models_file)
+    managed_dir = Path(args.managed_dir)
+    links_dir = Path(args.links_dir)
+    manifest_file = Path(args.manifest_file)
+    preset_file = Path(args.preset_file)
+    preset_global_file = Path(args.preset_global_file) if args.preset_global_file else None
+
+    managed_dir.mkdir(parents=True, exist_ok=True)
+    links_dir.mkdir(parents=True, exist_ok=True)
+    manifest_file.parent.mkdir(parents=True, exist_ok=True)
+    preset_file.parent.mkdir(parents=True, exist_ok=True)
+
+    models = _load_models(models_file)
+    global_options = _load_preset_global(preset_global_file)
+    api = HfApi()
+    entries: list[dict[str, Any]] = []
+
+    for model in models:
+        slug = _sanitize_slug(model["name"])
+        target_dir = managed_dir / slug
+        target_dir.mkdir(parents=True, exist_ok=True)
+        repo_file = _resolve_repo_file(api, model)
+
+        _download_model(model, target_dir, repo_file, args.dry_run)
+        selected_file = target_dir / repo_file
+        if not args.dry_run and not selected_file.exists():
+            raise RuntimeError(f"Expected downloaded file not found: {selected_file}")
+
+        link_path = links_dir / f"{slug}.gguf"
+        container_model_path = f"{args.container_links_dir.rstrip('/')}/{slug}.gguf"
+
+        if not args.dry_run:
+            if link_path.exists() or link_path.is_symlink():
+                link_path.unlink()
+            # Keep symlink targets relative so they remain valid inside the
+            # container-mounted /models tree.
+            relative_target = os.path.relpath(selected_file, start=link_path.parent)
+            link_path.symlink_to(relative_target)
+
+        entries.append(
+            {
+                "name": model["name"],
+                "slug": slug,
+                "model_id": model["model_id"],
+                "revision": model["revision"],
+                "quant": model["quant"],
+                "hf_file": model.get("hf_file"),
+                "preset": model.get("preset", {}),
+                "repo_file": repo_file,
+                "selected_file": str(selected_file),
+                "link_path": str(link_path),
+                "container_model_path": container_model_path,
+            }
+        )
+
+    pruned_dirs: list[str] = []
+    pruned_links: list[str] = []
+    if args.prune:
+        keep_slugs = {entry["slug"] for entry in entries}
+        pruned_dirs, pruned_links = _prune_unmanaged(managed_dir, links_dir, keep_slugs, args.dry_run)
+
+    result = {
+        "models": entries,
+        "dry_run": args.dry_run,
+        "prune": args.prune,
+        "pruned_dirs": pruned_dirs,
+        "pruned_links": pruned_links,
+    }
+
+    if not args.dry_run:
+        manifest_file.write_text(json.dumps(result, indent=2), encoding="utf-8")
+        _write_preset(preset_file, entries, global_options)
+
+    print(json.dumps(result))
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except Exception as exc:
+        print(f"ERROR: {exc}", file=sys.stderr)
+        raise SystemExit(1)

ansible/roles/llama_server/files/scripts/test_llama_exporter.py

@@ -0,0 +1,231 @@
+#!/usr/bin/env python3
+"""Tests for llama_exporter cache persistence and delta computation."""
+
+import json
+import os
+import sys
+import tempfile
+import shutil
+import unittest
+
+# Setup temp cache file before importing the module
+TEMP_DIR = tempfile.mkdtemp()
+os.environ["LLAMA_EXPORTER_CACHE"] = os.path.join(TEMP_DIR, "test_cache.json")
+
+# Import from parent directory (scripts)
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from llama_exporter import Cache
+
+
+class TestCachePersistenceAndDeltas(unittest.TestCase):
+
+    def setUp(self):
+        # Ensure temp dir exists (pytest may clean it between test methods)
+        os.makedirs(TEMP_DIR, exist_ok=True)
+        self.cache = Cache()
+
+    def tearDown(self):
+        shutil.rmtree(TEMP_DIR, ignore_errors=True)
+
+    def test_cold_start_no_deltas(self):
+        """First scrape should produce no deltas since there's no previous state."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+                {"id": "model-v2", "status": {"value": "unloaded"}},
+            ]
+        }
+        metric_lines = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 100.0),
+            ("llamacpp:prompt_seconds_total", "model-v1", 2.5),
+            ("llamacpp:n_tokens_max", "model-v1", 4096.0),
+        ]
+        known_models = {"model-v1": "http://localhost:11434", "model-v2": "http://localhost:11434"}
+
+        self.cache.update(models_data, metric_lines, known_models)
+
+        snap = self.cache.snapshot()
+        # No deltas on first scrape
+        self.assertEqual(snap["deltas"], {})
+        # Absolute values are correct
+        self.assertEqual(snap["data"]["model-v1"]["llamacpp:prompt_tokens_total"], 100.0)
+        # Unloaded model has zeros
+        self.assertEqual(snap["data"]["model-v2"]["llamacpp:prompt_tokens_total"], 0.0)
+        # Only loaded model is in loaded set
+        self.assertIn("model-v1", snap["loaded"])
+        self.assertNotIn("model-v2", snap["loaded"])
+
+    def test_delta_computation(self):
+        """Second scrape should produce deltas from first scrape."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+            ]
+        }
+
+        # First scrape
+        metric_lines_1 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 100.0),
+            ("llamacpp:prompt_seconds_total", "model-v1", 2.5),
+        ]
+        known_models_1 = {"model-v1": "http://localhost:11434"}
+        self.cache.update(models_data, metric_lines_1, known_models_1)
+
+        # Second scrape - values increased
+        metric_lines_2 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 250.0),
+            ("llamacpp:prompt_seconds_total", "model-v1", 5.0),
+        ]
+        self.cache.update(models_data, metric_lines_2, known_models_1)
+
+        snap = self.cache.snapshot()
+        # Deltas should be computed
+        self.assertIn("model-v1", snap["deltas"])
+        self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_tokens_total"], 150.0)
+        self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_seconds_total"], 2.5)
+
+    def test_counter_reset(self):
+        """If a counter value goes backward, delta should be 0."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+            ]
+        }
+
+        # First scrape
+        metric_lines_1 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 100.0),
+        ]
+        known_models_1 = {"model-v1": "http://localhost:11434"}
+        self.cache.update(models_data, metric_lines_1, known_models_1)
+
+        # Second scrape - counter reset (model was reloaded)
+        metric_lines_2 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 10.0),
+        ]
+        self.cache.update(models_data, metric_lines_2, known_models_1)
+
+        snap = self.cache.snapshot()
+        # On counter reset, delta is implicitly 0 (no entry in deltas)
+        self.assertNotIn("model-v1", snap["deltas"])
+
+    def test_gauge_metrics_no_delta(self):
+        """Gauge metrics should not get delta entries (only counters do)."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+            ]
+        }
+
+        # First scrape
+        metric_lines_1 = [
+            ("llamacpp:n_tokens_max", "model-v1", 4096.0),
+            ("llamacpp:requests_processing", "model-v1", 2.0),
+        ]
+        known_models_1 = {"model-v1": "http://localhost:11434"}
+        self.cache.update(models_data, metric_lines_1, known_models_1)
+
+        # Second scrape
+        metric_lines_2 = [
+            ("llamacpp:n_tokens_max", "model-v1", 8192.0),
+            ("llamacpp:requests_processing", "model-v1", 5.0),
+        ]
+        self.cache.update(models_data, metric_lines_2, known_models_1)
+
+        snap = self.cache.snapshot()
+        # Gauge metrics should not have deltas
+        self.assertNotIn("model-v1", snap["deltas"])
+
+    def test_persistence_across_restarts(self):
+        """Delta computation should work across cache reloads from disk."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+            ]
+        }
+
+        # First scrape (first process instance)
+        metric_lines_1 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 100.0),
+            ("llamacpp:prompt_seconds_total", "model-v1", 2.5),
+        ]
+        known_models_1 = {"model-v1": "http://localhost:11434"}
+        self.cache.update(models_data, metric_lines_1, known_models_1)
+
+        # Simulate restart: create new Cache instance (loads from disk)
+        self.cache._save_cache(self.cache._data, self.cache._known_models, self.cache._loaded)
+
+        # Create a new cache instance (simulates restart)
+        new_cache = Cache()
+
+        # Second scrape after restart
+        metric_lines_2 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 300.0),
+            ("llamacpp:prompt_seconds_total", "model-v1", 8.0),
+        ]
+        known_models_2 = {"model-v1": "http://localhost:11434"}
+        new_cache.update(models_data, metric_lines_2, known_models_2)
+
+        snap = new_cache.snapshot()
+        # Deltas should be computed using persisted previous state
+        self.assertIn("model-v1", snap["deltas"])
+        self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_tokens_total"], 200.0)
+        self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_seconds_total"], 5.5)
+
+    def test_unloaded_model_zero_values(self):
+        """Unloaded models should appear with zero values but never trigger /metrics calls."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+                {"id": "model-v2", "status": {"value": "unloaded"}},
+                {"id": "model-v3", "status": {"value": "unloaded"}},
+            ]
+        }
+        metric_lines = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 50.0),
+        ]
+        known_models = {
+            "model-v1": "http://localhost:11434",
+            "model-v2": "http://localhost:11434",
+            "model-v3": "http://localhost:11434",
+        }
+
+        self.cache.update(models_data, metric_lines, known_models)
+
+        snap = self.cache.snapshot()
+        # Loaded model has its actual value
+        self.assertEqual(snap["data"]["model-v1"]["llamacpp:prompt_tokens_total"], 50.0)
+        # Unloaded models have zeros for all metrics
+        for m in ["model-v2", "model-v3"]:
+            self.assertEqual(snap["data"][m]["llamacpp:prompt_tokens_total"], 0.0)
+            self.assertEqual(snap["data"][m]["llamacpp:n_tokens_max"], 0.0)
+
+    def test_removed_model_zeroed(self):
+        """Models removed from llama-server should be zeroed out."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+            ]
+        }
+
+        # First scrape with two models
+        metric_lines_1 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 100.0),
+        ]
+        known_models_1 = {"model-v1": "http://localhost:11434"}
+        self.cache.update(models_data, metric_lines_1, known_models_1)
+
+        # Second scrape with only model-v1 (model-v2 was removed)
+        metric_lines_2 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 200.0),
+        ]
+        known_models_2 = {"model-v1": "http://localhost:11434"}
+        self.cache.update(models_data, metric_lines_2, known_models_2)
+
+        snap = self.cache.snapshot()
+        # model-v1 delta should be 100
+        self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_tokens_total"], 100.0)
+
+
+if __name__ == "__main__":
+    unittest.main()

ansible/roles/llama_server/handlers/main.yml

@@ -0,0 +1,13 @@
+---
+- name: Reload systemd daemon
+  become: true
+  ansible.builtin.systemd:
+    daemon_reload: true
+  listen: Reload systemd daemon
+
+- name: Restart llama_exporter
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ llama_server_exporter_service_name }}"
+    state: restarted
+  listen: Restart llama_exporter

ansible/roles/llama_server/meta/main.yml

@@ -0,0 +1,13 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Sync llama models from Hugging Face
+  company: A13Labs
+  role_name: llama_models_sync
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: EL
+      versions:
+        - "9"

ansible/roles/llama_server/tasks/llama_exporter.yml

@@ -2,16 +2,16 @@
 - name: Create llama_exporter group
   become: true
   ansible.builtin.group:
-    name: "{{ llama_exporter_user }}"
+    name: "{{ llama_server_exporter_user }}"
     state: present
     system: true
 
 - name: Create llama_exporter user
   become: true
   ansible.builtin.user:
-    name: "{{ llama_exporter_user }}"
-    group: "{{ llama_exporter_user }}"
-    home: "{{ llama_exporter_user_home }}"
+    name: "{{ llama_server_exporter_user }}"
+    group: "{{ llama_server_exporter_user }}"
+    home: "{{ llama_server_exporter_user_home }}"
     create_home: true
     shell: /usr/sbin/nologin
     system: true
@@ -21,20 +21,20 @@
   ansible.builtin.file:
     path: "{{ item }}"
     state: directory
-    owner: "{{ llama_exporter_user }}"
-    group: "{{ llama_exporter_user }}"
+    owner: "{{ llama_server_exporter_user }}"
+    group: "{{ llama_server_exporter_user }}"
     mode: "0755"
   loop:
-    - "{{ llama_exporter_install_dir }}"
-    - "{{ llama_exporter_install_dir }}/logs"
+    - "{{ llama_server_exporter_install_dir }}"
+    - "{{ llama_server_exporter_install_dir }}/logs"
 
 - name: Copy llama_exporter script
   become: true
   ansible.builtin.copy:
-    src: llama_exporter.py
-    dest: "{{ llama_exporter_script_path }}"
-    owner: "{{ llama_exporter_user }}"
-    group: "{{ llama_exporter_user }}"
+    src: scripts/llama_exporter.py
+    dest: "{{ llama_server_exporter_script_path }}"
+    owner: "{{ llama_server_exporter_user }}"
+    group: "{{ llama_server_exporter_user }}"
     mode: "0755"
   notify:
     - Restart llama_exporter
@@ -42,31 +42,22 @@
 - name: Create Python virtual environment
   become: true
   ansible.builtin.command:
-    cmd: "{{ ansible_facts['python']['executable'] }} -m venv {{ llama_exporter_venv_path }}"
-    creates: "{{ llama_exporter_venv_path }}/bin/activate"
-
-- name: Install Python dependencies in llama_exporter venv
-  become: true
-  ansible.builtin.pip:
-    name:
-      - prometheus-client
-      - requests
-    virtualenv: "{{ llama_exporter_venv_path }}"
-    state: present
+    cmd: "{{ ansible_facts['python']['executable'] }} -m venv {{ llama_server_exporter_venv_path }}"
+    creates: "{{ llama_server_exporter_venv_path }}/bin/activate"
 
 - name: Fix ownership of llama_exporter directory
   become: true
   ansible.builtin.file:
-    path: "{{ llama_exporter_install_dir }}"
-    owner: "{{ llama_exporter_user }}"
-    group: "{{ llama_exporter_user }}"
+    path: "{{ llama_server_exporter_install_dir }}"
+    owner: "{{ llama_server_exporter_user }}"
+    group: "{{ llama_server_exporter_user }}"
     recurse: true
 
 - name: Generate llama_exporter systemd service
   become: true
   ansible.builtin.template:
     src: llama_exporter.service.j2
-    dest: "/etc/systemd/system/{{ llama_exporter_service_name }}.service"
+    dest: "/etc/systemd/system/{{ llama_server_exporter_service_name }}.service"
     owner: root
     group: root
     mode: "0644"
@@ -77,6 +68,6 @@
 - name: Enable llama_exporter service
   become: true
   ansible.builtin.systemd:
-    name: "{{ llama_exporter_service_name }}"
+    name: "{{ llama_server_exporter_service_name }}"
     enabled: true
     state: started

ansible/roles/llama_server/tasks/main.yml

@@ -0,0 +1,358 @@
+---
+- name: Validate llama_server_models schema
+  ansible.builtin.assert:
+    that:
+      - llama_server_models is iterable
+    fail_msg: "llama_server_models must be a list"
+
+- name: Create group
+  become: true
+  ansible.builtin.group:
+    name: "{{ llama_server_group }}"
+    system: true
+
+- name: Create user
+  become: true
+  ansible.builtin.user:
+    name: "{{ llama_server_user }}"
+    groups: "{{ podman_extra_groups | default([]) }}"
+    shell: /bin/bash
+    home: "{{ llama_server_home }}"
+    group: "{{ llama_server_group }}"
+
+- name: Disable password login for user
+  ansible.builtin.command: passwd -d "{{ llama_server_user }}"
+  become: true
+  changed_when: false
+
+- name: Create user home
+  become: true
+  ansible.builtin.file:
+    path: "{{ llama_server_home }}"
+    state: directory
+    owner: "{{ llama_server_user }}"
+    group: "{{ llama_server_group }}"
+    mode: "0750"
+
+- name: Set fact podman_binary (Debian family)
+  ansible.builtin.set_fact:
+    podman_binary: /usr/bin/podman
+  when: ansible_os_family == 'Debian'
+
+- name: Set fact podman_binary (Red Hat family)
+  ansible.builtin.set_fact:
+    podman_binary: /usr/sbin/podman
+  when: ansible_os_family == 'RedHat'
+
+- name: Create required folders
+  become: true
+  ansible.builtin.file:
+    path: "{{ llama_server_home }}/{{ item }}"
+    state: directory
+    owner: "{{ llama_server_user }}"
+    group: "{{ llama_server_group }}"
+    mode: "0750"
+  loop:
+    - models
+    - .cache
+    - .cache/llama.cpp
+
+- name: "Add SSH Authorized key"
+  become: true
+  ansible.posix.authorized_key:
+    user: "{{ llama_server_user }}"
+    key: "{{ llama_server_pubkey }}"
+    state: "{{ 'present' if llama_server_pubkey is defined and llama_server_pubkey != '' else 'absent' }}"
+  when: llama_server_pubkey is defined and llama_server_pubkey != ""
+
+- name: SELinux tasks (RedHat only)
+  when: ansible_os_family == 'RedHat'
+  become: true
+  block:
+    - name: Persistently set SELinux context
+      community.general.sefcontext:
+        target: "{{ llama_server_home }}(/.*)?"
+        setype: user_home_dir_t
+        state: present
+
+    - name: Persistently set SELinux context (ssh authorized keys)
+      community.general.sefcontext:
+        target: "{{ llama_server_home }}/.ssh/authorized_keys"
+        setype: ssh_home_t
+        state: present
+      when: llama_server_pubkey is defined and llama_server_pubkey != ""
+
+    - name: Apply SELinux context
+      ansible.builtin.command: restorecon -Rv {{ llama_server_home }}
+      changed_when: false
+
+- name: Enable sudo access to llama user (temporary)
+  become: true
+  ansible.builtin.lineinfile:
+    path: "/etc/sudoers.d/ansible_{{ llama_server_user }}_tmp"
+    create: true
+    mode: "0440"
+    line: "{{ ansible_facts['user_id'] }} ALL=({{ llama_server_user }}) NOPASSWD: ALL"
+    validate: "visudo -cf %s"
+
+- name: Enable lingering for llama user
+  become: true
+  ansible.builtin.command: loginctl enable-linger {{ llama_server_user }}
+  register: linger_status
+  changed_when: >-
+    'created' in linger_status.stdout or
+    (linger_status.rc == 0 and not
+      ('linger file already exists' in linger_status.stderr or
+      'linger file does not exist' in linger_status.stderr))
+  failed_when:
+    - linger_status.rc != 0 and not
+      ('linger file already exists' in linger_status.stderr)
+
+- name: Get current user's UID
+  ansible.builtin.command: id -u {{ llama_server_user }}
+  changed_when: false
+  register: uid
+
+- name: Get current user's GID
+  ansible.builtin.command: id -g {{ llama_server_user }}
+  changed_when: false
+  register: gid
+
+- name: Run tasks as podman user
+  become: true
+  become_user: "{{ llama_server_user }}"
+  block:
+    - name: Pull the requested images
+      containers.podman.podman_image:
+        name: "{{ llama_server_image }}"
+        tag: "{{ llama_server_tag }}"
+        state: present
+
+    - name: Create pods
+      containers.podman.podman_container:
+        name: "llama.cpp"
+        image: "{{ llama_server_image }}:{{ llama_server_tag }}"
+        state: started
+        device: "{{ llama_server_devices | default(omit) }}"
+        env: "{{ llama_server_env | default(omit) }}"
+        ports: "0.0.0.0:{{ llama_server_port }}:8080/tcp"
+        volumes:
+          - "{{ llama_server_home }}/models:/models:Z"
+          - "{{ llama_server_home }}/.cache:/app/.cache:Z"
+        command:
+          - "-t"
+          - "{{ llama_server_argv_threads | default(10) }}"
+          - "-np"
+          - "{{ llama_server_argv_parallel | default(1) }}"
+          - "-b"
+          - "{{ llama_server_argv_batch_size | default(1024) }}"
+          - "-ub"
+          - "{{ llama_server_argv_ubatch_size | default(512) }}"
+          - "-fa"
+          - "{{ llama_server_argv_flash_attn | default('on') }}"
+          - "-ctk"
+          - "{{ llama_server_argv_cache_type_k | default('q4_0') }}"
+          - "-ctv"
+          - "{{ llama_server_argv_cache_type_v | default('q8_0') }}"
+          - "-cram"
+          - "{{ llama_server_argv_cache_reuse | default(-1) }}"
+          - "-sm"
+          - "{{ llama_server_argv_split_mode | default('layer') }}"
+          - "-dev"
+          - "{{ llama_server_argv_devices | default('Vulkan1,Vulkan2') }}"
+          - "-ts"
+          - "{{ llama_server_argv_tensor_split | default('8,12') }}"
+          - "-fit"
+          - "{{ llama_server_argv_fit | default('off') }}"
+          - "-mg"
+          - "{{ llama_server_argv_main_gpu | default(1) }}"
+          - "--mmap"
+          - "--metrics"
+          - "--models-preset"
+          - "/models/.router/models.ini"
+          - "--models-max"
+          - "{{ llama_server_models_max }}"
+
+        cmd_args:
+          - "--userns=keep-id"
+          - "--security-opt=label=disable"
+
+- name: Create systemd service file for pods
+  become: true
+  ansible.builtin.template:
+    src: podman.service.j2
+    dest: "/etc/systemd/system/podman-llama.cpp.service"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Reload systemd daemon
+  become: true
+  ansible.builtin.systemd:
+    daemon_reload: true
+
+- name: Enable containers at boot
+  become: true
+  ansible.builtin.systemd:
+    name: "podman-llama.cpp"
+    enabled: true
+    state: "started"
+
+- name: Install Python packages required for llama server model sync
+  become: true
+  ansible.builtin.package:
+    name: "{{ __llama_sync_system_packages__ }}"
+    state: present
+  vars:
+    __llama_sync_system_packages__: >-
+      {{
+        {
+          'Debian': ['python3-pip', 'python3-packaging', 'python3-venv'],
+          'RedHat': ['python3-pip', 'python3-packaging'],
+          'Archlinux': ['python-pip', 'python-packaging']
+        }[ansible_os_family]
+      }}
+
+- name: Install Hugging Face hub client in llama server sync virtualenv
+  become: true
+  become_user: "{{ llama_server_user }}"
+  ansible.builtin.pip:
+    name:
+      - pip
+      - setuptools
+      - packaging
+      - huggingface_hub>=0.32.0
+    state: present
+    virtualenv: "{{ llama_server_home }}/.router/.venv"
+    virtualenv_command: "python3 -m venv"
+
+- name: Create llama server router model directories
+  become: true
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: directory
+    owner: "{{ llama_server_user }}"
+    group: "{{ llama_server_group }}"
+    mode: "{{ item.mode }}"
+  loop:
+    - { path: "{{ llama_server_home }}/models", mode: "0750" }
+    - { path: "{{ llama_server_home }}/models/managed", mode: "0750" }
+    - { path: "{{ llama_server_home }}/models/managed-links", mode: "0750" }
+    - { path: "{{ llama_server_home }}/models/.router", mode: "0750" }
+
+- name: Copy llama server Hugging Face sync helper
+  become: true
+  ansible.builtin.copy:
+    src: scripts/llama_hf_sync.py
+    dest: "{{ llama_server_home }}/models/.router/llama_hf_sync.py"
+    owner: "{{ llama_server_user }}"
+    group: "{{ llama_server_group }}"
+    mode: "0750"
+
+- name: Write desired llama server model manifest input
+  become: true
+  ansible.builtin.copy:
+    content: "{{ llama_server_models | to_nice_json }}"
+    dest: "{{ llama_server_home }}/models/.router/desired-models.json"
+    owner: "{{ llama_server_user }}"
+    group: "{{ llama_server_group }}"
+    mode: "0640"
+
+- name: Write global llama server preset options
+  become: true
+  ansible.builtin.copy:
+    content: "{{ llama_server_preset_global | to_nice_json }}"
+    dest: "{{ llama_server_home }}/models/.router/preset-global.json"
+    owner: "{{ llama_server_user }}"
+    group: "{{ llama_server_group }}"
+    mode: "0640"
+
+- name: Set arguments for llama server model sync
+  ansible.builtin.set_fact:
+    __llama_preset_file__:
+      - "{{ llama_server_home }}/.router/.venv/bin/python"
+      - "{{ llama_server_home }}/models/.router/llama_hf_sync.py"
+      - "--models-file"
+      - "{{ llama_server_home }}/models/.router/desired-models.json"
+      - "--managed-dir"
+      - "{{ llama_server_home }}/models/managed"
+      - "--links-dir"
+      - "{{ llama_server_home }}/models/managed-links"
+      - "--manifest-file"
+      - "{{ llama_server_home }}/models/.router/manifest.json"
+      - "--preset-file"
+      - "{{ llama_server_home }}/models/.router/models.ini"
+      - "--preset-global-file"
+      - "{{ llama_server_home }}/models/.router/preset-global.json"
+      - "--container-links-dir"
+      - "/models/managed-links"
+
+- name: Run llama server model sync from Hugging Face
+  vars:
+    __llama_sync_argv__: >-
+      {{ __llama_preset_file__
+        + (['--prune'] if llama_server_models_prune_enabled | default(true) else [])
+        + (['--dry-run'] if llama_server_models_dry_run | default(false) else [])
+      }}
+  become: true
+  become_user: "{{ llama_server_user }}"
+  ansible.builtin.command:
+    argv: "{{ __llama_sync_argv__ }}"
+  environment:
+    HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') | default('', true) }}"
+  register: __llama_sync_output__
+  changed_when: false
+
+- name: Parse llama server sync output
+  ansible.builtin.set_fact:
+    __llama_sync_result__: "{{ __llama_sync_output__.stdout | from_json }}"
+
+- name: Stat llama router preset file (when not in dry-run mode)
+  become: true
+  become_user: "{{ llama_server_user }}"
+  ansible.builtin.stat:
+    path: "{{ llama_server_home }}/models/.router/models.ini"
+  register: __llama_preset_stat__
+  when: not (llama_server_models_dry_run | default(false))
+
+- name: Ensure router preset file exists when not in dry-run mode
+  ansible.builtin.assert:
+    that:
+      - __llama_preset_stat__.stat.exists
+    fail_msg: "llama router preset file was not created"
+  when: not (llama_server_models_dry_run | default(false))
+
+- name: Query llama router model list
+  ansible.builtin.uri:
+    url: "http://127.0.0.1:{{ llama_server_port | default(8080) }}/models?reload=1"
+    method: GET
+    status_code: 200
+  register: __llama_router_models__
+  changed_when: false
+  when:
+    - llama_server_models_sync_enabled | default(true)
+    - not (llama_server_models_dry_run | default(false))
+
+- name: Validate expected model aliases are available in router
+  ansible.builtin.assert:
+    that:
+      - item.name in (__llama_router_models__.json.data | map(attribute='id') | list)
+    fail_msg: "Model name '{{ item.name }}' not present in llama router /models output"
+  loop: "{{ __llama_sync_result__.models | default([]) }}"
+  loop_control:
+    label: "{{ item.name }}"
+  when:
+    - llama_server_models_sync_enabled | default(true)
+    - not (llama_server_models_dry_run | default(false))
+
+- name: Include Llama Exporter setup
+  when: llama_server_exporter_enabled | default(true) | bool
+  ansible.builtin.include_tasks: llama_exporter.yml
+  tags:
+    - roles::llama_server::llama_exporter
+
+- name: Remove temporary sudo access
+  become: true
+  ansible.builtin.file:
+    path: "/etc/sudoers.d/ansible_{{ llama_server_user }}_tmp"
+    state: absent

ansible/roles/llama_server/templates/llama_exporter.service.j2

@@ -0,0 +1,35 @@
+[Unit]
+Description=LLM Inference Metrics Exporter (llama.cpp + Ollama)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User={{ llama_server_exporter_user }}
+Group={{ llama_server_exporter_user }}
+WorkingDirectory={{ llama_server_exporter_install_dir }}
+Environment="LLAMA_EXPORTER_PORT={{ llama_server_exporter_port }}"
+Environment="LLAMA_EXPORTER_BIND=0.0.0.0"
+Environment="LLAMA_EXPORTER_INTERVAL={{ llama_server_exporter_scrape_interval }}"
+Environment="LLAMA_EXPORTER_TIMEOUT={{ llama_server_exporter_scrape_timeout }}"
+{% if llama_server_exporter_targets and llama_server_exporter_targets != "" %}
+Environment="LLAMA_TARGETS={{ llama_server_exporter_targets | to_json }}"
+{% endif %}
+Environment="PATH={{ llama_server_exporter_venv_bin }}:/usr/bin:/bin"
+ExecStart={{ llama_server_exporter_venv_bin }}/python3 {{ llama_server_exporter_script_path }}
+Restart=always
+RestartSec=10
+TimeoutStopSec=10
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=llama_exporter
+
+# Security hardening
+ProtectSystem=strict
+ProtectHome=true
+ReadWritePaths={{ llama_server_exporter_install_dir }} /var/log
+NoNewPrivileges=true
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/llama_server/templates/podman.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=Podman file for llama.cpp server
+After=local-fs.target network-online.target nvidia-cdi-generator.service
+
+[Service]
+Type=oneshot
+User={{ llama_server_user }}
+Group={{ llama_server_group }}
+RemainAfterExit=true
+ExecStart={{ podman_binary }} start podman-llama.cpp
+ExecStop={{ podman_binary }} stop podman-llama.cpp
+
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/llama_server/vars/main.yml

@@ -0,0 +1,11 @@
+---
+# Internal role variables (non-overridable)
+
+llama_server_exporter_script_path: "{{ llama_server_exporter_install_dir }}/llama_exporter.py"
+llama_server_exporter_venv_bin: "{{ llama_server_exporter_venv_path }}/bin"
+llama_server_exporter_service_name: "llama_exporter"
+
+# Default llama.cpp targets if none provided
+llama_server_exporter_targets_default:
+  - name: "llama-cpp-11434"
+    url: "http://localhost:11434"

ansible/roles/nvidia_cdi_generator/meta/main.yml

@@ -0,0 +1,13 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Install and configure NVIDIA CDI generator service
+  company: A13Labs
+  role_name: nvidia_cdi_generator
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: EL
+      versions:
+        - "9"

ansible/roles/nvidia_cdi_generator/tasks/main.yml

@@ -0,0 +1,29 @@
+---
+- name: Install nvidia-cdi-generator
+  become: true
+  ansible.builtin.package:
+    name:
+      - nvidia-container-toolkit
+    state: present
+
+- name: Create systemd service file for nvidia-cdi-generator
+  become: true
+  ansible.builtin.copy:
+    src: services/nvidia-cdi-generator.service
+    dest: /etc/systemd/system/nvidia-cdi-generator.service
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Reload systemd daemon
+  become: true
+  ansible.builtin.systemd:
+    daemon_reload: true
+    name: nvidia-cdi-generator
+
+- name: Enable nvidia-cdi-generator service
+  become: true
+  ansible.builtin.systemd:
+    name: nvidia-cdi-generator
+    enabled: true
+    state: started

ansible/roles/podman/defaults/main.yml

@@ -0,0 +1,15 @@
+---
+# podman defaults
+
+podman_user: ""
+podman_group: ""
+podman_user_home: ""
+podman_extra_groups: []
+podman_user_folders: []
+podman_user_pubkey: ""
+podman_network_name: "none"
+podman_build_files: []
+podman_pods: []
+
+# Binary path - will be set by OS family in tasks
+# podman_binary is set as a fact in tasks/main.yml

ansible/roles/podman/meta/main.yml

@@ -0,0 +1,18 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Deploy and manage Podman users, pods, and containers
+  company: A13Labs
+  role_name: podman
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: Fedora
+      versions:
+        - "39"
+        - "40"
+        - "41"
+    - name: EL
+      versions:
+        - "9"

ansible/roles/podman/tasks/main.yml

@@ -0,0 +1,206 @@
+---
+- name: Set fact podman_binary (Debian family)
+  ansible.builtin.set_fact:
+    podman_binary: /usr/bin/podman
+  when: ansible_os_family == 'Debian'
+
+- name: Set fact podman_binary (Red Hat family)
+  ansible.builtin.set_fact:
+    podman_binary: /usr/sbin/podman
+  when: ansible_os_family == 'RedHat'
+
+- name: Create group
+  become: true
+  ansible.builtin.group:
+    name: "{{ podman_group }}"
+    system: true
+
+- name: Create user
+  become: true
+  ansible.builtin.user:
+    name: "{{ podman_user }}"
+    groups: "{{ podman_extra_groups | default([]) }}"
+    shell: /bin/bash
+    home: "{{ podman_user_home }}"
+    group: "{{ podman_group }}"
+
+- name: Disable password login for user
+  ansible.builtin.command: passwd -d "{{ podman_user }}"
+  become: true
+  changed_when: false
+
+- name: Create user home
+  become: true
+  ansible.builtin.file:
+    path: "{{ podman_user_home }}"
+    state: directory
+    owner: "{{ podman_user }}"
+    group: "{{ podman_group }}"
+    mode: "0750"
+
+- name: Create build folders
+  become: true
+  ansible.builtin.file:
+    path: "{{ podman_user_home }}/build"
+    state: directory
+    owner: "{{ podman_user }}"
+    group: "{{ podman_group }}"
+    mode: "0750"
+
+- name: Create folders
+  become: true
+  ansible.builtin.file:
+    path: "{{ podman_user_home }}/{{ item }}"
+    state: directory
+    owner: "{{ podman_user }}"
+    group: "{{ podman_group }}"
+    mode: "0750"
+  loop: "{{ podman_user_folders }}"
+
+- name: "Add SSH Authorized key"
+  become: true
+  ansible.posix.authorized_key:
+    user: "{{ podman_user }}"
+    key: "{{ podman_user_pubkey }}"
+    state: "{{ 'present' if podman_user_pubkey is defined and podman_user_pubkey != '' else 'absent' }}"
+  when: podman_user_pubkey is defined and podman_user_pubkey != ""
+
+- name: SELinux tasks (RedHat only)
+  when: ansible_os_family == 'RedHat'
+  become: true
+  block:
+    - name: Persistently set SELinux context
+      community.general.sefcontext:
+        target: "{{ podman_user_home }}(/.*)?"
+        setype: user_home_dir_t
+        state: present
+
+    - name: Persistently set SELinux context (ssh authorized keys)
+      community.general.sefcontext:
+        target: "{{ podman_user_home }}/.ssh/authorized_keys"
+        setype: ssh_home_t
+        state: present
+      when: podman_user_pubkey is defined and podman_user_pubkey != ""
+
+    - name: Apply SELinux context
+      ansible.builtin.command: restorecon -Rv {{ podman_user_home }}
+      changed_when: false
+
+- name: Enable sudo access to podman user (temporary)
+  become: true
+  ansible.builtin.lineinfile:
+    path: "/etc/sudoers.d/ansible_{{ podman_user }}_tmp"
+    create: true
+    mode: "0440"
+    line: "{{ ansible_facts['user_id'] }} ALL=({{ podman_user }}) NOPASSWD: ALL"
+    validate: "visudo -cf %s"
+
+- name: Enable lingering for podman user
+  become: true
+  ansible.builtin.command: loginctl enable-linger {{ podman_user }}
+  register: linger_status
+  changed_when: >-
+    'created' in linger_status.stdout or
+    (linger_status.rc == 0 and not
+      ('linger file already exists' in linger_status.stderr or
+      'linger file does not exist' in linger_status.stderr))
+  failed_when:
+    - linger_status.rc != 0 and not
+      ('linger file already exists' in linger_status.stderr)
+
+- name: Get current user's UID
+  ansible.builtin.command: id -u {{ podman_user }}
+  changed_when: false
+  register: uid
+
+- name: Get current user's GID
+  ansible.builtin.command: id -g {{ podman_user }}
+  changed_when: false
+  register: gid
+
+- name: Run tasks as podman user
+  become: true
+  become_user: "{{ podman_user }}"
+  block:
+    - name: Create a custom Podman network
+      when: podman_network_name is defined and podman_network_name != 'none'
+      containers.podman.podman_network:
+        name: "{{ podman_network_name }}"
+        state: present
+
+    - name: Copy build files to user home
+      ansible.builtin.copy:
+        src: "{{ item }}"
+        dest: "{{ podman_user_home }}/build/"
+        owner: "{{ podman_user }}"
+        group: "{{ podman_group }}"
+        mode: "0644"
+      loop: "{{ podman_build_files | default([]) }}"
+      when: podman_build_files is defined and podman_build_files | length > 0
+
+    - name: Build containers images
+      containers.podman.podman_image:
+        name: "{{ podman_user }}/{{ item.name }}"
+        path: "{{ podman_user_home }}/build"
+        build:
+          cache: false
+          force_rm: true
+          container_file: "{{ item.build.dockerfile }}"
+          extra_args: "--build-arg UID={{ uid.stdout }} --build-arg GID={{ gid.stdout }} --security-opt=label=disable {{ item.build.args | default('') }}"
+        tag: "latest"
+        state: present
+      loop: "{{ podman_pods }}"
+      when: item.build is defined and item.build.dockerfile is defined
+
+    - name: Pull the requested images
+      containers.podman.podman_image:
+        name: "{{ item.repo.image }}"
+        tag: "{{ item.repo.tag | default('latest') }}"
+        state: present
+      loop: "{{ podman_pods }}"
+      when: item.repo is defined and item.repo.image is defined
+
+    - name: Create pods
+      containers.podman.podman_container:
+        name: "{{ item.name }}"
+        image: "{{ item.repo.image | default(podman_user + '/' + item.name) }}"
+        state: "{{ item.state | default('started') }}"
+        device: "{{ item.device | default(omit) }}"
+        env: "{{ item.env | default(omit) }}"
+        network: "{{ podman_network_name | default(omit) }}"
+        ports: "{{ item.ports | default(omit) }}"
+        volumes: "{{ item.volumes | default(omit) }}"
+        command: "{{ item.command | default(omit) }}"
+        cmd_args:
+          - "--userns=keep-id"
+          - "--security-opt=label=disable"
+      loop: "{{ podman_pods }}"
+
+- name: Create systemd service file for pods
+  become: true
+  ansible.builtin.template:
+    src: podman.service.j2
+    dest: "/etc/systemd/system/podman-{{ item.name }}.service"
+    owner: root
+    group: root
+    mode: "0644"
+  loop: "{{ podman_pods }}"
+
+- name: Reload systemd daemon
+  become: true
+  ansible.builtin.systemd:
+    daemon_reload: true
+
+- name: Enable containers at boot
+  become: true
+  ansible.builtin.systemd:
+    name: "podman-{{ item.name }}"
+    enabled: "{{ item.systemd.enabled | default(true) }}"
+    state: "{{ item.systemd.state | default('started') }}"
+  loop: "{{ podman_pods }}"
+
+- name: Remove temporary sudo access
+  become: true
+  ansible.builtin.file:
+    path: "/etc/sudoers.d/ansible_{{ podman_user }}_tmp"
+    state: absent

ansible/roles/podman/templates/podman.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=Podman service for {{ item.name }}
+After=local-fs.target network-online.target
+
+[Service]
+Type=oneshot
+User={{ podman_user }}
+Group={{ podman_group }}
+RemainAfterExit=true
+ExecStart={{ podman_binary }} start {{ item.name }}
+ExecStop={{ podman_binary }} stop {{ item.name }}
+
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/windows_game/meta/main.yml

@@ -0,0 +1,11 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Windows game optimization (user creation, auto-login, power plan, service disable)
+  company: A13Labs
+  role_name: windows_game
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: windows

ansible/roles/windows_game/tasks/main.yml

@@ -0,0 +1,180 @@
+---
+- name: Create limited GameUser
+  ansible.windows.win_user:
+    name: "{{ game_user }}"
+    password: "{{ game_password }}"
+    groups: "{{ game_default_groups }}"
+    account_disabled: false
+    password_never_expires: true
+    user_cannot_change_password: true
+    state: present
+
+- name: Configure auto-login for GameUser
+  ansible.windows.win_regedit:
+    path: HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon
+    name: "{{ item.name }}"
+    data: "{{ item.data }}"
+    type: string
+  loop:
+    - { name: "AutoAdminLogon", data: "1" }
+    - { name: "DefaultUserName", data: "{{ game_user }}" }
+    - { name: "DefaultPassword", data: "{{ game_password }}" }
+    - { name: "DefaultDomainName", data: "{{ ansible_hostname }}" }
+
+- name: Create launcher script directory
+  ansible.windows.win_file:
+    path: C:\GameShell
+    state: directory
+
+- name: Deploy launcher.bat
+  ansible.windows.win_copy:
+    dest: C:\GameShell\launcher.bat
+    content: "{{ lookup('file', 'scripts/launcher.bat') }}"
+
+- name: Set High Performance Power Plan
+  ansible.windows.win_command: powercfg -setactive SCHEME_MIN
+
+- name: Disable auto suspend (sleep) for plugged in
+  ansible.windows.win_command: powercfg -change -standby-timeout-ac 0
+
+- name: Disable auto suspend (sleep) for battery
+  ansible.windows.win_command: powercfg -change -standby-timeout-dc 0
+
+- name: Disable Windows Defender Real-Time Protection
+  ansible.windows.win_shell: Set-MpPreference -DisableRealtimeMonitoring $true
+
+- name: Remove unnecessary startup items (HKLM)
+  ansible.windows.win_regedit:
+    path: HKLM:\Software\Microsoft\Windows\CurrentVersion\Run
+    name: SecurityHealth
+    state: absent
+
+- name: Remove startup folder shortcuts (user)
+  ansible.windows.win_file:
+    path: "{{ item }}"
+    state: absent
+  with_fileglob:
+    - "{{ ansible_env.APPDATA }}\\Microsoft\\Windows\\Start Menu\\Programs\\Startup\\*.lnk"
+
+- name: Remove startup folder shortcuts (all users)
+  ansible.windows.win_file:
+    path: "{{ item }}"
+    state: absent
+  with_fileglob:
+    - "C:\\ProgramData\\Microsoft\\Windows\\Start Menu\\Programs\\Startup\\*.lnk"
+
+- name: Disable Xbox Game Bar and DVR
+  ansible.windows.win_regedit:
+    path: HKCU:\Software\Microsoft\Windows\CurrentVersion\GameDVR
+    name: AppCaptureEnabled
+    data: 0
+    type: dword
+    state: present
+
+- name: Disable GameDVR_Enabled
+  ansible.windows.win_regedit:
+    path: HKCU:\System\GameConfigStore
+    name: GameDVR_Enabled
+    data: 0
+    type: dword
+    state: present
+
+- name: Disable GameDVR_FSEBehaviorMode
+  ansible.windows.win_regedit:
+    path: HKCU:\System\GameConfigStore
+    name: GameDVR_FSEBehaviorMode
+    data: 2
+    type: dword
+    state: present
+
+- name: Disable GameDVR_HonorUserFSEBehaviorMode
+  ansible.windows.win_regedit:
+    path: HKCU:\System\GameConfigStore
+    name: GameDVR_HonorUserFSEBehaviorMode
+    data: 1
+    type: dword
+    state: present
+
+- name: Disable Windows notifications
+  ansible.windows.win_regedit:
+    path: HKCU:\Software\Microsoft\Windows\CurrentVersion\PushNotifications
+    name: ToastEnabled
+    data: 0
+    type: dword
+    state: present
+
+- name: Disable background services
+  vars:
+    services_to_disable:
+      - WSearch
+      - SysMain
+      - DiagTrack
+      - XblAuthManager
+      - XblGameSave
+      - XboxGipSvc
+      - Fax
+      - MapsBroker
+      - RetailDemo
+      - RemoteRegistry
+      - WMPNetworkSvc
+      - edgeupdate
+      - WpnService
+      - BITS
+      - WerSvc
+      - Spooler
+      - SCardSvr
+  ansible.windows.win_service:
+    name: "{{ item }}"
+    start_mode: disabled
+    state: stopped
+  loop: "{{ services_to_disable }}"
+
+- name: Get all services
+  ansible.windows.win_service_info:
+  register: win_services
+
+- name: Disable scheduled telemetry tasks
+  vars:
+    scheduled_tasks_to_disable:
+      - "\\Microsoft\\Windows\\Application Experience\\ProgramDataUpdater"
+      - "\\Microsoft\\Windows\\Customer Experience Improvement Program\\Consolidator"
+      - "\\Microsoft\\Windows\\Customer Experience Improvement Program\\KernelCeipTask"
+      - "\\Microsoft\\Windows\\Feedback\\Siuf\\DmClient"
+      - "\\Microsoft\\Windows\\Windows Error Reporting\\QueueReporting"
+  ansible.windows.win_shell: |
+    schtasks /Change /TN "{{ item }}" /Disable
+  loop: "{{ scheduled_tasks_to_disable }}"
+  register: telemetry_task_result
+  failed_when: "{{ telemetry_task_result.rc != 0 and ('ERROR: The system cannot find the file specified.' not in telemetry_task_result.stderr) }}"
+
+- name: Disable Windows Search feature
+  ansible.windows.win_shell: DISM /Online /Disable-Feature /FeatureName:SearchEngine-Client-Package /NoRestart
+  register: disable_search_result
+  failed_when: >
+    disable_search_result.rc != 0 and
+    ('The specified package is not valid Windows package' not in disable_search_result.stderr) and
+    ('is already disabled' not in disable_search_result.stdout)
+
+- name: Disable Windows Media Player feature
+  ansible.windows.win_shell: DISM /Online /Disable-Feature /FeatureName:WindowsMediaPlayer /NoRestart
+  register: disable_wmp_result
+  failed_when: >
+    disable_wmp_result.rc != 0 and
+    ('The specified package is not valid Windows package' not in disable_wmp_result.stderr) and
+    ('is already disabled' not in disable_wmp_result.stdout)
+
+- name: Disable Print to PDF
+  ansible.windows.win_shell: DISM /Online /Disable-Feature /FeatureName:Printing-PrintToPDFServices-Features /NoRestart
+  register: disable_print_pdf_result
+  failed_when: >
+    disable_print_pdf_result.rc != 0 and
+    ('The specified package is not valid Windows package' not in disable_print_pdf_result.stderr) and
+    ('is already disabled' not in disable_print_pdf_result.stdout)
+
+- name: Reminder - NVIDIA Control Panel Tweaks (manual)
+  ansible.builtin.debug:
+    msg: |
+      Please manually set the following in NVIDIA Control Panel:
+      - Power Management: Prefer Maximum Performance
+      - Low Latency Mode: Ultra
+      - Vertical Sync: Off or Fast Sync (depending on monitor)

ansible/roles/windows_prometheus/meta/main.yml

@@ -0,0 +1,11 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Install Prometheus and Windows Exporter on Windows
+  company: A13Labs
+  role_name: windows_prometheus
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: windows

ansible/roles/windows_prometheus/tasks/main.yml

@@ -0,0 +1,199 @@
+---
+# Install Prometheus + Windows Exporter on Windows and open firewall for Grafana
+- name: Define Prometheus Versions and install dir
+  ansible.builtin.set_fact:
+    prometheus_version: "{{ prometheus_version | default('3.6.0') }}"
+    prometheus_install_dir: "{{ prometheus_install_dir | default('C:\\Prometheus') }}"
+    prometheus_listen_address: "{{ prometheus_listen_address | default('0.0.0.0:9090') }}"
+    nssm_dir: "{{ nssm_dir | default('C:\\nssm') }}"
+    nssm_version: "{{ nssm_version | default('2.24') }}"
+    windows_exporter_version: "{{ windows_exporter_version | default('0.31.3') }}"
+    windows_exporter_collectors: "{{ windows_exporter_collectors | default(['cache', 'cpu', 'cpu_info', 'diskdrive', 'memory', 'net', 'os', 'process', 'service', 'tcp', 'time', 'update']) }}"
+
+- name: Define Windows Exporter defaults
+  ansible.builtin.set_fact:
+    __nssm_url: "https://nssm.cc/release/nssm-{{ nssm_version }}.zip"
+    __prometheus_zip_url: "https://github.com/prometheus/prometheus/releases/download/v{{ prometheus_version }}/prometheus-{{ prometheus_version }}.windows-amd64.zip"
+    __prometheus_zip_path: "{{ (ansible_env.TEMP | default('C:\\Windows\\Temp')) }}\\prometheus-{{ prometheus_version }}.zip"
+    __prometheus_home: "{{ prometheus_install_dir }}\\prometheus-{{ prometheus_version }}.windows-amd64"
+    __windows_exporter_msi_url: "https://github.com/prometheus-community/windows_exporter/releases/download/v{{ windows_exporter_version }}/windows_exporter-{{ windows_exporter_version }}-amd64.msi"
+    __windows_exporter_msi_path: "{{ (ansible_env.TEMP | default('C:\\Windows\\Temp')) }}\\windows_exporter-{{ windows_exporter_version }}-amd64.msi"
+    __nssm_zip_path: "{{ (ansible_env.TEMP | default('C:\\Windows\\Temp')) }}\\nssm.zip"
+    __nssm_exe: "{{ nssm_dir }}\\nssm.exe"
+
+- name: Create Prometheus installation directories
+  ansible.windows.win_file:
+    path: "{{ item }}"
+    state: directory
+  loop:
+    - "{{ prometheus_install_dir }}"
+    - "{{ prometheus_install_dir }}\\data"
+    - "{{ nssm_dir }}"
+
+- name: Download required files (Prometheus, NSSM, Windows Exporter)
+  ansible.windows.win_get_url:
+    url: "{{ item.url }}"
+    dest: "{{ item.dest }}"
+    force: false
+  loop:
+    - url: "{{ __prometheus_zip_url }}"
+      dest: "{{ __prometheus_zip_path }}"
+    - url: "{{ __nssm_url }}"
+      dest: "{{ __nssm_zip_path }}"
+    - url: "{{ __windows_exporter_msi_url }}"
+      dest: "{{ __windows_exporter_msi_path }}"
+
+- name: Extract downloaded archives
+  community.windows.win_unzip:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    delete_archive: false
+    creates: "{{ item.creates | default(omit) }}"
+  loop:
+    - src: "{{ __prometheus_zip_path }}"
+      dest: "{{ prometheus_install_dir }}"
+      creates: "{{ __prometheus_home }}\\prometheus.exe"
+    - src: "{{ __nssm_zip_path }}"
+      dest: "{{ nssm_dir }}"
+
+- name: Verify Prometheus binary is present
+  ansible.windows.win_stat:
+    path: "{{ __prometheus_home }}\\prometheus.exe"
+  register: prometheus_bin
+
+- name: Fail if Prometheus binary is missing
+  ansible.builtin.fail:
+    msg: >-
+      Prometheus binary not found at {{ __prometheus_home }}\prometheus.exe.
+      Check download/unzip and prometheus_version.
+  when: not prometheus_bin.stat.exists
+
+- name: Deploy Prometheus configuration
+  ansible.windows.win_copy:
+    dest: "{{ prometheus_install_dir }}\\prometheus.yml"
+    content: |
+      global:
+        scrape_interval: 15s
+        evaluation_interval: 15s
+
+      scrape_configs:
+        - job_name: 'prometheus'
+          static_configs:
+            - targets: ['localhost:9090']
+
+        - job_name: 'windows'
+          static_configs:
+            - targets: ['localhost:9182']
+
+- name: Check if promtool exists
+  ansible.windows.win_stat:
+    path: "{{ __prometheus_home }}\\promtool.exe"
+  register: promtool_bin
+
+- name: Validate Prometheus config with promtool
+  ansible.windows.win_command: >-
+    "{{ __prometheus_home }}\promtool.exe"
+    check config
+    "{{ prometheus_install_dir }}\prometheus.yml"
+  register: promtool_check
+  changed_when: false
+  failed_when: promtool_check.rc != 0
+  when: promtool_bin.stat.exists
+
+- name: Copy NSSM executable (win64)
+  ansible.windows.win_copy:
+    src: "{{ nssm_dir }}\\nssm-2.24\\win64\\nssm.exe"
+    dest: "{{ __nssm_exe }}"
+    remote_src: true
+
+- name: Remove existing Prometheus service if present
+  ansible.windows.win_service:
+    name: Prometheus
+    state: absent
+  ignore_errors: true
+
+- name: Install Prometheus service via NSSM
+  ansible.windows.win_command: >-
+    "{{ __nssm_exe }}"
+    install Prometheus
+    "{{ __prometheus_home }}\prometheus.exe"
+    --config.file={{ prometheus_install_dir }}\prometheus.yml
+    --storage.tsdb.path={{ prometheus_install_dir }}\data
+    --web.listen-address={{ prometheus_listen_address }}
+    --web.console.templates={{ __prometheus_home }}\consoles
+    --web.console.libraries={{ __prometheus_home }}\console_libraries
+  args:
+    chdir: "{{ __prometheus_home }}"
+  register: nssm_install
+  changed_when: nssm_install.rc == 0
+
+- name: Set Prometheus AppDirectory via NSSM
+  ansible.windows.win_command: >-
+    "{{ __nssm_exe }}" set Prometheus AppDirectory "{{ __prometheus_home }}"
+
+- name: Configure Prometheus stdout log via NSSM
+  ansible.windows.win_command: >-
+    "{{ __nssm_exe }}" set Prometheus AppStdout
+    "{{ prometheus_install_dir }}\prometheus-service.out.log"
+
+- name: Configure Prometheus stderr log via NSSM
+  ansible.windows.win_command: >-
+    "{{ __nssm_exe }}" set Prometheus AppStderr
+    "{{ prometheus_install_dir }}\prometheus-service.err.log"
+
+- name: Set Prometheus Start mode to AUTO via NSSM
+  ansible.windows.win_command: >-
+    "{{ __nssm_exe }}" set Prometheus Start SERVICE_AUTO_START
+
+- name: Ensure Prometheus service is running (skip in check mode)
+  ansible.windows.win_service:
+    name: Prometheus
+    start_mode: auto
+    state: started
+  when: not ansible_check_mode | default(false)
+
+- name: Wait for Prometheus to listen on 9090 (skip in check mode)
+  ansible.windows.win_wait_for:
+    port: 9090
+    delay: 0
+    timeout: 30
+  when: not ansible_check_mode | default(false)
+
+- name: Install Windows Exporter
+  ansible.windows.win_package:
+    path: "{{ __windows_exporter_msi_path }}"
+    arguments: 'ENABLED_COLLECTORS={{ windows_exporter_collectors | join(",") }}'
+
+    state: present
+
+- name: Ensure windows_exporter service is running
+  ansible.windows.win_service:
+    name: windows_exporter
+    start_mode: auto
+    state: started
+
+- name: Set firewall remote IPs when string provided
+  ansible.builtin.set_fact:
+    grafana_remoteip: "{{ grafana_prometheus_sources }}"
+  when:
+    - grafana_prometheus_sources is defined
+    - grafana_prometheus_sources is string
+
+- name: Set firewall remote IPs when list provided
+  ansible.builtin.set_fact:
+    grafana_remoteip: "{{ grafana_prometheus_sources | join(',') }}"
+  when:
+    - grafana_prometheus_sources is defined
+    - not (grafana_prometheus_sources is string)
+
+- name: Allow inbound Prometheus (9090) from Grafana
+  community.windows.win_firewall_rule:
+    name: Prometheus 9090 (Grafana)
+    localport: 9090
+    protocol: tcp
+    direction: in
+    action: allow
+    enabled: true
+    # Provide one or more IPs/CIDRs in grafana_prometheus_sources (e.g. ['203.0.113.10'])
+    remoteip: "{{ grafana_remoteip | default(omit) }}"
+    state: present

ansible/roles/windows_ssh/meta/main.yml

@@ -0,0 +1,11 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Install and configure OpenSSH Server on Windows
+  company: A13Labs
+  role_name: windows_ssh
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: windows

ansible/roles/windows_ssh/tasks/main.yml

@@ -0,0 +1,370 @@
+# Expects variable `ssh_users` as list of objects:
+# ssh_users:
+#   - name: alice
+#     keys:
+#       - "ssh-ed25519 AAAA..."
+#       - "ssh-rsa AAAA..."
+- name: Setting defaults for Windows OpenSSH Server installation
+  ansible.builtin.set_fact:
+    __ssh_default_ciphers__:
+      - chacha20-poly1305@openssh.com
+      - aes256-gcm@openssh.com
+      - aes256-ctr
+    __ssh_default_hostkey_algorithms__:
+      - ssh-ed25519-cert-v01@openssh.com
+      - ssh-rsa-cert-v01@openssh.com
+      - ssh-ed25519
+      - ssh-rsa
+      - ecdsa-sha2-nistp521-cert-v01@openssh.com
+      - ecdsa-sha2-nistp384-cert-v01@openssh.com
+      - ecdsa-sha2-nistp256-cert-v01@openssh.com
+      - ecdsa-sha2-nistp521
+      - ecdsa-sha2-nistp384
+      - ecdsa-sha2-nistp256
+      - sk-ecdsa-sha2-nistp256@openssh.com
+    __ssh_default_kex_algorithms__:
+      - mlkem768x25519-sha256
+      - sntrup761x25519-sha512@openssh.com
+      - curve25519-sha256
+      - curve25519-sha256@libssh.org
+      - ecdh-sha2-nistp521
+      - ecdh-sha2-nistp384
+      - ecdh-sha2-nistp256
+      - diffie-hellman-group-exchange-sha256
+    __ssh_default_macs__:
+      - hmac-sha2-512-etm@openssh.com
+      - hmac-sha2-256-etm@openssh.com
+      - hmac-sha2-512
+      - hmac-sha2-256
+    __ssh_default_auth_pubkey_accepted_algorithms__:
+      - ecdsa-sha2-nistp521
+      - ecdsa-sha2-nistp384
+      - ecdsa-sha2-nistp256
+      - ssh-ed25519
+      - rsa-sha2-512
+      - rsa-sha2-256
+
+- name: Override defaults with any provided variables
+  ansible.builtin.set_fact:
+    __ssh_users__: "{{ ssh_users | default([]) }}"
+    __ssh_ciphers__: "{{ ssh_ciphers | default(__ssh_default_ciphers__) }}"
+    __ssh_hostkey_algorithms__: "{{ ssh_hostkey_algorithms | default(__ssh_default_hostkey_algorithms__) }}"
+    __ssh_kex_algorithms__: "{{ ssh_kex_algorithms | default(__ssh_default_kex_algorithms__) }}"
+    __ssh_macs__: "{{ ssh_macs | default(__ssh_default_macs__) }}"
+    __ssh_max_auth_tries__: "{{ ssh_max_auth_tries | default(10) }}"
+    __ssh_max_sessions__: "{{ ssh_max_sessions | default(3) }}"
+    __ssh_pubkey_accepted_algorithms__: "{{ ssh_pubkey_accepted_algorithms | default(__ssh_default_auth_pubkey_accepted_algorithms__) }}"
+
+- name: Get OpenSSH Server capability state
+  ansible.windows.win_shell: |
+    $cap = Get-WindowsCapability -Online | Where-Object { $_.Name -like 'OpenSSH.Server*' } | Select-Object -First 1
+    if ($cap) {
+      $cap.State.ToString().Trim()
+    } else {
+      'NotPresent'
+    }
+  args:
+    executable: powershell.exe
+  register: openssh_capability_state
+  changed_when: false
+
+- name: Install OpenSSH Server capability when not installed
+  ansible.windows.win_shell: |
+    $cap = Get-WindowsCapability -Online | Where-Object { $_.Name -like 'OpenSSH.Server*' } | Select-Object -First 1
+    if (-not $cap) {
+      throw 'OpenSSH.Server capability not found on this host.'
+    }
+
+    Add-WindowsCapability -Online -Name $cap.Name
+  args:
+    executable: powershell.exe
+  register: openssh_install
+  when: openssh_capability_state.stdout | trim != 'Installed'
+
+- name: Check whether OpenSSH host keys already exist
+  ansible.windows.win_shell: |
+    $keys = Get-ChildItem -Path 'C:\ProgramData\ssh' -Filter 'ssh_host_*_key' -File -ErrorAction SilentlyContinue
+    if ($keys -and $keys.Count -gt 0) {
+      Write-Output 'present'
+    } else {
+      Write-Output 'missing'
+    }
+  args:
+    executable: powershell.exe
+  register: ssh_hostkeys_state
+  changed_when: false
+  become: true
+  become_method: ansible.builtin.runas
+  become_user: "NT AUTHORITY\\SYSTEM"
+
+- name: Generate OpenSSH host keys when missing (ssh-keygen -A)
+  ansible.windows.win_shell: |
+    $exe = 'C:\Windows\System32\OpenSSH\ssh-keygen.exe'
+    if (Test-Path $exe) {
+      & $exe -A 2>&1
+      Write-Output 'hostkeys-generated'
+    } else {
+      Write-Output 'ssh-keygen-not-found'
+      exit(2)
+    }
+  register: ssh_keygen
+  changed_when: "'hostkeys-generated' in ssh_keygen.stdout"
+  when: ssh_hostkeys_state.stdout | trim == 'missing'
+  become: true
+  become_method: ansible.builtin.runas
+  become_user: "NT AUTHORITY\\SYSTEM"
+
+- name: Ensure OpenSSH private host keys are owned by SYSTEM
+  ansible.windows.win_shell: |
+    $changed = $false
+    $keys = Get-ChildItem -Path 'C:\ProgramData\ssh' -Filter 'ssh_host_*_key' -File -ErrorAction SilentlyContinue
+    if ($keys -and $keys.Count -gt 0) {
+      foreach ($k in $keys) {
+        $before = (Get-Acl -Path $k.FullName).Owner
+        icacls.exe $k.FullName /setowner "*S-1-5-18" /C | Out-Null
+        $after = (Get-Acl -Path $k.FullName).Owner
+        if ($before -ne $after) {
+          $changed = $true
+        }
+      }
+      if ($changed) {
+        Write-Output 'hostkey-owner-changed'
+      } else {
+        Write-Output 'hostkey-owner-unchanged'
+      }
+    } else {
+      Write-Output 'no-keys'
+    }
+  register: ssh_hostkey_owner
+  changed_when: "'hostkey-owner-changed' in ssh_hostkey_owner.stdout"
+  become: true
+  become_method: ansible.builtin.runas
+  become_user: "NT AUTHORITY\\SYSTEM"
+
+- name: Restrict private host key file permissions (icacls) and remove provision user access
+  ansible.windows.win_shell: |
+    $changed = $false
+    $removePrincipal = "{{ ansible_user | default('') }}"
+    $keys = Get-ChildItem -Path 'C:\ProgramData\ssh' -Filter 'ssh_host_*_key' -File -ErrorAction SilentlyContinue
+    if ($keys -and $keys.Count -gt 0) {
+      foreach ($k in $keys) {
+        $before = (Get-Acl -Path $k.FullName).Sddl
+        icacls.exe $k.FullName /inheritance:r
+        icacls.exe $k.FullName /grant:r "S-1-5-18:(F)"
+        if ($removePrincipal -and $removePrincipal -notmatch '^(?i:NT AUTHORITY\\SYSTEM|SYSTEM)$') {
+          icacls.exe $k.FullName /remove "$removePrincipal" 2>$null
+        }
+        $after = (Get-Acl -Path $k.FullName).Sddl
+        if ($before -ne $after) {
+          $changed = $true
+        }
+      }
+      if ($changed) {
+        Write-Output 'restricted-changed'
+      } else {
+        Write-Output 'restricted-unchanged'
+      }
+    } else {
+      Write-Output 'no-keys'
+    }
+  args:
+    executable: powershell.exe
+  register: restrict_hostkey_perms
+  changed_when: "'restricted-changed' in restrict_hostkey_perms.stdout"
+  become: true
+  become_method: ansible.builtin.runas
+  become_user: "NT AUTHORITY\\SYSTEM"
+
+- name: Restrict C:\ProgramData\ssh folder write permissions
+  ansible.windows.win_shell: |
+    $p = 'C:\ProgramData\ssh'
+    if (Test-Path $p) {
+      $before = (Get-Acl -Path $p).Sddl
+      icacls.exe $p /inheritance:r | Out-Null
+      icacls.exe $p /grant:r "S-1-5-18:(OI)(CI)(F)" "S-1-5-32-544:(OI)(CI)(F)" | Out-Null
+      icacls.exe $p /remove "S-1-5-32-545" 2>$null
+      $after = (Get-Acl -Path $p).Sddl
+      if ($before -ne $after) {
+        Write-Output 'ssh-dir-acl-changed'
+      } else {
+        Write-Output 'ssh-dir-acl-unchanged'
+      }
+    } else {
+      Write-Output 'no-dir'
+    }
+  args:
+    executable: powershell.exe
+  register: ssh_dir_acl
+  changed_when: "'ssh-dir-acl-changed' in ssh_dir_acl.stdout"
+  become: true
+  become_method: ansible.builtin.runas
+  become_user: "NT AUTHORITY\\SYSTEM"
+
+- name: Ensure sshd service is enabled and running
+  ansible.windows.win_service:
+    name: sshd
+    start_mode: auto
+    state: started
+
+- name: Ensure firewall allows SSH (TCP/22)
+  community.windows.win_firewall_rule:
+    name: "OpenSSH Server (sshd)"
+    enable: true
+    direction: in
+    action: allow
+    localport: 22
+    protocol: TCP
+    profile: "Domain,Private"
+
+- name: Ensure SSH banner file is present
+  ansible.windows.win_copy:
+    src: "windows/issue.net"
+    dest: 'C:\ProgramData\ssh\issue.net'
+    force: true
+  become: true
+  become_method: ansible.builtin.runas
+  become_user: "NT AUTHORITY\\SYSTEM"
+
+- name: Detect available PowerShell for ssh subsystem
+  ansible.windows.win_shell: |
+    if (Test-Path 'C:\Program Files\PowerShell\7\pwsh.exe') { Write-Output 'C:\Program Files\PowerShell\7\pwsh.exe' }
+    elseif (Test-Path 'C:\Program Files\PowerShell\6\pwsh.exe') { Write-Output 'C:\Program Files\PowerShell\6\pwsh.exe' }
+    elseif (Test-Path 'C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe') { Write-Output 'C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe' }
+    else { Write-Output '' }
+  register: ssh_subsystem_path
+  changed_when: false
+
+- name: Set ssh subsystem executable path fact
+  ansible.builtin.set_fact:
+    __ssh_subsystem_path__: "{{ ssh_subsystem_path.stdout | trim }}"
+
+- name: Detect supported SSH key exchange algorithms
+  ansible.windows.win_shell: |
+    $ssh = 'C:\Windows\System32\OpenSSH\ssh.exe'
+    if (Test-Path $ssh) {
+      & $ssh -Q kex
+    } else {
+      Write-Output ''
+    }
+  args:
+    executable: powershell.exe
+  register: ssh_supported_kex
+  changed_when: false
+
+- name: Render sshd_config from template
+  ansible.windows.win_template:
+    src: windows_sshd_config.j2
+    dest: 'C:\ProgramData\ssh\sshd_config'
+  register: sshd_config_changed
+  become: true
+  become_method: ansible.builtin.runas
+  become_user: "NT AUTHORITY\\SYSTEM"
+
+- name: Restart sshd if config changed # noqa no-handler
+  ansible.windows.win_service:
+    name: sshd
+    state: restarted
+  when: sshd_config_changed.changed
+
+- name: Ensure users for SSH keys exist (no-op if already present)
+  ansible.windows.win_user:
+    name: "{{ item.name }}"
+    state: present
+  loop: "{{ __ssh_users__ | default([]) }}"
+  loop_control:
+    label: "{{ item.name }}"
+  become: true
+  become_method: ansible.builtin.runas
+  become_user: "NT AUTHORITY\\SYSTEM"
+
+- name: Ensure user .ssh directory exists
+  ansible.windows.win_file:
+    path: "C:\\Users\\{{ item.name }}\\.ssh"
+    state: directory
+  loop: "{{ __ssh_users__ | default([]) }}"
+  loop_control:
+    label: "{{ item.name }}"
+  become: true
+  become_method: ansible.builtin.runas
+  become_user: "NT AUTHORITY\\SYSTEM"
+
+- name: Deploy authorized_keys for each user
+  ansible.windows.win_copy:
+    dest: "C:\\Users\\{{ item.name }}\\.ssh\\authorized_keys"
+    content: "{{ (item['keys'] | default([])) | join('\r\n') }}\r\n"
+    force: true
+  loop: "{{ __ssh_users__ | default([]) }}"
+  loop_control:
+    label: "{{ item.name }}"
+  become: true
+  become_method: ansible.builtin.runas
+  become_user: "NT AUTHORITY\\SYSTEM"
+
+- name: Set ACL on .ssh directory to allow only the user and Administradores (directory)
+  ansible.windows.win_acl:
+    path: "C:\\Users\\{{ item.name }}\\.ssh"
+    user: "{{ item.name }}"
+    rights: FullControl
+    type: allow
+    inheritance: false
+    propagate: false
+    state: present
+  loop: "{{ __ssh_users__ | default([]) }}"
+  loop_control:
+    label: "{{ item.name }}"
+  become: true
+  become_method: ansible.builtin.runas
+  become_user: "NT AUTHORITY\\SYSTEM"
+
+- name: Set ACL on authorized_keys file to allow only the user and Administradores (file)
+  ansible.windows.win_acl:
+    path: "C:\\Users\\{{ item.name }}\\.ssh\\authorized_keys"
+    user: "{{ item.name }}"
+    rights: FullControl
+    type: allow
+    inheritance: false
+    propagate: false
+    state: present
+  loop: "{{ __ssh_users__ | default([]) }}"
+  loop_control:
+    label: "{{ item.name }}"
+  become: true
+  become_method: ansible.builtin.runas
+  become_user: "NT AUTHORITY\\SYSTEM"
+
+- name: Create administrators_authorized_keys from admin users' keys (if any)
+  ansible.windows.win_copy:
+    dest: 'C:\ProgramData\ssh\administrators_authorized_keys'
+    content: >-
+      {{ (__ssh_users__ | default([]) | selectattr('admin', 'equalto', true)
+      | map(attribute='keys') | list | flatten | default([])) | join('\r\n') }}\r\n
+    force: true
+  when: (__ssh_users__ | default([]) | selectattr('admin','equalto',true) | list) | length > 0
+  become: true
+  become_method: ansible.builtin.runas
+  become_user: "NT AUTHORITY\\SYSTEM"
+
+- name: Restrict administrators_authorized_keys permissions
+  ansible.windows.win_shell: |
+    $path = 'C:\ProgramData\ssh\administrators_authorized_keys'
+    if (Test-Path $path) {
+      $before = (Get-Acl -Path $path).Sddl
+      icacls.exe $path /inheritance:r | Out-Null
+      icacls.exe $path /grant:r "S-1-5-18:(F)" "S-1-5-32-544:(F)" | Out-Null
+      $after = (Get-Acl -Path $path).Sddl
+      if ($before -ne $after) {
+        Write-Output 'admin-auth-keys-acl-changed'
+      } else {
+        Write-Output 'admin-auth-keys-unchanged'
+      }
+    } else {
+      Write-Output 'no-admin-auth-keys'
+    }
+  args:
+    executable: powershell.exe
+  register: restrict_admin_auth_perms
+  changed_when: "'admin-auth-keys-acl-changed' in restrict_admin_auth_perms.stdout"
+  when: (__ssh_users__ | default([]) | selectattr('admin','equalto',true) | list) | length > 0
+  become: true
+  become_method: ansible.builtin.runas
+  become_user: "NT AUTHORITY\\SYSTEM"

ansible/roles/wol/meta/main.yml

@@ -0,0 +1,13 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Configure Wake-on-LAN via systemd-networkd
+  company: A13Labs
+  role_name: wol
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: EL
+      versions:
+        - "9"

ansible/roles/wol/tasks/main.yml

@@ -0,0 +1,59 @@
+---
+- name: Install ethtool
+  become: true
+  ansible.builtin.package:
+    name: ethtool
+    state: present
+
+- name: Enable WOL on the network interface
+  become: true
+  ansible.builtin.command: ethtool -s {{ network_interface }} wol g
+  changed_when: false
+
+- name: Create systemd-networkd WOL config
+  become: true
+  ansible.builtin.copy:
+    dest: "/etc/systemd/network/10-wol.network"
+    content: |
+      [Match]
+      Name={{ network_interface }}
+
+      [Link]
+      WakeOnLan=magic
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Create systemd WOL config
+  become: true
+  ansible.builtin.copy:
+    dest: "/etc/systemd/system/wol.service"
+    content: |
+      [Unit]
+      Description=Enable Wake-on-LAN
+      After=network-online.target
+      [Service]
+      Type=oneshot
+      ExecStart=/sbin/ethtool --change {{ network_interface }} wol g
+      [Install]
+      WantedBy=network-online.target
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Enable and restart systemd-networkd
+  become: true
+  ansible.builtin.systemd:
+    name: wol.service
+    state: started
+    enabled: true
+
+- name: Confirm WOL setting applied
+  become: true
+  ansible.builtin.command: ethtool {{ network_interface }}
+  register: wol_status
+  changed_when: false
+
+- name: Show WOL setting
+  ansible.builtin.debug:
+    var: wol_status.stdout_lines