feat(vaultwarden): add SMTP configuration options and enhance signup settings

- Introduced SMTP settings for Vaultwarden including host, port, security, and authentication details.
- Added variables for signup verification, 2FA settings, password hints, and logging options.
- Updated Vaultwarden deployment to utilize new SMTP configurations.
- Enhanced Grafana module to support dynamic dashboard and datasource provisioning.
- Added LLM proxy configuration for Open Web UI with necessary environment variables.

ansible/amd-rocm-container.yml

@@ -1,31 +0,0 @@
-- name: Add ROCm repository
-  become: true
-  ansible.builtin.dnf_repository:
-    name: rocm
-    description: ROCm Repository
-    baseurl: https://repo.radeon.com/rocm/el9/latest/main/
-    gpgcheck: yes
-    gpgkey: https://repo.radeon.com/rocm/rocm.gpg.key
-    enabled: yes
-
-- name: Add AMD Graphics repository
-  become: true
-  ansible.builtin.dnf_repository:
-    name: rocmgraphics
-    description: AMD Graphics Repository
-    baseurl: https://repo.radeon.com/graphics/latest/el/9.6/main/x86_64/
-    gpgcheck: yes
-    gpgkey: https://repo.radeon.com/rocm/rocm.gpg.key
-    enabled: yes
-
-- name: Install rocm-dkms and dependencies
-  become: true
-  ansible.builtin.package:
-    name:
-      - rocm
-      - rocm-developer-tools
-      - hipblas-devel
-      - hip-devel
-      - rocwmma-devel
-      - rocm-opencl-devel
-    state: present

ansible/host_vars/gpu-01.lab.alexpires.me.yml

@@ -88,8 +88,6 @@ duo_users: ["*", "!provision", "!comfyui", "!kokoro", "!ollama"]
 duo_api_hostname: api-7cda1654.duosecurity.com
 
 # podman services settings
-ollama_home: "/mnt/data/ollama"
-llama_home: "/mnt/data/llamacpp"
 podman_user_pubkey: "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAHHOPBR9p9kq5Cqzpe4cr3jHnweaYrHPXv5sXNzt+sCXP54uc5rWUBhxW2OQVvQzJ47dEVhEKi4WSC7LcuKS2G5AQDzWXNiasHvYIYQU3F/EknVCZnsiXYqXphYkJA6rJCNRnISZCIC1mocq6PB7J08ONdRFCvjfUBuVRT8BNGXNmQ/zQ=="
 
 # ComfyUI settings
@@ -108,39 +106,18 @@ certbot_domains:
   - "gpu-01.lab.alexpires.me"
 
 certbot_cockpit_domain: "gpu-01.lab.alexpires.me"
-ollama_models:
-  # Thinking models
-  - gpt-oss:latest
-  - gemma4:e4b
-  - qwen3.5:9b
-  - qwen3.5:4b
-  # Non-thinking models
-  - ministral-3:3b
-  - ministral-3:8b
-  - ministral-3:14b
-  - granite4.1:3b
-  - granite4.1:8b
-  - deepseek-r1:latest
-  - qwen2.5-coder:3b
-  # Embedding models
-  - embeddinggemma:latest
-  - nomic-embed-text:latest
-  # Whisper models
-  - dimavz/whisper-tiny:latest
-ollama_model_files:
-  - file: "ollama/gpt-oss-latest-32k.modelfile"
-    name: "gpt-oss-32k:latest"
-  - file: "ollama/gemma4-e4b-32k.modelfile"
-    name: "gemma4-e4b-32k:latest"
-  - file: "ollama/qwen3.5-9b-32k.modelfile"
-    name: "qwen3.5-9b-32k:latest"
-  - file: "ollama/qwen3.5-9b-64k.modelfile"
-    name: "qwen3.5-9b-64k:latest"
-  - file: "ollama/qwen3.5-4b-32k.modelfile"
-    name: "qwen3.5-4b-32k:latest"
-  - file: "ollama/qwen3.5-4b-64k.modelfile"
-    name: "qwen3.5-4b-64k:latest"
-llama_models:
+
+# Llama.cpp server settings
+llama_server_home: "/mnt/data/llamacpp"
+llama_server_user_pubkey: "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAHHOPBR9p9kq5Cqzpe4cr3jHnweaYrHPXv5sXNzt+sCXP54uc5rWUBhxW2OQVvQzJ47dEVhEKi4WSC7LcuKS2G5AQDzWXNiasHvYIYQU3F/EknVCZnsiXYqXphYkJA6rJCNRnISZCIC1mocq6PB7J08ONdRFCvjfUBuVRT8BNGXNmQ/zQ=="
+
+llama_server_port: 11434
+llama_server_devices:
+  - "nvidia.com/gpu=all"
+  - "/dev/kfd:/dev/kfd:rw"
+  - "/dev/dri:/dev/dri:rw"
+
+llama_server_models:
   # TODO: replace revision values with pinned commit SHAs for strict reproducibility.
   - name: "gpt-oss-20b-q4-0"
     model_id: "unsloth/gpt-oss-20b-GGUF"
@@ -186,9 +163,13 @@ llama_models:
     model_id: "unsloth/Devstral-Small-2-24B-Instruct-2512-GGUF"
     quant: "IQ4_XS"
     revision: "main"
+  - name: "qwen3-6-27b-mtp-iq4-xs"
+    model_id: "unsloth/Qwen3.6-27B-MTP-GGUF"
+    quant: "IQ4_XS"
+    revision: "main"
 
-# Optional global model preset options applied under [*] in models.ini.
-# Keys map to llama.cpp CLI args without leading dashes, short args, or LLAMA_ARG_* env names.
-llama_preset_global:
-  c: 131072
-  n-gpu-layers: all
+# # Optional global model preset options applied under [*] in models.ini.
+# # Keys map to llama.cpp CLI args without leading dashes, short args, or LLAMA_ARG_* env names.
+# llama_preset_global:
+#   c: 131072
+#   n-gpu-layers: all

ansible/playbook_amd_rocm.yml

@@ -1,7 +1,5 @@
 - name: Setup AMD ROCm Container Environment
   hosts: amd
 
-  tasks:
-    - name: Setup AMD ROCm Container Environment
-      ansible.builtin.include_tasks:
-        file: tasks/amd-rocm-container.yml
+  roles:
+    - amd_rocm

ansible/playbook_auto_suspend.yml

@@ -1,7 +1,5 @@
 - name: Setup Auto Suspend
   hosts: auto_suspend
 
-  tasks:
-    - name: Setup Auto Suspend
-      ansible.builtin.include_tasks:
-        file: tasks/auto_suspend.yml
+  roles:
+    - auto_suspend

ansible/playbook_llama_server.yml

@@ -0,0 +1,4 @@
+- name: Setup llama.cpp - Vulkan
+  hosts: llamacpp
+  roles:
+    - llama_server

ansible/playbook_nvidia_cdi_generator.yml

@@ -1,7 +1,5 @@
 - name: Setup NVidia CDI Generator
   hosts: nvidia
 
-  tasks:
-    - name: Setup nvidia-cdi-generator
-      ansible.builtin.include_tasks:
-        file: tasks/nvidia-cdi-generator.yml
+  roles:
+    - nvidia_cdi_generator

ansible/playbook_podman_comfyui.yml

@@ -45,7 +45,5 @@
         device:
           - "nvidia.com/gpu=all"
 
-  tasks:
-    - name: Setup Pods
-      ansible.builtin.include_tasks:
-        file: tasks/podman.yml
+  roles:
+    - podman

ansible/playbook_podman_kokoro.yml

@@ -26,7 +26,5 @@
         # device:
         #   - "nvidia.com/gpu=all"
 
-  tasks:
-    - name: Setup Pods
-      ansible.builtin.include_tasks:
-        file: tasks/podman.yml
+  roles:
+    - podman

ansible/playbook_podman_llama_vulkan.yml

@@ -5,24 +5,11 @@
     podman_group: "{{ llama_group | default('llama') }}"
     podman_extra_groups: "users,video"
     podman_user_home: "{{ llama_home | default('/home/llama') }}"
-    __llama_models__: "{{ llama_models | default([]) }}"
-    __llama_router_dir__: "{{ podman_user_home }}/models/.router"
-    __llama_managed_dir__: "{{ podman_user_home }}/models/managed"
-    __llama_links_dir__: "{{ podman_user_home }}/models/managed-links"
-    __llama_manifest_file__: "{{ __llama_router_dir__ }}/manifest.json"
-    __llama_models_input_file__: "{{ __llama_router_dir__ }}/desired-models.json"
-    __llama_preset_global__: "{{ llama_preset_global | default({}) }}"
-    __llama_preset_global_input_file__: "{{ __llama_router_dir__ }}/preset-global.json"
-    __llama_preset_file__: "{{ __llama_router_dir__ }}/models.ini"
-    __llama_sync_script__: "{{ __llama_router_dir__ }}/llama_hf_sync.py"
-    __llama_sync_venv__: "{{ llama_sync_venv | default(__llama_router_dir__ ~ '/.venv') }}"
-    __llama_sync_python__: "{{ __llama_sync_venv__ }}/bin/python"
-    __llama_models_max__: "{{ llama_models_max_loaded | default(1) }}"
     podman_user_folders:
       - models
       - .cache
       - .cache/llama.cpp
-    pods:
+    podman_pods:
       - name: llama.cpp
         repo:
           image: ghcr.io/ggml-org/llama.cpp:server-vulkan
@@ -67,16 +54,11 @@
           - "nvidia.com/gpu=all"
           - "/dev/kfd:/dev/kfd:rw"
           - "/dev/dri:/dev/dri:rw"
+  roles:
+    - llama_models_sync
+    - podman
+
   tasks:
-    - name: Sync llama models from Hugging Face
-      ansible.builtin.include_tasks:
-        file: tasks/llama_models_sync.yml
-      when: llama_models_sync_enabled | default(true)
-
-    - name: Setup Pods
-      ansible.builtin.include_tasks:
-        file: tasks/podman.yml
-
     - name: Query llama router model list
       ansible.builtin.uri:
         url: "http://127.0.0.1:{{ llama_port | default(8012) }}/models?reload=1"

ansible/playbook_podman_ollama.yml

@@ -27,7 +27,5 @@
           - "nvidia.com/gpu=all"
           - "/dev/kfd:/dev/kfd:rw"
           - "/dev/dri:/dev/dri:rw"
-  tasks:
-    - name: Setup Pods
-      ansible.builtin.include_tasks:
-        file: tasks/podman.yml
+  roles:
+    - podman

ansible/playbook_podman_ollama_rocm.yml

@@ -28,11 +28,10 @@
           - "/dev/kfd:/dev/kfd:rw"
           - "/dev/dri:/dev/dri:rw"
 
-  tasks:
-    - name: Setup Pods
-      ansible.builtin.include_tasks:
-        file: tasks/podman.yml
+  roles:
+    - podman
 
+  tasks:
     - name: Enable sudo access to podman user (temporary)
       become: true
       ansible.builtin.lineinfile:

ansible/playbook_podman_qwen2.yml

@@ -31,7 +31,5 @@
         device:
           - "/dev/kfd:/dev/kfd:rw"
           - "/dev/dri:/dev/dri:rw"
-  tasks:
-    - name: Setup Pods
-      ansible.builtin.include_tasks:
-        file: tasks/podman.yml
+  roles:
+    - podman

ansible/playbook_podman_qwen3.6.yml

@@ -58,7 +58,5 @@
           - "nvidia.com/gpu=all"
           - "/dev/kfd:/dev/kfd:rw"
           - "/dev/dri:/dev/dri:rw"
-  tasks:
-    - name: Setup Pods
-      ansible.builtin.include_tasks:
-        file: tasks/podman.yml
+  roles:
+    - podman

ansible/playbook_windows_game.yml

@@ -1,7 +1,5 @@
 - name: Setup Windows Game Optimization
   hosts: windows_game
 
-  tasks:
-    - name: Setup Optimizations
-      ansible.builtin.include_tasks:
-        file: tasks/windows_game.yml
+  roles:
+    - windows_game

ansible/playbook_windows_prometheus.yml

@@ -1,7 +1,5 @@
 - name: Setup Prometheus on Windows
   hosts: windows_prometheus
 
-  tasks:
-    - name: Setup Prometheus
-      ansible.builtin.include_tasks:
-        file: tasks/windows_prometheus.yml
+  roles:
+    - windows_prometheus

ansible/playbook_windows_ssh.yml

@@ -1,7 +1,5 @@
 - name: Setup ssh on Windows
   hosts: windows
 
-  tasks:
-    - name: Setup ssh
-      ansible.builtin.include_tasks:
-        file: tasks/windows_ssh.yml
+  roles:
+    - windows_ssh

ansible/playbook_wol.yml

@@ -1,7 +1,5 @@
 - name: Setup WakeOnLan
   hosts: wol
 
-  tasks:
-    - name: Setup WakeOnLan
-      ansible.builtin.include_tasks:
-        file: tasks/wol.yml
+  roles:
+    - wol

ansible/roles/amd_rocm/defaults/main.yml

@@ -0,0 +1,13 @@
+---
+# amd_rocm defaults
+
+rocm_repo_baseurl: "https://repo.radeon.com/rocm/el9/latest/main/"
+rocm_repo_gpgkey: "https://repo.radeon.com/rocm/rocm.gpg.key"
+rocmgraphics_repo_baseurl: "https://repo.radeon.com/graphics/latest/el/9.6/main/x86_64/"
+rocm_packages:
+  - rocm
+  - rocm-developer-tools
+  - hipblas-devel
+  - hip-devel
+  - rocwmma-devel
+  - rocm-opencl-devel

ansible/roles/amd_rocm/meta/main.yml

@@ -0,0 +1,18 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Install AMD ROCm repository and packages for GPU computing
+  company: A13Labs
+  role_name: amd_rocm
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: Fedora
+      versions:
+        - "39"
+        - "40"
+        - "41"
+    - name: EL
+      versions:
+        - "9"

ansible/roles/amd_rocm/tasks/main.yml

@@ -0,0 +1,52 @@
+---
+- name: Add ROCm repository (yum)
+  when: ansible_pkg_mgr == 'yum'
+  become: true
+  ansible.builtin.yum_repository:
+    name: rocm
+    description: ROCm Repository
+    baseurl: "{{ rocm_repo_baseurl }}"
+    gpgcheck: true
+    gpgkey: "{{ rocm_repo_gpgkey }}"
+    enabled: true
+    state: present
+
+- name: Add ROCm repository (dnf)
+  when: ansible_pkg_mgr == 'dnf'
+  become: true
+  ansible.builtin.dnf_repository:
+    name: rocm
+    description: ROCm Repository
+    baseurl: "{{ rocm_repo_baseurl }}"
+    gpgcheck: yes
+    gpgkey: "{{ rocm_repo_gpgkey }}"
+    enabled: yes
+
+- name: Add AMD Graphics repository (yum)
+  when: ansible_pkg_mgr == 'yum'
+  become: true
+  ansible.builtin.yum_repository:
+    name: rocmgraphics
+    description: AMD Graphics Repository
+    baseurl: "{{ rocmgraphics_repo_baseurl }}"
+    gpgkey: "{{ rocm_repo_gpgkey }}"
+    gpgcheck: true
+    enabled: true
+    state: present
+
+- name: Add AMD Graphics repository (dnf)
+  when: ansible_pkg_mgr == 'dnf'
+  become: true
+  ansible.builtin.dnf_repository:
+    name: rocmgraphics
+    description: AMD Graphics Repository
+    baseurl: "{{ rocmgraphics_repo_baseurl }}"
+    gpgkey: "{{ rocm_repo_gpgkey }}"
+    gpgcheck: yes
+    enabled: yes
+
+- name: Install rocm-dkms and dependencies
+  become: true
+  ansible.builtin.package:
+    name: "{{ rocm_packages }}"
+    state: present

ansible/roles/auto_suspend/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+auto_suspend_grub_config_path_bios: /boot/grub2/grub.cfg

ansible/roles/auto_suspend/meta/main.yml

@@ -0,0 +1,13 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Configure automatic system suspend based on idle time
+  company: A13Labs
+  role_name: auto_suspend
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: EL
+      versions:
+        - "9"

ansible/roles/auto_suspend/tasks/main.yml

@@ -1,3 +1,4 @@
+---
 - name: Install dependencies
   become: true
   ansible.builtin.package:

ansible/roles/gpu_monitoring/defaults/main.yml

@@ -46,18 +46,6 @@ amd_exporter_python_version: "python3"
 amd_exporter_sysfs_gpu_pattern: "/sys/class/drm/card?/device/"
 
 # Llama Exporter configuration
-llama_exporter_enabled: true
-llama_exporter_port: 9550
-llama_exporter_user: llama_exporter
-llama_exporter_user_home: "/home/llama_exporter"
-llama_exporter_install_dir: "/opt/llama_exporter"
-llama_exporter_venv_path: "/opt/llama_exporter/.venv"
-llama_exporter_scrape_interval: 15
-llama_exporter_scrape_timeout: 5
-
-# Llama.cpp scrape targets
-llama_exporter_targets: []
-
 # Firewall configuration
 prometheus_fw_port: "{{ prometheus_port }}"
 prometheus_fw_proto: "tcp"

ansible/roles/gpu_monitoring/files/llama_exporter.py

@@ -1,176 +0,0 @@
-#!/usr/bin/env python3
-"""LLM inference metrics exporter for Prometheus.
-
-Scrape targets:
-    - llama.cpp HTTP API servers (standard /metrics or custom health)
-"""
-
-import json
-import os
-import time
-import logging
-import urllib.request
-import urllib.error
-from http.server import HTTPServer, BaseHTTPRequestHandler
-
-logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
-logger = logging.getLogger(__name__)
-
-POLL_INTERVAL = int(os.environ.get("LLAMA_EXPORTER_INTERVAL", "15"))
-SCRAPE_TIMEOUT = int(os.environ.get("LLAMA_EXPORTER_TIMEOUT", "5"))
-
-LLAMA_CPP_DEFAULTS = [
-    {"name": "llama-cpp-8012", "url": "http://localhost:8012"},
-]
-
-
-def _fetch_json(url, timeout=SCRAPE_TIMEOUT):
-    try:
-        req = urllib.request.Request(url, headers={"Accept": "application/json"})
-        with urllib.request.urlopen(req, timeout=timeout) as resp:
-            return json.loads(resp.read().decode("utf-8"))
-    except Exception as e:
-        logger.debug("Failed to fetch %s: %s", url, e)
-        return None
-
-
-def _collect_llama_cpp(target):
-    metrics = {}
-
-    def _set_metric(metric_name, labels, value):
-        label_key = tuple(sorted(labels.items()))
-        metrics[(metric_name, label_key)] = value
-
-    base_url = target["url"].rstrip("/")
-
-    # Check health endpoint
-    health_url = f"{base_url}/health"
-    health = _fetch_json(health_url)
-    if health and isinstance(health, dict):
-        status = health.get("status", "unknown")
-        _set_metric("llama_server_health", {"server": target["name"], "model": health.get("model", "unknown")}, 1.0 if status == "ok" else 0.0)
-    else:
-        _set_metric("llama_server_health", {"server": target["name"], "model": "unknown"}, 0.0)
-
-    # Check /metrics endpoint for llama.cpp built-in metrics
-    metrics_url = f"{base_url}/metrics"
-    try:
-        req = urllib.request.Request(metrics_url, headers={"Accept": "text/plain"})
-        with urllib.request.urlopen(req, timeout=SCRAPE_TIMEOUT) as resp:
-            body = resp.read().decode("utf-8")
-            for line in body.splitlines():
-                line = line.strip()
-                if not line or line.startswith("#"):
-                    continue
-                # Parse simple key value pairs
-                if "{" in line:
-                    key_part = line.split("{")[0]
-                    labels_part = line.split("{")[1].rstrip("}").split("}")[0] if "}" in line else ""
-                    value = line.split()[-1] if line.split() else "1"
-                    label_str = ""
-                    if labels_part:
-                        label_str = ",".join(f'{{{k}="{v}"}}' for k, v in (
-                            pair.split("=") for pair in labels_part.split(",") if "=" in pair
-                        ))
-                    _set_metric(key_part.strip(), {"server": target["name"], "raw": labels_part}, float(value) if value.replace(".", "").replace("-", "").isdigit() else value)
-                else:
-                    parts = line.split()
-                    if len(parts) >= 2:
-                        try:
-                            float(parts[1])
-                            _set_metric(parts[0], {"server": target["name"]}, parts[1])
-                        except ValueError:
-                            pass
-    except Exception as e:
-        logger.debug("Failed to scrape %s/metrics: %s", metrics_url, e)
-
-    # Check /model endpoint for model info
-    model_url = f"{base_url}/models"
-    models = _fetch_json(model_url)
-    if models and isinstance(models, dict):
-        model_list = models.get("data", [models]) if "data" in models else [models]
-        for m in model_list:
-            if not isinstance(m, dict):
-                continue
-            model_id = m.get("id", "unknown")
-            _set_metric("llama_models_loaded", {"server": target["name"], "model": model_id}, 1.0)
-            ctx_max = m.get("context_size", m.get("n_ctx", 0))
-            if ctx_max:
-                _set_metric("llama_model_context_max", {"server": target["name"], "model": model_id}, int(ctx_max))
-            trained = m.get("train_tokens", m.get("n_train", 0))
-            if trained:
-                _set_metric("llama_model_tokens_trained", {"server": target["name"], "model": model_id}, int(trained))
-
-    return metrics
-
-
-
-class MetricsHandler(BaseHTTPRequestHandler):
-    def do_GET(self):
-        if self.path != "/metrics":
-            self.send_response(404)
-            self.end_headers()
-            return
-
-        all_metrics = {}
-
-        # Scrape llama.cpp targets
-        targets = []
-        env_targets = os.environ.get("LLAMA_TARGETS", "")
-        if env_targets:
-            try:
-                targets = json.loads(env_targets)
-            except json.JSONDecodeError:
-                logger.warning("Invalid LLAMA_TARGETS JSON, using defaults")
-                targets = []
-
-        if not targets:
-            targets = LLAMA_CPP_DEFAULTS
-
-        for target in targets:
-            if "url" not in target or "name" not in target:
-                target["name"] = target.get("name", "default")
-                target["url"] = target.get("url", "http://localhost:8012")
-            try:
-                all_metrics.update(_collect_llama_cpp(target))
-            except Exception as e:
-                logger.error("Error scraping llama.cpp target %s: %s", target.get("name", "unknown"), e)
-
-        lines = []
-        for (metric_type, label_pairs), value in sorted(all_metrics.items()):
-            labels = dict(label_pairs)
-            if not labels:
-                line = f"{metric_type} {value}"
-            else:
-                label_str = ",".join(f'{k}="{v}"' for k, v in sorted(labels.items()))
-                line = f"{metric_type}{{{label_str}}} {value}"
-            lines.append(line)
-
-        body = "\n".join(lines) + "\n" if lines else "# no metrics available\n"
-
-        self.send_response(200)
-        self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
-        self.end_headers()
-        self.wfile.write(body.encode("utf-8"))
-
-    def log_message(self, format, *args):
-        logger.debug("%s - - %s", self.address_string(), format % args)
-
-
-def main():
-    port = int(os.environ.get("LLAMA_EXPORTER_PORT", "9550"))
-    host = os.environ.get("LLAMA_EXPORTER_BIND", "0.0.0.0")
-
-    server = HTTPServer((host, port), MetricsHandler)
-    logger.info("Starting Llama Exporter on %s:%d", host, port)
-    logger.info("Polling every %d seconds, timeout %ds", POLL_INTERVAL, SCRAPE_TIMEOUT)
-
-    try:
-        server.serve_forever()
-    except KeyboardInterrupt:
-        logger.info("Shutting down")
-        server.shutdown()
-
-
-if __name__ == "__main__":
-    main()

ansible/roles/gpu_monitoring/handlers/main.yml

@@ -18,10 +18,3 @@
     name: "{{ amd_exporter_service_name }}"
     state: restarted
   listen: Restart amd_exporter
-
-- name: Restart llama_exporter
-  become: true
-  ansible.builtin.systemd:
-    name: "{{ llama_exporter_service_name }}"
-    state: restarted
-  listen: Restart llama_exporter

ansible/roles/gpu_monitoring/molecule/default/converge.yml

@@ -8,4 +8,3 @@
         name: "gpu_monitoring"
       vars:
         amd_exporter_enabled: false
-        llama_exporter_enabled: false

ansible/roles/gpu_monitoring/tasks/main.yml

@@ -11,12 +11,6 @@
   tags:
     - roles::gpu_monitoring::amd_exporter
 
-- name: Include Llama Exporter setup
-  when: llama_exporter_enabled | default(true) | bool
-  ansible.builtin.include_tasks: llama_exporter.yml
-  tags:
-    - roles::gpu_monitoring::llama_exporter
-
 - name: Include Prometheus setup
   ansible.builtin.include_tasks: prometheus.yml
   tags:

ansible/roles/gpu_monitoring/templates/llama_exporter.service.j2

@@ -1,35 +0,0 @@
-[Unit]
-Description=LLM Inference Metrics Exporter (llama.cpp + Ollama)
-After=network-online.target
-Wants=network-online.target
-
-[Service]
-Type=simple
-User={{ llama_exporter_user }}
-Group={{ llama_exporter_user }}
-WorkingDirectory={{ llama_exporter_install_dir }}
-Environment="LLAMA_EXPORTER_PORT={{ llama_exporter_port }}"
-Environment="LLAMA_EXPORTER_BIND=0.0.0.0"
-Environment="LLAMA_EXPORTER_INTERVAL={{ llama_exporter_scrape_interval }}"
-Environment="LLAMA_EXPORTER_TIMEOUT={{ llama_exporter_scrape_timeout }}"
-{% if llama_exporter_targets and llama_exporter_targets != "" %}
-Environment="LLAMA_TARGETS={{ llama_exporter_targets | to_json }}"
-{% endif %}
-Environment="PATH={{ llama_exporter_venv_bin }}:/usr/bin:/bin"
-ExecStart={{ llama_exporter_venv_bin }}/python3 {{ llama_exporter_script_path }}
-Restart=always
-RestartSec=10
-TimeoutStopSec=10
-StandardOutput=journal
-StandardError=journal
-SyslogIdentifier=llama_exporter
-
-# Security hardening
-ProtectSystem=strict
-ProtectHome=true
-ReadWritePaths={{ llama_exporter_install_dir }} /var/log
-NoNewPrivileges=true
-PrivateTmp=true
-
-[Install]
-WantedBy=multi-user.target

ansible/roles/gpu_monitoring/templates/prometheus.yml.j2

@@ -21,10 +21,3 @@ scrape_configs:
       - targets: ["{{ prometheus_host_gateway }}:{{ amd_exporter_port }}"]
         labels:
           datacenter: "{{ inventory_hostname }}"
-
-  - job_name: llama_exporter
-    metrics_path: /metrics
-    static_configs:
-      - targets: ["{{ prometheus_host_gateway }}:{{ llama_exporter_port }}"]
-        labels:
-          datacenter: "{{ inventory_hostname }}"

ansible/roles/gpu_monitoring/vars/main.yml

@@ -12,12 +12,3 @@ nvidia_exporter_service_name: "nvidia_exporter"
 amd_exporter_script_path: "{{ amd_exporter_install_dir }}/amd_gpu_exporter.py"
 amd_exporter_venv_bin: "{{ amd_exporter_venv_path }}/bin"
 amd_exporter_service_name: "amdgpu_exporter"
-
-llama_exporter_script_path: "{{ llama_exporter_install_dir }}/llama_exporter.py"
-llama_exporter_venv_bin: "{{ llama_exporter_venv_path }}/bin"
-llama_exporter_service_name: "llama_exporter"
-
-# Default llama.cpp targets if none provided
-llama_exporter_targets_default:
-  - name: "llama-cpp-8012"
-    url: "http://localhost:8012"

ansible/roles/llama_server/defaults/main.yml

@@ -0,0 +1,26 @@
+llama_server_user: "llama"
+llama_server_group: "llama"
+llama_server_home: "/home/llama"
+llama_server_pubkey: ""
+llama_server_models: []
+llama_server_models_max: 1
+llama_server_image: "ghcr.io/ggml-org/llama.cpp"
+llama_server_tag: "server-vulkan"
+llama_server_extra_groups: "users,video"
+llama_server_devices: []
+llama_server_env: {}
+llama_server_port: 8080
+llama_server_preset_global: {}
+
+# Llama Exporter configuration
+llama_server_exporter_enabled: true
+llama_server_exporter_port: 9550
+llama_server_exporter_user: llama_exporter
+llama_server_exporter_user_home: "/opt/llama_exporter"
+llama_server_exporter_install_dir: "/opt/llama_exporter"
+llama_server_exporter_venv_path: "/opt/llama_exporter/.venv"
+llama_server_exporter_scrape_interval: 15
+llama_server_exporter_scrape_timeout: 5
+
+# Llama.cpp scrape targets
+llama_server_exporter_targets: []

ansible/roles/llama_server/files/scripts/llama_exporter.py

@@ -0,0 +1,389 @@
+#!/usr/bin/env python3
+"""LLM inference metrics exporter for Prometheus.
+
+Background thread scrapes only loaded models to avoid triggering model loads
+in llama-server, then the HTTP handler serves the cached data immediately.
+
+Exposes metrics for every known model from /models; unloaded models show
+zero values without ever requesting /metrics?model=<unloaded>.
+
+Uses persistent JSON cache on disk to survive restarts and compute counter
+deltas for Prometheus rate/irate queries.
+"""
+
+import json
+import os
+import time
+import threading
+import logging
+import urllib.request
+import urllib.error
+from http.server import HTTPServer, BaseHTTPRequestHandler
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
+logger = logging.getLogger(__name__)
+
+POLL_INTERVAL = int(os.environ.get("LLAMA_EXPORTER_INTERVAL", "15"))
+SCRAPE_TIMEOUT = int(os.environ.get("LLAMA_EXPORTER_TIMEOUT", "5"))
+CACHE_FILE = os.environ.get(
+    "LLAMA_EXPORTER_CACHE",
+    os.path.join(os.path.dirname(os.path.abspath(__file__)), "llama_exporter_cache.json"),
+)
+
+LLAMA_CPP_DEFAULTS = [
+    {"name": "llama.cpp", "url": "http://localhost:11434"},
+]
+
+# All llama.cpp metrics we expose per model (counters + gauges).
+ALL_METRICS = [
+    "llamacpp:prompt_tokens_total",
+    "llamacpp:prompt_seconds_total",
+    "llamacpp:tokens_predicted_total",
+    "llamacpp:tokens_predicted_seconds_total",
+    "llamacpp:n_decode_total",
+    "llamacpp:n_tokens_max",
+    "llamacpp:prompt_tokens_seconds",
+    "llamacpp:predicted_tokens_seconds",
+    "llamacpp:requests_processing",
+    "llamacpp:requests_deferred",
+    "llamacpp:n_busy_slots_per_decode",
+]
+
+# Counter metrics that accumulate over time — exposed as _delta for rate() queries.
+COUNTER_METRICS = {
+    "llamacpp:prompt_tokens_total",
+    "llamacpp:prompt_seconds_total",
+    "llamacpp:tokens_predicted_total",
+    "llamacpp:tokens_predicted_seconds_total",
+    "llamacpp:n_decode_total",
+}
+
+
+class Cache:
+    """Persistent, thread-safe metrics cache with delta computation."""
+
+    def __init__(self):
+        self._lock = threading.Lock()
+        self._data = {}  # { model_id: { metric_name: value } }
+        self._known_models = set()
+        self._health = None
+        self._loaded = {}
+
+        # Previous state for delta computation: { model_id: { metric_name: value } }
+        self._prev = {}
+        self._prev_timestamp = 0.0
+        self._deltas = {}
+
+        # Load persisted state
+        self._load_cache()
+
+    def _load_cache(self):
+        """Load previous scrape state from disk."""
+        try:
+            with open(CACHE_FILE, "r") as f:
+                state = json.load(f)
+            self._prev = state.get("models", {})
+            self._prev_timestamp = state.get("timestamp", 0.0)
+            logger.info("Loaded cache from %s (timestamp=%s, models=%d)",
+                        CACHE_FILE, self._prev_timestamp, len(self._prev))
+        except FileNotFoundError:
+            logger.info("No cache file found at %s", CACHE_FILE)
+        except Exception as e:
+            logger.warning("Failed to load cache: %s", e)
+
+    def _save_cache(self, current_data, known_models, loaded):
+        """Save current scrape state to disk."""
+        try:
+            state = {
+                "timestamp": time.time(),
+                "known": list(known_models),
+                "loaded": list(loaded),
+                "models": {},
+            }
+            for mid, metrics in current_data.items():
+                state["models"][mid] = dict(metrics)
+            tmp = CACHE_FILE + ".tmp"
+            with open(tmp, "w") as f:
+                json.dump(state, f)
+            os.replace(tmp, CACHE_FILE)
+        except Exception as e:
+            logger.error("Failed to save cache: %s", e)
+
+    def _compute_deltas_for_data(self, current_data):
+        """Compute deltas given current data and stored previous state.
+
+        Only counter metrics get deltas. On counter reset (value went backward),
+        delta is silently 0 (no entry added).
+        """
+        deltas = {}
+        if not self._prev:
+            return deltas
+
+        for mid, prev_metrics in self._prev.items():
+            mid_deltas = {}
+            cur_metrics = current_data.get(mid, {})
+            for mname, prev_val in prev_metrics.items():
+                # Only compute deltas for counter metrics
+                if mname not in COUNTER_METRICS:
+                    continue
+                cur_val = cur_metrics.get(mname)
+                if cur_val is None:
+                    continue
+                diff = cur_val - prev_val
+                # Counter reset: value went backward, delta is 0 (wrapped)
+                if diff < 0:
+                    continue
+                if diff > 0:
+                    mid_deltas[mname] = diff
+            if mid_deltas:
+                deltas[mid] = mid_deltas
+        return deltas
+
+    def update(self, models_data, metric_lines, known_models=None):
+        """Called by the background thread after a successful scrape cycle."""
+        with self._lock:
+            # Discover model IDs and loaded status from /models endpoint
+            known = set()
+            loaded = {}
+            if models_data and isinstance(models_data, dict):
+                model_list = models_data.get("data", [])
+                for m in model_list:
+                    if not isinstance(m, dict):
+                        continue
+                    mid = m.get("id", "unknown")
+                    known.add(mid)
+                    status_data = m.get("status", {})
+                    if isinstance(status_data, dict) and status_data.get("value") == "loaded":
+                        loaded[mid] = 1.0
+                    self._health = {"status": "ok", "model": mid}
+
+            if not known:
+                self._health = {"status": "error", "model": "unknown"}
+
+            # Build new metrics dict for all known models
+            new_data = {}
+            for mid in known:
+                new_data[mid] = {}
+
+            # Apply parsed metric values from /metrics?model=<id>
+            for metric_name, model_id, value in metric_lines:
+                if model_id in known:
+                    new_data[model_id][metric_name] = value
+
+            # Ensure every known model has all ALL_METRICS entries
+            for mid in known:
+                for mname in ALL_METRICS:
+                    if mname not in new_data[mid]:
+                        new_data[mid][mname] = 0.0
+
+            # Previously known models no longer in the list get zeroed out
+            for mid in self._known_models - known:
+                new_data[mid] = {m: 0.0 for m in ALL_METRICS}
+
+            # Compute deltas before updating previous state
+            deltas = self._compute_deltas_for_data(new_data)
+
+            # Save previous state for next cycle
+            self._save_cache(new_data, known, loaded)
+
+            # Update state
+            self._prev = {mid: dict(metrics) for mid, metrics in new_data.items()}
+            self._known_models = known
+            self._data = new_data
+            self._loaded = loaded
+            self._deltas = deltas
+
+    def snapshot(self):
+        """Return a frozen copy of the current cache state including deltas."""
+        with self._lock:
+            return {
+                "data": {k: dict(v) for k, v in self._data.items()},
+                "known": set(self._known_models),
+                "health": dict(self._health) if self._health else {"status": "error", "model": "unknown"},
+                "loaded": dict(self._loaded),
+                "deltas": {k: dict(v) for k, v in self._deltas.items()},
+            }
+
+
+cache = Cache()
+
+
+def _fetch_json(url, timeout=SCRAPE_TIMEOUT):
+    try:
+        req = urllib.request.Request(url, headers={"Accept": "application/json"})
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return json.loads(resp.read().decode("utf-8"))
+    except Exception as e:
+        logger.debug("Failed to fetch %s: %s", url, e)
+        return None
+
+
+def _fetch_text(url, timeout=SCRAPE_TIMEOUT):
+    try:
+        req = urllib.request.Request(url, headers={"Accept": "text/plain"})
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return resp.read().decode("utf-8")
+    except Exception as e:
+        logger.debug("Failed to fetch %s: %s", url, e)
+        return None
+
+
+def _scrape_cycle():
+    """One full scrape cycle: discover models, then scrape metrics per model."""
+    targets = []
+    env_targets = os.environ.get("LLAMA_TARGETS", "")
+    if env_targets:
+        try:
+            targets = json.loads(env_targets)
+        except json.JSONDecodeError:
+            targets = []
+    if not targets:
+        targets = LLAMA_CPP_DEFAULTS
+
+    all_metric_lines = []
+    models_data = None
+    known_models = {}  # { model_id: base_url, ... }
+
+    for target in targets:
+        base_url = target["url"].rstrip("/")
+
+        # Fetch /models to discover models and their status
+        models_data = _fetch_json(f"{base_url}/models")
+
+        if models_data and isinstance(models_data, dict):
+            model_list = models_data.get("data", [])
+            for m in model_list:
+                if not isinstance(m, dict) or "id" not in m:
+                    continue
+                model_id = m["id"]
+                known_models[model_id] = base_url
+                # Only scrape metrics from loaded models to avoid triggering loads
+                status_data = m.get("status", {})
+                if isinstance(status_data, dict) and status_data.get("value") != "loaded":
+                    continue
+                # Scrape /metrics for loaded models only
+                metrics_url = f"{base_url}/metrics?model={model_id}"
+                body = _fetch_text(metrics_url)
+                if body:
+                    for line in body.splitlines():
+                        parsed = _parse_metric_line(line)
+                        if parsed:
+                            metric_name, metric_value = parsed
+                            if metric_name in ALL_METRICS:
+                                all_metric_lines.append((metric_name, model_id, metric_value))
+                else:
+                    logger.debug("No metrics body for model %s", model_id)
+
+    # Update the shared cache
+    cache.update(models_data, all_metric_lines, known_models)
+
+
+def _parse_metric_line(line):
+    """Parse a single Prometheus metric line. Returns (name, value) or None."""
+    line = line.strip()
+    if not line or line.startswith("#"):
+        return None
+    try:
+        # Handle lines with labels: metric_name{label="val"} value
+        if "{" in line:
+            parts = line.split("{")
+            name = parts[0].strip()
+            rest = parts[1]
+            # value is the last token after the closing }
+            value = rest.rsplit("}", 1)[-1].strip().split()[-1] if "}" in rest else rest.strip()
+        else:
+            parts = line.split()
+            name = parts[0]
+            value = parts[1] if len(parts) >= 2 else "1"
+        # Try to parse as float
+        float(value)
+        return (name, float(value))
+    except (ValueError, IndexError):
+        return None
+
+
+def _background_scrape():
+    """Background thread: periodically scrape and update cache."""
+    logger.info("Background scraper started (interval=%ds)", POLL_INTERVAL)
+    # Do one immediate scrape on startup
+    _scrape_cycle()
+    while True:
+        try:
+            time.sleep(POLL_INTERVAL)
+            _scrape_cycle()
+        except Exception as e:
+            logger.error("Scrape cycle error: %s", e)
+
+
+class MetricsHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path != "/metrics":
+            self.send_response(404)
+            self.end_headers()
+            return
+
+        snap = cache.snapshot()
+        lines = []
+
+        def _fmt(metric_name, model_id, value):
+            return metric_name + '{' + 'server="llama-cpp-11434",model="' + model_id + '"} ' + str(value)
+
+        # Health metric
+        health = snap["health"]
+        status = health.get("status", "error")
+        health_model = health.get("model", "unknown")
+        health_val = 1.0 if status == "ok" else 0.0
+        lines.append(_fmt("llama_server_health", health_model, health_val))
+
+        # Loaded metrics
+        for mid in snap["loaded"]:
+            lines.append(_fmt("llama_models_loaded", mid, snap["loaded"][mid]))
+
+        # Per-model metrics from cache (absolute values)
+        for mid in sorted(snap["data"]):
+            metrics = snap["data"][mid]
+            for mname in ALL_METRICS:
+                value = metrics.get(mname, 0.0)
+                lines.append(_fmt(mname, mid, value))
+
+        # Per-model delta metrics (counters as change since last scrape)
+        deltas = snap.get("deltas", {})
+        for mid in sorted(deltas):
+            delta_metrics = deltas[mid]
+            for mname in sorted(delta_metrics):
+                value = delta_metrics[mname]
+                lines.append(_fmt(mname + "_delta", mid, value))
+
+        body = "\n".join(lines) + "\n" if lines else "# no metrics available\n"
+
+        self.send_response(200)
+        self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+        self.end_headers()
+        self.wfile.write(body.encode("utf-8"))
+
+    def log_message(self, format, *args):
+        logger.debug("%s - - %s", self.address_string(), format % args)
+
+
+def main():
+    port = int(os.environ.get("LLAMA_EXPORTER_PORT", "9550"))
+    host = os.environ.get("LLAMA_EXPORTER_BIND", "0.0.0.0")
+
+    # Start background scraper thread
+    scraper = threading.Thread(target=_background_scrape, daemon=True)
+    scraper.start()
+
+    # Start HTTP server
+    server = HTTPServer((host, port), MetricsHandler)
+    logger.info("Starting Llama Exporter on %s:%d (interval=%ds, timeout=%ds, cache=%s)",
+                host, port, POLL_INTERVAL, SCRAPE_TIMEOUT, CACHE_FILE)
+
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        logger.info("Shutting down")
+        server.shutdown()
+
+
+if __name__ == "__main__":
+    main()

ansible/roles/llama_server/files/scripts/test_llama_exporter.py

@@ -0,0 +1,231 @@
+#!/usr/bin/env python3
+"""Tests for llama_exporter cache persistence and delta computation."""
+
+import json
+import os
+import sys
+import tempfile
+import shutil
+import unittest
+
+# Setup temp cache file before importing the module
+TEMP_DIR = tempfile.mkdtemp()
+os.environ["LLAMA_EXPORTER_CACHE"] = os.path.join(TEMP_DIR, "test_cache.json")
+
+# Import from parent directory (scripts)
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from llama_exporter import Cache
+
+
+class TestCachePersistenceAndDeltas(unittest.TestCase):
+
+    def setUp(self):
+        # Ensure temp dir exists (pytest may clean it between test methods)
+        os.makedirs(TEMP_DIR, exist_ok=True)
+        self.cache = Cache()
+
+    def tearDown(self):
+        shutil.rmtree(TEMP_DIR, ignore_errors=True)
+
+    def test_cold_start_no_deltas(self):
+        """First scrape should produce no deltas since there's no previous state."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+                {"id": "model-v2", "status": {"value": "unloaded"}},
+            ]
+        }
+        metric_lines = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 100.0),
+            ("llamacpp:prompt_seconds_total", "model-v1", 2.5),
+            ("llamacpp:n_tokens_max", "model-v1", 4096.0),
+        ]
+        known_models = {"model-v1": "http://localhost:11434", "model-v2": "http://localhost:11434"}
+
+        self.cache.update(models_data, metric_lines, known_models)
+
+        snap = self.cache.snapshot()
+        # No deltas on first scrape
+        self.assertEqual(snap["deltas"], {})
+        # Absolute values are correct
+        self.assertEqual(snap["data"]["model-v1"]["llamacpp:prompt_tokens_total"], 100.0)
+        # Unloaded model has zeros
+        self.assertEqual(snap["data"]["model-v2"]["llamacpp:prompt_tokens_total"], 0.0)
+        # Only loaded model is in loaded set
+        self.assertIn("model-v1", snap["loaded"])
+        self.assertNotIn("model-v2", snap["loaded"])
+
+    def test_delta_computation(self):
+        """Second scrape should produce deltas from first scrape."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+            ]
+        }
+
+        # First scrape
+        metric_lines_1 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 100.0),
+            ("llamacpp:prompt_seconds_total", "model-v1", 2.5),
+        ]
+        known_models_1 = {"model-v1": "http://localhost:11434"}
+        self.cache.update(models_data, metric_lines_1, known_models_1)
+
+        # Second scrape - values increased
+        metric_lines_2 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 250.0),
+            ("llamacpp:prompt_seconds_total", "model-v1", 5.0),
+        ]
+        self.cache.update(models_data, metric_lines_2, known_models_1)
+
+        snap = self.cache.snapshot()
+        # Deltas should be computed
+        self.assertIn("model-v1", snap["deltas"])
+        self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_tokens_total"], 150.0)
+        self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_seconds_total"], 2.5)
+
+    def test_counter_reset(self):
+        """If a counter value goes backward, delta should be 0."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+            ]
+        }
+
+        # First scrape
+        metric_lines_1 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 100.0),
+        ]
+        known_models_1 = {"model-v1": "http://localhost:11434"}
+        self.cache.update(models_data, metric_lines_1, known_models_1)
+
+        # Second scrape - counter reset (model was reloaded)
+        metric_lines_2 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 10.0),
+        ]
+        self.cache.update(models_data, metric_lines_2, known_models_1)
+
+        snap = self.cache.snapshot()
+        # On counter reset, delta is implicitly 0 (no entry in deltas)
+        self.assertNotIn("model-v1", snap["deltas"])
+
+    def test_gauge_metrics_no_delta(self):
+        """Gauge metrics should not get delta entries (only counters do)."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+            ]
+        }
+
+        # First scrape
+        metric_lines_1 = [
+            ("llamacpp:n_tokens_max", "model-v1", 4096.0),
+            ("llamacpp:requests_processing", "model-v1", 2.0),
+        ]
+        known_models_1 = {"model-v1": "http://localhost:11434"}
+        self.cache.update(models_data, metric_lines_1, known_models_1)
+
+        # Second scrape
+        metric_lines_2 = [
+            ("llamacpp:n_tokens_max", "model-v1", 8192.0),
+            ("llamacpp:requests_processing", "model-v1", 5.0),
+        ]
+        self.cache.update(models_data, metric_lines_2, known_models_1)
+
+        snap = self.cache.snapshot()
+        # Gauge metrics should not have deltas
+        self.assertNotIn("model-v1", snap["deltas"])
+
+    def test_persistence_across_restarts(self):
+        """Delta computation should work across cache reloads from disk."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+            ]
+        }
+
+        # First scrape (first process instance)
+        metric_lines_1 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 100.0),
+            ("llamacpp:prompt_seconds_total", "model-v1", 2.5),
+        ]
+        known_models_1 = {"model-v1": "http://localhost:11434"}
+        self.cache.update(models_data, metric_lines_1, known_models_1)
+
+        # Simulate restart: create new Cache instance (loads from disk)
+        self.cache._save_cache(self.cache._data, self.cache._known_models, self.cache._loaded)
+
+        # Create a new cache instance (simulates restart)
+        new_cache = Cache()
+
+        # Second scrape after restart
+        metric_lines_2 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 300.0),
+            ("llamacpp:prompt_seconds_total", "model-v1", 8.0),
+        ]
+        known_models_2 = {"model-v1": "http://localhost:11434"}
+        new_cache.update(models_data, metric_lines_2, known_models_2)
+
+        snap = new_cache.snapshot()
+        # Deltas should be computed using persisted previous state
+        self.assertIn("model-v1", snap["deltas"])
+        self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_tokens_total"], 200.0)
+        self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_seconds_total"], 5.5)
+
+    def test_unloaded_model_zero_values(self):
+        """Unloaded models should appear with zero values but never trigger /metrics calls."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+                {"id": "model-v2", "status": {"value": "unloaded"}},
+                {"id": "model-v3", "status": {"value": "unloaded"}},
+            ]
+        }
+        metric_lines = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 50.0),
+        ]
+        known_models = {
+            "model-v1": "http://localhost:11434",
+            "model-v2": "http://localhost:11434",
+            "model-v3": "http://localhost:11434",
+        }
+
+        self.cache.update(models_data, metric_lines, known_models)
+
+        snap = self.cache.snapshot()
+        # Loaded model has its actual value
+        self.assertEqual(snap["data"]["model-v1"]["llamacpp:prompt_tokens_total"], 50.0)
+        # Unloaded models have zeros for all metrics
+        for m in ["model-v2", "model-v3"]:
+            self.assertEqual(snap["data"][m]["llamacpp:prompt_tokens_total"], 0.0)
+            self.assertEqual(snap["data"][m]["llamacpp:n_tokens_max"], 0.0)
+
+    def test_removed_model_zeroed(self):
+        """Models removed from llama-server should be zeroed out."""
+        models_data = {
+            "data": [
+                {"id": "model-v1", "status": {"value": "loaded"}},
+            ]
+        }
+
+        # First scrape with two models
+        metric_lines_1 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 100.0),
+        ]
+        known_models_1 = {"model-v1": "http://localhost:11434"}
+        self.cache.update(models_data, metric_lines_1, known_models_1)
+
+        # Second scrape with only model-v1 (model-v2 was removed)
+        metric_lines_2 = [
+            ("llamacpp:prompt_tokens_total", "model-v1", 200.0),
+        ]
+        known_models_2 = {"model-v1": "http://localhost:11434"}
+        self.cache.update(models_data, metric_lines_2, known_models_2)
+
+        snap = self.cache.snapshot()
+        # model-v1 delta should be 100
+        self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_tokens_total"], 100.0)
+
+
+if __name__ == "__main__":
+    unittest.main()

ansible/roles/llama_server/handlers/main.yml

@@ -0,0 +1,13 @@
+---
+- name: Reload systemd daemon
+  become: true
+  ansible.builtin.systemd:
+    daemon_reload: true
+  listen: Reload systemd daemon
+
+- name: Restart llama_exporter
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ llama_server_exporter_service_name }}"
+    state: restarted
+  listen: Restart llama_exporter

ansible/roles/llama_server/meta/main.yml

@@ -0,0 +1,13 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Sync llama models from Hugging Face
+  company: A13Labs
+  role_name: llama_models_sync
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: EL
+      versions:
+        - "9"

ansible/roles/llama_server/tasks/llama_exporter.yml

@@ -2,16 +2,16 @@
 - name: Create llama_exporter group
   become: true
   ansible.builtin.group:
-    name: "{{ llama_exporter_user }}"
+    name: "{{ llama_server_exporter_user }}"
     state: present
     system: true
 
 - name: Create llama_exporter user
   become: true
   ansible.builtin.user:
-    name: "{{ llama_exporter_user }}"
-    group: "{{ llama_exporter_user }}"
-    home: "{{ llama_exporter_user_home }}"
+    name: "{{ llama_server_exporter_user }}"
+    group: "{{ llama_server_exporter_user }}"
+    home: "{{ llama_server_exporter_user_home }}"
     create_home: true
     shell: /usr/sbin/nologin
     system: true
@@ -21,20 +21,20 @@
   ansible.builtin.file:
     path: "{{ item }}"
     state: directory
-    owner: "{{ llama_exporter_user }}"
-    group: "{{ llama_exporter_user }}"
+    owner: "{{ llama_server_exporter_user }}"
+    group: "{{ llama_server_exporter_user }}"
     mode: "0755"
   loop:
-    - "{{ llama_exporter_install_dir }}"
-    - "{{ llama_exporter_install_dir }}/logs"
+    - "{{ llama_server_exporter_install_dir }}"
+    - "{{ llama_server_exporter_install_dir }}/logs"
 
 - name: Copy llama_exporter script
   become: true
   ansible.builtin.copy:
-    src: llama_exporter.py
-    dest: "{{ llama_exporter_script_path }}"
-    owner: "{{ llama_exporter_user }}"
-    group: "{{ llama_exporter_user }}"
+    src: scripts/llama_exporter.py
+    dest: "{{ llama_server_exporter_script_path }}"
+    owner: "{{ llama_server_exporter_user }}"
+    group: "{{ llama_server_exporter_user }}"
     mode: "0755"
   notify:
     - Restart llama_exporter
@@ -42,31 +42,22 @@
 - name: Create Python virtual environment
   become: true
   ansible.builtin.command:
-    cmd: "{{ ansible_facts['python']['executable'] }} -m venv {{ llama_exporter_venv_path }}"
-    creates: "{{ llama_exporter_venv_path }}/bin/activate"
-
-- name: Install Python dependencies in llama_exporter venv
-  become: true
-  ansible.builtin.pip:
-    name:
-      - prometheus-client
-      - requests
-    virtualenv: "{{ llama_exporter_venv_path }}"
-    state: present
+    cmd: "{{ ansible_facts['python']['executable'] }} -m venv {{ llama_server_exporter_venv_path }}"
+    creates: "{{ llama_server_exporter_venv_path }}/bin/activate"
 
 - name: Fix ownership of llama_exporter directory
   become: true
   ansible.builtin.file:
-    path: "{{ llama_exporter_install_dir }}"
-    owner: "{{ llama_exporter_user }}"
-    group: "{{ llama_exporter_user }}"
+    path: "{{ llama_server_exporter_install_dir }}"
+    owner: "{{ llama_server_exporter_user }}"
+    group: "{{ llama_server_exporter_user }}"
     recurse: true
 
 - name: Generate llama_exporter systemd service
   become: true
   ansible.builtin.template:
     src: llama_exporter.service.j2
-    dest: "/etc/systemd/system/{{ llama_exporter_service_name }}.service"
+    dest: "/etc/systemd/system/{{ llama_server_exporter_service_name }}.service"
     owner: root
     group: root
     mode: "0644"
@@ -77,6 +68,6 @@
 - name: Enable llama_exporter service
   become: true
   ansible.builtin.systemd:
-    name: "{{ llama_exporter_service_name }}"
+    name: "{{ llama_server_exporter_service_name }}"
     enabled: true
     state: started

ansible/roles/llama_server/tasks/main.yml

@@ -0,0 +1,358 @@
+---
+- name: Validate llama_server_models schema
+  ansible.builtin.assert:
+    that:
+      - llama_server_models is iterable
+    fail_msg: "llama_server_models must be a list"
+
+- name: Create group
+  become: true
+  ansible.builtin.group:
+    name: "{{ llama_server_group }}"
+    system: true
+
+- name: Create user
+  become: true
+  ansible.builtin.user:
+    name: "{{ llama_server_user }}"
+    groups: "{{ podman_extra_groups | default([]) }}"
+    shell: /bin/bash
+    home: "{{ llama_server_home }}"
+    group: "{{ llama_server_group }}"
+
+- name: Disable password login for user
+  ansible.builtin.command: passwd -d "{{ llama_server_user }}"
+  become: true
+  changed_when: false
+
+- name: Create user home
+  become: true
+  ansible.builtin.file:
+    path: "{{ llama_server_home }}"
+    state: directory
+    owner: "{{ llama_server_user }}"
+    group: "{{ llama_server_group }}"
+    mode: "0750"
+
+- name: Set fact podman_binary (Debian family)
+  ansible.builtin.set_fact:
+    podman_binary: /usr/bin/podman
+  when: ansible_os_family == 'Debian'
+
+- name: Set fact podman_binary (Red Hat family)
+  ansible.builtin.set_fact:
+    podman_binary: /usr/sbin/podman
+  when: ansible_os_family == 'RedHat'
+
+- name: Create required folders
+  become: true
+  ansible.builtin.file:
+    path: "{{ llama_server_home }}/{{ item }}"
+    state: directory
+    owner: "{{ llama_server_user }}"
+    group: "{{ llama_server_group }}"
+    mode: "0750"
+  loop:
+    - models
+    - .cache
+    - .cache/llama.cpp
+
+- name: "Add SSH Authorized key"
+  become: true
+  ansible.posix.authorized_key:
+    user: "{{ llama_server_user }}"
+    key: "{{ llama_server_pubkey }}"
+    state: "{{ 'present' if llama_server_pubkey is defined and llama_server_pubkey != '' else 'absent' }}"
+  when: llama_server_pubkey is defined and llama_server_pubkey != ""
+
+- name: SELinux tasks (RedHat only)
+  when: ansible_os_family == 'RedHat'
+  become: true
+  block:
+    - name: Persistently set SELinux context
+      community.general.sefcontext:
+        target: "{{ llama_server_home }}(/.*)?"
+        setype: user_home_dir_t
+        state: present
+
+    - name: Persistently set SELinux context (ssh authorized keys)
+      community.general.sefcontext:
+        target: "{{ llama_server_home }}/.ssh/authorized_keys"
+        setype: ssh_home_t
+        state: present
+      when: llama_server_pubkey is defined and llama_server_pubkey != ""
+
+    - name: Apply SELinux context
+      ansible.builtin.command: restorecon -Rv {{ llama_server_home }}
+      changed_when: false
+
+- name: Enable sudo access to llama user (temporary)
+  become: true
+  ansible.builtin.lineinfile:
+    path: "/etc/sudoers.d/ansible_{{ llama_server_user }}_tmp"
+    create: true
+    mode: "0440"
+    line: "{{ ansible_facts['user_id'] }} ALL=({{ llama_server_user }}) NOPASSWD: ALL"
+    validate: "visudo -cf %s"
+
+- name: Enable lingering for llama user
+  become: true
+  ansible.builtin.command: loginctl enable-linger {{ llama_server_user }}
+  register: linger_status
+  changed_when: >-
+    'created' in linger_status.stdout or
+    (linger_status.rc == 0 and not
+      ('linger file already exists' in linger_status.stderr or
+      'linger file does not exist' in linger_status.stderr))
+  failed_when:
+    - linger_status.rc != 0 and not
+      ('linger file already exists' in linger_status.stderr)
+
+- name: Get current user's UID
+  ansible.builtin.command: id -u {{ llama_server_user }}
+  changed_when: false
+  register: uid
+
+- name: Get current user's GID
+  ansible.builtin.command: id -g {{ llama_server_user }}
+  changed_when: false
+  register: gid
+
+- name: Run tasks as podman user
+  become: true
+  become_user: "{{ llama_server_user }}"
+  block:
+    - name: Pull the requested images
+      containers.podman.podman_image:
+        name: "{{ llama_server_image }}"
+        tag: "{{ llama_server_tag }}"
+        state: present
+
+    - name: Create pods
+      containers.podman.podman_container:
+        name: "llama.cpp"
+        image: "{{ llama_server_image }}:{{ llama_server_tag }}"
+        state: started
+        device: "{{ llama_server_devices | default(omit) }}"
+        env: "{{ llama_server_env | default(omit) }}"
+        ports: "0.0.0.0:{{ llama_server_port }}:8080/tcp"
+        volumes:
+          - "{{ llama_server_home }}/models:/models:Z"
+          - "{{ llama_server_home }}/.cache:/app/.cache:Z"
+        command:
+          - "-t"
+          - "{{ llama_server_argv_threads | default(10) }}"
+          - "-np"
+          - "{{ llama_server_argv_parallel | default(1) }}"
+          - "-b"
+          - "{{ llama_server_argv_batch_size | default(1024) }}"
+          - "-ub"
+          - "{{ llama_server_argv_ubatch_size | default(512) }}"
+          - "-fa"
+          - "{{ llama_server_argv_flash_attn | default('on') }}"
+          - "-ctk"
+          - "{{ llama_server_argv_cache_type_k | default('q4_0') }}"
+          - "-ctv"
+          - "{{ llama_server_argv_cache_type_v | default('q8_0') }}"
+          - "-cram"
+          - "{{ llama_server_argv_cache_reuse | default(-1) }}"
+          - "-sm"
+          - "{{ llama_server_argv_split_mode | default('layer') }}"
+          - "-dev"
+          - "{{ llama_server_argv_devices | default('Vulkan1,Vulkan2') }}"
+          - "-ts"
+          - "{{ llama_server_argv_tensor_split | default('8,12') }}"
+          - "-fit"
+          - "{{ llama_server_argv_fit | default('off') }}"
+          - "-mg"
+          - "{{ llama_server_argv_main_gpu | default(1) }}"
+          - "--mmap"
+          - "--metrics"
+          - "--models-preset"
+          - "/models/.router/models.ini"
+          - "--models-max"
+          - "{{ llama_server_models_max }}"
+
+        cmd_args:
+          - "--userns=keep-id"
+          - "--security-opt=label=disable"
+
+- name: Create systemd service file for pods
+  become: true
+  ansible.builtin.template:
+    src: podman.service.j2
+    dest: "/etc/systemd/system/podman-llama.cpp.service"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Reload systemd daemon
+  become: true
+  ansible.builtin.systemd:
+    daemon_reload: true
+
+- name: Enable containers at boot
+  become: true
+  ansible.builtin.systemd:
+    name: "podman-llama.cpp"
+    enabled: true
+    state: "started"
+
+- name: Install Python packages required for llama server model sync
+  become: true
+  ansible.builtin.package:
+    name: "{{ __llama_sync_system_packages__ }}"
+    state: present
+  vars:
+    __llama_sync_system_packages__: >-
+      {{
+        {
+          'Debian': ['python3-pip', 'python3-packaging', 'python3-venv'],
+          'RedHat': ['python3-pip', 'python3-packaging'],
+          'Archlinux': ['python-pip', 'python-packaging']
+        }[ansible_os_family]
+      }}
+
+- name: Install Hugging Face hub client in llama server sync virtualenv
+  become: true
+  become_user: "{{ llama_server_user }}"
+  ansible.builtin.pip:
+    name:
+      - pip
+      - setuptools
+      - packaging
+      - huggingface_hub>=0.32.0
+    state: present
+    virtualenv: "{{ llama_server_home }}/.router/.venv"
+    virtualenv_command: "python3 -m venv"
+
+- name: Create llama server router model directories
+  become: true
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: directory
+    owner: "{{ llama_server_user }}"
+    group: "{{ llama_server_group }}"
+    mode: "{{ item.mode }}"
+  loop:
+    - { path: "{{ llama_server_home }}/models", mode: "0750" }
+    - { path: "{{ llama_server_home }}/models/managed", mode: "0750" }
+    - { path: "{{ llama_server_home }}/models/managed-links", mode: "0750" }
+    - { path: "{{ llama_server_home }}/models/.router", mode: "0750" }
+
+- name: Copy llama server Hugging Face sync helper
+  become: true
+  ansible.builtin.copy:
+    src: scripts/llama_hf_sync.py
+    dest: "{{ llama_server_home }}/models/.router/llama_hf_sync.py"
+    owner: "{{ llama_server_user }}"
+    group: "{{ llama_server_group }}"
+    mode: "0750"
+
+- name: Write desired llama server model manifest input
+  become: true
+  ansible.builtin.copy:
+    content: "{{ llama_server_models | to_nice_json }}"
+    dest: "{{ llama_server_home }}/models/.router/desired-models.json"
+    owner: "{{ llama_server_user }}"
+    group: "{{ llama_server_group }}"
+    mode: "0640"
+
+- name: Write global llama server preset options
+  become: true
+  ansible.builtin.copy:
+    content: "{{ llama_server_preset_global | to_nice_json }}"
+    dest: "{{ llama_server_home }}/models/.router/preset-global.json"
+    owner: "{{ llama_server_user }}"
+    group: "{{ llama_server_group }}"
+    mode: "0640"
+
+- name: Set arguments for llama server model sync
+  ansible.builtin.set_fact:
+    __llama_preset_file__:
+      - "{{ llama_server_home }}/.router/.venv/bin/python"
+      - "{{ llama_server_home }}/models/.router/llama_hf_sync.py"
+      - "--models-file"
+      - "{{ llama_server_home }}/models/.router/desired-models.json"
+      - "--managed-dir"
+      - "{{ llama_server_home }}/models/managed"
+      - "--links-dir"
+      - "{{ llama_server_home }}/models/managed-links"
+      - "--manifest-file"
+      - "{{ llama_server_home }}/models/.router/manifest.json"
+      - "--preset-file"
+      - "{{ llama_server_home }}/models/.router/models.ini"
+      - "--preset-global-file"
+      - "{{ llama_server_home }}/models/.router/preset-global.json"
+      - "--container-links-dir"
+      - "/models/managed-links"
+
+- name: Run llama server model sync from Hugging Face
+  vars:
+    __llama_sync_argv__: >-
+      {{ __llama_preset_file__
+        + (['--prune'] if llama_server_models_prune_enabled | default(true) else [])
+        + (['--dry-run'] if llama_server_models_dry_run | default(false) else [])
+      }}
+  become: true
+  become_user: "{{ llama_server_user }}"
+  ansible.builtin.command:
+    argv: "{{ __llama_sync_argv__ }}"
+  environment:
+    HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') | default('', true) }}"
+  register: __llama_sync_output__
+  changed_when: false
+
+- name: Parse llama server sync output
+  ansible.builtin.set_fact:
+    __llama_sync_result__: "{{ __llama_sync_output__.stdout | from_json }}"
+
+- name: Stat llama router preset file (when not in dry-run mode)
+  become: true
+  become_user: "{{ llama_server_user }}"
+  ansible.builtin.stat:
+    path: "{{ llama_server_home }}/models/.router/models.ini"
+  register: __llama_preset_stat__
+  when: not (llama_server_models_dry_run | default(false))
+
+- name: Ensure router preset file exists when not in dry-run mode
+  ansible.builtin.assert:
+    that:
+      - __llama_preset_stat__.stat.exists
+    fail_msg: "llama router preset file was not created"
+  when: not (llama_server_models_dry_run | default(false))
+
+- name: Query llama router model list
+  ansible.builtin.uri:
+    url: "http://127.0.0.1:{{ llama_server_port | default(8080) }}/models?reload=1"
+    method: GET
+    status_code: 200
+  register: __llama_router_models__
+  changed_when: false
+  when:
+    - llama_server_models_sync_enabled | default(true)
+    - not (llama_server_models_dry_run | default(false))
+
+- name: Validate expected model aliases are available in router
+  ansible.builtin.assert:
+    that:
+      - item.name in (__llama_router_models__.json.data | map(attribute='id') | list)
+    fail_msg: "Model name '{{ item.name }}' not present in llama router /models output"
+  loop: "{{ __llama_sync_result__.models | default([]) }}"
+  loop_control:
+    label: "{{ item.name }}"
+  when:
+    - llama_server_models_sync_enabled | default(true)
+    - not (llama_server_models_dry_run | default(false))
+
+- name: Include Llama Exporter setup
+  when: llama_server_exporter_enabled | default(true) | bool
+  ansible.builtin.include_tasks: llama_exporter.yml
+  tags:
+    - roles::llama_server::llama_exporter
+
+- name: Remove temporary sudo access
+  become: true
+  ansible.builtin.file:
+    path: "/etc/sudoers.d/ansible_{{ llama_server_user }}_tmp"
+    state: absent

ansible/roles/llama_server/templates/llama_exporter.service.j2

@@ -0,0 +1,35 @@
+[Unit]
+Description=LLM Inference Metrics Exporter (llama.cpp + Ollama)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User={{ llama_server_exporter_user }}
+Group={{ llama_server_exporter_user }}
+WorkingDirectory={{ llama_server_exporter_install_dir }}
+Environment="LLAMA_EXPORTER_PORT={{ llama_server_exporter_port }}"
+Environment="LLAMA_EXPORTER_BIND=0.0.0.0"
+Environment="LLAMA_EXPORTER_INTERVAL={{ llama_server_exporter_scrape_interval }}"
+Environment="LLAMA_EXPORTER_TIMEOUT={{ llama_server_exporter_scrape_timeout }}"
+{% if llama_server_exporter_targets and llama_server_exporter_targets != "" %}
+Environment="LLAMA_TARGETS={{ llama_server_exporter_targets | to_json }}"
+{% endif %}
+Environment="PATH={{ llama_server_exporter_venv_bin }}:/usr/bin:/bin"
+ExecStart={{ llama_server_exporter_venv_bin }}/python3 {{ llama_server_exporter_script_path }}
+Restart=always
+RestartSec=10
+TimeoutStopSec=10
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=llama_exporter
+
+# Security hardening
+ProtectSystem=strict
+ProtectHome=true
+ReadWritePaths={{ llama_server_exporter_install_dir }} /var/log
+NoNewPrivileges=true
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/llama_server/templates/podman.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=Podman file for llama.cpp server
+After=local-fs.target network-online.target nvidia-cdi-generator.service
+
+[Service]
+Type=oneshot
+User={{ llama_server_user }}
+Group={{ llama_server_group }}
+RemainAfterExit=true
+ExecStart={{ podman_binary }} start podman-llama.cpp
+ExecStop={{ podman_binary }} stop podman-llama.cpp
+
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/llama_server/vars/main.yml

@@ -0,0 +1,11 @@
+---
+# Internal role variables (non-overridable)
+
+llama_server_exporter_script_path: "{{ llama_server_exporter_install_dir }}/llama_exporter.py"
+llama_server_exporter_venv_bin: "{{ llama_server_exporter_venv_path }}/bin"
+llama_server_exporter_service_name: "llama_exporter"
+
+# Default llama.cpp targets if none provided
+llama_server_exporter_targets_default:
+  - name: "llama-cpp-11434"
+    url: "http://localhost:11434"

ansible/roles/nvidia_cdi_generator/meta/main.yml

@@ -0,0 +1,13 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Install and configure NVIDIA CDI generator service
+  company: A13Labs
+  role_name: nvidia_cdi_generator
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: EL
+      versions:
+        - "9"

ansible/roles/nvidia_cdi_generator/tasks/main.yml

@@ -1,3 +1,4 @@
+---
 - name: Install nvidia-cdi-generator
   become: true
   ansible.builtin.package:

ansible/roles/podman/defaults/main.yml

@@ -0,0 +1,15 @@
+---
+# podman defaults
+
+podman_user: ""
+podman_group: ""
+podman_user_home: ""
+podman_extra_groups: []
+podman_user_folders: []
+podman_user_pubkey: ""
+podman_network_name: "none"
+podman_build_files: []
+podman_pods: []
+
+# Binary path - will be set by OS family in tasks
+# podman_binary is set as a fact in tasks/main.yml

ansible/roles/podman/meta/main.yml

@@ -0,0 +1,18 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Deploy and manage Podman users, pods, and containers
+  company: A13Labs
+  role_name: podman
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: Fedora
+      versions:
+        - "39"
+        - "40"
+        - "41"
+    - name: EL
+      versions:
+        - "9"

ansible/roles/podman/tasks/main.yml

@@ -1,32 +1,90 @@
-- name: Check if required vars are set
-  ansible.builtin.fail:
-    msg: "Required variables are not set for Podman setup. Please check your playbook variables.)"
-  when:
-    - podman_user is not defined
-    - podman_group is not defined
-    - podman_user_home is not defined
-
+---
 - name: Set fact podman_binary (Debian family)
-  when: ansible_os_family == 'Debian'
   ansible.builtin.set_fact:
     podman_binary: /usr/bin/podman
+  when: ansible_os_family == 'Debian'
 
 - name: Set fact podman_binary (Red Hat family)
-  when: ansible_os_family == 'RedHat'
   ansible.builtin.set_fact:
     podman_binary: /usr/sbin/podman
+  when: ansible_os_family == 'RedHat'
 
-- name: Install Podman and its dependencies
+- name: Create group
   become: true
-  ansible.builtin.package:
-    name:
-      - podman
-      - acl
-    state: present
+  ansible.builtin.group:
+    name: "{{ podman_group }}"
+    system: true
 
-- name: Setup user
-  ansible.builtin.include_tasks:
-    file: podman/user.yml
+- name: Create user
+  become: true
+  ansible.builtin.user:
+    name: "{{ podman_user }}"
+    groups: "{{ podman_extra_groups | default([]) }}"
+    shell: /bin/bash
+    home: "{{ podman_user_home }}"
+    group: "{{ podman_group }}"
+
+- name: Disable password login for user
+  ansible.builtin.command: passwd -d "{{ podman_user }}"
+  become: true
+  changed_when: false
+
+- name: Create user home
+  become: true
+  ansible.builtin.file:
+    path: "{{ podman_user_home }}"
+    state: directory
+    owner: "{{ podman_user }}"
+    group: "{{ podman_group }}"
+    mode: "0750"
+
+- name: Create build folders
+  become: true
+  ansible.builtin.file:
+    path: "{{ podman_user_home }}/build"
+    state: directory
+    owner: "{{ podman_user }}"
+    group: "{{ podman_group }}"
+    mode: "0750"
+
+- name: Create folders
+  become: true
+  ansible.builtin.file:
+    path: "{{ podman_user_home }}/{{ item }}"
+    state: directory
+    owner: "{{ podman_user }}"
+    group: "{{ podman_group }}"
+    mode: "0750"
+  loop: "{{ podman_user_folders }}"
+
+- name: "Add SSH Authorized key"
+  become: true
+  ansible.posix.authorized_key:
+    user: "{{ podman_user }}"
+    key: "{{ podman_user_pubkey }}"
+    state: "{{ 'present' if podman_user_pubkey is defined and podman_user_pubkey != '' else 'absent' }}"
+  when: podman_user_pubkey is defined and podman_user_pubkey != ""
+
+- name: SELinux tasks (RedHat only)
+  when: ansible_os_family == 'RedHat'
+  become: true
+  block:
+    - name: Persistently set SELinux context
+      community.general.sefcontext:
+        target: "{{ podman_user_home }}(/.*)?"
+        setype: user_home_dir_t
+        state: present
+
+    - name: Persistently set SELinux context (ssh authorized keys)
+      community.general.sefcontext:
+        target: "{{ podman_user_home }}/.ssh/authorized_keys"
+        setype: ssh_home_t
+        state: present
+      when: podman_user_pubkey is defined and podman_user_pubkey != ""
+
+    - name: Apply SELinux context
+      ansible.builtin.command: restorecon -Rv {{ podman_user_home }}
+      changed_when: false
 
 - name: Enable sudo access to podman user (temporary)
   become: true
@@ -91,7 +149,7 @@
           extra_args: "--build-arg UID={{ uid.stdout }} --build-arg GID={{ gid.stdout }} --security-opt=label=disable {{ item.build.args | default('') }}"
         tag: "latest"
         state: present
-      loop: "{{ pods }}"
+      loop: "{{ podman_pods }}"
       when: item.build is defined and item.build.dockerfile is defined
 
     - name: Pull the requested images
@@ -99,7 +157,7 @@
         name: "{{ item.repo.image }}"
         tag: "{{ item.repo.tag | default('latest') }}"
         state: present
-      loop: "{{ pods }}"
+      loop: "{{ podman_pods }}"
       when: item.repo is defined and item.repo.image is defined
 
     - name: Create pods
@@ -116,7 +174,7 @@
         cmd_args:
           - "--userns=keep-id"
           - "--security-opt=label=disable"
-      loop: "{{ pods }}"
+      loop: "{{ podman_pods }}"
 
 - name: Create systemd service file for pods
   become: true
@@ -126,7 +184,7 @@
     owner: root
     group: root
     mode: "0644"
-  loop: "{{ pods }}"
+  loop: "{{ podman_pods }}"
 
 - name: Reload systemd daemon
   become: true
@@ -139,7 +197,7 @@
     name: "podman-{{ item.name }}"
     enabled: "{{ item.systemd.enabled | default(true) }}"
     state: "{{ item.systemd.state | default('started') }}"
-  loop: "{{ pods }}"
+  loop: "{{ podman_pods }}"
 
 - name: Remove temporary sudo access
   become: true

ansible/roles/podman/templates/podman.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=Podman service for {{ item.name }}
+After=local-fs.target network-online.target
+
+[Service]
+Type=oneshot
+User={{ podman_user }}
+Group={{ podman_group }}
+RemainAfterExit=true
+ExecStart={{ podman_binary }} start {{ item.name }}
+ExecStop={{ podman_binary }} stop {{ item.name }}
+
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/windows_game/meta/main.yml

@@ -0,0 +1,11 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Windows game optimization (user creation, auto-login, power plan, service disable)
+  company: A13Labs
+  role_name: windows_game
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: windows

ansible/roles/windows_game/tasks/main.yml

@@ -1,3 +1,4 @@
+---
 - name: Create limited GameUser
   ansible.windows.win_user:
     name: "{{ game_user }}"
@@ -177,10 +178,3 @@
       - Power Management: Prefer Maximum Performance
       - Low Latency Mode: Ultra
       - Vertical Sync: Off or Fast Sync (depending on monitor)
-
-# - name: Set custom shell to launcher.bat
-#   ansible.windows.win_regedit:
-#     path: HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon
-#     name: Shell
-#     data: C:\GameShell\launcher.bat
-#     type: string

ansible/roles/windows_prometheus/meta/main.yml

@@ -0,0 +1,11 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Install Prometheus and Windows Exporter on Windows
+  company: A13Labs
+  role_name: windows_prometheus
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: windows

ansible/roles/windows_ssh/meta/main.yml

@@ -0,0 +1,11 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Install and configure OpenSSH Server on Windows
+  company: A13Labs
+  role_name: windows_ssh
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: windows

ansible/roles/windows_ssh/tasks/main.yml

@@ -4,9 +4,6 @@
 #     keys:
 #       - "ssh-ed25519 AAAA..."
 #       - "ssh-rsa AAAA..."
-#   - name: bob
-#     keys:
-#       - "ssh-rsa AAAA..."
 - name: Setting defaults for Windows OpenSSH Server installation
   ansible.builtin.set_fact:
     __ssh_default_ciphers__:
@@ -108,7 +105,7 @@
       Write-Output 'hostkeys-generated'
     } else {
       Write-Output 'ssh-keygen-not-found'
-      exit 2
+      exit(2)
     }
   register: ssh_keygen
   changed_when: "'hostkeys-generated' in ssh_keygen.stdout"
@@ -358,7 +355,7 @@
       if ($before -ne $after) {
         Write-Output 'admin-auth-keys-acl-changed'
       } else {
-        Write-Output 'admin-auth-keys-acl-unchanged'
+        Write-Output 'admin-auth-keys-unchanged'
       }
     } else {
       Write-Output 'no-admin-auth-keys'

ansible/roles/wol/meta/main.yml

@@ -0,0 +1,13 @@
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Configure Wake-on-LAN via systemd-networkd
+  company: A13Labs
+  role_name: wol
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+  platforms:
+    - name: EL
+      versions:
+        - "9"

ansible/roles/wol/tasks/main.yml

@@ -1,3 +1,4 @@
+---
 - name: Install ethtool
   become: true
   ansible.builtin.package:

ansible/tasks/amd-rocm-container.yml

@@ -1,33 +0,0 @@
-- name: Add ROCm repository
-  become: true
-  ansible.builtin.yum_repository:
-    name: rocm
-    description: ROCm Repository
-    baseurl: https://repo.radeon.com/rocm/el9/latest/main/
-    gpgcheck: true
-    gpgkey: https://repo.radeon.com/rocm/rocm.gpg.key
-    enabled: true
-    state: present
-
-- name: Add AMD Graphics repository
-  become: true
-  ansible.builtin.yum_repository:
-    name: rocmgraphics
-    description: AMD Graphics Repository
-    baseurl: https://repo.radeon.com/graphics/latest/el/9.6/main/x86_64/
-    gpgkey: https://repo.radeon.com/rocm/rocm.gpg.key
-    gpgcheck: true
-    enabled: true
-    state: present
-
-- name: Install rocm-dkms and dependencies
-  become: true
-  ansible.builtin.package:
-    name:
-      - rocm
-      - rocm-developer-tools
-      - hipblas-devel
-      - hip-devel
-      - rocwmma-devel
-      - rocm-opencl-devel
-    state: present

ansible/tasks/llama_models_sync.yml

@@ -1,146 +0,0 @@
-- name: Ensure podman user exists before model sync
-  ansible.builtin.include_tasks:
-    file: podman/user.yml
-
-- name: Enable sudo access to podman user (temporary)
-  become: true
-  ansible.builtin.lineinfile:
-    path: "/etc/sudoers.d/ansible_{{ podman_user }}_tmp"
-    create: true
-    mode: "0440"
-    line: "{{ ansible_user_id }} ALL=({{ podman_user }}) NOPASSWD: ALL"
-    validate: "visudo -cf %s"
-
-- name: Validate llama_models schema
-  ansible.builtin.assert:
-    that:
-      - __llama_models__ is iterable
-    fail_msg: "llama_models must be a list"
-
-- name: Install Python packages required for llama model sync
-  become: true
-  ansible.builtin.package:
-    name: "{{ __llama_sync_system_packages__ }}"
-    state: present
-  vars:
-    __llama_sync_system_packages__: >-
-      {{
-        {
-          'Debian': ['python3-pip', 'python3-packaging', 'python3-venv'],
-          'RedHat': ['python3-pip', 'python3-packaging'],
-          'Archlinux': ['python-pip', 'python-packaging']
-        }[ansible_os_family]
-      }}
-
-- name: Install Hugging Face hub client in llama model sync virtualenv
-  become: true
-  become_user: "{{ podman_user }}"
-  ansible.builtin.pip:
-    name:
-      - pip
-      - setuptools
-      - packaging
-      - huggingface_hub>=0.32.0
-    state: present
-    virtualenv: "{{ __llama_sync_venv__ }}"
-    virtualenv_command: "python3 -m venv"
-
-- name: Create llama router model directories
-  become: true
-  ansible.builtin.file:
-    path: "{{ item.path }}"
-    state: directory
-    owner: "{{ podman_user }}"
-    group: "{{ podman_group }}"
-    mode: "{{ item.mode }}"
-  loop:
-    - { path: "{{ podman_user_home }}/models", mode: "0750" }
-    - { path: "{{ __llama_managed_dir__ }}", mode: "0750" }
-    - { path: "{{ __llama_links_dir__ }}", mode: "0750" }
-    - { path: "{{ __llama_router_dir__ }}", mode: "0750" }
-
-- name: Copy llama Hugging Face sync helper
-  become: true
-  ansible.builtin.copy:
-    src: scripts/llama_hf_sync.py
-    dest: "{{ __llama_sync_script__ }}"
-    owner: "{{ podman_user }}"
-    group: "{{ podman_group }}"
-    mode: "0750"
-
-- name: Write desired llama model manifest input
-  become: true
-  ansible.builtin.copy:
-    content: "{{ __llama_models__ | to_nice_json }}"
-    dest: "{{ __llama_models_input_file__ }}"
-    owner: "{{ podman_user }}"
-    group: "{{ podman_group }}"
-    mode: "0640"
-
-- name: Write global llama preset options
-  become: true
-  ansible.builtin.copy:
-    content: "{{ __llama_preset_global__ | to_nice_json }}"
-    dest: "{{ __llama_preset_global_input_file__ }}"
-    owner: "{{ podman_user }}"
-    group: "{{ podman_group }}"
-    mode: "0640"
-
-- name: Run llama model sync from Hugging Face
-  vars:
-    __llama_sync_argv__: >-
-      {{
-        [
-          __llama_sync_python__,
-          __llama_sync_script__,
-          '--models-file',
-          __llama_models_input_file__,
-          '--managed-dir',
-          __llama_managed_dir__,
-          '--links-dir',
-          __llama_links_dir__,
-          '--manifest-file',
-          __llama_manifest_file__,
-          '--preset-file',
-          __llama_preset_file__,
-          '--preset-global-file',
-          __llama_preset_global_input_file__,
-          '--container-links-dir',
-          '/models/managed-links'
-        ]
-        + (['--prune'] if llama_models_prune_enabled | default(true) else [])
-        + (['--dry-run'] if llama_models_dry_run | default(false) else [])
-      }}
-  become: true
-  become_user: "{{ podman_user }}"
-  ansible.builtin.command:
-    argv: "{{ __llama_sync_argv__ }}"
-  environment:
-    HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') | default('', true) }}"
-  register: __llama_sync_output__
-  changed_when: false
-
-- name: Parse llama sync output
-  ansible.builtin.set_fact:
-    __llama_sync_result__: "{{ __llama_sync_output__.stdout | from_json }}"
-
-- name: Ensure router preset file exists when not in dry-run mode
-  become: true
-  become_user: "{{ podman_user }}"
-  ansible.builtin.stat:
-    path: "{{ __llama_preset_file__ }}"
-  register: __llama_preset_stat__
-  when: not (llama_models_dry_run | default(false))
-
-- name: Ensure router preset file exists when not in dry-run mode
-  ansible.builtin.assert:
-    that:
-      - __llama_preset_stat__.stat.exists
-    fail_msg: "llama router preset file was not created"
-  when: not (llama_models_dry_run | default(false))
-
-- name: Remove temporary sudo access
-  become: true
-  ansible.builtin.file:
-    path: "/etc/sudoers.d/ansible_{{ podman_user }}_tmp"
-    state: absent

ansible/tasks/podman/user.yml

@@ -1,84 +0,0 @@
-- name: Check if required vars are set
-  ansible.builtin.fail:
-    msg: "Required variables are not set for Podman setup. Please check your playbook variables.)"
-  when:
-    - podman_user is not defined
-    - podman_group is not defined
-    - podman_user_home is not defined
-
-- name: Create group
-  become: true
-  ansible.builtin.group:
-    name: "{{ podman_group }}"
-    system: true
-
-- name: Create user
-  become: true
-  ansible.builtin.user:
-    name: "{{ podman_user }}"
-    groups: "{{ podman_extra_groups | default([]) }}"
-    shell: /bin/bash
-    home: "{{ podman_user_home }}"
-    group: "{{ podman_group }}"
-
-- name: Disable password login for user
-  ansible.builtin.command: passwd -d "{{ podman_user }}"
-  become: true
-  changed_when: false
-
-- name: Create user home
-  become: true
-  ansible.builtin.file:
-    path: "{{ podman_user_home }}"
-    state: directory
-    owner: "{{ podman_user }}"
-    group: "{{ podman_group }}"
-    mode: "0750"
-
-- name: Create build folders
-  become: true
-  ansible.builtin.file:
-    path: "{{ podman_user_home }}/build"
-    state: directory
-    owner: "{{ podman_user }}"
-    group: "{{ podman_group }}"
-    mode: "0750"
-
-- name: Create folders
-  become: true
-  ansible.builtin.file:
-    path: "{{ podman_user_home }}/{{ item }}"
-    state: directory
-    owner: "{{ podman_user }}"
-    group: "{{ podman_group }}"
-    mode: "0750"
-  loop: "{{ podman_user_folders }}"
-
-- name: "Add SSH Authorized key"
-  become: true
-  ansible.posix.authorized_key:
-    user: "{{ podman_user }}"
-    key: "{{ podman_user_pubkey }}"
-    state: "{{ 'present' if podman_user_pubkey is defined else 'absent' }}"
-  when: podman_user_pubkey is defined and podman_user_pubkey != ""
-
-- name: SELinux tasks (RedHat only)
-  when: ansible_os_family == 'RedHat'
-  become: true
-  block:
-    - name: Persistently set SELinux context
-      community.general.sefcontext:
-        target: "{{ podman_user_home }}(/.*)?"
-        setype: user_home_dir_t
-        state: present
-
-    - name: Persistently set SELinux context (ssh authorized keys)
-      community.general.sefcontext:
-        target: "{{ podman_user_home }}/.ssh/authorized_keys"
-        setype: ssh_home_t
-        state: present
-      when: podman_user_pubkey is defined and podman_user_pubkey != ""
-
-    - name: Apply SELinux context
-      ansible.builtin.command: restorecon -Rv {{ podman_user_home }}
-      changed_when: false

ansible/tmp/gpu-realtime-dashboard.json

@@ -338,14 +338,14 @@
             "targets": [
                 {
                     "editorMode": "code",
-                    "expr": "llama_server_health",
+                    "expr": "llama_server_health{model=~\"$model\"}",
                     "legendFormat": "health {{server}}",
                     "range": true,
                     "refId": "A"
                 },
                 {
                     "editorMode": "code",
-                    "expr": "llama_models_loaded",
+                    "expr": "llama_models_loaded{model=~\"$model\"}",
                     "legendFormat": "models {{server}} {{model}}",
                     "range": true,
                     "refId": "B"
@@ -354,6 +354,156 @@
             "title": "Llama Server Metrics",
             "type": "timeseries"
         },
+        {
+            "datasource": "$datasource",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "unit": "reqps"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 8,
+                "x": 0,
+                "y": 30
+            },
+            "id": 8,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "table",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "pluginVersion": "11.0.0",
+            "targets": [
+                {
+                    "editorMode": "code",
+                    "expr": "rate(llamacpp:tokens_predicted_total{model=~\"$model\"}[5m])",
+                    "legendFormat": "decode {{model}}",
+                    "range": true,
+                    "refId": "A"
+                },
+                {
+                    "editorMode": "code",
+                    "expr": "rate(llamacpp:prompt_tokens_total{model=~\"$model\"}[5m])",
+                    "legendFormat": "prompt {{model}}",
+                    "range": true,
+                    "refId": "B"
+                }
+            ],
+            "title": "Token Throughput",
+            "type": "timeseries"
+        },
+        {
+            "datasource": "$datasource",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "unit": "s"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 8,
+                "x": 8,
+                "y": 30
+            },
+            "id": 9,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "table",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "pluginVersion": "11.0.0",
+            "targets": [
+                {
+                    "editorMode": "code",
+                    "expr": "rate(llamacpp:tokens_predicted_seconds_total{model=~\"$model\"}[5m])",
+                    "legendFormat": "decode time {{model}}",
+                    "range": true,
+                    "refId": "A"
+                },
+                {
+                    "editorMode": "code",
+                    "expr": "rate(llamacpp:prompt_seconds_total{model=~\"$model\"}[5m])",
+                    "legendFormat": "prompt time {{model}}",
+                    "range": true,
+                    "refId": "B"
+                }
+            ],
+            "title": "Processing Time",
+            "type": "timeseries"
+        },
+        {
+            "datasource": "$datasource",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "unit": "short"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 8,
+                "x": 16,
+                "y": 30
+            },
+            "id": 10,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "table",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "pluginVersion": "11.0.0",
+            "targets": [
+                {
+                    "editorMode": "code",
+                    "expr": "llamacpp:requests_processing{model=~\"$model\"}",
+                    "legendFormat": "processing {{model}}",
+                    "range": true,
+                    "refId": "A"
+                },
+                {
+                    "editorMode": "code",
+                    "expr": "llamacpp:requests_deferred{model=~\"$model\"}",
+                    "legendFormat": "deferred {{model}}",
+                    "range": true,
+                    "refId": "B"
+                }
+            ],
+            "title": "Request Queue",
+            "type": "timeseries"
+        },
         {
             "datasource": "$datasource",
             "fieldConfig": {
@@ -369,9 +519,95 @@
                 "h": 8,
                 "w": 12,
                 "x": 0,
-                "y": 30
+                "y": 38
             },
-            "id": 8,
+            "id": 11,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "table",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "pluginVersion": "11.0.0",
+            "targets": [
+                {
+                    "editorMode": "code",
+                    "expr": "llamacpp:n_busy_slots_per_decode{model=~\"$model\"}",
+                    "legendFormat": "busy slots {{model}}",
+                    "range": true,
+                    "refId": "A"
+                }
+            ],
+            "title": "Busy Slots Per Decode",
+            "type": "timeseries"
+        },
+        {
+            "datasource": "$datasource",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "unit": "short"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 38
+            },
+            "id": 12,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "table",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "pluginVersion": "11.0.0",
+            "targets": [
+                {
+                    "editorMode": "code",
+                    "expr": "llamacpp:n_tokens_max{model=~\"$model\"}",
+                    "legendFormat": "max tokens {{model}}",
+                    "range": true,
+                    "refId": "A"
+                }
+            ],
+            "title": "Max Tokens (Context Window)",
+            "type": "timeseries"
+        },
+        {
+            "datasource": "$datasource",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "unit": "short"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 46
+            },
+            "id": 13,
             "options": {
                 "legend": {
                     "calcs": [],
@@ -534,6 +770,30 @@
                 "regex": "",
                 "skipUrlSync": false,
                 "type": "datasource"
+            },
+            {
+                "current": {
+                    "selected": false,
+                    "text": "All",
+                    "value": "$__all"
+                },
+                "datasource": {
+                    "type": "prometheus",
+                    "uid": "$datasource"
+                },
+                "definition": "label_values(llamacpp:tokens_predicted_total, model)",
+                "hide": 0,
+                "includeAll": true,
+                "label": "Model",
+                "multi": false,
+                "name": "model",
+                "options": [],
+                "query": "label_values(llamacpp:tokens_predicted_total, model)",
+                "refresh": 2,
+                "regex": "",
+                "skipUrlSync": false,
+                "sort": 1,
+                "type": "query"
             }
         ]
     },

terraform/apps/dev-01/apps.tf

@@ -3,7 +3,7 @@ module "open-webui" {
 
   persistent_folder = local.persistent_folder
   fqdn              = local.open_webui_fqdn
-  ollama_proxy      = local.ollama_proxy
+  llm_proxy         = local.llm_proxy
   secret_key        = var.webui_secret_key
   tag               = "v0.9.2-slim"
   log_level         = local.open_webui_log_level
@@ -43,13 +43,13 @@ module "nextcloud" {
   issuer      = local.internal_issuer
 }
 
-module "comfyui_proxy" {
-  source = "../../modules/utils/wol_proxy"
+# module "comfyui_proxy" {
+#   source = "../../modules/utils/wol_proxy"
 
-  upstream   = local.comfyui_proxy
-  proxy_port = local.comfyui_proxy.port
-  namespace  = "comfyui"
-}
+#   upstream   = local.comfyui_proxy
+#   proxy_port = local.comfyui_proxy.port
+#   namespace  = "comfyui"
+# }
 
 module "prometheus" {
   source = "../../modules/utils/prometheus"
@@ -73,6 +73,15 @@ module "grafana" {
 
   persistent_folder = "${local.persistent_folder}/monitoring"
   fqdn              = local.grafana_fqdn
+  dashboard_folder = {
+    System     = "${path.module}/resources/grafana/system"
+    Kubernetes = "${path.module}/resources/grafana/kubernetes"
+    Security   = "${path.module}/resources/grafana/security"
+    GPU        = "${path.module}/resources/grafana/gpu"
+    LLM        = "${path.module}/resources/grafana/llm"
+    Windows    = "${path.module}/resources/grafana/windows"
+  }
+  datasource_folder = "${path.module}/resources/grafana"
 }
 
 module "mstream" {
@@ -99,6 +108,16 @@ module "vaultwarden" {
   persistent_folder = local.vaultwarden_folder
   fqdn              = local.vaultwarden_fqdn
   issuer            = local.vaultwarden_issuer
-  tag               = "1.33.4"
+  tag               = local.vaultwarden_version
   admin_token       = var.vaultwarden_admin_token
+  smtp_host         = local.vaultwarden_smtp_host
+  smtp_port         = local.vaultwarden_smtp_port
+  smtp_security     = local.vaultwarden_smtp_security
+  smtp_from         = local.vaultwarden_smtp_from
+  smtp_from_name    = local.vaultwarden_smtp_from_name
+  smtp_username     = local.vaultwarden_smtp_username
+
+  # We temporarily enable signup to create the first admin user, then we will disable it and rotate the admin token
+  # signup_enabled = true
+  # signup_verify  = false
 }

terraform/apps/dev-01/config.tf

@@ -132,22 +132,22 @@ locals {
   ]
 
   proxy_services = {
-    "comfyui" = {
-      http_port  = 8080
-      https_port = 8443
-      tls        = true
-      fqdn       = ["comfyui.lab.alexpires.me"]
-      upstream   = "http://upstream.comfyui.svc.cluster.local:8188"
-      resolver   = "kube-dns.kube-system.svc.cluster.local"
+    # "comfyui" = {
+    #   http_port  = 8080
+    #   https_port = 8443
+    #   tls        = true
+    #   fqdn       = ["comfyui.lab.alexpires.me"]
+    #   upstream   = "http://upstream.comfyui.svc.cluster.local:8188"
+    #   resolver   = "kube-dns.kube-system.svc.cluster.local"
 
-      locations = [
-        {
-          path         = "/"
-          "Upgrade"    = "$http_upgrade"
-          "Connection" = "upgrade"
-        },
-      ]
-    }
+    #   locations = [
+    #     {
+    #       path         = "/"
+    #       "Upgrade"    = "$http_upgrade"
+    #       "Connection" = "upgrade"
+    #     },
+    #   ]
+    # }
   }
 
   streams = {
@@ -171,20 +171,20 @@ locals {
     }
   }
 
-  ollama_proxy = {
+  llm_proxy = {
     enabled = true
     mac     = "04:7c:16:d6:63:66"
     ip      = "10.19.4.106"
-    port    = "11435"
+    port    = 11434
     scheme  = "http"
   }
 
-  comfyui_proxy = {
-    mac    = "04:7c:16:d6:63:66"
-    ip     = "10.19.4.106"
-    port   = "8188"
-    scheme = "http"
-  }
+  # comfyui_proxy = {
+  #   mac    = "04:7c:16:d6:63:66"
+  #   ip     = "10.19.4.106"
+  #   port   = "8188"
+  #   scheme = "http"
+  # }
 
   scaleway_smtp_host = "smtp.tem.scw.cloud"
 
@@ -235,16 +235,16 @@ locals {
 
   grafana_fqdn = "grafana.${local.lab_domain}"
 
-  devolo_devices = {
-    devolo_620 = {
-      ip    = "192.168.0.199"
-      model = "620"
-    }
-    # devolo_326 = {
-    #   ip    = "192.168.0.200"
-    #   model = "326"
-    # }
-  }
+  # devolo_devices = {
+  #   devolo_620 = {
+  #     ip    = "192.168.0.199"
+  #     model = "620"
+  #   }
+  #   # devolo_326 = {
+  #   #   ip    = "192.168.0.200"
+  #   #   model = "326"
+  #   # }
+  # }
 
   # mstream
   mstream_music_folder = "/mnt/storage/Music"
@@ -256,9 +256,17 @@ locals {
   searxng_issuer = "letsencrypt-prod"
 
   # vaultwarden
-  vaultwarden_folder = "/mnt/md0/vaultwarden"
-  vaultwarden_fqdn   = "vaultwarden.${local.lab_domain}"
-  vaultwarden_issuer = "internal-ca"
+  vaultwarden_folder         = "/mnt/md0/vaultwarden"
+  vaultwarden_fqdn           = "vaultwarden.${local.lab_domain}"
+  vaultwarden_issuer         = "letsencrypt-prod"
+  vaultwarden_version        = "1.36.0"
+  vaultwarden_smtp_host      = local.scaleway_smtp_host
+  vaultwarden_smtp_port      = 465
+  vaultwarden_smtp_security  = "force_tls"
+  vaultwarden_smtp_from      = "vaultwarden@${local.lab_domain}"
+  vaultwarden_smtp_from_name = "Vaultwarden"
+  vaultwarden_smtp_username  = var.scaleway_project_id
+  vaultwarden_smtp_password  = local.mail_app_api_secret_key
 }
 
 data "terraform_remote_state" "scaleway" {

terraform/apps/dev-01/resources/grafana/datasource.yml

@@ -0,0 +1,35 @@
+apiVersion: 1
+
+datasources:
+  - name: prometheus-dev-01
+    type: prometheus
+    url: http://prometheus.prometheus.svc.cluster.local:9090
+    isDefault: true
+    editable: false
+    jsonData:
+      httpMethod: POST
+      timeInterval: 15s
+  - name: prometheus-ci-01
+    type: prometheus
+    url: http://ci-01.lab.alexpires.me:9090
+    isDefault: false
+    editable: false
+    jsonData:
+      httpMethod: POST
+      timeInterval: 15s
+  - name: prometheus-gpu-01
+    type: prometheus
+    url: http://gpu-01.lab.alexpires.me:9091
+    isDefault: false
+    editable: false
+    jsonData:
+      httpMethod: POST
+      timeInterval: 15s
+  - name: prometheus-prod-01
+    type: prometheus
+    url: http://10.5.5.2:9090
+    isDefault: false
+    editable: false
+    jsonData:
+      httpMethod: POST
+      timeInterval: 15s

terraform/apps/dev-01/resources/grafana/gpu/gpu-realtime-dashboard.json

@@ -0,0 +1,500 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "liveNow": true,
+  "panels": [
+    {
+      "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 1
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "up{job=~\"prometheus|nvidia-gpu|amdgpu_exporter\"}",
+          "instant": true,
+          "legendFormat": "{{job}}",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "GPU Exporter Health",
+      "type": "stat"
+    },
+    {
+      "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 6
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "{__name__=~\"gpu_.*_gpu_util_percent|gpu_util_percent\"}",
+          "legendFormat": "{{__name__}} {{gpu}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "GPU Utilization",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "celsius"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 6
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "{__name__=~\"gpu_.*_temperature_celsius|temperature_celsius\"}",
+          "legendFormat": "{{__name__}} {{gpu}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "GPU Temperature",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "watt"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 14
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "{__name__=~\"gpu_.*_power_watts|power_watts\"}",
+          "legendFormat": "{{__name__}} {{gpu}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "GPU Power Draw",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 14
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "{__name__=~\"gpu_.*_memory_used_percent\"}",
+          "legendFormat": "nvidia",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "editorMode": "code",
+          "expr": "100 * mem_used_bytes / clamp_min(mem_used_bytes + mem_avail_bytes, 1)",
+          "legendFormat": "amd {{gpu}}",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "GPU Memory Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 22
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "scrape_duration_seconds{job=~\"nvidia-gpu|amdgpu_exporter|prometheus\"}",
+          "legendFormat": "{{job}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "GPU Scrape Duration",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 22
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "{__name__=~\"gpu_.*_pstate\"}",
+          "legendFormat": "{{__name__}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "NVIDIA P-State",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 30
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "{__name__=~\"gpu_.*_pcie_link_gen_current\"}",
+          "legendFormat": "pcie gen",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "editorMode": "code",
+          "expr": "{__name__=~\"gpu_.*_pcie_link_width_current\"}",
+          "legendFormat": "pcie width",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "NVIDIA PCIe Link",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "max": 1,
+          "min": 0,
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 30
+      },
+      "id": 9,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "{__name__=~\"gpu_.*_throttle_sw_power_cap\"}",
+          "legendFormat": "sw power cap",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "editorMode": "code",
+          "expr": "{__name__=~\"gpu_.*_throttle_hw_thermal_slowdown\"}",
+          "legendFormat": "hw thermal slowdown",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "editorMode": "code",
+          "expr": "{__name__=~\"gpu_.*_throttle_hw_power_brake_slowdown\"}",
+          "legendFormat": "hw power brake slowdown",
+          "range": true,
+          "refId": "C"
+        }
+      ],
+      "title": "NVIDIA Throttle Flags",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 39,
+  "style": "dark",
+  "tags": [
+    "gpu",
+    "monitoring",
+    "realtime"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": true,
+          "text": "Prometheus",
+          "value": "Prometheus"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datasource",
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "GPU Monitoring Realtime",
+  "uid": "gpu-monitoring-realtime",
+  "version": 2,
+  "weekStart": ""
+}

terraform/apps/dev-01/resources/grafana/llm/llm-realtime-dashboard.json

@@ -0,0 +1,436 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "liveNow": true,
+  "panels": [
+    {
+      "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 1
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "up{job=~\"prometheus|llama_exporter\"}",
+          "instant": true,
+          "legendFormat": "{{job}}",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "LLM Exporter Health",
+      "type": "stat"
+    },
+    {
+      "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 6
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "llama_server_health{model=~\"$model\"}",
+          "legendFormat": "health {{server}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "editorMode": "code",
+          "expr": "llama_models_loaded{model=~\"$model\"}",
+          "legendFormat": "models {{server}} {{model}}",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Llama Server Metrics",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 6
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "rate(llamacpp:tokens_predicted_total{model=~\"$model\"}[5m])",
+          "legendFormat": "decode {{model}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "editorMode": "code",
+          "expr": "rate(llamacpp:prompt_tokens_total{model=~\"$model\"}[5m])",
+          "legendFormat": "prompt {{model}}",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Token Throughput",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 14
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "rate(llamacpp:tokens_predicted_seconds_total{model=~\"$model\"}[5m])",
+          "legendFormat": "decode time {{model}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "editorMode": "code",
+          "expr": "rate(llamacpp:prompt_seconds_total{model=~\"$model\"}[5m])",
+          "legendFormat": "prompt time {{model}}",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Processing Time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 14
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "llamacpp:requests_processing{model=~\"$model\"}",
+          "legendFormat": "processing {{model}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "editorMode": "code",
+          "expr": "llamacpp:requests_deferred{model=~\"$model\"}",
+          "legendFormat": "deferred {{model}}",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Request Queue",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 22
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "llamacpp:n_busy_slots_per_decode{model=~\"$model\"}",
+          "legendFormat": "busy slots {{model}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Busy Slots Per Decode",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 22
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "llamacpp:n_tokens_max{model=~\"$model\"}",
+          "legendFormat": "max tokens {{model}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Max Tokens (Context Window)",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 39,
+  "style": "dark",
+  "tags": [
+    "llm",
+    "monitoring",
+    "realtime"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": true,
+          "text": "Prometheus",
+          "value": "Prometheus"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datasource",
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "$datasource"
+        },
+        "definition": "label_values(llamacpp:tokens_predicted_total, model)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Model",
+        "multi": false,
+        "name": "model",
+        "options": [],
+        "query": "label_values(llamacpp:tokens_predicted_total, model)",
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "LLM Monitoring Realtime",
+  "uid": "llm-monitoring-realtime",
+  "version": 1,
+  "weekStart": ""
+}

terraform/apps/dev-01/secrets.tf

@@ -39,3 +39,4 @@ variable "vaultwarden_admin_token" {
   type        = string
   sensitive   = true
 }
+

terraform/apps/prod-01/apps.tf

@@ -161,9 +161,20 @@ module "auth_exporter" {
 module "vaultwarden" {
   source = "../../modules/apps/vaultwarden"
 
-  persistent_folder = local.vaultwarden_folder
-  fqdn              = local.vaultwarden_fqdn
-  issuer            = local.internal_issuer
-  tag               = "1.33.4"
-  admin_token       = var.vaultwarden_admin_token
+  persistent_folder   = local.vaultwarden_folder
+  fqdn                = local.vaultwarden_fqdn
+  issuer              = local.vaultwarden_issuer
+  tag                 = local.vaultwarden_version
+  admin_token         = var.vaultwarden_admin_token
+  smtp_host           = local.vaultwarden_smtp_host
+  smtp_port           = local.vaultwarden_smtp_port
+  smtp_security       = local.vaultwarden_smtp_security
+  smtp_from           = local.vaultwarden_smtp_from
+  smtp_from_name      = local.vaultwarden_smtp_from_name
+  smtp_username       = local.vaultwarden_smtp_username
+  smtp_password       = var.vaultwarden_smtp_password
+  smtp_timeout        = local.vaultwarden_smtp_timeout
+  smtp_auth_mechanism = local.vaultwarden_smtp_auth_mechanism
+  smtp_helo_name      = local.vaultwarden_smtp_helo_name
+  smtp_embed_images   = local.vaultwarden_smtp_embed_images
 }

terraform/apps/prod-01/config.tf

@@ -211,9 +211,20 @@ locals {
   telegram_message = file("${path.module}/resources/telegram.tpl")
 
   # vaultwarden
-  vaultwarden_folder = "${local.persistent_folder}/vaultwarden"
-  vaultwarden_fqdn   = "vaultwarden.${local.primary_domain}"
-  internal_issuer    = "internal-ca"
+  vaultwarden_folder              = "${local.persistent_folder}/vaultwarden"
+  vaultwarden_fqdn                = "vaultwarden.${local.primary_domain}"
+  vaultwarden_issuer              = "internal-ca"
+  vaultwarden_version             = "1.36.0"
+  vaultwarden_smtp_host           = ""
+  vaultwarden_smtp_port           = 587
+  vaultwarden_smtp_security       = "starttls"
+  vaultwarden_smtp_from           = ""
+  vaultwarden_smtp_from_name      = "Vaultwarden"
+  vaultwarden_smtp_username       = ""
+  vaultwarden_smtp_timeout        = 15
+  vaultwarden_smtp_auth_mechanism = ""
+  vaultwarden_smtp_helo_name      = ""
+  vaultwarden_smtp_embed_images   = true
 
   # get secrets from remote state
   backup_bucket_name        = data.terraform_remote_state.scaleway.outputs.backup_bucket_name

terraform/apps/prod-01/secrets.tf

@@ -140,3 +140,10 @@ variable "vaultwarden_admin_token" {
   type        = string
   sensitive   = true
 }
+
+variable "vaultwarden_smtp_password" {
+  description = "SMTP password for VaultWarden email"
+  type        = string
+  sensitive   = true
+  default     = ""
+}

terraform/apps/prod-01/sectool.env

@@ -34,3 +34,4 @@ TF_VAR_wireguard_1_pre_shared_key=$WIREGUARD_1_PRE_SHARED_KEY
 TF_VAR_nginx_token_bearer=$NGINX_TOKEN_BEARER
 TF_VAR_telegram_bot_token=$TELEGRAM_BOT_TOKEN
 TF_VAR_vaultwarden_admin_token=$VAULTWARDEN_ADMIN_TOKEN
+TF_VAR_vaultwarden_smtp_password=$VAULTWARDEN_SMTP_PASSWORD

terraform/modules/apps/grafana/config.tf

@@ -2,8 +2,47 @@ locals {
   app_version       = var.tag
   monitoring_folder = "${var.persistent_folder}/grafana"
 
-  grafana_datasource_config      = file("${path.module}/resources/default.yml")
-  grafana_dashboard_provider     = file("${path.module}/resources/dashboard-provider.yml")
-  grafana_trivy_dashboard        = file("${path.module}/resources/trivy-vulnerabilities-dashboard.json")
-  grafana_gpu_realtime_dashboard = file("${path.module}/resources/gpu-realtime-dashboard.json")
+  dashboard_resource_names = {
+    for folder_name in keys(var.dashboard_folder) :
+    folder_name => "${substr(replace(lower(folder_name), "/[^a-z0-9-]/", "-"), 0, 40)}-${substr(md5(folder_name), 0, 8)}"
+  }
+
+  dashboard_mount_paths = {
+    for folder_name in keys(var.dashboard_folder) :
+    folder_name => "/var/lib/grafana/dashboards/${local.dashboard_resource_names[folder_name]}"
+  }
+
+  # Load datasource YAMLs from app root if datasource_folder is provided
+  datasource_files = {
+    for f in fileset(var.datasource_folder, "*.{yml,yaml}") : f => file("${var.datasource_folder}/${f}")
+  }
+
+  # Load dashboard JSONs from app root folders grouped by Grafana category name
+  dashboard_files_by_category = {
+    for folder_name, folder_path in var.dashboard_folder :
+    folder_name => {
+      for f in fileset(folder_path, "*.json") : f => file("${folder_path}/${f}")
+    }
+  }
+
+  grafana_dashboard_provider = yamlencode({
+    apiVersion = 1
+    providers = [
+      for folder_name in keys(var.dashboard_folder) : {
+        name                  = "local-${local.dashboard_resource_names[folder_name]}"
+        orgId                 = 1
+        folder                = folder_name
+        type                  = "file"
+        disableDeletion       = false
+        updateIntervalSeconds = 30
+        allowUiUpdates        = true
+        options = {
+          path = local.dashboard_mount_paths[folder_name]
+        }
+      }
+    ]
+  })
+
+  checksum_datasources = md5(join("", values(local.datasource_files)))
+  checksum_dashboards  = md5("${local.grafana_dashboard_provider}${jsonencode(local.dashboard_files_by_category)}")
 }

terraform/modules/apps/grafana/grafana.tf

@@ -4,9 +4,7 @@ resource "kubernetes_config_map" "grafana_datasources" {
     namespace = kubernetes_namespace.grafana.metadata[0].name
   }
 
-  data = {
-    "default.yml" = replace(local.grafana_datasource_config, "__PROMETHEUS_URL__", var.prometheus_url)
-  }
+  data = local.datasource_files
 }
 
 resource "kubernetes_config_map" "grafana_dashboard_provider" {
@@ -21,15 +19,14 @@ resource "kubernetes_config_map" "grafana_dashboard_provider" {
 }
 
 resource "kubernetes_config_map" "grafana_dashboards" {
+  for_each = local.dashboard_files_by_category
+
   metadata {
-    name      = "grafana-dashboards"
+    name      = "grafana-dashboards-${local.dashboard_resource_names[each.key]}"
     namespace = kubernetes_namespace.grafana.metadata[0].name
   }
 
-  data = {
-    "trivy-vulnerabilities-dashboard.json" = local.grafana_trivy_dashboard
-    "gpu-realtime-dashboard.json"          = local.grafana_gpu_realtime_dashboard
-  }
+  data = each.value
 }
 
 resource "kubernetes_deployment" "grafana" {
@@ -50,7 +47,9 @@ resource "kubernetes_deployment" "grafana" {
     template {
       metadata {
         labels = {
-          app = "grafana"
+          app                  = "grafana"
+          checksum_datasources = local.checksum_datasources
+          checksum_dashboards  = local.checksum_dashboards
         }
       }
       spec {
@@ -111,10 +110,13 @@ resource "kubernetes_deployment" "grafana" {
             mount_path = "/etc/grafana/provisioning/dashboards"
             read_only  = true
           }
-          volume_mount {
-            name       = "grafana-dashboards"
-            mount_path = "/var/lib/grafana/dashboards"
-            read_only  = true
+          dynamic "volume_mount" {
+            for_each = local.dashboard_mount_paths
+            content {
+              name       = "grafana-dashboards-${local.dashboard_resource_names[volume_mount.key]}"
+              mount_path = volume_mount.value
+              read_only  = true
+            }
           }
         }
         volume {
@@ -135,10 +137,13 @@ resource "kubernetes_deployment" "grafana" {
             name = kubernetes_config_map.grafana_dashboard_provider.metadata[0].name
           }
         }
-        volume {
-          name = "grafana-dashboards"
-          config_map {
-            name = kubernetes_config_map.grafana_dashboards.metadata[0].name
+        dynamic "volume" {
+          for_each = local.dashboard_mount_paths
+          content {
+            name = "grafana-dashboards-${local.dashboard_resource_names[volume.key]}"
+            config_map {
+              name = kubernetes_config_map.grafana_dashboards[volume.key].metadata[0].name
+            }
           }
         }
       }

terraform/modules/apps/grafana/resources/dashboard-provider.yml

@@ -1,9 +1,9 @@
 apiVersion: 1
 
 providers:
-  - name: trivy-vulnerabilities
+  - name: local
     orgId: 1
-    folder: Security
+    folder: Default
     type: file
     disableDeletion: false
     updateIntervalSeconds: 30

terraform/modules/apps/grafana/resources/default.yml

@@ -1,9 +0,0 @@
-apiVersion: 1
-
-datasources:
-  - name: Prometheus
-    type: prometheus
-    access: proxy
-    url: __PROMETHEUS_URL__
-    isDefault: true
-    editable: true

terraform/modules/apps/grafana/variables.tf

@@ -21,8 +21,13 @@ variable "persistent_folder" {
   type        = string
 }
 
-variable "prometheus_url" {
-  description = "Prometheus service URL used by Grafana datasource provisioning"
-  type        = string
-  default     = "http://prometheus.prometheus.svc.cluster.local:9090"
+variable "dashboard_folder" {
+  description = "Map of Grafana folder name => local folder path containing dashboard JSON files. Each entry becomes a Grafana provider/category and loads all .json files from its path."
+  type        = map(string)
+  default     = {}
+}
+
+variable "datasource_folder" {
+  description = "Path to a folder containing Grafana datasource provisioning YAML files. When set, all .yml/.yaml files in this folder will be loaded and have __PROMETHEUS_URL__ replaced with the actual prometheus_url. If not set, falls back to the default single-prometheus datasource."
+  type        = string
 }

terraform/modules/apps/open-webui/llm_proxy.tf

@@ -1,22 +1,22 @@
-resource "kubernetes_service_account" "ollama_proxy" {
+resource "kubernetes_service_account" "llm_proxy" {
 
-  count = var.ollama_proxy.enabled ? 1 : 0
+  count = var.llm_proxy.enabled ? 1 : 0
   metadata {
-    name      = "ollama-proxy-sa"
+    name      = "llm-proxy-sa"
     namespace = kubernetes_namespace.openwebui.metadata[0].name
   }
 
   automount_service_account_token = false
 }
 
-resource "kubernetes_deployment" "ollama_proxy" {
+resource "kubernetes_deployment" "llm_proxy" {
 
-  count = var.ollama_proxy.enabled ? 1 : 0
+  count = var.llm_proxy.enabled ? 1 : 0
   metadata {
-    name      = "ollama-proxy"
+    name      = "llm-proxy"
     namespace = kubernetes_namespace.openwebui.metadata[0].name
     labels = {
-      app = "ollama-proxy"
+      app = "llm-proxy"
     }
   }
 
@@ -24,7 +24,7 @@ resource "kubernetes_deployment" "ollama_proxy" {
     replicas = 1
     selector {
       match_labels = {
-        app = "ollama-proxy"
+        app = "llm-proxy"
       }
     }
 
@@ -35,7 +35,7 @@ resource "kubernetes_deployment" "ollama_proxy" {
     template {
       metadata {
         labels = {
-          app = "ollama-proxy"
+          app = "llm-proxy"
         }
       }
 
@@ -57,37 +57,37 @@ resource "kubernetes_deployment" "ollama_proxy" {
 
           port {
             name           = "http"
-            container_port = var.ollama_proxy.port
+            container_port = var.llm_proxy.port
           }
 
           env {
             name  = "UPSTREAM_MAC"
-            value = var.ollama_proxy.mac
+            value = var.llm_proxy.mac
           }
 
           env {
             name  = "UPSTREAM_IP"
-            value = var.ollama_proxy.ip
+            value = var.llm_proxy.ip
           }
 
           env {
             name  = "UPSTREAM_PORT"
-            value = var.ollama_proxy.port
+            value = try(var.llm_proxy.upstream_port, var.llm_proxy.port)
           }
 
           env {
             name  = "UPSTREAM_SCHEME"
-            value = var.ollama_proxy.scheme
+            value = var.llm_proxy.scheme
           }
 
           env {
             name  = "PROXY_PORT"
-            value = var.ollama_proxy.port
+            value = var.llm_proxy.port
           }
 
           readiness_probe {
             tcp_socket {
-              port = 11434
+              port = var.llm_proxy.port
             }
             initial_delay_seconds = 5
             period_seconds        = 10

terraform/modules/apps/open-webui/variables.tf

@@ -20,14 +20,15 @@ variable "issuer" {
   default     = "internal-ca"
 }
 
-variable "ollama_proxy" {
-  description = "Configuration for the Ollama proxy"
+variable "llm_proxy" {
+  description = "Configuration for the llm proxy"
   type = object({
-    enabled = bool
-    mac     = optional(string, "")
-    ip      = optional(string, "")
-    port    = optional(string, "11434")
-    scheme  = optional(string, "http")
+    enabled       = bool
+    mac           = optional(string, "")
+    ip            = optional(string, "")
+    port          = optional(string, "11434")
+    scheme        = optional(string, "http")
+    upstream_port = optional(string, "11434")
   })
   default = {
     enabled = false
@@ -54,3 +55,9 @@ variable "log_level" {
     error_message = "log_level must be one of: DEBUG, INFO, WARNING, ERROR, CRITICAL"
   }
 }
+
+variable "aiohttp_client_timeout" {
+  description = "The timeout for aiohttp client in seconds (e.g., 300, 600 etc.) (By default the timeout will be set to None, effectively disabling the timeout and allowing the client to wait indefinitely.)"
+  type        = string
+  default     = ""
+}

terraform/modules/apps/open-webui/webui.tf

@@ -77,7 +77,6 @@ resource "kubernetes_deployment" "openwebui" {
           }
         }
 
-
         container {
           name              = "app"
           image             = "ghcr.io/open-webui/open-webui:${var.tag}"
@@ -115,6 +114,10 @@ resource "kubernetes_deployment" "openwebui" {
             value = var.log_level
           }
 
+          env {
+            name  = "AIOHTTP_CLIENT_TIMEOUT"
+            value = var.aiohttp_client_timeout
+          }
 
           readiness_probe {
             tcp_socket {

terraform/modules/apps/vaultwarden/config.tf

@@ -1,6 +1,23 @@
 locals {
-  app_folder = var.persistent_folder
-  app_version = var.tag
-  app_fqdn    = var.fqdn
-  db_path     = var.db_folder != "" ? var.db_folder : local.app_folder
+  app_folder                        = var.persistent_folder
+  app_version                       = var.tag
+  app_fqdn                          = var.fqdn
+  db_path                           = var.db_folder != "" ? var.db_folder : local.app_folder
+  signup                            = var.signup_enabled ? "true" : "false"
+  signup_verify                     = var.signup_verify ? "true" : "false"
+  disable_2fa_remember              = var.disable_2fa_remember ? "true" : "false"
+  show_password_hint                = var.show_password_hint ? "true" : "false"
+  password_hints_allowed            = var.password_hints_allowed ? "true" : "false"
+  disable_icon_download             = var.disable_icon_download ? "true" : "false"
+  icon_service                      = var.icon_service
+  http_request_block_non_global_ips = var.http_request_block_non_global_ips ? "true" : "false"
+  extended_logging                  = var.extended_logging ? "true" : "false"
+  websocket_enabled                 = var.websocket_enabled ? "true" : "false"
+  sends_allowed                     = var.sends_allowed ? "true" : "false"
+  trash_auto_delete_days            = var.trash_auto_delete_days
+  events_days_retain                = var.events_days_retain
+  admin_session_lifetime            = tostring(var.admin_session_lifetime)
+
+  # SMTP locals — only enable mail if host or from is configured
+  smtp_enabled = var.smtp_host != "" || var.smtp_from != ""
 }

terraform/modules/apps/vaultwarden/main.tf

@@ -19,11 +19,38 @@ resource "kubernetes_config_map" "vaultwarden_config" {
     namespace = kubernetes_namespace.vaultwarden.metadata[0].name
   }
 
-  data = {
-    DOMAIN       = "https://${local.app_fqdn}/"
-    SIGNUP       = var.signup_enabled ? "true" : "false"
-    DATABASE_URL = "file:///data/db.sqlite3"
-  }
+  data = merge({
+    DOMAIN                            = "https://${local.app_fqdn}/"
+    SIGNUPS_ALLOWED                   = local.signup
+    SIGNUPS_VERIFY                    = local.signup_verify
+    DATABASE_URL                      = "/data/db.sqlite3"
+    ENABLE_WEBSOCKET                  = local.websocket_enabled
+    DISABLE_2FA_REMEMBER              = local.disable_2fa_remember
+    SHOW_PASSWORD_HINT                = local.show_password_hint
+    PASSWORD_HINTS_ALLOWED            = local.password_hints_allowed
+    DISABLE_ICON_DOWNLOAD             = local.disable_icon_download
+    ICON_SERVICE                      = local.icon_service
+    HTTP_REQUEST_BLOCK_NON_GLOBAL_IPS = local.http_request_block_non_global_ips
+    EXTENDED_LOGGING                  = local.extended_logging
+    SENDS_ALLOWED                     = local.sends_allowed
+    TRASH_AUTO_DELETE_DAYS            = local.trash_auto_delete_days
+    EVENTS_DAYS_RETAIN                = local.events_days_retain
+    ADMIN_SESSION_LIFETIME            = local.admin_session_lifetime
+    USE_SENDMAIL                      = "false"
+    },
+    local.smtp_enabled ? {
+      SMTP_HOST                     = var.smtp_host
+      SMTP_PORT                     = tostring(var.smtp_port)
+      SMTP_SECURITY                 = var.smtp_security
+      SMTP_FROM                     = var.smtp_from
+      SMTP_FROM_NAME                = var.smtp_from_name
+      SMTP_TIMEOUT                  = tostring(var.smtp_timeout)
+      SMTP_EMBED_IMAGES             = var.smtp_embed_images ? "true" : "false"
+      SMTP_DEBUG                    = var.smtp_debug ? "true" : "false"
+      SMTP_ACCEPT_INVALID_CERTS     = var.smtp_accept_invalid_certs ? "true" : "false"
+      SMTP_ACCEPT_INVALID_HOSTNAMES = var.smtp_accept_invalid_hostnames ? "true" : "false"
+    } : {}
+  )
 }
 
 resource "kubernetes_secret" "vaultwarden_secrets" {
@@ -32,9 +59,14 @@ resource "kubernetes_secret" "vaultwarden_secrets" {
     namespace = kubernetes_namespace.vaultwarden.metadata[0].name
   }
 
-  data = {
+  data = merge({
     ADMIN_TOKEN = var.admin_token
-  }
+    },
+    local.smtp_enabled ? {
+      SMTP_USERNAME = var.smtp_username
+      SMTP_PASSWORD = var.smtp_password
+    } : {}
+  )
 }
 
 resource "kubernetes_deployment" "vaultwarden" {
@@ -62,7 +94,8 @@ resource "kubernetes_deployment" "vaultwarden" {
     template {
       metadata {
         labels = {
-          app = "vaultwarden"
+          app             = "vaultwarden"
+          config_checksum = md5(join("", values(kubernetes_config_map.vaultwarden_config.data)))
         }
       }
 
@@ -101,6 +134,11 @@ resource "kubernetes_deployment" "vaultwarden" {
             }
           }
 
+          env {
+            name  = "ROCKET_PORT"
+            value = "8080"
+          }
+
           security_context {
             allow_privilege_escalation = false
             privileged                 = false
@@ -114,7 +152,7 @@ resource "kubernetes_deployment" "vaultwarden" {
 
           port {
             name           = "http"
-            container_port = 80
+            container_port = 8080
             protocol       = "TCP"
           }
 
@@ -151,6 +189,7 @@ resource "kubernetes_deployment" "vaultwarden" {
           name = "vaultwarden-data"
           host_path {
             path = local.db_path
+            type = "DirectoryOrCreate"
           }
         }
 
@@ -177,7 +216,7 @@ resource "kubernetes_service" "vaultwarden" {
     port {
       name        = "http"
       port        = 80
-      target_port = 80
+      target_port = 8080
     }
 
     type = "ClusterIP"

terraform/modules/apps/vaultwarden/variables.tf

@@ -32,6 +32,84 @@ variable "signup_enabled" {
   default     = false
 }
 
+variable "signup_verify" {
+  description = "Require email verification on signup (defense-in-depth)"
+  type        = bool
+  default     = true
+}
+
+variable "disable_2fa_remember" {
+  description = "Force 2FA on every login, no 'remember this browser'"
+  type        = bool
+  default     = true
+}
+
+variable "show_password_hint" {
+  description = "Show password hints on web login (no SMTP configured)"
+  type        = bool
+  default     = false
+}
+
+variable "password_hints_allowed" {
+  description = "Allow users to set password hints"
+  type        = bool
+  default     = false
+}
+
+variable "disable_icon_download" {
+  description = "Disable external icon downloads to prevent network leaks"
+  type        = bool
+  default     = true
+}
+
+variable "icon_service" {
+  description = "Icon service to use (internal, bitwarden, duckduckgo, google)"
+  type        = string
+  default     = "internal"
+}
+
+variable "http_request_block_non_global_ips" {
+  description = "Block requests to private/reserved IP addresses"
+  type        = bool
+  default     = true
+}
+
+variable "extended_logging" {
+  description = "Enable extended logging with timestamps for audit"
+  type        = bool
+  default     = true
+}
+
+variable "websocket_enabled" {
+  description = "Enable WebSocket notifications for real-time sync"
+  type        = bool
+  default     = true
+}
+
+variable "sends_allowed" {
+  description = "Allow users to create Bitwarden Sends"
+  type        = bool
+  default     = true
+}
+
+variable "trash_auto_delete_days" {
+  description = "Days before auto-deleting trashed items (empty = never)"
+  type        = string
+  default     = "30"
+}
+
+variable "events_days_retain" {
+  description = "Days to retain event logs (empty = indefinitely)"
+  type        = string
+  default     = "90"
+}
+
+variable "admin_session_lifetime" {
+  description = "Admin session lifetime in minutes"
+  type        = number
+  default     = 20
+}
+
 variable "admin_token" {
   description = "Admin token for VaultWarden"
   type        = string
@@ -43,3 +121,90 @@ variable "db_folder" {
   type        = string
   default     = ""
 }
+
+# --- SMTP / Email settings ---
+
+variable "smtp_host" {
+  description = "SMTP host for sending emails"
+  type        = string
+  default     = ""
+}
+
+variable "smtp_port" {
+  description = "SMTP port (default: 587 for starttls, 465 for force_tls, 25 for off)"
+  type        = number
+  default     = 587
+}
+
+variable "smtp_security" {
+  description = "SMTP security mode: starttls, force_tls, or off"
+  type        = string
+  default     = "starttls"
+}
+
+variable "smtp_from" {
+  description = "Email address to send from"
+  type        = string
+  default     = ""
+}
+
+variable "smtp_from_name" {
+  description = "Sender name for outgoing emails"
+  type        = string
+  default     = "Vaultwarden"
+}
+
+variable "smtp_username" {
+  description = "SMTP username for authentication"
+  type        = string
+  default     = ""
+}
+
+variable "smtp_password" {
+  description = "SMTP password for authentication"
+  type        = string
+  sensitive   = true
+  default     = ""
+}
+
+variable "smtp_timeout" {
+  description = "SMTP connection timeout in seconds"
+  type        = number
+  default     = 15
+}
+
+variable "smtp_auth_mechanism" {
+  description = "SMTP authentication mechanisms (e.g., Plain, Login, Xoauth2, comma-separated)"
+  type        = string
+  default     = ""
+}
+
+variable "smtp_helo_name" {
+  description = "Server name sent during SMTP HELO"
+  type        = string
+  default     = ""
+}
+
+variable "smtp_embed_images" {
+  description = "Embed images as email attachments"
+  type        = bool
+  default     = true
+}
+
+variable "smtp_debug" {
+  description = "Enable detailed SMTP debug output (warning: may contain sensitive data)"
+  type        = bool
+  default     = false
+}
+
+variable "smtp_accept_invalid_certs" {
+  description = "Accept invalid SMTP server certificates (dangerous, not recommended)"
+  type        = bool
+  default     = false
+}
+
+variable "smtp_accept_invalid_hostnames" {
+  description = "Accept invalid SMTP server hostnames (dangerous, not recommended)"
+  type        = bool
+  default     = false
+}