diff --git a/.opencode/skills/ansible-develop/SKILL.md b/.opencode/skills/ansible-develop/SKILL.md new file mode 100644 index 0000000..0aa264d --- /dev/null +++ b/.opencode/skills/ansible-develop/SKILL.md @@ -0,0 +1,486 @@ +--- +name: ansible-develop +description: > + Use when developing, writing, or debugging Ansible playbooks and roles in this + repository. Covers playbook and role conventions, inventory group targeting, + molecule-podman testing, variable scoping (defaults/vars/host_vars), Jinja2 + templates, tag-based execution, and known patterns such as the podman shared + tasks helper and dual Debian/RedHat OS-family blocks. +--- + +# Ansible Playbook & Role Skill + +## Purpose + +Use this skill when creating new playbooks or roles in `ansible/`. It covers +project conventions for structure, testing, variable management, secrets, and +execution patterns used across all existing automation. + +--- + +## Repository Layout + +``` +ansible/ + ansible.cfg # inventory = ../inventory/hosts + requirements.yml # collections: ansible.posix, community.general, + # containers.podman, community.libvirt, ... + .yamllint.yml # 120-char line limit, extended default rules + playbook_*.yml # top-level entry points (run from repo root) + tasks/ # shared task fragments (e.g. tasks/podman.yml) + roles/ + / + defaults/main.yml # public defaults (overridable by callers) + vars/main.yml # internal values (non-overridable) + tasks/main.yml # entry point; include_ or block per sub-feature + handlers/main.yml # notify handlers (services, reloads) + templates/ # Jinja2 files (.j2 extension) + files/ # static files copied as-is + meta/main.yml # galaxy metadata, min_ansible_version, dependencies + molecule/default/ # tests: converge.yml, prepare.yml, verify.yml, molecule.yml + tests/inventory # small static inventory for role tests + tests/test.yml # molecule verifier + .yamllint.yml # (optional) role-level yamllint overrides + README.md # role documentation +inventory/ + hosts # host groups (ini-format inventory) + host_vars/ # per-host YAML variable files +``` + +Playbooks live in `ansible/` and are run from the repository root. The +inventory path is relative: `../inventory/hosts` (as declared in +`ansible/ansible.cfg`). + +--- + +## Inventory & Group Targeting + +The inventory at `inventory/hosts` uses **ini-format groups**. Playbooks target +groups via `hosts: `. Group names follow kebab-case convention. + +Existing groups: + +| Group | Hosts | +|-------------------|------------------------------------------------| +| `all` | prod-01, dev-01, vh-01, gpu-01 | +| `hardening` | prod-01, dev-01, vh-01, gpu-01 | +| `ubuntu` | prod-01, dev-01 | +| `fedora` | vh-01, gpu-01 | +| `microk8s` | prod-01, dev-01 | +| `monitoring` | prod-01 | +| `duo` | prod-01, dev-01, vh-01, gpu-01 | +| `virtualization` | vh-01 | +| `ollama` | gpu-01 | +| `comfyui` | gpu-01 | +| `microk8s` | prod-01, dev-01 | +| `certbot` | vh-01, gpu-01 | +| `nvidia` | gpu-01 | +| `amd` | gpu-01 | +| `public` | prod-01 | +| `private` | dev-01, vh-01, gpu-01 | +| `windows` | ci-01 | + +When adding a new playbook, **add a new group** in `inventory/hosts` and target +it from the playbook. + +--- + +## Playbook Patterns + +Two patterns are used. Choose the one that fits the task. + +### Pattern 1 — Role-based (preferred for reusable logic) + +Use when the task is encapsulated in a role or may be reused across playbooks. + +```yaml +--- +- name: My feature setup + hosts: mygroup + gather_facts: true + + roles: + - role: "my_role" + tags: + - roles + - roles::my_role +``` + +### Pattern 2 — Inline tasks + +Use for single-shot playbooks that configure one specific host with custom logic +(e.g., virt_host, encryption). + +```yaml +--- +- name: My inline setup + hosts: mygroup + become: true + gather_facts: true + + tasks: + - name: Ensure the system is Debian + ansible.builtin.assert: + that: + - ansible_os_family == 'Debian' + fail_msg: "This playbook only supports Debian hosts." + + - name: Install required packages + become: true + ansible.builtin.apt: + name: + - package-one + - package-two + state: present + update_cache: true +``` + +### Shared task fragments + +Complex operations (e.g. podman pod/container lifecycle) go into `tasks/` and +are included via `ansible.builtin.include_tasks`. See `tasks/podman.yml` for +the pattern: required-var checks, OS-family `set_fact`, `become_user` blocks, +and loop-based resource creation. + +--- + +## Role Development + +### Minimal role directory structure + +``` +roles/my_role/ + defaults/main.yml # public defaults + tasks/main.yml # entry point + handlers/main.yml # notify handlers + templates/ # Jinja2 files + meta/main.yml # galaxy metadata + molecule/default/ # tests +``` + +### defaults/main.yml + +Declare **all** public variables here with safe defaults. This is how callers +override behavior. + +```yaml +--- +my_feature_enabled: true +my_feature_port: 8080 +my_feature_users: [] +``` + +Use `lookup('env', 'VAR_NAME')` for secrets with empty default: + +```yaml +my_api_key: "{{ lookup('env', 'MY_API_KEY') | default('') }}" +``` + +### tasks/main.yml + +Use `block` + `when` for OS-family conditional logic: + +```yaml +--- +- name: Install packages (Debian) + when: ansible_os_family == 'Debian' + block: + - name: Install deps + become: true + ansible.builtin.apt: + name: "{{ item }}" + state: present + update_cache: true + loop: + - package-one + +- name: Install packages (RedHat) + when: ansible_os_family == 'RedHat' + block: + - name: Install deps + become: true + ansible.builtin.dnf: + name: "{{ item }}" + state: present + loop: + - package-one +``` + +Use `include_tasks` for sub-features, tagged separately: + +```yaml +- name: Include volume provisioning + ansible.builtin.include_tasks: volume.yml + tags: + - roles::my_role::volume + +- name: Include KMS setup + ansible.builtin.include_tasks: kms.yml + tags: + - roles::my_role::kms +``` + +### meta/main.yml + +Always declare `galaxy_info` with namespace `a13labs`, author +`Alexandre Pires`, and company `A13Labs`: + +```yaml +--- +galaxy_info: + author: Alexandre Pires + description: Role to do something + company: A13Labs + role_name: my_role + namespace: a13labs + license: MIT + min_ansible_version: "2.1" +``` + +### Tag conventions + +Use nested tags following the pattern `roles::::`. Top-level +playbooks use `roles` and `roles::`. + +--- + +## Molecule Testing (Podman) + +Every role should have a `molecule/default/` test scenario using the Podman +driver. + +### molecule.yml + +```yaml +--- +driver: + name: podman +platforms: + - name: instance + image: ubuntu:latest + pre_build_image: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true +provisioner: + name: ansible + env: + MY_ROLE_VAR: "test_value" + playbooks: + converge: converge.yml + prepare: prepare.yml +verifier: + name: ansible +``` + +### prepare.yml + +Must install python3 and sudo using raw (facts are unavailable in containers): + +```yaml +--- +- name: Prepare + hosts: all + gather_facts: false + tasks: + - name: Update cache + ansible.builtin.raw: apt update + + - name: Install required packages + ansible.builtin.raw: apt install -y python3 sudo +``` + +### converge.yml + +```yaml +--- +- name: Converge + hosts: all + gather_facts: true + tasks: + - name: Include my_role + ansible.builtin.include_role: + name: "my_role" +``` + +### Testing + +Run from the role directory: + +```bash +cd ansible/roles/my_role +molecule test +``` + +This runs: prepare -> converge -> verify -> destroy. + +**Important**: Most containers lack systemd. Guard service-related tasks: +`when: ansible_service_mgr == "systemd"`. + +--- + +## Secrets & Environment Variables + +Secrets are injected via `sectool` using `sectool.json` at the repository root. +The `ANSIBLE_USER` env var must be set before opening opencode. + +Secrets in roles use `lookup('env', 'VAR_NAME')` — `sectool` injects them +automatically when playbooks are run through `sectool exec`. + +Pattern in role defaults: + +```yaml +api_key: "{{ lookup('env', 'MY_API_KEY') | default('') }}" +``` + +Pattern in host_vars (for host-level secrets): + +```yaml +cloudns_auth_id: "{{ lookup('env', 'CLOUDNS_AUTH_ID') }}" +cloudns_auth_password: "{{ lookup('env', 'CLOUDNS_PASSWORD') }}" +``` + +Never hardcode secrets. Always use `lookup('env', ...)` with a `| default('')` +fallback. + +--- + +## Execution + +All Ansible commands must be run from the **`ansible/` directory** using +`sectool` to inject secrets: + +```bash +cd ansible/ + +# Role-based playbook +sectool -f ../sectool.json exec ansible-playbook -u $ANSIBLE_USER playbook_myfeature.yml + +# Tag-based partial run +sectool -f ../sectool.json exec ansible-playbook -u $ANSIBLE_USER playbook_myfeature.yml --tags roles::my_role::volume + +# Limit to specific group +sectool -f ../sectool.json exec ansible-playbook -u $ANSIBLE_USER playbook_myfeature.yml --limit mygroup + +# Role tests (molecule runs independently, no sectool wrapper needed) +cd ansible/roles/my_role && molecule test +``` + +`$ANSIBLE_USER` must be set in the shell before opening opencode. + +--- + +## Linting + +### yamllint + +```bash +cd ansible +yamllint -c .yamllint.yml . +``` + +Rules: 120-char line width (warning), forbid implicit/explicit octal, extended +default ruleset. + +### ansible-lint + +If available, run from the `ansible/` directory: + +```bash +cd ansible +ansible-lint +``` + +--- + +## Known Patterns + +### Dual OS-family support + +Always support both Debian and RedHat families when packages differ: + +```yaml +- name: Set binary path (Debian) + when: ansible_os_family == 'Debian' + ansible.builtin.set_fact: + podman_binary: /usr/bin/podman + +- name: Set binary path (RedHat) + when: ansible_os_family == 'RedHat' + ansible.builtin.set_fact: + podman_binary: /usr/sbin/podman +``` + +### become_user blocks + +When a task needs to run as a different user (e.g. podman operations), use a +`become: true` + `become_user` block: + +```yaml +- name: Run tasks as podman user + become: true + become_user: "{{ podman_user }}" + block: + - name: Build images + containers.podman.podman_image: + ... +``` + +### systemd service generation + +Use `ansible.builtin.template` to generate `.service` files from `.j2` +templates, then `daemon_reload` + `systemd` enable: + +```yaml +- name: Create systemd service file + become: true + ansible.builtin.template: + src: myservice.service.j2 + dest: "/etc/systemd/system/my-service.service" + mode: "0644" + notify: Reload daemon + +handlers: + - name: Reload daemon + changed_when: false + ansible.builtin.systemd: + daemon_reload: true +``` + +### nmcli bridge creation + +Network bridge configuration uses `nmcli connection add` with `async` + +`wait_for` to handle reconnection after interface changes: + +```yaml +- name: Create bridge + ansible.builtin.command: nmcli connection add ... + register: bridge_created + async: 10 + poll: 0 + +- name: Wait for reconnection + ansible.builtin.wait_for: + host: "{{ bridge_ip }}" + port: 22 + timeout: 30 + delegate_to: localhost +``` + +--- + +## Review Checklist + +Before proposing a new playbook or role: + +- [ ] Playbook targets a defined group in `inventory/hosts` (add one if needed). +- [ ] Role has `defaults/main.yml` with all public variables + safe defaults. +- [ ] Secrets use `lookup('env', 'VAR') | default('')`, never hardcoded. +- [ ] Dual OS-family support when package names differ. +- [ ] Tasks tagged with `roles::::`. +- [ ] Molecule `molecule/default/` scenario with prepare + converge. +- [ ] `molecule.yml` uses Podman driver with `privileged: true` + cgroup mount. +- [ ] Line length within 120 characters. +- [ ] `yamllint -c ansible/.yamllint.yml ansible/` passes. +- [ ] Service tasks guarded by `when: ansible_service_mgr == "systemd"` in + molecule containers. diff --git a/ansible/files/scripts/llama_hf_sync.py b/ansible/files/scripts/llama_hf_sync.py new file mode 100644 index 0000000..3606e30 --- /dev/null +++ b/ansible/files/scripts/llama_hf_sync.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +"""Synchronize llama.cpp models from Hugging Face into a managed local directory.""" + +from __future__ import annotations + +import argparse +import configparser +import json +import os +import re +import shutil +import sys +from pathlib import Path +from typing import Any + +from huggingface_hub import HfApi, snapshot_download + + +def _sanitize_slug(value: str) -> str: + slug = re.sub(r"[^a-zA-Z0-9._-]+", "-", value.strip()) + slug = slug.strip("-._") + return slug.lower() or "model" + + +def _resolve_repo_file(api: HfApi, model: dict[str, Any]) -> str: + repo_files = api.list_repo_files(repo_id=model["model_id"], revision=model["revision"], repo_type="model") + + hf_file = model.get("hf_file") + if hf_file: + if hf_file in repo_files: + return hf_file + raise RuntimeError( + f"Requested hf_file '{hf_file}' was not found in repo " + f"{model['model_id']}@{model['revision']}" + ) + + quant_upper = (model.get("quant") or "").upper() + ggufs = [item for item in repo_files if item.lower().endswith(".gguf")] + matches = [item for item in ggufs if quant_upper in Path(item).name.upper()] + if matches: + return sorted(matches, key=lambda name: (len(Path(name).name), name))[0] + + raise RuntimeError( + f"No GGUF file containing quant '{model.get('quant')}' found in " + f"repo {model['model_id']}@{model['revision']}. Set hf_file explicitly if needed." + ) + + +def _load_models(models_file: Path) -> list[dict[str, Any]]: + data = json.loads(models_file.read_text(encoding="utf-8")) + if not isinstance(data, list): + raise RuntimeError("models file must contain a JSON array") + + normalized: list[dict[str, Any]] = [] + for item in data: + if not isinstance(item, dict): + raise RuntimeError("each model entry must be an object") + if item.get("enable", True) is False: + continue + + model_id = str(item.get("model_id", "")).strip() + quant = str(item.get("quant", "")).strip() + revision = str(item.get("revision", "")).strip() + hf_file = str(item.get("hf_file", "")).strip() or None + name = str(item.get("name", "")).strip() + preset = item.get("preset", {}) + + if preset is None: + preset = {} + if not isinstance(preset, dict): + raise RuntimeError(f"preset must be an object for model '{name or model_id or 'unknown'}'") + + if not name: + raise RuntimeError(f"name is required for model '{model_id or 'unknown'}'") + if not model_id: + raise RuntimeError("model_id is required for all models") + if not quant and not hf_file: + raise RuntimeError(f"quant or hf_file is required for model '{model_id}'") + if not revision: + raise RuntimeError(f"revision is required for model '{model_id}'") + + normalized.append( + { + "model_id": model_id, + "quant": quant, + "revision": revision, + "hf_file": hf_file, + "name": name, + "preset": preset, + } + ) + + return normalized + + +def _load_preset_global(preset_global_file: Path | None) -> dict[str, Any]: + if preset_global_file is None: + return {} + + if not preset_global_file.exists(): + return {} + + data = json.loads(preset_global_file.read_text(encoding="utf-8")) + if data is None: + return {} + if not isinstance(data, dict): + raise RuntimeError("global preset options must be a JSON object") + return data + + +def _to_preset_value(value: Any) -> str: + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float, str)): + return str(value) + return json.dumps(value, separators=(",", ":")) + + +def _normalize_preset_options(options: dict[str, Any], scope: str) -> dict[str, str]: + normalized: dict[str, str] = {} + for key, value in options.items(): + key_str = str(key).strip() + if not key_str: + raise RuntimeError(f"{scope} preset option keys must be non-empty") + if value is None: + continue + normalized[key_str] = _to_preset_value(value) + return normalized + + +def _download_model(model: dict[str, Any], target_dir: Path, repo_file: str, dry_run: bool) -> None: + if dry_run: + return + + snapshot_download( + repo_id=model["model_id"], + revision=model["revision"], + local_dir=str(target_dir), + allow_patterns=[repo_file], + local_dir_use_symlinks=False, + ) + + +def _write_preset(preset_file: Path, entries: list[dict[str, Any]], global_options: dict[str, Any]) -> None: + parser = configparser.ConfigParser(interpolation=None) + parser.optionxform = str + parser["*"] = _normalize_preset_options(global_options, "global") + + for entry in entries: + model_options = _normalize_preset_options(entry.get("preset", {}), f"model '{entry['name']}'") + model_options["model"] = entry["container_model_path"] + parser[entry["name"]] = model_options + + preset_file.parent.mkdir(parents=True, exist_ok=True) + with preset_file.open("w", encoding="utf-8") as fh: + # Router preset format supports top-level version key. + fh.write("version = 1\n\n") + parser.write(fh) + + +def _prune_unmanaged(managed_dir: Path, links_dir: Path, keep_slugs: set[str], dry_run: bool) -> tuple[list[str], list[str]]: + pruned_dirs: list[str] = [] + pruned_links: list[str] = [] + + if managed_dir.exists(): + for child in managed_dir.iterdir(): + if not child.is_dir(): + continue + if child.name == ".router": + continue + if child.name not in keep_slugs: + pruned_dirs.append(child.name) + if not dry_run: + shutil.rmtree(child) + + if links_dir.exists(): + for child in links_dir.iterdir(): + if child.suffix != ".gguf": + continue + slug = child.stem + if slug not in keep_slugs: + pruned_links.append(child.name) + if not dry_run: + child.unlink(missing_ok=True) + + return pruned_dirs, pruned_links + + +def main() -> int: + parser = argparse.ArgumentParser(description="Sync Hugging Face GGUF models for llama.cpp router mode") + parser.add_argument("--models-file", required=True) + parser.add_argument("--managed-dir", required=True) + parser.add_argument("--links-dir", required=True) + parser.add_argument("--manifest-file", required=True) + parser.add_argument("--preset-file", required=True) + parser.add_argument("--preset-global-file") + parser.add_argument("--container-links-dir", required=True) + parser.add_argument("--prune", action="store_true") + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + models_file = Path(args.models_file) + managed_dir = Path(args.managed_dir) + links_dir = Path(args.links_dir) + manifest_file = Path(args.manifest_file) + preset_file = Path(args.preset_file) + preset_global_file = Path(args.preset_global_file) if args.preset_global_file else None + + managed_dir.mkdir(parents=True, exist_ok=True) + links_dir.mkdir(parents=True, exist_ok=True) + manifest_file.parent.mkdir(parents=True, exist_ok=True) + preset_file.parent.mkdir(parents=True, exist_ok=True) + + models = _load_models(models_file) + global_options = _load_preset_global(preset_global_file) + api = HfApi() + entries: list[dict[str, Any]] = [] + + for model in models: + slug = _sanitize_slug(model["name"]) + target_dir = managed_dir / slug + target_dir.mkdir(parents=True, exist_ok=True) + repo_file = _resolve_repo_file(api, model) + + _download_model(model, target_dir, repo_file, args.dry_run) + selected_file = target_dir / repo_file + if not args.dry_run and not selected_file.exists(): + raise RuntimeError(f"Expected downloaded file not found: {selected_file}") + + link_path = links_dir / f"{slug}.gguf" + container_model_path = f"{args.container_links_dir.rstrip('/')}/{slug}.gguf" + + if not args.dry_run: + if link_path.exists() or link_path.is_symlink(): + link_path.unlink() + # Keep symlink targets relative so they remain valid inside the + # container-mounted /models tree. + relative_target = os.path.relpath(selected_file, start=link_path.parent) + link_path.symlink_to(relative_target) + + entries.append( + { + "name": model["name"], + "slug": slug, + "model_id": model["model_id"], + "revision": model["revision"], + "quant": model["quant"], + "hf_file": model.get("hf_file"), + "preset": model.get("preset", {}), + "repo_file": repo_file, + "selected_file": str(selected_file), + "link_path": str(link_path), + "container_model_path": container_model_path, + } + ) + + pruned_dirs: list[str] = [] + pruned_links: list[str] = [] + if args.prune: + keep_slugs = {entry["slug"] for entry in entries} + pruned_dirs, pruned_links = _prune_unmanaged(managed_dir, links_dir, keep_slugs, args.dry_run) + + result = { + "models": entries, + "dry_run": args.dry_run, + "prune": args.prune, + "pruned_dirs": pruned_dirs, + "pruned_links": pruned_links, + } + + if not args.dry_run: + manifest_file.write_text(json.dumps(result, indent=2), encoding="utf-8") + _write_preset(preset_file, entries, global_options) + + print(json.dumps(result)) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as exc: + print(f"ERROR: {exc}", file=sys.stderr) + raise SystemExit(1) diff --git a/ansible/host_vars/gpu-01.lab.alexpires.me.yml b/ansible/host_vars/gpu-01.lab.alexpires.me.yml index 69a9b09..463d6fa 100644 --- a/ansible/host_vars/gpu-01.lab.alexpires.me.yml +++ b/ansible/host_vars/gpu-01.lab.alexpires.me.yml @@ -47,7 +47,8 @@ fw_allowed_ports: - { rule: "allow", port: "8999", proto: "tcp", from: "10.19.4.0/24" } # ComfyUI - SimpleHttpServer (local network) - { rule: "allow", port: "8012", proto: "tcp", from: "10.19.4.0/24" } # Qwen2 (local network) - { rule: "allow", port: "8012", proto: "tcp", from: "10.5.5.5/32" } # Qwen2 (Laptop) - - { rule: "allow", port: "9090", proto: "tcp", from: "10.19.4.0/24" } # local network + - { rule: "allow", port: "9091", proto: "tcp", from: "10.19.4.0/24" } # Prometheus (local network) + - { rule: "allow", port: "9091", proto: "tcp", from: "10.5.5.5/32" } # Prometheus (Laptop) ufw_outgoing_traffic: - 22 # SSH - 53 # DNS @@ -139,3 +140,55 @@ ollama_model_files: name: "qwen3.5-4b-32k:latest" - file: "ollama/qwen3.5-4b-64k.modelfile" name: "qwen3.5-4b-64k:latest" +llama_models: + # TODO: replace revision values with pinned commit SHAs for strict reproducibility. + - name: "gpt-oss-20b-q4-0" + model_id: "unsloth/gpt-oss-20b-GGUF" + quant: "Q4_0" + revision: "main" + - name: "gemma-4-26b-a4b-ud-iq4-xs" + model_id: "unsloth/gemma-4-26B-A4B-it-GGUF" + quant: "UD-IQ4_XS" + revision: "main" + - name: "gemma-4-31b-iq4-xs" + model_id: "unsloth/gemma-4-31B-it-GGUF" + quant: "IQ4_XS" + revision: "main" + - name: "qwen3-6-35b-a3b-mtp-ud-iq4-xs" + model_id: "unsloth/Qwen3.6-35B-A3B-MTP-GGUF" + quant: "UD-IQ4_XS" + revision: "main" + - name: "qwen3-5-27b-mtp-iq4-xs" + model_id: "unsloth/Qwen3.5-27B-MTP-GGUF" + quant: "IQ4_XS" + revision: "main" + - name: "qwen3-5-35b-a3b-ud-iq4-xs" + model_id: "unsloth/Qwen3.5-35B-A3B-GGUF" + quant: "UD-IQ4_XS" + revision: "main" + - name: "ministral-3-14b-instruct-iq4-xs" + model_id: "unsloth/Ministral-3-14B-Instruct-2512-GGUF" + quant: "IQ4_XS" + revision: "main" + - name: "granite-4-1-30b-iq4-xs" + model_id: "unsloth/granite-4.1-30b-GGUF" + quant: "IQ4_XS" + revision: "main" + - name: "qwen3-5-9b-iq4-xs" + model_id: "unsloth/Qwen3.5-9B-GGUF" + quant: "IQ4_XS" + revision: "main" + - name: "qwen3-6-35b-a3b-claude47-distilled-iq4-xs" + model_id: "lordx64/Qwen3.6-35B-A3B-Claude-4.7-Opus-Reasoning-Distilled-IQ4_XS-GGUF" + quant: "IQ4_XS" + revision: "main" + - name: "devstral-small-2-24b-instruct-iq4-xs" + model_id: "unsloth/Devstral-Small-2-24B-Instruct-2512-GGUF" + quant: "IQ4_XS" + revision: "main" + +# Optional global model preset options applied under [*] in models.ini. +# Keys map to llama.cpp CLI args without leading dashes, short args, or LLAMA_ARG_* env names. +llama_preset_global: + c: 131072 + n-gpu-layers: all diff --git a/ansible/playbook_gpu_monitoring.yml b/ansible/playbook_gpu_monitoring.yml new file mode 100644 index 0000000..0f84952 --- /dev/null +++ b/ansible/playbook_gpu_monitoring.yml @@ -0,0 +1,18 @@ +--- +- name: GPU Monitoring Stack Setup + hosts: gpu_monitoring + gather_facts: true + become: true + + pre_tasks: + - name: Check that host is Fedora + ansible.builtin.assert: + that: + - ansible_os_family == "RedHat" + fail_msg: "This playbook only supports RedHat-family OS (Fedora)." + + roles: + - role: "gpu_monitoring" + tags: + - roles + - roles::gpu_monitoring diff --git a/ansible/playbook_podman_llama_rocm.yml b/ansible/playbook_podman_llama_rocm.yml deleted file mode 100644 index 575786a..0000000 --- a/ansible/playbook_podman_llama_rocm.yml +++ /dev/null @@ -1,30 +0,0 @@ -- name: Setup llama.cpp - ROCm - hosts: llamacpp - vars: - podman_user: "{{ llama_user | default('llama') }}" - podman_group: "{{ llama_group | default('llama') }}" - podman_extra_groups: "users,video" - podman_user_home: "{{ llama_home | default('/home/llama') }}" - podman_user_folders: - - data - pods: - - name: llamacpp-rocm - build: - dockerfile: "{{ lookup('file', 'dockerfiles/Dockerfile.llamacpp.rocm') }}" - env: - LLAMA_ARG_MODELS_DIR: "/home/worker/.llamacpp" - LLAMA_ARG_THREADS: 6 - LLAMA_ARG_PARALLEL: 1 - LLAMA_ARG_CACHE: "true" - ports: - - "0.0.0.0:{{ llama_port | default(11435) }}:11434/tcp" - volumes: - - "{{ llama_home }}/data:/home/worker/.llamacpp:Z" - device: - - "/dev/kfd:/dev/kfd:rw" - - "/dev/dri:/dev/dri:rw" - - tasks: - - name: Setup Pods - ansible.builtin.include_tasks: - file: tasks/podman.yml diff --git a/ansible/playbook_podman_llama_vulkan.yml b/ansible/playbook_podman_llama_vulkan.yml new file mode 100644 index 0000000..73329a5 --- /dev/null +++ b/ansible/playbook_podman_llama_vulkan.yml @@ -0,0 +1,101 @@ +- name: Setup llama.cpp - Vulkan + hosts: llamacpp + vars: + podman_user: "{{ llama_user | default('llama') }}" + podman_group: "{{ llama_group | default('llama') }}" + podman_extra_groups: "users,video" + podman_user_home: "{{ llama_home | default('/home/llama') }}" + __llama_models__: "{{ llama_models | default([]) }}" + __llama_router_dir__: "{{ podman_user_home }}/models/.router" + __llama_managed_dir__: "{{ podman_user_home }}/models/managed" + __llama_links_dir__: "{{ podman_user_home }}/models/managed-links" + __llama_manifest_file__: "{{ __llama_router_dir__ }}/manifest.json" + __llama_models_input_file__: "{{ __llama_router_dir__ }}/desired-models.json" + __llama_preset_global__: "{{ llama_preset_global | default({}) }}" + __llama_preset_global_input_file__: "{{ __llama_router_dir__ }}/preset-global.json" + __llama_preset_file__: "{{ __llama_router_dir__ }}/models.ini" + __llama_sync_script__: "{{ __llama_router_dir__ }}/llama_hf_sync.py" + __llama_sync_venv__: "{{ llama_sync_venv | default(__llama_router_dir__ ~ '/.venv') }}" + __llama_sync_python__: "{{ __llama_sync_venv__ }}/bin/python" + __llama_models_max__: "{{ llama_models_max_loaded | default(1) }}" + podman_user_folders: + - models + - .cache + - .cache/llama.cpp + pods: + - name: llama.cpp + repo: + image: ghcr.io/ggml-org/llama.cpp:server-vulkan + ports: + - "0.0.0.0:{{ llama_port | default(8012) }}:8080/tcp" + command: + - "-t" + - "{{ llama_cmd_threads | default('12') }}" + - "-np" + - "{{ llama_cmd_parallel | default('1') }}" + - "-b" + - "{{ llama_cmd_batch_size | default('1024') }}" + - "-ub" + - "{{ llama_cmd_ubatch_size | default('512') }}" + - "-fa" + - "{{ llama_cmd_flash_attn | default('on') }}" + - "-ctk" + - "{{ llama_cmd_cache_type_k | default('q4_0') }}" + - "-ctv" + - "{{ llama_cmd_cache_type_v | default('q4_0') }}" + - "-cram" + - "{{ llama_cmd_cache_reuse | default('-1') }}" + - "-sm" + - "{{ llama_cmd_split_mode | default('layer') }}" + - "-dev" + - "{{ llama_cmd_devices | default('Vulkan1,Vulkan2') }}" + - "-ts" + - "{{ llama_cmd_tensor_split | default('8,12') }}" + - "-fit" + - "{{ llama_cmd_fit | default('off') }}" + - "-mg" + - "{{ llama_cmd_main_gpu | default('1') }}" + - "--mmap" + - "--models-preset" + - "{{ llama_cmd_models_preset_path | default('/models/.router/models.ini') }}" + - "--models-max" + - "{{ llama_cmd_models_max | default(__llama_models_max__) }}" + volumes: + - "{{ podman_user_home }}/models:/models:Z" + - "{{ podman_user_home }}/.cache:/app/.cache:Z" + device: + - "nvidia.com/gpu=all" + - "/dev/kfd:/dev/kfd:rw" + - "/dev/dri:/dev/dri:rw" + tasks: + - name: Sync llama models from Hugging Face + ansible.builtin.include_tasks: + file: tasks/llama_models_sync.yml + when: llama_models_sync_enabled | default(true) + + - name: Setup Pods + ansible.builtin.include_tasks: + file: tasks/podman.yml + + - name: Query llama router model list + ansible.builtin.uri: + url: "http://127.0.0.1:{{ llama_port | default(8012) }}/models?reload=1" + method: GET + status_code: 200 + register: __llama_router_models__ + changed_when: false + when: + - llama_models_sync_enabled | default(true) + - not (llama_models_dry_run | default(false)) + + - name: Validate expected model aliases are available in router + ansible.builtin.assert: + that: + - item.name in (__llama_router_models__.json.data | map(attribute='id') | list) + fail_msg: "Model name '{{ item.name }}' not present in llama router /models output" + loop: "{{ __llama_sync_result__.models | default([]) }}" + loop_control: + label: "{{ item.name }}" + when: + - llama_models_sync_enabled | default(true) + - not (llama_models_dry_run | default(false)) diff --git a/ansible/playbook_podman_qwen3.6.yml b/ansible/playbook_podman_qwen3.6.yml index ca175a4..bb789d5 100644 --- a/ansible/playbook_podman_qwen3.6.yml +++ b/ansible/playbook_podman_qwen3.6.yml @@ -1,4 +1,4 @@ -- name: Setup Qwen3.6 - Coder (FIM - LLAMA.cpp) +- name: Setup Qwen3.6 - Coder hosts: qwen3_6_host vars: podman_user: "{{ qwen3_Z6_user | default('qwen36') }}" @@ -6,7 +6,7 @@ podman_extra_groups: "users,video" podman_user_home: "{{ qwen3_6_home | default('/home/qwen36') }}" llama_model: "{{ qwen3_6_model | default('bartowski/Qwen_Qwen3.6-35B-A3B-GGUF:IQ4_XS') }}" - __qwen3_6_context_length__: "{{ qwen3_6_context_length | default(65537) }}" + __qwen3_6_context_length__: "{{ qwen3_6_context_length | default(262144) }}" podman_user_folders: - models - .cache @@ -14,38 +14,50 @@ pods: - name: qwen3.6 repo: - image: ghcr.io/ggml-org/llama.cpp:server-cuda + image: ghcr.io/ggml-org/llama.cpp:server-vulkan ports: - "0.0.0.0:{{ qwen3_6_port | default(8012) }}:8080/tcp" command: + - "-t" + - "12" - "-hf" - "{{ llama_model }}" - "-ngl" - - "40" - - "-ncmoe" - - "32" + - "999" - "-c" - "{{ __qwen3_6_context_length__ }}" - "-np" - "1" - - "-ub" - - "512" - "-b" - "1024" + - "-ub" + - "512" - "-fa" - "on" - - "--no-mmap" - "-ctk" - - "iq4_nl" + - "q4_0" - "-ctv" - - "iq4_nl" + - "q8_0" - "-cram" - "{{ __qwen3_6_context_length__ }}" + - "-sm" + - "layer" + - "-dev" + - "Vulkan1,Vulkan2" + - "-ts" + - "8,12" + - "-fit" + - "off" + - "-mg" + - 1 + - "--mmap" volumes: - "{{ podman_user_home }}/models:/models:Z" - "{{ podman_user_home }}/.cache:/app/.cache:Z" device: - "nvidia.com/gpu=all" + - "/dev/kfd:/dev/kfd:rw" + - "/dev/dri:/dev/dri:rw" tasks: - name: Setup Pods ansible.builtin.include_tasks: diff --git a/ansible/roles/gpu_monitoring/defaults/main.yml b/ansible/roles/gpu_monitoring/defaults/main.yml new file mode 100644 index 0000000..409472e --- /dev/null +++ b/ansible/roles/gpu_monitoring/defaults/main.yml @@ -0,0 +1,68 @@ +--- +# gpu_monitoring defaults + +# Prometheus configuration +prometheus_image: "docker.io/prom/prometheus:latest" +prometheus_image_tag: "latest" +prometheus_version: "latest" +prometheus_port: 9091 +prometheus_container_port: 9090 +prometheus_data_dir: "/mnt/data/prometheus" +prometheus_retention_days: "30" +prometheus_retention_size: "10GB" +prometheus_scrape_interval: "15s" +prometheus_evaluation_interval: "15s" +prometheus_scrape_timeout: "10s" + +# Prometheus container +prometheus_container_memory: "1g" +prometheus_user: prometheus +prometheus_group: prometheus +prometheus_user_home: "/mnt/data/prometheus" +prometheus_network_name: prometheus-net +prometheus_host_port: "{{ prometheus_port }}" +prometheus_host_gateway: "host.containers.internal" +prometheus_extra_groups: [] +prometheus_user_folders: + - .prometheus + - .cache + +# NVIDIA GPU Exporter configuration (NVML-based, works with consumer GPUs) +nvidia_exporter_enabled: true +nvidia_exporter_port: 9400 +nvidia_exporter_user: nvidia_exporter +nvidia_exporter_user_home: "/home/nvidia_exporter" +nvidia_exporter_install_dir: "/opt/nvidia_exporter" +nvidia_exporter_venv_path: "/opt/nvidia_exporter/.venv" + +# AMD GPU Exporter configuration +amd_exporter_enabled: true +amd_exporter_port: 9500 +amd_exporter_user: amd_exporter +amd_exporter_user_home: "/home/amd_exporter" +amd_exporter_install_dir: "/opt/amdgpu_exporter" +amd_exporter_venv_path: "/opt/amdgpu_exporter/.venv" +amd_exporter_python_version: "python3" +amd_exporter_sysfs_gpu_pattern: "/sys/class/drm/card?/device/" + +# Llama Exporter configuration +llama_exporter_enabled: true +llama_exporter_port: 9550 +llama_exporter_user: llama_exporter +llama_exporter_user_home: "/home/llama_exporter" +llama_exporter_install_dir: "/opt/llama_exporter" +llama_exporter_venv_path: "/opt/llama_exporter/.venv" +llama_exporter_scrape_interval: 15 +llama_exporter_scrape_timeout: 5 + +# Llama.cpp scrape targets +llama_exporter_targets: [] + +# Firewall configuration +prometheus_fw_port: "{{ prometheus_port }}" +prometheus_fw_proto: "tcp" +prometheus_fw_allowed_networks: + - cidr: "10.19.4.0/24" + comment: "local network" + - cidr: "10.5.5.5/32" + comment: "Laptop Wireguard" diff --git a/ansible/roles/gpu_monitoring/files/amd_gpu_exporter.py b/ansible/roles/gpu_monitoring/files/amd_gpu_exporter.py new file mode 100644 index 0000000..59cbe0f --- /dev/null +++ b/ansible/roles/gpu_monitoring/files/amd_gpu_exporter.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +"""AMD GPU sysfs metrics exporter for Prometheus.""" + +import glob +import os +import re +import time +import logging +import json +import subprocess +from http.server import HTTPServer, BaseHTTPRequestHandler + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") +logger = logging.getLogger(__name__) + +AMD_VENDOR_ID = 0x1002 + +POLL_INTERVAL = 10 + + +def _read_sysfs(path): + try: + with open(path, "r") as f: + raw = f.read().strip() + return int(raw, 0) + except (OSError, ValueError): + return None + + +def _read_first_int(paths): + for path in paths: + val = _read_sysfs(path) + if val is not None: + return val + return None + + +def _metric_value(entry): + if isinstance(entry, dict): + value = entry.get("value") + if isinstance(value, (int, float)): + return float(value) + return None + + +def _collect_amd_smi_metrics(results, set_metric): + """Populate metrics from amd-smi JSON output. Returns True when data is collected.""" + cmd = ["amd-smi", "metric", "-m", "-u", "-p", "-t", "--json", "--gpu", "all"] + try: + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=8, check=True) + except (OSError, subprocess.SubprocessError): + return False + + try: + payload = json.loads(proc.stdout) + except (json.JSONDecodeError, TypeError): + return False + + gpu_rows = payload.get("gpu_data", []) if isinstance(payload, dict) else [] + if not gpu_rows: + return False + + got_any = False + for row in gpu_rows: + if not isinstance(row, dict): + continue + gpu_id = row.get("gpu") + if gpu_id is None: + continue + + labels = {"gpu": f"amd{gpu_id}", "gpu_name": f"AMD_GPU_{gpu_id}"} + set_metric("gpu_exists", labels, 1.0) + got_any = True + + usage = row.get("usage", {}) if isinstance(row.get("usage"), dict) else {} + gfx_activity = _metric_value(usage.get("gfx_activity")) + if gfx_activity is not None: + set_metric("gpu_util_percent", labels, gfx_activity) + + umc_activity = _metric_value(usage.get("umc_activity")) + if umc_activity is not None: + set_metric("memory_used_percent", labels, umc_activity) + + power = row.get("power", {}) if isinstance(row.get("power"), dict) else {} + socket_power = _metric_value(power.get("socket_power")) + if socket_power is not None: + set_metric("power_watts", labels, socket_power) + + temp = row.get("temperature", {}) if isinstance(row.get("temperature"), dict) else {} + edge_temp = _metric_value(temp.get("edge")) + if edge_temp is not None: + set_metric("temperature_celsius", labels, edge_temp) + + mem_usage = row.get("mem_usage", {}) if isinstance(row.get("mem_usage"), dict) else {} + used_vram = _metric_value(mem_usage.get("used_vram")) + free_vram = _metric_value(mem_usage.get("free_vram")) + total_vram = _metric_value(mem_usage.get("total_vram")) + + if used_vram is not None: + set_metric("mem_used_bytes", labels, used_vram * 1024 * 1024) + if free_vram is not None: + set_metric("mem_avail_bytes", labels, free_vram * 1024 * 1024) + if total_vram is not None and used_vram is not None and total_vram > 0: + set_metric("memory_used_percent", labels, (used_vram / total_vram) * 100.0) + + return got_any + + +def _get_card_ids(): + paths = sorted(glob.glob("/sys/class/drm/card[0-9]*")) + seen = set() + card_ids = [] + for p in paths: + name = os.path.basename(p) + dev_match = re.match(r"card(\d+)", name) + if dev_match: + idx = int(dev_match.group(1)) + if idx not in seen: + seen.add(idx) + card_ids.append(idx) + return card_ids + + +def _is_amd_gpu(idx): + vendor_path = f"/sys/class/drm/card{idx}/device/vendor" + vendor = _read_sysfs(vendor_path) + return vendor == AMD_VENDOR_ID + + +def _get_gpu_name(idx): + if _is_amd_gpu(idx): + device_id = _read_sysfs(f"/sys/class/drm/card{idx}/device/device") + if device_id is not None: + return f"AMD_GPU_{device_id:#06x}" + return f"AMD_GPU_card{idx}" + return f"UNKNOWN_card{idx}" + + +def _collect_metrics(): + results = {} + + def _set_metric(metric_name, labels, value): + label_key = tuple(sorted(labels.items())) + results[(metric_name, label_key)] = value + + # Prefer amd-smi for robust telemetry on kernels where some sysfs nodes are busy. + if _collect_amd_smi_metrics(results, _set_metric): + return results + + card_ids = _get_card_ids() + + if not card_ids: + logger.warning("No AMD GPU cards found in sysfs") + return results + + for idx in card_ids: + if not _is_amd_gpu(idx): + continue + + labels = {"gpu": f"card{idx}", "gpu_name": _get_gpu_name(idx)} + + # Power (microwatts in hwmon on many kernels, milliwatts on older path) + power = _read_first_int([ + f"/sys/class/drm/card{idx}/device/power_average", + f"/sys/class/drm/card{idx}/device/hwmon/hwmon0/power1_average", + f"/sys/class/drm/card{idx}/device/hwmon/hwmon1/power1_average", + f"/sys/class/drm/card{idx}/device/hwmon/hwmon2/power1_average", + f"/sys/class/drm/card{idx}/device/hwmon/hwmon3/power1_average", + ]) + if power is not None: + # Heuristic: hwmon power1_average is usually microwatts. + if power > 1_000_000: + _set_metric("power_watts", labels, power / 1_000_000.0) + else: + _set_metric("power_watts", labels, power / 1000.0) + + # Memory used (bytes) - support both gpu_mem_* and mem_info_vram_* layouts + mem_used = _read_first_int([ + f"/sys/class/drm/card{idx}/device/gpu_mem_used_bytes", + f"/sys/class/drm/card{idx}/device/mem_info_vram_used", + ]) + if mem_used is not None: + _set_metric("mem_used_bytes", labels, mem_used) + + # Memory available (bytes) + mem_avail = _read_first_int([ + f"/sys/class/drm/card{idx}/device/gpu_mem_avail_bytes", + ]) + + # If total VRAM is available, derive avail = total - used. + if mem_avail is None and mem_used is not None: + mem_total = _read_first_int([ + f"/sys/class/drm/card{idx}/device/mem_info_vram_total", + ]) + if mem_total is not None and mem_total >= mem_used: + mem_avail = mem_total - mem_used + + if mem_avail is not None: + _set_metric("mem_avail_bytes", labels, mem_avail) + + # Some kernels expose direct busy percentage. + mem_busy = _read_first_int([ + f"/sys/class/drm/card{idx}/device/mem_busy_percent", + ]) + if mem_busy is not None: + _set_metric("memory_used_percent", labels, float(mem_busy)) + + if mem_used is not None and mem_avail is not None and (mem_used + mem_avail) > 0: + _set_metric("memory_used_percent", labels, (mem_used / (mem_used + mem_avail)) * 100.0) + + # Temperature (milliCelsius) + temp = _read_first_int([ + f"/sys/class/drm/card{idx}/device/temp_edge_input", + f"/sys/class/drm/card{idx}/device/hwmon/hwmon0/temp1_input", + f"/sys/class/drm/card{idx}/device/hwmon/hwmon1/temp1_input", + f"/sys/class/drm/card{idx}/device/hwmon/hwmon2/temp1_input", + f"/sys/class/drm/card{idx}/device/hwmon/hwmon3/temp1_input", + ]) + if temp is not None: + _set_metric("temperature_celsius", labels, temp / 1000.0) + + # Clock speed (MHz) - clock[1] = performance state + clock_path = f"/sys/class/drm/card{idx}/device/pp_cur_clk" + try: + with open(clock_path, "r") as f: + clocks_raw = f.read().strip() + clocks = [int(c) for c in clocks_raw.split()] + if len(clocks) >= 2: + _set_metric("gpu_clock_mhz", labels, clocks[1]) + except (OSError, ValueError): + pass + + # Existence info + _set_metric("gpu_exists", labels, 1.0) + + return results + + +class MetricsHandler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path != "/metrics": + self.send_response(404) + self.end_headers() + return + + metrics = _collect_metrics() + lines = [] + + for (metric_type, label_pairs), value in sorted(metrics.items()): + labels = dict(label_pairs) + label_str = ",".join(f'{k}="{v}"' for k, v in sorted(labels.items())) + if label_str: + line = f'{metric_type}{{{label_str}}} {value}' + else: + line = f"{metric_type} {value}" + lines.append(line) + + body = "\n".join(lines) + "\n" if lines else "# no metrics available\n" + + self.send_response(200) + self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + self.end_headers() + self.wfile.write(body.encode("utf-8")) + + def log_message(self, format, *args): + logger.debug("%s - - %s", self.address_string(), format % args) + + +def main(): + port = int(os.environ.get("AMD_EXPORTER_PORT", "9500")) + host = os.environ.get("AMD_EXPORTER_BIND", "0.0.0.0") + + server = HTTPServer((host, port), MetricsHandler) + logger.info("Starting AMD GPU Exporter on %s:%d", host, port) + logger.info("Polling sysfs every %d seconds", POLL_INTERVAL) + + def _update_loop(): + while True: + _collect_metrics() + time.sleep(POLL_INTERVAL) + + try: + server.serve_forever() + except KeyboardInterrupt: + logger.info("Shutting down") + server.shutdown() + + +if __name__ == "__main__": + main() diff --git a/ansible/roles/gpu_monitoring/files/llama_exporter.py b/ansible/roles/gpu_monitoring/files/llama_exporter.py new file mode 100644 index 0000000..599b106 --- /dev/null +++ b/ansible/roles/gpu_monitoring/files/llama_exporter.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""LLM inference metrics exporter for Prometheus. + +Scrape targets: + - llama.cpp HTTP API servers (standard /metrics or custom health) +""" + +import json +import os +import time +import logging +import urllib.request +import urllib.error +from http.server import HTTPServer, BaseHTTPRequestHandler + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") +logger = logging.getLogger(__name__) + +POLL_INTERVAL = int(os.environ.get("LLAMA_EXPORTER_INTERVAL", "15")) +SCRAPE_TIMEOUT = int(os.environ.get("LLAMA_EXPORTER_TIMEOUT", "5")) + +LLAMA_CPP_DEFAULTS = [ + {"name": "llama-cpp-8012", "url": "http://localhost:8012"}, +] + + +def _fetch_json(url, timeout=SCRAPE_TIMEOUT): + try: + req = urllib.request.Request(url, headers={"Accept": "application/json"}) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + except Exception as e: + logger.debug("Failed to fetch %s: %s", url, e) + return None + + +def _collect_llama_cpp(target): + metrics = {} + + def _set_metric(metric_name, labels, value): + label_key = tuple(sorted(labels.items())) + metrics[(metric_name, label_key)] = value + + base_url = target["url"].rstrip("/") + + # Check health endpoint + health_url = f"{base_url}/health" + health = _fetch_json(health_url) + if health and isinstance(health, dict): + status = health.get("status", "unknown") + _set_metric("llama_server_health", {"server": target["name"], "model": health.get("model", "unknown")}, 1.0 if status == "ok" else 0.0) + else: + _set_metric("llama_server_health", {"server": target["name"], "model": "unknown"}, 0.0) + + # Check /metrics endpoint for llama.cpp built-in metrics + metrics_url = f"{base_url}/metrics" + try: + req = urllib.request.Request(metrics_url, headers={"Accept": "text/plain"}) + with urllib.request.urlopen(req, timeout=SCRAPE_TIMEOUT) as resp: + body = resp.read().decode("utf-8") + for line in body.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + # Parse simple key value pairs + if "{" in line: + key_part = line.split("{")[0] + labels_part = line.split("{")[1].rstrip("}").split("}")[0] if "}" in line else "" + value = line.split()[-1] if line.split() else "1" + label_str = "" + if labels_part: + label_str = ",".join(f'{{{k}="{v}"}}' for k, v in ( + pair.split("=") for pair in labels_part.split(",") if "=" in pair + )) + _set_metric(key_part.strip(), {"server": target["name"], "raw": labels_part}, float(value) if value.replace(".", "").replace("-", "").isdigit() else value) + else: + parts = line.split() + if len(parts) >= 2: + try: + float(parts[1]) + _set_metric(parts[0], {"server": target["name"]}, parts[1]) + except ValueError: + pass + except Exception as e: + logger.debug("Failed to scrape %s/metrics: %s", metrics_url, e) + + # Check /model endpoint for model info + model_url = f"{base_url}/models" + models = _fetch_json(model_url) + if models and isinstance(models, dict): + model_list = models.get("data", [models]) if "data" in models else [models] + for m in model_list: + if not isinstance(m, dict): + continue + model_id = m.get("id", "unknown") + _set_metric("llama_models_loaded", {"server": target["name"], "model": model_id}, 1.0) + ctx_max = m.get("context_size", m.get("n_ctx", 0)) + if ctx_max: + _set_metric("llama_model_context_max", {"server": target["name"], "model": model_id}, int(ctx_max)) + trained = m.get("train_tokens", m.get("n_train", 0)) + if trained: + _set_metric("llama_model_tokens_trained", {"server": target["name"], "model": model_id}, int(trained)) + + return metrics + + + +class MetricsHandler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path != "/metrics": + self.send_response(404) + self.end_headers() + return + + all_metrics = {} + + # Scrape llama.cpp targets + targets = [] + env_targets = os.environ.get("LLAMA_TARGETS", "") + if env_targets: + try: + targets = json.loads(env_targets) + except json.JSONDecodeError: + logger.warning("Invalid LLAMA_TARGETS JSON, using defaults") + targets = [] + + if not targets: + targets = LLAMA_CPP_DEFAULTS + + for target in targets: + if "url" not in target or "name" not in target: + target["name"] = target.get("name", "default") + target["url"] = target.get("url", "http://localhost:8012") + try: + all_metrics.update(_collect_llama_cpp(target)) + except Exception as e: + logger.error("Error scraping llama.cpp target %s: %s", target.get("name", "unknown"), e) + + lines = [] + for (metric_type, label_pairs), value in sorted(all_metrics.items()): + labels = dict(label_pairs) + if not labels: + line = f"{metric_type} {value}" + else: + label_str = ",".join(f'{k}="{v}"' for k, v in sorted(labels.items())) + line = f"{metric_type}{{{label_str}}} {value}" + lines.append(line) + + body = "\n".join(lines) + "\n" if lines else "# no metrics available\n" + + self.send_response(200) + self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + self.end_headers() + self.wfile.write(body.encode("utf-8")) + + def log_message(self, format, *args): + logger.debug("%s - - %s", self.address_string(), format % args) + + +def main(): + port = int(os.environ.get("LLAMA_EXPORTER_PORT", "9550")) + host = os.environ.get("LLAMA_EXPORTER_BIND", "0.0.0.0") + + server = HTTPServer((host, port), MetricsHandler) + logger.info("Starting Llama Exporter on %s:%d", host, port) + logger.info("Polling every %d seconds, timeout %ds", POLL_INTERVAL, SCRAPE_TIMEOUT) + + try: + server.serve_forever() + except KeyboardInterrupt: + logger.info("Shutting down") + server.shutdown() + + +if __name__ == "__main__": + main() diff --git a/ansible/roles/gpu_monitoring/files/nvidia_gpu_exporter.py b/ansible/roles/gpu_monitoring/files/nvidia_gpu_exporter.py new file mode 100644 index 0000000..21177b4 --- /dev/null +++ b/ansible/roles/gpu_monitoring/files/nvidia_gpu_exporter.py @@ -0,0 +1,436 @@ +# nvidia_gpu_exporter - NVIDIA GPU Prometheus exporter using NVML (nvidia-ml-py) +# Works with consumer GPUs (RTX series, GTX series) and datacenter GPUs +# +# Exposes metrics via HTTP at /metrics +# Required package: nvidia-ml-py (pip) + +import http.server +import json +import threading +import time +import sys +import signal +import traceback +import subprocess +import csv + +try: + import pynvml +except ImportError: + print("ERROR: pynvml not installed. Run: pip install nvidia-ml-py", file=sys.stderr) + sys.exit(1) + +# Globals +port = 9400 +stop_event = threading.Event() +nvidia_initialized = False +gpus_detected = 0 +gpu_handles = {} +gpu_names = {} +last_metrics = {} +last_metrics_time = 0 +metrics_lock = threading.Lock() + + +def signal_handler(sig, frame): + stop_event.set() + + +def format_value(value, dtype): + """Format a value based on its NVML type.""" + if value is None: + return "0" + try: + if dtype == pynvml.NVML_FEATURE_COUNTER_DATA_TYPE_UINT32: + return str(int(value)) + elif dtype == pynvml.NVML_FEATURE_COUNTER_DATA_TYPE_UINT64: + return str(int(value)) + elif dtype == pynvml.NVML_FEATURE_COUNTER_DATA_TYPE_FLOAT: + return f"{float(value):.6f}" + else: + return str(value) + except (ValueError, TypeError): + return "0" + + +def extract_nvml_stats_pair(stats): + """Return (session_count, avg_fps) from NVML stats object or tuple.""" + if stats is None: + return 0.0, 0.0 + + # Newer bindings may return an object with named fields. + if hasattr(stats, "sessionCount") and hasattr(stats, "avgFps"): + return float(stats.sessionCount), float(stats.avgFps) + + # Older bindings may return a tuple/list. + if isinstance(stats, (tuple, list)) and len(stats) >= 2: + return float(stats[0]), float(stats[1]) + + return 0.0, 0.0 + + +def _to_float_or_none(value): + if value is None: + return None + raw = str(value).strip() + if raw == "" or raw.upper() == "N/A": + return None + try: + return float(raw) + except ValueError: + return None + + +def _to_bool_float_or_none(value): + if value is None: + return None + raw = str(value).strip().lower() + if raw in {"1", "true", "yes", "active", "enabled"}: + return 1.0 + if raw in {"0", "false", "no", "not active", "disabled", "n/a"}: + return 0.0 + return None + + +def collect_nvidia_smi_metrics(): + """Collect supplemental metrics via nvidia-smi keyed by GPU index.""" + fields = [ + "index", + "pstate", + "pcie.link.gen.current", + "pcie.link.width.current", + "clocks_event_reasons.sw_power_cap", + "clocks_event_reasons.hw_thermal_slowdown", + "clocks_event_reasons.hw_power_brake_slowdown", + ] + + cmd = [ + "nvidia-smi", + f"--query-gpu={','.join(fields)}", + "--format=csv,noheader,nounits", + ] + + try: + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=5, check=True) + except (OSError, subprocess.SubprocessError): + return {} + + metrics_by_index = {} + reader = csv.reader(proc.stdout.splitlines()) + for row in reader: + if len(row) < len(fields): + continue + + idx_raw = row[0].strip() + if not idx_raw.isdigit(): + continue + idx = int(idx_raw) + + smi_metrics = {} + + pstate_raw = row[1].strip() + if pstate_raw.startswith("P") and pstate_raw[1:].isdigit(): + smi_metrics["pstate"] = float(int(pstate_raw[1:])) + + pcie_gen = _to_float_or_none(row[2]) + if pcie_gen is not None: + smi_metrics["pcie_link_gen_current"] = pcie_gen + + pcie_width = _to_float_or_none(row[3]) + if pcie_width is not None: + smi_metrics["pcie_link_width_current"] = pcie_width + + sw_power_cap = _to_bool_float_or_none(row[4]) + if sw_power_cap is not None: + smi_metrics["throttle_sw_power_cap"] = sw_power_cap + + hw_thermal = _to_bool_float_or_none(row[5]) + if hw_thermal is not None: + smi_metrics["throttle_hw_thermal_slowdown"] = hw_thermal + + hw_power_brake = _to_bool_float_or_none(row[6]) + if hw_power_brake is not None: + smi_metrics["throttle_hw_power_brake_slowdown"] = hw_power_brake + + if smi_metrics: + metrics_by_index[idx] = smi_metrics + + return metrics_by_index + + +def get_gpu_metrics(): + """Collect all GPU metrics from NVML.""" + global gpus_detected, gpu_handles, gpu_names, last_metrics, last_metrics_time, nvidia_initialized + + if not nvidia_initialized: + return {} + + metrics = {} + now = time.time() + current_time = int(now) + + # Global metrics + metrics["gpu_count"] = float(gpus_detected) + metrics["instance"] = "0" + + smi_metrics_by_index = collect_nvidia_smi_metrics() + + for i in range(gpus_detected): + try: + handle = gpu_handles.get(i) + name = gpu_names.get(i, "Unknown") + gpu_prefix = f"gpu_{i}" + + # Basic info (set once) + if f"{gpu_prefix}_info" not in last_metrics or current_time == 0: + metrics[f"{gpu_prefix}_info"] = 1.0 + metrics[f"{gpu_prefix}_name"] = float(name_to_float(name)) + + # Power + try: + power_usage = pynvml.nvmlDeviceGetPowerUsage(handle) + metrics[f"{gpu_prefix}_power_watts"] = power_usage / 1000.0 + except pynvml.NVMLError: + metrics[f"{gpu_prefix}_power_watts"] = 0.0 + + try: + power_limit = pynvml.nvmlDeviceGetPowerManagementLimit(handle) + if power_limit: + metrics[f"{gpu_prefix}_power_limit_watts"] = power_limit / 1000.0 + except pynvml.NVMLError: + pass + + # Temperature + try: + temp = pynvml.nvmlDeviceGetTemperature(handle, pynvml.NVML_TEMPERATURE_GPU) + metrics[f"{gpu_prefix}_temperature_celsius"] = float(temp) + except pynvml.NVMLError: + metrics[f"{gpu_prefix}_temperature_celsius"] = 0.0 + + # Memory + try: + mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle) + metrics[f"{gpu_prefix}_memory_used_bytes"] = mem_info.used + metrics[f"{gpu_prefix}_memory_total_bytes"] = mem_info.total + if mem_info.total > 0: + metrics[f"{gpu_prefix}_memory_used_percent"] = (mem_info.used / mem_info.total) * 100.0 + except pynvml.NVMLError: + metrics[f"{gpu_prefix}_memory_used_bytes"] = 0.0 + metrics[f"{gpu_prefix}_memory_total_bytes"] = 0.0 + metrics[f"{gpu_prefix}_memory_used_percent"] = 0.0 + + # Utilization + try: + util = pynvml.nvmlDeviceGetUtilizationRates(handle) + metrics[f"{gpu_prefix}_gpu_util_percent"] = float(util.gpu) + metrics[f"{gpu_prefix}_memory_util_percent"] = float(util.memory) + except pynvml.NVMLError: + metrics[f"{gpu_prefix}_gpu_util_percent"] = 0.0 + metrics[f"{gpu_prefix}_memory_util_percent"] = 0.0 + + # Clock speeds + try: + clocks = pynvml.nvmlDeviceGetClockInfo(handle, pynvml.NVML_CLOCK_GRAPHICS) + metrics[f"{gpu_prefix}_clock_graphics_hz"] = float(clocks) * 1000000 + except pynvml.NVMLError: + metrics[f"{gpu_prefix}_clock_graphics_hz"] = 0.0 + + try: + # pynvml exposes memory clock as NVML_CLOCK_MEM in some versions. + mem_clock_type = getattr(pynvml, "NVML_CLOCK_MEMORY", None) + if mem_clock_type is None: + mem_clock_type = getattr(pynvml, "NVML_CLOCK_MEM", None) + if mem_clock_type is None: + raise AttributeError("No NVML memory clock constant available") + mem_clock = pynvml.nvmlDeviceGetClockInfo(handle, mem_clock_type) + metrics[f"{gpu_prefix}_clock_memory_hz"] = float(mem_clock) * 1000000 + except (pynvml.NVMLError, AttributeError): + metrics[f"{gpu_prefix}_clock_memory_hz"] = 0.0 + + try: + vol_clock = pynvml.nvmlDeviceGetClockInfo(handle, pynvml.NVML_CLOCK_VIDEO) + metrics[f"{gpu_prefix}_clock_video_hz"] = float(vol_clock) * 1000000 + except pynvml.NVMLError: + metrics[f"{gpu_prefix}_clock_video_hz"] = 0.0 + + # Fan speed + try: + fan = pynvml.nvmlDeviceGetFanSpeed(handle) + metrics[f"{gpu_prefix}_fan_speed_percent"] = float(fan) + except pynvml.NVMLError: + metrics[f"{gpu_prefix}_fan_speed_percent"] = 0.0 + + # PCIe stats + try: + tx = pynvml.nvmlDeviceGetPcieThroughputStats(handle) + if hasattr(tx, 'tx_bytes'): + metrics[f"{gpu_prefix}_pcie_tx_bytes_total"] = float(tx.tx_bytes) + metrics[f"{gpu_prefix}_pcie_rx_bytes_total"] = float(tx.rx_bytes) + except (pynvml.NVMLError, AttributeError): + pass + + # Compute/pending processes + try: + processes = pynvml.nvmlDeviceGetComputeRunningProcesses(handle) + metrics[f"{gpu_prefix}_processes"] = float(len(processes)) + except pynvml.NVMLError: + metrics[f"{gpu_prefix}_processes"] = 0.0 + + # Encoder/Decoder stats + try: + enc_stats = pynvml.nvmlDeviceGetEncoderStats(handle) + enc_sessions, enc_avg_fps = extract_nvml_stats_pair(enc_stats) + metrics[f"{gpu_prefix}_encoder_sessions"] = enc_sessions + metrics[f"{gpu_prefix}_encoder_avg_fps"] = enc_avg_fps + except (pynvml.NVMLError, AttributeError, TypeError, ValueError): + metrics[f"{gpu_prefix}_encoder_sessions"] = 0.0 + metrics[f"{gpu_prefix}_encoder_avg_fps"] = 0.0 + + try: + dec_stats = pynvml.nvmlDeviceGetDecoderStats(handle) + dec_sessions, dec_avg_fps = extract_nvml_stats_pair(dec_stats) + metrics[f"{gpu_prefix}_decoder_sessions"] = dec_sessions + metrics[f"{gpu_prefix}_decoder_avg_fps"] = dec_avg_fps + except (pynvml.NVMLError, AttributeError, TypeError, ValueError): + metrics[f"{gpu_prefix}_decoder_sessions"] = 0.0 + metrics[f"{gpu_prefix}_decoder_avg_fps"] = 0.0 + + # Reboot/Pwr events + try: + inforom_fn = getattr(pynvml, "nvmlDeviceGetInforomConfigurationVersion", None) + if inforom_fn is None: + raise AttributeError("Inforom API not available in this pynvml version") + events = inforom_fn(handle) + if events: + metrics[f"{gpu_prefix}_firmware_version"] = 1.0 + except (pynvml.NVMLError, AttributeError): + pass + + # Supplemental nvidia-smi metrics (when available). + smi_metrics = smi_metrics_by_index.get(i, {}) + for name, value in smi_metrics.items(): + metrics[f"{gpu_prefix}_{name}"] = value + + except pynvml.NVMLError: + traceback.print_exc() + continue + + last_metrics = metrics + last_metrics_time = current_time + return metrics + + +def name_to_float(name): + """Convert GPU name to a numeric value for prometheus.""" + # Prometheus metrics don't support string values, so we use hash + return float(hash(name)) % 1000000 + + +def generate_prometheus_output(metrics): + """Format metrics as Prometheus text exposition format.""" + lines = [] + lines.append("# HELP gpu_info NVIDIA GPU information") + lines.append("# TYPE gpu_info gauge") + for i in range(gpus_detected): + lines.append(f'gpu_info{{gpu="{gpu_names.get(i, "Unknown")}",gpu_id="{i}"}} 1') + + for key, value in sorted(metrics.items()): + # Skip info metric (handled above) + if key == "gpu_count": + lines.append(f"# HELP gpu_total Total number of NVIDIA GPUs") + lines.append("# TYPE gpu_total gauge") + lines.append(f"gpu_total {value}") + continue + + if key == "instance": + continue + + if key.endswith("_name"): + continue + + if key.endswith("_info"): + continue + + lines.append(f"# HELP {key} {key} metric") + lines.append("# TYPE " + key + " gauge") + if isinstance(value, float): + lines.append(f"{key} {value:.6f}") + else: + lines.append(f"{key} {value}") + + return "\n".join(lines) + "\n" + + +class MetricsHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/metrics": + with metrics_lock: + metrics = get_gpu_metrics() + output = generate_prometheus_output(metrics) + self.send_response(200) + self.send_header("Content-Type", "text/plain; charset=utf-8") + self.end_headers() + self.wfile.write(output.encode()) + elif self.path == "/health": + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write("OK\n".encode()) + else: + self.send_response(404) + self.end_headers() + + def log_message(self, format, *args): + # Suppress default logging + pass + + +def run_server(): + global port, stop_event, nvidia_initialized, gpus_detected, gpu_handles, gpu_names + + try: + pynvml.nvmlInit() + nvidia_initialized = True + except pynvml.NVMLError as e: + print(f"ERROR: Failed to initialize NVML: {e}", file=sys.stderr) + print("Make sure NVIDIA drivers are properly installed.", file=sys.stderr) + sys.exit(1) + + try: + gpus_detected = pynvml.nvmlDeviceGetCount() + print(f"Found {gpus_detected} NVIDIA GPU(s)") + + for i in range(gpus_detected): + try: + handle = pynvml.nvmlDeviceGetHandleByIndex(i) + gpu_handles[i] = handle + name = pynvml.nvmlDeviceGetName(handle) + if isinstance(name, bytes): + name = name.decode("utf-8") + gpu_names[i] = name + print(f" GPU {i}: {name}") + except pynvml.NVMLError: + pass + except pynvml.NVMLError as e: + print(f"ERROR: Failed to enumerate devices: {e}", file=sys.stderr) + sys.exit(1) + + server = http.server.HTTPServer(("0.0.0.0", port), MetricsHandler) + print(f"NVIDIA GPU Exporter listening on port {port}") + + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + + # Poll so SIGTERM can stop the loop quickly instead of waiting on long timeouts. + server.timeout = 1 + + try: + while not stop_event.is_set(): + server.handle_request() + finally: + server.server_close() + pynvml.nvmlShutdown() + + +if __name__ == "__main__": + run_server() diff --git a/ansible/roles/gpu_monitoring/handlers/main.yml b/ansible/roles/gpu_monitoring/handlers/main.yml new file mode 100644 index 0000000..885b52f --- /dev/null +++ b/ansible/roles/gpu_monitoring/handlers/main.yml @@ -0,0 +1,27 @@ +--- +- name: Reload systemd daemon + become: true + ansible.builtin.systemd: + daemon_reload: true + listen: Reload systemd daemon + +- name: Restart nvidia_exporter + become: true + ansible.builtin.systemd: + name: "{{ nvidia_exporter_service_name }}" + state: restarted + listen: Restart nvidia_exporter + +- name: Restart amd_exporter + become: true + ansible.builtin.systemd: + name: "{{ amd_exporter_service_name }}" + state: restarted + listen: Restart amd_exporter + +- name: Restart llama_exporter + become: true + ansible.builtin.systemd: + name: "{{ llama_exporter_service_name }}" + state: restarted + listen: Restart llama_exporter diff --git a/ansible/roles/gpu_monitoring/meta/main.yml b/ansible/roles/gpu_monitoring/meta/main.yml new file mode 100644 index 0000000..8c24c7a --- /dev/null +++ b/ansible/roles/gpu_monitoring/meta/main.yml @@ -0,0 +1,15 @@ +--- +galaxy_info: + author: Alexandre Pires + description: Deploy Prometheus monitoring stack on gpu-01 for GPU and LLM metrics + company: A13Labs + role_name: gpu_monitoring + namespace: a13labs + license: MIT + min_ansible_version: "2.1" + platforms: + - name: Fedora + versions: + - "39" + - "40" + - "41" diff --git a/ansible/roles/gpu_monitoring/molecule/default/converge.yml b/ansible/roles/gpu_monitoring/molecule/default/converge.yml new file mode 100644 index 0000000..798aa60 --- /dev/null +++ b/ansible/roles/gpu_monitoring/molecule/default/converge.yml @@ -0,0 +1,11 @@ +--- +- name: Converge + hosts: all + gather_facts: true + tasks: + - name: Include gpu_monitoring role + ansible.builtin.include_role: + name: "gpu_monitoring" + vars: + amd_exporter_enabled: false + llama_exporter_enabled: false diff --git a/ansible/roles/gpu_monitoring/molecule/default/molecule.yml b/ansible/roles/gpu_monitoring/molecule/default/molecule.yml new file mode 100644 index 0000000..dd8ab78 --- /dev/null +++ b/ansible/roles/gpu_monitoring/molecule/default/molecule.yml @@ -0,0 +1,17 @@ +--- +driver: + name: podman +platforms: + - name: instance + image: fedora:latest + pre_build_image: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true +provisioner: + name: ansible + playbooks: + converge: converge.yml + prepare: prepare.yml +verifier: + name: ansible diff --git a/ansible/roles/gpu_monitoring/molecule/default/prepare.yml b/ansible/roles/gpu_monitoring/molecule/default/prepare.yml new file mode 100644 index 0000000..460a6cb --- /dev/null +++ b/ansible/roles/gpu_monitoring/molecule/default/prepare.yml @@ -0,0 +1,10 @@ +--- +- name: Prepare + hosts: all + gather_facts: false + tasks: + - name: Update cache + ansible.builtin.raw: dnf update -y + + - name: Install required packages + ansible.builtin.raw: dnf install -y python3 diff --git a/ansible/roles/gpu_monitoring/tasks/amd_exporter.yml b/ansible/roles/gpu_monitoring/tasks/amd_exporter.yml new file mode 100644 index 0000000..d3b76a6 --- /dev/null +++ b/ansible/roles/gpu_monitoring/tasks/amd_exporter.yml @@ -0,0 +1,82 @@ +--- +- name: Create amd_exporter group + become: true + ansible.builtin.group: + name: "{{ amd_exporter_user }}" + state: present + system: true + +- name: Create amd_exporter user + become: true + ansible.builtin.user: + name: "{{ amd_exporter_user }}" + group: "{{ amd_exporter_user }}" + home: "{{ amd_exporter_user_home }}" + create_home: true + shell: /usr/sbin/nologin + system: true + +- name: Create amd_exporter install directory + become: true + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: "{{ amd_exporter_user }}" + group: "{{ amd_exporter_user }}" + mode: "0755" + loop: + - "{{ amd_exporter_install_dir }}" + - "{{ amd_exporter_install_dir }}/logs" + +- name: Copy amd_gpu_exporter script + become: true + ansible.builtin.copy: + src: amd_gpu_exporter.py + dest: "{{ amd_exporter_script_path }}" + owner: "{{ amd_exporter_user }}" + group: "{{ amd_exporter_user }}" + mode: "0755" + notify: + - Restart amd_exporter + +- name: Create Python virtual environment + become: true + ansible.builtin.command: + cmd: "{{ amd_exporter_python_version }} -m venv {{ amd_exporter_venv_path }}" + creates: "{{ amd_exporter_venv_path }}/bin/activate" + +- name: Install Python dependencies in venv + become: true + ansible.builtin.pip: + name: + - prometheus-client + - requests + virtualenv: "{{ amd_exporter_venv_path }}" + state: present + +- name: Fix ownership of amd_exporter directory + become: true + ansible.builtin.file: + path: "{{ amd_exporter_install_dir }}" + owner: "{{ amd_exporter_user }}" + group: "{{ amd_exporter_user }}" + recurse: true + +- name: Generate amd_exporter systemd service + become: true + ansible.builtin.template: + src: amdgpu_exporter.service.j2 + dest: "/etc/systemd/system/{{ amd_exporter_service_name }}.service" + owner: root + group: root + mode: "0644" + notify: + - Reload systemd daemon + - Restart amd_exporter + +- name: Enable amd_exporter service + become: true + ansible.builtin.systemd: + name: "{{ amd_exporter_service_name }}" + enabled: true + state: started diff --git a/ansible/roles/gpu_monitoring/tasks/firewall.yml b/ansible/roles/gpu_monitoring/tasks/firewall.yml new file mode 100644 index 0000000..d2559f2 --- /dev/null +++ b/ansible/roles/gpu_monitoring/tasks/firewall.yml @@ -0,0 +1,13 @@ +--- +- name: Add Prometheus firewall rules + become: true + community.general.ufw: + rule: allow + src: "{{ item.cidr }}" + port: "{{ prometheus_fw_port | string }}" + proto: "{{ prometheus_fw_proto }}" + comment: "{{ item.comment | default(omit) }}" + loop: "{{ prometheus_fw_allowed_networks }}" + loop_control: + label: "{{ item.cidr }} -> port {{ prometheus_fw_port }}" + when: item.cidr is defined and item.cidr | length > 0 diff --git a/ansible/roles/gpu_monitoring/tasks/llama_exporter.yml b/ansible/roles/gpu_monitoring/tasks/llama_exporter.yml new file mode 100644 index 0000000..5633ac4 --- /dev/null +++ b/ansible/roles/gpu_monitoring/tasks/llama_exporter.yml @@ -0,0 +1,82 @@ +--- +- name: Create llama_exporter group + become: true + ansible.builtin.group: + name: "{{ llama_exporter_user }}" + state: present + system: true + +- name: Create llama_exporter user + become: true + ansible.builtin.user: + name: "{{ llama_exporter_user }}" + group: "{{ llama_exporter_user }}" + home: "{{ llama_exporter_user_home }}" + create_home: true + shell: /usr/sbin/nologin + system: true + +- name: Create llama_exporter install directory + become: true + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: "{{ llama_exporter_user }}" + group: "{{ llama_exporter_user }}" + mode: "0755" + loop: + - "{{ llama_exporter_install_dir }}" + - "{{ llama_exporter_install_dir }}/logs" + +- name: Copy llama_exporter script + become: true + ansible.builtin.copy: + src: llama_exporter.py + dest: "{{ llama_exporter_script_path }}" + owner: "{{ llama_exporter_user }}" + group: "{{ llama_exporter_user }}" + mode: "0755" + notify: + - Restart llama_exporter + +- name: Create Python virtual environment + become: true + ansible.builtin.command: + cmd: "{{ ansible_facts['python']['executable'] }} -m venv {{ llama_exporter_venv_path }}" + creates: "{{ llama_exporter_venv_path }}/bin/activate" + +- name: Install Python dependencies in llama_exporter venv + become: true + ansible.builtin.pip: + name: + - prometheus-client + - requests + virtualenv: "{{ llama_exporter_venv_path }}" + state: present + +- name: Fix ownership of llama_exporter directory + become: true + ansible.builtin.file: + path: "{{ llama_exporter_install_dir }}" + owner: "{{ llama_exporter_user }}" + group: "{{ llama_exporter_user }}" + recurse: true + +- name: Generate llama_exporter systemd service + become: true + ansible.builtin.template: + src: llama_exporter.service.j2 + dest: "/etc/systemd/system/{{ llama_exporter_service_name }}.service" + owner: root + group: root + mode: "0644" + notify: + - Reload systemd daemon + - Restart llama_exporter + +- name: Enable llama_exporter service + become: true + ansible.builtin.systemd: + name: "{{ llama_exporter_service_name }}" + enabled: true + state: started diff --git a/ansible/roles/gpu_monitoring/tasks/main.yml b/ansible/roles/gpu_monitoring/tasks/main.yml new file mode 100644 index 0000000..e7bd0e9 --- /dev/null +++ b/ansible/roles/gpu_monitoring/tasks/main.yml @@ -0,0 +1,28 @@ +--- +- name: Include NVIDIA GPU Exporter setup + when: nvidia_exporter_enabled | default(true) | bool + ansible.builtin.include_tasks: nvidia_exporter.yml + tags: + - roles::gpu_monitoring::nvidia_exporter + +- name: Include AMD GPU Exporter setup + when: amd_exporter_enabled | default(true) | bool + ansible.builtin.include_tasks: amd_exporter.yml + tags: + - roles::gpu_monitoring::amd_exporter + +- name: Include Llama Exporter setup + when: llama_exporter_enabled | default(true) | bool + ansible.builtin.include_tasks: llama_exporter.yml + tags: + - roles::gpu_monitoring::llama_exporter + +- name: Include Prometheus setup + ansible.builtin.include_tasks: prometheus.yml + tags: + - roles::gpu_monitoring::prometheus + +- name: Include firewall rules + ansible.builtin.include_tasks: firewall.yml + tags: + - roles::gpu_monitoring::firewall diff --git a/ansible/roles/gpu_monitoring/tasks/nvidia_exporter.yml b/ansible/roles/gpu_monitoring/tasks/nvidia_exporter.yml new file mode 100644 index 0000000..499ab6f --- /dev/null +++ b/ansible/roles/gpu_monitoring/tasks/nvidia_exporter.yml @@ -0,0 +1,84 @@ +--- +# NVIDIA GPU Exporter using NVML (pynvml) +# Works with consumer GPUs (RTX/GTX) and datacenter GPUs +# No DCGM required - uses pynvml Python library + +- name: Create NVIDIA exporter group + ansible.builtin.group: + name: "{{ nvidia_exporter_user }}" + state: present + system: true + become: true + +- name: Create NVIDIA exporter user + ansible.builtin.user: + name: "{{ nvidia_exporter_user }}" + group: "{{ nvidia_exporter_user }}" + home: "{{ nvidia_exporter_user_home }}" + system: true + create_home: true + shell: /usr/sbin/nologin + become: true + +- name: Create NVIDIA exporter install directory + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: "{{ nvidia_exporter_user }}" + group: "{{ nvidia_exporter_user }}" + mode: "0755" + loop: + - "{{ nvidia_exporter_install_dir }}" + - "{{ nvidia_exporter_install_dir }}/logs" + become: true + +- name: Copy NVIDIA exporter script + ansible.builtin.copy: + src: nvidia_gpu_exporter.py + dest: "{{ nvidia_exporter_script_path }}" + owner: "{{ nvidia_exporter_user }}" + group: "{{ nvidia_exporter_user }}" + mode: "0755" + become: true + notify: + - Restart nvidia_exporter + +- name: Create Python virtual environment for NVIDIA exporter + ansible.builtin.command: + cmd: "{{ ansible_facts['python']['executable'] }} -m venv {{ nvidia_exporter_venv_path }}" + creates: "{{ nvidia_exporter_venv_path }}/bin/activate" + become: true + +- name: Install nvidia-ml-py in NVIDIA exporter venv + ansible.builtin.pip: + name: "nvidia-ml-py" + virtualenv: "{{ nvidia_exporter_venv_path }}" + state: present + become: true + +- name: Fix ownership of NVIDIA exporter directory + ansible.builtin.file: + path: "{{ nvidia_exporter_install_dir }}" + owner: "{{ nvidia_exporter_user }}" + group: "{{ nvidia_exporter_user }}" + recurse: true + become: true + +- name: Copy NVIDIA exporter systemd service + ansible.builtin.template: + src: nvidia_exporter.service.j2 + dest: "/etc/systemd/system/{{ nvidia_exporter_service_name }}.service" + owner: root + group: root + mode: "0644" + become: true + notify: + - Reload systemd daemon + - Restart nvidia_exporter + +- name: Ensure NVIDIA exporter is enabled and started + ansible.builtin.systemd: + name: "{{ nvidia_exporter_service_name }}" + enabled: true + state: started + become: true diff --git a/ansible/roles/gpu_monitoring/tasks/prometheus.yml b/ansible/roles/gpu_monitoring/tasks/prometheus.yml new file mode 100644 index 0000000..d9fca1d --- /dev/null +++ b/ansible/roles/gpu_monitoring/tasks/prometheus.yml @@ -0,0 +1,206 @@ +--- +# Prometheus - Podman container with standard podman.yml pattern +# Follows the pattern in playbook_podman_llama_vulkan.yml + +- name: Prometheus | Check if required vars are set + ansible.builtin.fail: + msg: "Required variables are not set for Podman setup. Please check your playbook variables.)" + when: prometheus_user is not defined or prometheus_group is not defined or prometheus_user_home is not defined + tags: + - prometheus + +- name: Prometheus | Create prometheus group + become: true + ansible.builtin.group: + name: "{{ prometheus_group }}" + system: true + tags: + - prometheus + +- name: Prometheus | Create prometheus user + become: true + ansible.builtin.user: + name: "{{ prometheus_user }}" + groups: "{{ prometheus_extra_groups | default([]) }}" + shell: /bin/bash + home: "{{ prometheus_user_home }}" + group: "{{ prometheus_group }}" + tags: + - prometheus + +- name: Prometheus | Disable password login for prometheus user + ansible.builtin.command: passwd -d "{{ prometheus_user }}" + become: true + changed_when: false + tags: + - prometheus + +- name: Prometheus | Create prometheus user home + become: true + ansible.builtin.file: + path: "{{ prometheus_user_home }}" + state: directory + owner: "{{ prometheus_user }}" + group: "{{ prometheus_group }}" + mode: "0750" + tags: + - prometheus + +- name: Prometheus | Create build folders + become: true + ansible.builtin.file: + path: "{{ prometheus_user_home }}/build" + state: directory + owner: "{{ prometheus_user }}" + group: "{{ prometheus_group }}" + mode: "0750" + tags: + - prometheus + +- name: Prometheus | Create folders + become: true + ansible.builtin.file: + path: "{{ prometheus_user_home }}/{{ item }}" + state: directory + owner: "{{ prometheus_user }}" + group: "{{ prometheus_group }}" + mode: "0750" + loop: "{{ prometheus_user_folders | default([]) }}" + tags: + - prometheus + +- name: Prometheus | Add SSH Authorized key + become: true + ansible.posix.authorized_key: + user: "{{ prometheus_user }}" + key: "{{ prometheus_user_pubkey }}" + state: "{{ 'present' if prometheus_user_pubkey is defined else 'absent' }}" + when: prometheus_user_pubkey is defined and prometheus_user_pubkey != "" + tags: + - prometheus + +- name: Prometheus | SELinux tasks (RedHat only) + become: true + when: ansible_os_family == 'RedHat' + tags: + - prometheus + block: + - name: Prometheus | Persistently set SELinux context + community.general.sefcontext: + target: "{{ prometheus_user_home }}(/.*)?" + setype: user_home_dir_t + state: present + + - name: Prometheus | Persistently set SELinux context (ssh authorized keys) + community.general.sefcontext: + target: "{{ prometheus_user_home }}/.ssh/authorized_keys" + setype: ssh_home_t + state: present + when: prometheus_user_pubkey is defined and prometheus_user_pubkey != "" + + - name: Prometheus | Apply SELinux context + ansible.builtin.command: restorecon -Rv {{ prometheus_user_home }} + changed_when: false + +- name: Prometheus | Enable lingering for prometheus user + become: true + ansible.builtin.command: loginctl enable-linger {{ prometheus_user }} + register: linger_status + changed_when: >- + 'created' in linger_status.stdout or + (linger_status.rc == 0 and not + ('linger file already exists' in linger_status.stderr or + 'linger file does not exist' in linger_status.stderr)) + failed_when: + - linger_status.rc != 0 and not + ('linger file already exists' in linger_status.stderr) + tags: + - prometheus + +- name: Prometheus | Get prometheus user UID + ansible.builtin.command: id -u {{ prometheus_user }} + changed_when: false + register: prometheus_uid + tags: + - prometheus + +- name: Prometheus | Deploy Prometheus configuration + ansible.builtin.template: + src: prometheus.yml.j2 + dest: "{{ prometheus_user_home }}/prometheus.yml" + owner: "{{ prometheus_user }}" + group: "{{ prometheus_group }}" + mode: "0644" + tags: + - prometheus + +- name: Prometheus | Create Prometheus data directory + ansible.builtin.file: + path: "{{ prometheus_data_dir }}" + state: directory + owner: "{{ prometheus_user }}" + group: "{{ prometheus_group }}" + mode: "0700" + tags: + - prometheus + +- name: Prometheus | Run Podman tasks as prometheus user + become: true + become_user: "{{ prometheus_user }}" + environment: + XDG_RUNTIME_DIR: "/run/user/{{ prometheus_uid.stdout }}" + tags: + - prometheus + block: + - name: Prometheus | Pull Prometheus image + containers.podman.podman_image: + name: "docker.io/prom/prometheus:{{ prometheus_image_tag }}" + force: false + + - name: Prometheus | Create podman network + containers.podman.podman_network: + name: "{{ prometheus_network_name }}" + + - name: Prometheus | Create Podman container + containers.podman.podman_container: + name: prometheus + image: "docker.io/prom/prometheus:{{ prometheus_image_tag }}" + state: started + recreate: true + user: "{{ prometheus_uid.stdout }}" + network: "{{ prometheus_network_name }}" + network_alias: + - prometheus + ports: + - "0.0.0.0:{{ prometheus_host_port }}:{{ prometheus_container_port }}/tcp" + volumes: + - "{{ prometheus_data_dir }}:/prometheus" + - "{{ prometheus_user_home }}/prometheus.yml:/etc/prometheus/prometheus.yml" + memory: "{{ prometheus_container_memory }}" + memory_swap: "0" + env: + TZ: "{{ timezone | default('UTC') }}" + PROMETHEUS_PORT: "{{ prometheus_port | string }}" + + systemd: "true" + restart: false + detach: true + label: + a13labs.prometheus: "true" + a13labs.service: "prometheus" + a13labs.team: "infra" + cmd_args: + - "--userns=keep-id" + - "--security-opt=label=disable" + restart_policy: unless-stopped + stop_signal: 1 +- name: Prometheus | Ensure systemd is reloaded after container creation + become: true + become_user: "{{ prometheus_user }}" + environment: + XDG_RUNTIME_DIR: "/run/user/{{ prometheus_uid.stdout }}" + ansible.builtin.systemd: + daemon_reload: true + scope: user + tags: + - prometheus diff --git a/ansible/roles/gpu_monitoring/templates/amdgpu_exporter.service.j2 b/ansible/roles/gpu_monitoring/templates/amdgpu_exporter.service.j2 new file mode 100644 index 0000000..aa05e35 --- /dev/null +++ b/ansible/roles/gpu_monitoring/templates/amdgpu_exporter.service.j2 @@ -0,0 +1,30 @@ +[Unit] +Description=AMD GPU Sysfs Metrics Exporter +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User={{ amd_exporter_user }} +Group={{ amd_exporter_user }} +WorkingDirectory={{ amd_exporter_install_dir }} +Environment="AMD_EXPORTER_PORT={{ amd_exporter_port }}" +Environment="AMD_EXPORTER_BIND=0.0.0.0" +Environment="PATH={{ amd_exporter_venv_bin }}:/usr/bin:/bin" +ExecStart={{ amd_exporter_venv_bin }}/python3 {{ amd_exporter_script_path }} +Restart=always +RestartSec=10 +TimeoutStopSec=10 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=amdgpu_exporter + +# Security hardening +ProtectSystem=strict +ProtectHome=true +ReadWritePaths={{ amd_exporter_install_dir }} /var/log +NoNewPrivileges=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/gpu_monitoring/templates/llama_exporter.service.j2 b/ansible/roles/gpu_monitoring/templates/llama_exporter.service.j2 new file mode 100644 index 0000000..38cc393 --- /dev/null +++ b/ansible/roles/gpu_monitoring/templates/llama_exporter.service.j2 @@ -0,0 +1,35 @@ +[Unit] +Description=LLM Inference Metrics Exporter (llama.cpp + Ollama) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User={{ llama_exporter_user }} +Group={{ llama_exporter_user }} +WorkingDirectory={{ llama_exporter_install_dir }} +Environment="LLAMA_EXPORTER_PORT={{ llama_exporter_port }}" +Environment="LLAMA_EXPORTER_BIND=0.0.0.0" +Environment="LLAMA_EXPORTER_INTERVAL={{ llama_exporter_scrape_interval }}" +Environment="LLAMA_EXPORTER_TIMEOUT={{ llama_exporter_scrape_timeout }}" +{% if llama_exporter_targets and llama_exporter_targets != "" %} +Environment="LLAMA_TARGETS={{ llama_exporter_targets | to_json }}" +{% endif %} +Environment="PATH={{ llama_exporter_venv_bin }}:/usr/bin:/bin" +ExecStart={{ llama_exporter_venv_bin }}/python3 {{ llama_exporter_script_path }} +Restart=always +RestartSec=10 +TimeoutStopSec=10 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=llama_exporter + +# Security hardening +ProtectSystem=strict +ProtectHome=true +ReadWritePaths={{ llama_exporter_install_dir }} /var/log +NoNewPrivileges=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/gpu_monitoring/templates/nvidia_exporter.service.j2 b/ansible/roles/gpu_monitoring/templates/nvidia_exporter.service.j2 new file mode 100644 index 0000000..5bc58c7 --- /dev/null +++ b/ansible/roles/gpu_monitoring/templates/nvidia_exporter.service.j2 @@ -0,0 +1,34 @@ +# nvidia_gpu_exporter - Prometheus exporter for NVIDIA GPUs via NVML +# +# Exposes GPU metrics at /metrics port {{ nvidia_exporter_port }} + +[Unit] +Description=NVIDIA GPU Prometheus Exporter +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User={{ nvidia_exporter_user }} +Group={{ nvidia_exporter_user }} +WorkingDirectory={{ nvidia_exporter_install_dir }} +Environment="PYTHONUNBUFFERED=1" +ExecStart={{ nvidia_exporter_venv_path }}/bin/python3 {{ nvidia_exporter_script_path }} +Restart=on-failure +RestartSec=5 +TimeoutStopSec=10 +StandardOutput=journal +StandardError=journal + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +ReadOnlyPaths=/sys +PrivateTmp=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/gpu_monitoring/templates/podman.service.j2 b/ansible/roles/gpu_monitoring/templates/podman.service.j2 new file mode 100644 index 0000000..cd8a2d0 --- /dev/null +++ b/ansible/roles/gpu_monitoring/templates/podman.service.j2 @@ -0,0 +1,17 @@ +[Unit] +Description=Podman service for {{ item.name }} +After=local-fs.target network-online.target + +[Service] +Type=oneshot +User={{ prometheus_user }} +Group={{ prometheus_group }} +RemainAfterExit=true +ExecStart={{ podman_binary | default('/usr/bin/podman') }} start {{ item.name }} +ExecStop={{ podman_binary | default('/usr/bin/podman') }} stop {{ item.name }} + +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/gpu_monitoring/templates/prometheus.yml.j2 b/ansible/roles/gpu_monitoring/templates/prometheus.yml.j2 new file mode 100644 index 0000000..5ef9609 --- /dev/null +++ b/ansible/roles/gpu_monitoring/templates/prometheus.yml.j2 @@ -0,0 +1,30 @@ +global: + scrape_interval: {{ prometheus_scrape_interval }} + evaluation_interval: {{ prometheus_evaluation_interval }} + scrape_timeout: {{ prometheus_scrape_timeout }} + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["localhost:{{ prometheus_container_port }}"] + + - job_name: nvidia-gpu + metrics_path: /metrics + static_configs: + - targets: ["{{ prometheus_host_gateway }}:{{ nvidia_exporter_port }}"] + labels: + datacenter: "{{ inventory_hostname }}" + + - job_name: amdgpu_exporter + metrics_path: /metrics + static_configs: + - targets: ["{{ prometheus_host_gateway }}:{{ amd_exporter_port }}"] + labels: + datacenter: "{{ inventory_hostname }}" + + - job_name: llama_exporter + metrics_path: /metrics + static_configs: + - targets: ["{{ prometheus_host_gateway }}:{{ llama_exporter_port }}"] + labels: + datacenter: "{{ inventory_hostname }}" diff --git a/ansible/roles/gpu_monitoring/vars/main.yml b/ansible/roles/gpu_monitoring/vars/main.yml new file mode 100644 index 0000000..416105e --- /dev/null +++ b/ansible/roles/gpu_monitoring/vars/main.yml @@ -0,0 +1,23 @@ +--- +# Internal role variables (non-overridable) + +# Container names +gpu_monitoring_prometheus_container_name: "prometheus" + +# Derived paths +nvidia_exporter_script_path: "{{ nvidia_exporter_install_dir }}/nvidia_gpu_exporter.py" +nvidia_exporter_venv_bin: "{{ nvidia_exporter_venv_path }}/bin" +nvidia_exporter_service_name: "nvidia_exporter" + +amd_exporter_script_path: "{{ amd_exporter_install_dir }}/amd_gpu_exporter.py" +amd_exporter_venv_bin: "{{ amd_exporter_venv_path }}/bin" +amd_exporter_service_name: "amdgpu_exporter" + +llama_exporter_script_path: "{{ llama_exporter_install_dir }}/llama_exporter.py" +llama_exporter_venv_bin: "{{ llama_exporter_venv_path }}/bin" +llama_exporter_service_name: "llama_exporter" + +# Default llama.cpp targets if none provided +llama_exporter_targets_default: + - name: "llama-cpp-8012" + url: "http://localhost:8012" diff --git a/ansible/tasks/llama_models_sync.yml b/ansible/tasks/llama_models_sync.yml new file mode 100644 index 0000000..86806df --- /dev/null +++ b/ansible/tasks/llama_models_sync.yml @@ -0,0 +1,146 @@ +- name: Ensure podman user exists before model sync + ansible.builtin.include_tasks: + file: podman/user.yml + +- name: Enable sudo access to podman user (temporary) + become: true + ansible.builtin.lineinfile: + path: "/etc/sudoers.d/ansible_{{ podman_user }}_tmp" + create: true + mode: "0440" + line: "{{ ansible_user_id }} ALL=({{ podman_user }}) NOPASSWD: ALL" + validate: "visudo -cf %s" + +- name: Validate llama_models schema + ansible.builtin.assert: + that: + - __llama_models__ is iterable + fail_msg: "llama_models must be a list" + +- name: Install Python packages required for llama model sync + become: true + ansible.builtin.package: + name: "{{ __llama_sync_system_packages__ }}" + state: present + vars: + __llama_sync_system_packages__: >- + {{ + { + 'Debian': ['python3-pip', 'python3-packaging', 'python3-venv'], + 'RedHat': ['python3-pip', 'python3-packaging'], + 'Archlinux': ['python-pip', 'python-packaging'] + }[ansible_os_family] + }} + +- name: Install Hugging Face hub client in llama model sync virtualenv + become: true + become_user: "{{ podman_user }}" + ansible.builtin.pip: + name: + - pip + - setuptools + - packaging + - huggingface_hub>=0.32.0 + state: present + virtualenv: "{{ __llama_sync_venv__ }}" + virtualenv_command: "python3 -m venv" + +- name: Create llama router model directories + become: true + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + owner: "{{ podman_user }}" + group: "{{ podman_group }}" + mode: "{{ item.mode }}" + loop: + - { path: "{{ podman_user_home }}/models", mode: "0750" } + - { path: "{{ __llama_managed_dir__ }}", mode: "0750" } + - { path: "{{ __llama_links_dir__ }}", mode: "0750" } + - { path: "{{ __llama_router_dir__ }}", mode: "0750" } + +- name: Copy llama Hugging Face sync helper + become: true + ansible.builtin.copy: + src: scripts/llama_hf_sync.py + dest: "{{ __llama_sync_script__ }}" + owner: "{{ podman_user }}" + group: "{{ podman_group }}" + mode: "0750" + +- name: Write desired llama model manifest input + become: true + ansible.builtin.copy: + content: "{{ __llama_models__ | to_nice_json }}" + dest: "{{ __llama_models_input_file__ }}" + owner: "{{ podman_user }}" + group: "{{ podman_group }}" + mode: "0640" + +- name: Write global llama preset options + become: true + ansible.builtin.copy: + content: "{{ __llama_preset_global__ | to_nice_json }}" + dest: "{{ __llama_preset_global_input_file__ }}" + owner: "{{ podman_user }}" + group: "{{ podman_group }}" + mode: "0640" + +- name: Run llama model sync from Hugging Face + vars: + __llama_sync_argv__: >- + {{ + [ + __llama_sync_python__, + __llama_sync_script__, + '--models-file', + __llama_models_input_file__, + '--managed-dir', + __llama_managed_dir__, + '--links-dir', + __llama_links_dir__, + '--manifest-file', + __llama_manifest_file__, + '--preset-file', + __llama_preset_file__, + '--preset-global-file', + __llama_preset_global_input_file__, + '--container-links-dir', + '/models/managed-links' + ] + + (['--prune'] if llama_models_prune_enabled | default(true) else []) + + (['--dry-run'] if llama_models_dry_run | default(false) else []) + }} + become: true + become_user: "{{ podman_user }}" + ansible.builtin.command: + argv: "{{ __llama_sync_argv__ }}" + environment: + HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') | default('', true) }}" + register: __llama_sync_output__ + changed_when: false + +- name: Parse llama sync output + ansible.builtin.set_fact: + __llama_sync_result__: "{{ __llama_sync_output__.stdout | from_json }}" + +- name: Ensure router preset file exists when not in dry-run mode + become: true + become_user: "{{ podman_user }}" + ansible.builtin.stat: + path: "{{ __llama_preset_file__ }}" + register: __llama_preset_stat__ + when: not (llama_models_dry_run | default(false)) + +- name: Ensure router preset file exists when not in dry-run mode + ansible.builtin.assert: + that: + - __llama_preset_stat__.stat.exists + fail_msg: "llama router preset file was not created" + when: not (llama_models_dry_run | default(false)) + +- name: Remove temporary sudo access + become: true + ansible.builtin.file: + path: "/etc/sudoers.d/ansible_{{ podman_user }}_tmp" + state: absent diff --git a/ansible/tasks/podman.yml b/ansible/tasks/podman.yml index 25e9ca8..9494e40 100644 --- a/ansible/tasks/podman.yml +++ b/ansible/tasks/podman.yml @@ -34,7 +34,7 @@ path: "/etc/sudoers.d/ansible_{{ podman_user }}_tmp" create: true mode: "0440" - line: "{{ ansible_user_id }} ALL=({{ podman_user }}) NOPASSWD: ALL" + line: "{{ ansible_facts['user_id'] }} ALL=({{ podman_user }}) NOPASSWD: ALL" validate: "visudo -cf %s" - name: Enable lingering for podman user @@ -42,10 +42,10 @@ ansible.builtin.command: loginctl enable-linger {{ podman_user }} register: linger_status changed_when: >- - "'created' in linger_status.stdout or + 'created' in linger_status.stdout or (linger_status.rc == 0 and not ('linger file already exists' in linger_status.stderr or - 'linger file does not exist' in linger_status.stderr))" + 'linger file does not exist' in linger_status.stderr)) failed_when: - linger_status.rc != 0 and not ('linger file already exists' in linger_status.stderr) diff --git a/inventory/hosts b/inventory/hosts index 85a4d9e..0cce085 100644 --- a/inventory/hosts +++ b/inventory/hosts @@ -125,3 +125,6 @@ gpu-01.lab.alexpires.me [qwen3_6_host] gpu-01.lab.alexpires.me +[gpu_monitoring] +gpu-01.lab.alexpires.me + diff --git a/terraform/apps/dev-01/apps.tf b/terraform/apps/dev-01/apps.tf index a6c4915..359becd 100644 --- a/terraform/apps/dev-01/apps.tf +++ b/terraform/apps/dev-01/apps.tf @@ -92,3 +92,13 @@ module "searxng" { tag = "2026.5.2-aefc3c316" secret_key = var.searxng_secret_key } + +module "vaultwarden" { + source = "../../modules/apps/vaultwarden" + + persistent_folder = local.vaultwarden_folder + fqdn = local.vaultwarden_fqdn + issuer = local.vaultwarden_issuer + tag = "1.33.4" + admin_token = var.vaultwarden_admin_token +} diff --git a/terraform/apps/dev-01/config.tf b/terraform/apps/dev-01/config.tf index ce2d10f..6d93a6a 100644 --- a/terraform/apps/dev-01/config.tf +++ b/terraform/apps/dev-01/config.tf @@ -121,6 +121,11 @@ locals { name = "ci-01" type = "A" content = "10.19.4.207" + }, + { + name = "vaultwarden" + type = "CNAME" + content = "dev-01.${local.lab_domain}." } ] } @@ -170,7 +175,7 @@ locals { enabled = true mac = "04:7c:16:d6:63:66" ip = "10.19.4.106" - port = "11434" + port = "11435" scheme = "http" } @@ -249,6 +254,11 @@ locals { searxng_folder = "/mnt/md0/searxng" searxng_fqdn = "search.${local.lab_domain}" searxng_issuer = "letsencrypt-prod" + + # vaultwarden + vaultwarden_folder = "/mnt/md0/vaultwarden" + vaultwarden_fqdn = "vaultwarden.${local.lab_domain}" + vaultwarden_issuer = "internal-ca" } data "terraform_remote_state" "scaleway" { diff --git a/terraform/apps/dev-01/secrets.tf b/terraform/apps/dev-01/secrets.tf index eea8a33..c34bdcf 100644 --- a/terraform/apps/dev-01/secrets.tf +++ b/terraform/apps/dev-01/secrets.tf @@ -33,3 +33,9 @@ variable "searxng_secret_key" { type = string sensitive = true } + +variable "vaultwarden_admin_token" { + description = "The admin token for VaultWarden" + type = string + sensitive = true +} diff --git a/terraform/apps/dev-01/sectool.env b/terraform/apps/dev-01/sectool.env index 6c088bb..de9f07e 100644 --- a/terraform/apps/dev-01/sectool.env +++ b/terraform/apps/dev-01/sectool.env @@ -7,3 +7,4 @@ TF_VAR_scaleway_project_id=$SCALEWAY_PROJECT_ID TF_VAR_telegram_bot_token=$TELEGRAM_BOT_TOKEN TF_VAR_webui_secret_key=$WEBUI_SECRET_KEY TF_VAR_searxng_secret_key=$SEARXNG_SECRET_KEY +TF_VAR_vaultwarden_admin_token=$VAULTWARDEN_ADMIN_TOKEN diff --git a/terraform/apps/prod-01/apps.tf b/terraform/apps/prod-01/apps.tf index be41687..60a66de 100644 --- a/terraform/apps/prod-01/apps.tf +++ b/terraform/apps/prod-01/apps.tf @@ -157,3 +157,13 @@ module "auth_exporter" { source = "../../modules/utils/auth-exporter" persistent_folder = "${local.persistent_folder}/monitoring" } + +module "vaultwarden" { + source = "../../modules/apps/vaultwarden" + + persistent_folder = local.vaultwarden_folder + fqdn = local.vaultwarden_fqdn + issuer = local.internal_issuer + tag = "1.33.4" + admin_token = var.vaultwarden_admin_token +} diff --git a/terraform/apps/prod-01/config.tf b/terraform/apps/prod-01/config.tf index e9b994d..3c27112 100644 --- a/terraform/apps/prod-01/config.tf +++ b/terraform/apps/prod-01/config.tf @@ -210,6 +210,11 @@ locals { # telegram_chat_id = "-950480346" # alexpires.me group telegram_message = file("${path.module}/resources/telegram.tpl") + # vaultwarden + vaultwarden_folder = "${local.persistent_folder}/vaultwarden" + vaultwarden_fqdn = "vaultwarden.${local.primary_domain}" + internal_issuer = "internal-ca" + # get secrets from remote state backup_bucket_name = data.terraform_remote_state.scaleway.outputs.backup_bucket_name backup_app_api_secret_key = data.terraform_remote_state.scaleway.outputs.backup_app_api_secret_key diff --git a/terraform/apps/prod-01/secrets.tf b/terraform/apps/prod-01/secrets.tf index 4b4cc0f..c8b369d 100644 --- a/terraform/apps/prod-01/secrets.tf +++ b/terraform/apps/prod-01/secrets.tf @@ -134,3 +134,9 @@ variable "telegram_bot_token" { type = string sensitive = true } + +variable "vaultwarden_admin_token" { + description = "The admin token for VaultWarden" + type = string + sensitive = true +} diff --git a/terraform/apps/prod-01/sectool.env b/terraform/apps/prod-01/sectool.env index 4cf4889..5afd3f4 100644 --- a/terraform/apps/prod-01/sectool.env +++ b/terraform/apps/prod-01/sectool.env @@ -33,3 +33,4 @@ TF_VAR_wireguard_1_public_key=$WIREGUARD_1_PUBLIC_KEY TF_VAR_wireguard_1_pre_shared_key=$WIREGUARD_1_PRE_SHARED_KEY TF_VAR_nginx_token_bearer=$NGINX_TOKEN_BEARER TF_VAR_telegram_bot_token=$TELEGRAM_BOT_TOKEN +TF_VAR_vaultwarden_admin_token=$VAULTWARDEN_ADMIN_TOKEN diff --git a/terraform/modules/apps/grafana/config.tf b/terraform/modules/apps/grafana/config.tf index e65d009..02d805f 100644 --- a/terraform/modules/apps/grafana/config.tf +++ b/terraform/modules/apps/grafana/config.tf @@ -2,7 +2,8 @@ locals { app_version = var.tag monitoring_folder = "${var.persistent_folder}/grafana" - grafana_datasource_config = file("${path.module}/resources/default.yml") - grafana_dashboard_provider = file("${path.module}/resources/dashboard-provider.yml") - grafana_trivy_dashboard = file("${path.module}/resources/trivy-vulnerabilities-dashboard.json") + grafana_datasource_config = file("${path.module}/resources/default.yml") + grafana_dashboard_provider = file("${path.module}/resources/dashboard-provider.yml") + grafana_trivy_dashboard = file("${path.module}/resources/trivy-vulnerabilities-dashboard.json") + grafana_gpu_realtime_dashboard = file("${path.module}/resources/gpu-realtime-dashboard.json") } diff --git a/terraform/modules/apps/grafana/grafana.tf b/terraform/modules/apps/grafana/grafana.tf index 234c285..47c5ae9 100644 --- a/terraform/modules/apps/grafana/grafana.tf +++ b/terraform/modules/apps/grafana/grafana.tf @@ -28,6 +28,7 @@ resource "kubernetes_config_map" "grafana_dashboards" { data = { "trivy-vulnerabilities-dashboard.json" = local.grafana_trivy_dashboard + "gpu-realtime-dashboard.json" = local.grafana_gpu_realtime_dashboard } } diff --git a/terraform/modules/apps/grafana/resources/gpu-realtime-dashboard.json b/terraform/modules/apps/grafana/resources/gpu-realtime-dashboard.json new file mode 100644 index 0000000..a67345a --- /dev/null +++ b/terraform/modules/apps/grafana/resources/gpu-realtime-dashboard.json @@ -0,0 +1,550 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": true, + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "up{job=~\"prometheus|nvidia-gpu|amdgpu_exporter|llama_exporter\"}", + "instant": true, + "legendFormat": "{{job}}", + "range": false, + "refId": "A" + } + ], + "title": "Exporter Health", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "{__name__=~\"gpu_.*_gpu_util_percent|gpu_util_percent\"}", + "legendFormat": "{{__name__}} {{gpu}}", + "range": true, + "refId": "A" + } + ], + "title": "GPU Utilization", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "celsius" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "{__name__=~\"gpu_.*_temperature_celsius|temperature_celsius\"}", + "legendFormat": "{{__name__}} {{gpu}}", + "range": true, + "refId": "A" + } + ], + "title": "GPU Temperature", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "{__name__=~\"gpu_.*_power_watts|power_watts\"}", + "legendFormat": "{{__name__}} {{gpu}}", + "range": true, + "refId": "A" + } + ], + "title": "GPU Power Draw", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "{__name__=~\"gpu_.*_memory_used_percent\"}", + "legendFormat": "nvidia", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "100 * mem_used_bytes / clamp_min(mem_used_bytes + mem_avail_bytes, 1)", + "legendFormat": "amd {{gpu}}", + "range": true, + "refId": "B" + } + ], + "title": "GPU Memory Usage", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "scrape_duration_seconds{job=~\"nvidia-gpu|amdgpu_exporter|llama_exporter|prometheus\"}", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Scrape Duration", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "llama_server_health", + "legendFormat": "health {{server}}", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "llama_models_loaded", + "legendFormat": "models {{server}} {{model}}", + "range": true, + "refId": "B" + } + ], + "title": "Llama Server Metrics", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "{__name__=~\"gpu_.*_pstate\"}", + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "NVIDIA P-State", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "{__name__=~\"gpu_.*_pcie_link_gen_current\"}", + "legendFormat": "pcie gen", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "{__name__=~\"gpu_.*_pcie_link_width_current\"}", + "legendFormat": "pcie width", + "range": true, + "refId": "B" + } + ], + "title": "NVIDIA PCIe Link", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "max": 1, + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "{__name__=~\"gpu_.*_throttle_sw_power_cap\"}", + "legendFormat": "sw power cap", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "{__name__=~\"gpu_.*_throttle_hw_thermal_slowdown\"}", + "legendFormat": "hw thermal slowdown", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "expr": "{__name__=~\"gpu_.*_throttle_hw_power_brake_slowdown\"}", + "legendFormat": "hw power brake slowdown", + "range": true, + "refId": "C" + } + ], + "title": "NVIDIA Throttle Flags", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "gpu", + "monitoring", + "realtime" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GPU Monitoring Realtime", + "uid": "gpu-monitoring-realtime", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/terraform/modules/apps/vaultwarden/config.tf b/terraform/modules/apps/vaultwarden/config.tf new file mode 100644 index 0000000..4f1de71 --- /dev/null +++ b/terraform/modules/apps/vaultwarden/config.tf @@ -0,0 +1,6 @@ +locals { + app_folder = var.persistent_folder + app_version = var.tag + app_fqdn = var.fqdn + db_path = var.db_folder != "" ? var.db_folder : local.app_folder +} diff --git a/terraform/modules/apps/vaultwarden/ingress.tf b/terraform/modules/apps/vaultwarden/ingress.tf new file mode 100644 index 0000000..9db52bf --- /dev/null +++ b/terraform/modules/apps/vaultwarden/ingress.tf @@ -0,0 +1,39 @@ +resource "kubernetes_ingress_v1" "vaultwarden" { + metadata { + name = "ingress" + namespace = kubernetes_namespace.vaultwarden.metadata[0].name + + annotations = { + "kubernetes.io/ingress.class" = "public" + "cert-manager.io/cluster-issuer" = var.issuer + "kubernetes.io/tls-acme" = "true" + "nginx.ingress.kubernetes.io/ssl-redirect" = "true" + } + } + + spec { + dynamic "tls" { + for_each = var.tls_enabled ? [1] : [] + content { + hosts = [var.fqdn] + secret_name = "vaultwarden-tls" + } + } + + rule { + host = var.fqdn + http { + path { + backend { + service { + name = kubernetes_service.vaultwarden.metadata[0].name + port { + number = 80 + } + } + } + } + } + } + } +} diff --git a/terraform/modules/apps/vaultwarden/main.tf b/terraform/modules/apps/vaultwarden/main.tf new file mode 100644 index 0000000..deb6a4a --- /dev/null +++ b/terraform/modules/apps/vaultwarden/main.tf @@ -0,0 +1,191 @@ +resource "kubernetes_namespace" "vaultwarden" { + metadata { + name = "vaultwarden" + } +} + +resource "kubernetes_service_account" "vaultwarden" { + metadata { + name = "vaultwarden-sa" + namespace = kubernetes_namespace.vaultwarden.metadata[0].name + } + + automount_service_account_token = false +} + +resource "kubernetes_config_map" "vaultwarden_config" { + metadata { + name = "vaultwarden-config" + namespace = kubernetes_namespace.vaultwarden.metadata[0].name + } + + data = { + DOMAIN = "https://${local.app_fqdn}/" + SIGNUP = var.signup_enabled ? "true" : "false" + DATABASE_URL = "file:///data/db.sqlite3" + } +} + +resource "kubernetes_secret" "vaultwarden_secrets" { + metadata { + name = "vaultwarden-keys" + namespace = kubernetes_namespace.vaultwarden.metadata[0].name + } + + data = { + ADMIN_TOKEN = var.admin_token + } +} + +resource "kubernetes_deployment" "vaultwarden" { + metadata { + name = "vaultwarden" + namespace = kubernetes_namespace.vaultwarden.metadata[0].name + labels = { + app = "vaultwarden" + } + } + + spec { + replicas = 1 + + selector { + match_labels = { + app = "vaultwarden" + } + } + + strategy { + type = "Recreate" + } + + template { + metadata { + labels = { + app = "vaultwarden" + } + } + + spec { + service_account_name = kubernetes_service_account.vaultwarden.metadata[0].name + automount_service_account_token = false + + security_context { + run_as_non_root = true + run_as_user = 1000 + run_as_group = 1000 + fs_group = 1000 + + seccomp_profile { + type = "RuntimeDefault" + } + } + + container { + name = "vaultwarden" + image = "vaultwarden/server:${local.app_version}" + + env_from { + config_map_ref { + name = kubernetes_config_map.vaultwarden_config.metadata[0].name + } + } + + env { + name = "ADMIN_TOKEN" + value_from { + secret_key_ref { + name = kubernetes_secret.vaultwarden_secrets.metadata[0].name + key = "ADMIN_TOKEN" + } + } + } + + security_context { + allow_privilege_escalation = false + privileged = false + + capabilities { + drop = ["ALL"] + } + + read_only_root_filesystem = true + } + + port { + name = "http" + container_port = 80 + protocol = "TCP" + } + + liveness_probe { + http_get { + path = "/" + port = "http" + } + initial_delay_seconds = 30 + period_seconds = 15 + } + + readiness_probe { + http_get { + path = "/" + port = "http" + } + initial_delay_seconds = 10 + period_seconds = 10 + } + + volume_mount { + name = "vaultwarden-data" + mount_path = "/data" + } + + volume_mount { + name = "vaultwarden-tmp" + mount_path = "/tmp" + } + } + + volume { + name = "vaultwarden-data" + host_path { + path = local.db_path + } + } + + volume { + name = "vaultwarden-tmp" + empty_dir {} + } + } + } + } +} + +resource "kubernetes_service" "vaultwarden" { + metadata { + name = "vaultwarden-service" + namespace = kubernetes_namespace.vaultwarden.metadata[0].name + } + + spec { + selector = { + app = "vaultwarden" + } + + port { + name = "http" + port = 80 + target_port = 80 + } + + type = "ClusterIP" + } + + lifecycle { + ignore_changes = [ + metadata[0].annotations + ] + } +} diff --git a/terraform/modules/apps/vaultwarden/provider.tf b/terraform/modules/apps/vaultwarden/provider.tf new file mode 100644 index 0000000..2eaa508 --- /dev/null +++ b/terraform/modules/apps/vaultwarden/provider.tf @@ -0,0 +1,9 @@ +terraform { + required_version = "~>1.8" + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.36.0" + } + } +} diff --git a/terraform/modules/apps/vaultwarden/variables.tf b/terraform/modules/apps/vaultwarden/variables.tf new file mode 100644 index 0000000..f558dcc --- /dev/null +++ b/terraform/modules/apps/vaultwarden/variables.tf @@ -0,0 +1,45 @@ +variable "persistent_folder" { + description = "Path for persistent data on the host" + type = string +} + +variable "fqdn" { + description = "FQDN for the service" + type = string +} + +variable "tag" { + description = "Image tag to deploy" + type = string + default = "latest" +} + +variable "issuer" { + description = "The cert-manager cluster issuer" + type = string + default = "letsencrypt-prod" +} + +variable "tls_enabled" { + description = "Enable TLS in ingress" + type = bool + default = true +} + +variable "signup_enabled" { + description = "Allow new user signups" + type = bool + default = false +} + +variable "admin_token" { + description = "Admin token for VaultWarden" + type = string + sensitive = true +} + +variable "db_folder" { + description = "Path for the database folder (default: persistent_folder)" + type = string + default = "" +}