Add GPU monitoring services and configurations

- Create systemd service templates for llama_exporter, nvidia_exporter, and podman. - Add Prometheus configuration template for GPU metrics scraping. - Introduce variables for GPU monitoring role in main.yml. - Implement tasks for syncing llama models, including user setup and package installation. - Update podman tasks to ensure proper sudo access and lingering for the podman user. - Modify inventory to include gpu_monitoring group. - Add Terraform modules for deploying Vaultwarden, including ingress and service configurations. - Create Grafana dashboard for real-time GPU monitoring metrics. - Update secrets and environment files to include Vaultwarden admin token.
2026-05-29 23:42:14 +02:00
parent e9e28a5ca8
commit 350650ecc2
47 changed files with 3727 additions and 49 deletions
@@ -0,0 +1,101 @@
+- name: Setup llama.cpp - Vulkan
+  hosts: llamacpp
+  vars:
+    podman_user: "{{ llama_user | default('llama') }}"
+    podman_group: "{{ llama_group | default('llama') }}"
+    podman_extra_groups: "users,video"
+    podman_user_home: "{{ llama_home | default('/home/llama') }}"
+    __llama_models__: "{{ llama_models | default([]) }}"
+    __llama_router_dir__: "{{ podman_user_home }}/models/.router"
+    __llama_managed_dir__: "{{ podman_user_home }}/models/managed"
+    __llama_links_dir__: "{{ podman_user_home }}/models/managed-links"
+    __llama_manifest_file__: "{{ __llama_router_dir__ }}/manifest.json"
+    __llama_models_input_file__: "{{ __llama_router_dir__ }}/desired-models.json"
+    __llama_preset_global__: "{{ llama_preset_global | default({}) }}"
+    __llama_preset_global_input_file__: "{{ __llama_router_dir__ }}/preset-global.json"
+    __llama_preset_file__: "{{ __llama_router_dir__ }}/models.ini"
+    __llama_sync_script__: "{{ __llama_router_dir__ }}/llama_hf_sync.py"
+    __llama_sync_venv__: "{{ llama_sync_venv | default(__llama_router_dir__ ~ '/.venv') }}"
+    __llama_sync_python__: "{{ __llama_sync_venv__ }}/bin/python"
+    __llama_models_max__: "{{ llama_models_max_loaded | default(1) }}"
+    podman_user_folders:
+      - models
+      - .cache
+      - .cache/llama.cpp
+    pods:
+      - name: llama.cpp
+        repo:
+          image: ghcr.io/ggml-org/llama.cpp:server-vulkan
+        ports:
+          - "0.0.0.0:{{ llama_port | default(8012) }}:8080/tcp"
+        command:
+          - "-t"
+          - "{{ llama_cmd_threads | default('12') }}"
+          - "-np"
+          - "{{ llama_cmd_parallel | default('1') }}"
+          - "-b"
+          - "{{ llama_cmd_batch_size | default('1024') }}"
+          - "-ub"
+          - "{{ llama_cmd_ubatch_size | default('512') }}"
+          - "-fa"
+          - "{{ llama_cmd_flash_attn | default('on') }}"
+          - "-ctk"
+          - "{{ llama_cmd_cache_type_k | default('q4_0') }}"
+          - "-ctv"
+          - "{{ llama_cmd_cache_type_v | default('q4_0') }}"
+          - "-cram"
+          - "{{ llama_cmd_cache_reuse | default('-1') }}"
+          - "-sm"
+          - "{{ llama_cmd_split_mode | default('layer') }}"
+          - "-dev"
+          - "{{ llama_cmd_devices | default('Vulkan1,Vulkan2') }}"
+          - "-ts"
+          - "{{ llama_cmd_tensor_split | default('8,12') }}"
+          - "-fit"
+          - "{{ llama_cmd_fit | default('off') }}"
+          - "-mg"
+          - "{{ llama_cmd_main_gpu | default('1') }}"
+          - "--mmap"
+          - "--models-preset"
+          - "{{ llama_cmd_models_preset_path | default('/models/.router/models.ini') }}"
+          - "--models-max"
+          - "{{ llama_cmd_models_max | default(__llama_models_max__) }}"
+        volumes:
+          - "{{ podman_user_home }}/models:/models:Z"
+          - "{{ podman_user_home }}/.cache:/app/.cache:Z"
+        device:
+          - "nvidia.com/gpu=all"
+          - "/dev/kfd:/dev/kfd:rw"
+          - "/dev/dri:/dev/dri:rw"
+  tasks:
+    - name: Sync llama models from Hugging Face
+      ansible.builtin.include_tasks:
+        file: tasks/llama_models_sync.yml
+      when: llama_models_sync_enabled | default(true)
+
+    - name: Setup Pods
+      ansible.builtin.include_tasks:
+        file: tasks/podman.yml
+
+    - name: Query llama router model list
+      ansible.builtin.uri:
+        url: "http://127.0.0.1:{{ llama_port | default(8012) }}/models?reload=1"
+        method: GET
+        status_code: 200
+      register: __llama_router_models__
+      changed_when: false
+      when:
+        - llama_models_sync_enabled | default(true)
+        - not (llama_models_dry_run | default(false))
+
+    - name: Validate expected model aliases are available in router
+      ansible.builtin.assert:
+        that:
+          - item.name in (__llama_router_models__.json.data | map(attribute='id') | list)
+        fail_msg: "Model name '{{ item.name }}' not present in llama router /models output"
+      loop: "{{ __llama_sync_result__.models | default([]) }}"
+      loop_control:
+        label: "{{ item.name }}"
+      when:
+        - llama_models_sync_enabled | default(true)
+        - not (llama_models_dry_run | default(false))