Add GPU monitoring services and configurations

- Create systemd service templates for llama_exporter, nvidia_exporter, and podman. - Add Prometheus configuration template for GPU metrics scraping. - Introduce variables for GPU monitoring role in main.yml. - Implement tasks for syncing llama models, including user setup and package installation. - Update podman tasks to ensure proper sudo access and lingering for the podman user. - Modify inventory to include gpu_monitoring group. - Add Terraform modules for deploying Vaultwarden, including ingress and service configurations. - Create Grafana dashboard for real-time GPU monitoring metrics. - Update secrets and environment files to include Vaultwarden admin token.
2026-05-29 23:42:14 +02:00
parent e9e28a5ca8
commit 350650ecc2
47 changed files with 3727 additions and 49 deletions
@@ -0,0 +1,146 @@
+- name: Ensure podman user exists before model sync
+  ansible.builtin.include_tasks:
+    file: podman/user.yml
+
+- name: Enable sudo access to podman user (temporary)
+  become: true
+  ansible.builtin.lineinfile:
+    path: "/etc/sudoers.d/ansible_{{ podman_user }}_tmp"
+    create: true
+    mode: "0440"
+    line: "{{ ansible_user_id }} ALL=({{ podman_user }}) NOPASSWD: ALL"
+    validate: "visudo -cf %s"
+
+- name: Validate llama_models schema
+  ansible.builtin.assert:
+    that:
+      - __llama_models__ is iterable
+    fail_msg: "llama_models must be a list"
+
+- name: Install Python packages required for llama model sync
+  become: true
+  ansible.builtin.package:
+    name: "{{ __llama_sync_system_packages__ }}"
+    state: present
+  vars:
+    __llama_sync_system_packages__: >-
+      {{
+        {
+          'Debian': ['python3-pip', 'python3-packaging', 'python3-venv'],
+          'RedHat': ['python3-pip', 'python3-packaging'],
+          'Archlinux': ['python-pip', 'python-packaging']
+        }[ansible_os_family]
+      }}
+
+- name: Install Hugging Face hub client in llama model sync virtualenv
+  become: true
+  become_user: "{{ podman_user }}"
+  ansible.builtin.pip:
+    name:
+      - pip
+      - setuptools
+      - packaging
+      - huggingface_hub>=0.32.0
+    state: present
+    virtualenv: "{{ __llama_sync_venv__ }}"
+    virtualenv_command: "python3 -m venv"
+
+- name: Create llama router model directories
+  become: true
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: directory
+    owner: "{{ podman_user }}"
+    group: "{{ podman_group }}"
+    mode: "{{ item.mode }}"
+  loop:
+    - { path: "{{ podman_user_home }}/models", mode: "0750" }
+    - { path: "{{ __llama_managed_dir__ }}", mode: "0750" }
+    - { path: "{{ __llama_links_dir__ }}", mode: "0750" }
+    - { path: "{{ __llama_router_dir__ }}", mode: "0750" }
+
+- name: Copy llama Hugging Face sync helper
+  become: true
+  ansible.builtin.copy:
+    src: scripts/llama_hf_sync.py
+    dest: "{{ __llama_sync_script__ }}"
+    owner: "{{ podman_user }}"
+    group: "{{ podman_group }}"
+    mode: "0750"
+
+- name: Write desired llama model manifest input
+  become: true
+  ansible.builtin.copy:
+    content: "{{ __llama_models__ | to_nice_json }}"
+    dest: "{{ __llama_models_input_file__ }}"
+    owner: "{{ podman_user }}"
+    group: "{{ podman_group }}"
+    mode: "0640"
+
+- name: Write global llama preset options
+  become: true
+  ansible.builtin.copy:
+    content: "{{ __llama_preset_global__ | to_nice_json }}"
+    dest: "{{ __llama_preset_global_input_file__ }}"
+    owner: "{{ podman_user }}"
+    group: "{{ podman_group }}"
+    mode: "0640"
+
+- name: Run llama model sync from Hugging Face
+  vars:
+    __llama_sync_argv__: >-
+      {{
+        [
+          __llama_sync_python__,
+          __llama_sync_script__,
+          '--models-file',
+          __llama_models_input_file__,
+          '--managed-dir',
+          __llama_managed_dir__,
+          '--links-dir',
+          __llama_links_dir__,
+          '--manifest-file',
+          __llama_manifest_file__,
+          '--preset-file',
+          __llama_preset_file__,
+          '--preset-global-file',
+          __llama_preset_global_input_file__,
+          '--container-links-dir',
+          '/models/managed-links'
+        ]
+        + (['--prune'] if llama_models_prune_enabled | default(true) else [])
+        + (['--dry-run'] if llama_models_dry_run | default(false) else [])
+      }}
+  become: true
+  become_user: "{{ podman_user }}"
+  ansible.builtin.command:
+    argv: "{{ __llama_sync_argv__ }}"
+  environment:
+    HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') | default('', true) }}"
+  register: __llama_sync_output__
+  changed_when: false
+
+- name: Parse llama sync output
+  ansible.builtin.set_fact:
+    __llama_sync_result__: "{{ __llama_sync_output__.stdout | from_json }}"
+
+- name: Ensure router preset file exists when not in dry-run mode
+  become: true
+  become_user: "{{ podman_user }}"
+  ansible.builtin.stat:
+    path: "{{ __llama_preset_file__ }}"
+  register: __llama_preset_stat__
+  when: not (llama_models_dry_run | default(false))
+
+- name: Ensure router preset file exists when not in dry-run mode
+  ansible.builtin.assert:
+    that:
+      - __llama_preset_stat__.stat.exists
+    fail_msg: "llama router preset file was not created"
+  when: not (llama_models_dry_run | default(false))
+
+- name: Remove temporary sudo access
+  become: true
+  ansible.builtin.file:
+    path: "/etc/sudoers.d/ansible_{{ podman_user }}_tmp"
+    state: absent