alexandre.pires
350650ecc2
- Create systemd service templates for llama_exporter, nvidia_exporter, and podman. - Add Prometheus configuration template for GPU metrics scraping. - Introduce variables for GPU monitoring role in main.yml. - Implement tasks for syncing llama models, including user setup and package installation. - Update podman tasks to ensure proper sudo access and lingering for the podman user. - Modify inventory to include gpu_monitoring group. - Add Terraform modules for deploying Vaultwarden, including ingress and service configurations. - Create Grafana dashboard for real-time GPU monitoring metrics. - Update secrets and environment files to include Vaultwarden admin token.
102 lines
4.1 KiB
YAML
102 lines
4.1 KiB
YAML
- name: Setup llama.cpp - Vulkan
|
|
hosts: llamacpp
|
|
vars:
|
|
podman_user: "{{ llama_user | default('llama') }}"
|
|
podman_group: "{{ llama_group | default('llama') }}"
|
|
podman_extra_groups: "users,video"
|
|
podman_user_home: "{{ llama_home | default('/home/llama') }}"
|
|
__llama_models__: "{{ llama_models | default([]) }}"
|
|
__llama_router_dir__: "{{ podman_user_home }}/models/.router"
|
|
__llama_managed_dir__: "{{ podman_user_home }}/models/managed"
|
|
__llama_links_dir__: "{{ podman_user_home }}/models/managed-links"
|
|
__llama_manifest_file__: "{{ __llama_router_dir__ }}/manifest.json"
|
|
__llama_models_input_file__: "{{ __llama_router_dir__ }}/desired-models.json"
|
|
__llama_preset_global__: "{{ llama_preset_global | default({}) }}"
|
|
__llama_preset_global_input_file__: "{{ __llama_router_dir__ }}/preset-global.json"
|
|
__llama_preset_file__: "{{ __llama_router_dir__ }}/models.ini"
|
|
__llama_sync_script__: "{{ __llama_router_dir__ }}/llama_hf_sync.py"
|
|
__llama_sync_venv__: "{{ llama_sync_venv | default(__llama_router_dir__ ~ '/.venv') }}"
|
|
__llama_sync_python__: "{{ __llama_sync_venv__ }}/bin/python"
|
|
__llama_models_max__: "{{ llama_models_max_loaded | default(1) }}"
|
|
podman_user_folders:
|
|
- models
|
|
- .cache
|
|
- .cache/llama.cpp
|
|
pods:
|
|
- name: llama.cpp
|
|
repo:
|
|
image: ghcr.io/ggml-org/llama.cpp:server-vulkan
|
|
ports:
|
|
- "0.0.0.0:{{ llama_port | default(8012) }}:8080/tcp"
|
|
command:
|
|
- "-t"
|
|
- "{{ llama_cmd_threads | default('12') }}"
|
|
- "-np"
|
|
- "{{ llama_cmd_parallel | default('1') }}"
|
|
- "-b"
|
|
- "{{ llama_cmd_batch_size | default('1024') }}"
|
|
- "-ub"
|
|
- "{{ llama_cmd_ubatch_size | default('512') }}"
|
|
- "-fa"
|
|
- "{{ llama_cmd_flash_attn | default('on') }}"
|
|
- "-ctk"
|
|
- "{{ llama_cmd_cache_type_k | default('q4_0') }}"
|
|
- "-ctv"
|
|
- "{{ llama_cmd_cache_type_v | default('q4_0') }}"
|
|
- "-cram"
|
|
- "{{ llama_cmd_cache_reuse | default('-1') }}"
|
|
- "-sm"
|
|
- "{{ llama_cmd_split_mode | default('layer') }}"
|
|
- "-dev"
|
|
- "{{ llama_cmd_devices | default('Vulkan1,Vulkan2') }}"
|
|
- "-ts"
|
|
- "{{ llama_cmd_tensor_split | default('8,12') }}"
|
|
- "-fit"
|
|
- "{{ llama_cmd_fit | default('off') }}"
|
|
- "-mg"
|
|
- "{{ llama_cmd_main_gpu | default('1') }}"
|
|
- "--mmap"
|
|
- "--models-preset"
|
|
- "{{ llama_cmd_models_preset_path | default('/models/.router/models.ini') }}"
|
|
- "--models-max"
|
|
- "{{ llama_cmd_models_max | default(__llama_models_max__) }}"
|
|
volumes:
|
|
- "{{ podman_user_home }}/models:/models:Z"
|
|
- "{{ podman_user_home }}/.cache:/app/.cache:Z"
|
|
device:
|
|
- "nvidia.com/gpu=all"
|
|
- "/dev/kfd:/dev/kfd:rw"
|
|
- "/dev/dri:/dev/dri:rw"
|
|
tasks:
|
|
- name: Sync llama models from Hugging Face
|
|
ansible.builtin.include_tasks:
|
|
file: tasks/llama_models_sync.yml
|
|
when: llama_models_sync_enabled | default(true)
|
|
|
|
- name: Setup Pods
|
|
ansible.builtin.include_tasks:
|
|
file: tasks/podman.yml
|
|
|
|
- name: Query llama router model list
|
|
ansible.builtin.uri:
|
|
url: "http://127.0.0.1:{{ llama_port | default(8012) }}/models?reload=1"
|
|
method: GET
|
|
status_code: 200
|
|
register: __llama_router_models__
|
|
changed_when: false
|
|
when:
|
|
- llama_models_sync_enabled | default(true)
|
|
- not (llama_models_dry_run | default(false))
|
|
|
|
- name: Validate expected model aliases are available in router
|
|
ansible.builtin.assert:
|
|
that:
|
|
- item.name in (__llama_router_models__.json.data | map(attribute='id') | list)
|
|
fail_msg: "Model name '{{ item.name }}' not present in llama router /models output"
|
|
loop: "{{ __llama_sync_result__.models | default([]) }}"
|
|
loop_control:
|
|
label: "{{ item.name }}"
|
|
when:
|
|
- llama_models_sync_enabled | default(true)
|
|
- not (llama_models_dry_run | default(false))
|