53 lines
1.5 KiB
YAML
53 lines
1.5 KiB
YAML
|
- name: Setup Qwen3.6 - Coder (FIM - LLAMA.cpp)
|
||
|
|
hosts: qwen3_6_host
|
||
|
|
vars:
|
||
|
|
podman_user: "{{ qwen3_Z6_user | default('qwen36') }}"
|
||
|
|
podman_group: "{{ qwen3_6_group | default('qwen36') }}"
|
||
|
|
podman_extra_groups: "users,video"
|
||
|
|
podman_user_home: "{{ qwen3_6_home | default('/home/qwen36') }}"
|
||
|
|
llama_model: "{{ qwen3_6_model | default('bartowski/Qwen_Qwen3.6-35B-A3B-GGUF:IQ4_XS') }}"
|
||
|
|
__qwen3_6_context_length__: "{{ qwen3_6_context_length | default(65537) }}"
|
||
|
|
podman_user_folders:
|
||
|
|
- models
|
||
|
|
- .cache
|
||
|
|
- .cache/llama.cpp
|
||
|
|
pods:
|
||
|
|
- name: qwen3.6
|
||
|
|
repo:
|
||
|
|
image: ghcr.io/ggml-org/llama.cpp:server-cuda
|
||
|
|
ports:
|
||
|
|
- "0.0.0.0:{{ qwen3_6_port | default(8012) }}:8080/tcp"
|
||
|
|
command:
|
||
|
|
- "-hf"
|
||
|
|
- "{{ llama_model }}"
|
||
|
|
- "-ngl"
|
||
|
|
- "40"
|
||
|
|
- "-ncmoe"
|
||
|
|
- "32"
|
||
|
|
- "-c"
|
||
|
|
- "{{ __qwen3_6_context_length__ }}"
|
||
|
|
- "-np"
|
||
|
|
- "1"
|
||
|
|
- "-ub"
|
||
|
|
- "512"
|
||
|
|
- "-b"
|
||
|
|
- "1024"
|
||
|
|
- "-fa"
|
||
|
|
- "on"
|
||
|
|
- "--no-mmap"
|
||
|
|
- "-ctk"
|
||
|
|
- "iq4_nl"
|
||
|
|
- "-ctv"
|
||
|
|
- "iq4_nl"
|
||
|
|
- "-cram"
|
||
|
|
- "{{ __qwen3_6_context_length__ }}"
|
||
|
|
volumes:
|
||
|
|
- "{{ podman_user_home }}/models:/models:Z"
|
||
|
|
- "{{ podman_user_home }}/.cache:/app/.cache:Z"
|
||
|
|
device:
|
||
|
|
- "nvidia.com/gpu=all"
|
||
|
|
tasks:
|
||
|
|
- name: Setup Pods
|
||
|
|
ansible.builtin.include_tasks:
|
||
|
|
file: tasks/podman.yml
|