From e9e28a5ca8b3a8a3ac90f6a11c0d4a05945666b1 Mon Sep 17 00:00:00 2001 From: Alexandre Pires Date: Wed, 27 May 2026 23:53:00 +0200 Subject: [PATCH] Add new model files and update Ansible playbooks for Qwen3.5 and Qwen3.6 - Created model files for Qwen3.5 with different context lengths. - Added a new Ansible host variable file for CI server configuration. - Updated GPU server host variables to include Ollama models and model files. - Enhanced playbooks for Podman to manage Ollama models, including copying model files and checking existing models. - Introduced a new playbook for setting up Qwen3.6 with specific configurations. - Updated Windows SSH tasks to improve security and configuration management. - Added a Python utility to parse Ollama model names from command output. - Modified Terraform configurations to include new DNS entries and proxy settings. - Improved Nginx proxy configurations with timeout settings. --- .../dockerfiles/Dockerfile.llamacpp.cuda | 11 + ansible/files/ollama/gemma4-e4b-32k.modelfile | 2 + .../files/ollama/gpt-oss-latest-32k.modelfile | 2 + ansible/files/ollama/qwen3.5-4b-32k.modelfile | 2 + ansible/files/ollama/qwen3.5-4b-64k.modelfile | 2 + ansible/files/ollama/qwen3.5-9b-32k.modelfile | 2 + ansible/files/ollama/qwen3.5-9b-64k.modelfile | 2 + ansible/files/windows/issue.net | 5 + ...ires.me.yml => ci-01.lab.alexpires.me.yml} | 8 +- ansible/host_vars/gpu-01.lab.alexpires.me.yml | 32 ++ ansible/playbook_podman_ollama_rocm.yml | 81 ++++- ansible/playbook_podman_qwen3.6.yml | 52 +++ ansible/sectool.env | 2 +- ansible/tasks/windows_prometheus.yml | 23 +- ansible/tasks/windows_ssh.yml | 300 ++++++++++++++---- ansible/templates/windows_sshd_config.j2 | 49 +++ bin/parse_ollama_models.py | 57 ++++ inventory/hosts | 9 +- terraform/apps/dev-01/config.tf | 5 + terraform/apps/prod-01/config.tf | 14 +- terraform/modules/apps/open-webui/ingress.tf | 10 +- terraform/modules/apps/searxng/config.tf | 3 + terraform/modules/apps/searxng/main.tf | 27 ++ .../apps/searxng/resources/settings.yml.tpl | 5 + terraform/modules/utils/proxy/config.tf | 3 + .../resources/nginx/conf.d/default.conf.tpl | 3 + terraform/modules/utils/proxy/variables.tf | 3 + 27 files changed, 611 insertions(+), 103 deletions(-) create mode 100644 ansible/files/dockerfiles/Dockerfile.llamacpp.cuda create mode 100644 ansible/files/ollama/gemma4-e4b-32k.modelfile create mode 100644 ansible/files/ollama/gpt-oss-latest-32k.modelfile create mode 100644 ansible/files/ollama/qwen3.5-4b-32k.modelfile create mode 100644 ansible/files/ollama/qwen3.5-4b-64k.modelfile create mode 100644 ansible/files/ollama/qwen3.5-9b-32k.modelfile create mode 100644 ansible/files/ollama/qwen3.5-9b-64k.modelfile create mode 100644 ansible/files/windows/issue.net rename ansible/host_vars/{hpz440.lab.alexpires.me.yml => ci-01.lab.alexpires.me.yml} (72%) create mode 100644 ansible/playbook_podman_qwen3.6.yml create mode 100644 ansible/templates/windows_sshd_config.j2 create mode 100755 bin/parse_ollama_models.py create mode 100644 terraform/modules/apps/searxng/resources/settings.yml.tpl diff --git a/ansible/files/dockerfiles/Dockerfile.llamacpp.cuda b/ansible/files/dockerfiles/Dockerfile.llamacpp.cuda new file mode 100644 index 0000000..57892a0 --- /dev/null +++ b/ansible/files/dockerfiles/Dockerfile.llamacpp.cuda @@ -0,0 +1,11 @@ +FROM ghcr.io/ggml-org/llama.cpp:server-cuda + +ARG UID=1000 +ARG GID=1000 +RUN groupadd --system --gid ${GID} worker && \ + useradd --system --gid ${GID} --uid ${UID} --home /home/worker worker && \ + mkdir -p /home/worker && chown worker:worker /home/worker + +WORKDIR /home/worker +USER worker +ENTRYPOINT ["/app/llama-server"] \ No newline at end of file diff --git a/ansible/files/ollama/gemma4-e4b-32k.modelfile b/ansible/files/ollama/gemma4-e4b-32k.modelfile new file mode 100644 index 0000000..55832b0 --- /dev/null +++ b/ansible/files/ollama/gemma4-e4b-32k.modelfile @@ -0,0 +1,2 @@ +FROM gemma4:e4b +PARAMETER num_ctx 32359 \ No newline at end of file diff --git a/ansible/files/ollama/gpt-oss-latest-32k.modelfile b/ansible/files/ollama/gpt-oss-latest-32k.modelfile new file mode 100644 index 0000000..635c81c --- /dev/null +++ b/ansible/files/ollama/gpt-oss-latest-32k.modelfile @@ -0,0 +1,2 @@ +FROM gpt-oss:latest +PARAMETER num_ctx 32359 \ No newline at end of file diff --git a/ansible/files/ollama/qwen3.5-4b-32k.modelfile b/ansible/files/ollama/qwen3.5-4b-32k.modelfile new file mode 100644 index 0000000..12e76c4 --- /dev/null +++ b/ansible/files/ollama/qwen3.5-4b-32k.modelfile @@ -0,0 +1,2 @@ +FROM qwen3.5:9b +PARAMETER num_ctx 32359 \ No newline at end of file diff --git a/ansible/files/ollama/qwen3.5-4b-64k.modelfile b/ansible/files/ollama/qwen3.5-4b-64k.modelfile new file mode 100644 index 0000000..5d1db3f --- /dev/null +++ b/ansible/files/ollama/qwen3.5-4b-64k.modelfile @@ -0,0 +1,2 @@ +FROM qwen3.5:9b +PARAMETER num_ctx 65359 \ No newline at end of file diff --git a/ansible/files/ollama/qwen3.5-9b-32k.modelfile b/ansible/files/ollama/qwen3.5-9b-32k.modelfile new file mode 100644 index 0000000..12e76c4 --- /dev/null +++ b/ansible/files/ollama/qwen3.5-9b-32k.modelfile @@ -0,0 +1,2 @@ +FROM qwen3.5:9b +PARAMETER num_ctx 32359 \ No newline at end of file diff --git a/ansible/files/ollama/qwen3.5-9b-64k.modelfile b/ansible/files/ollama/qwen3.5-9b-64k.modelfile new file mode 100644 index 0000000..a3679fa --- /dev/null +++ b/ansible/files/ollama/qwen3.5-9b-64k.modelfile @@ -0,0 +1,2 @@ +FROM qwen3.5:9b +PARAMETER num_ctx 65537 \ No newline at end of file diff --git a/ansible/files/windows/issue.net b/ansible/files/windows/issue.net new file mode 100644 index 0000000..2b42af2 --- /dev/null +++ b/ansible/files/windows/issue.net @@ -0,0 +1,5 @@ + +By accessing this system, you consent to the following conditions: +- This system is for authorized use only. +- Any or all uses of this system and all files on this system may be monitored. +- Communications using, or data stored on, this system are not private. diff --git a/ansible/host_vars/hpz440.lab.alexpires.me.yml b/ansible/host_vars/ci-01.lab.alexpires.me.yml similarity index 72% rename from ansible/host_vars/hpz440.lab.alexpires.me.yml rename to ansible/host_vars/ci-01.lab.alexpires.me.yml index afe8be5..8d2cad6 100644 --- a/ansible/host_vars/hpz440.lab.alexpires.me.yml +++ b/ansible/host_vars/ci-01.lab.alexpires.me.yml @@ -1,6 +1,6 @@ -hostname: hpz440.lab.alexpires.me +hostname: ci-01.lab.alexpires.me ansible_user: "{{ lookup('env', 'ANSIBLE_USER') }}" -ansible_password: "{{ lookup('env', 'WIN_HPZ440_PASSWORD') }}" +ansible_password: "{{ lookup('env', 'WIN_CI_01_PASSWORD') }}" ansible_connection: winrm ansible_winrm_transport: basic ansible_winrm_server_cert_validation: ignore @@ -10,10 +10,6 @@ ssh_users: admin: true keys: - "{{ lookup('file', 'keys/ansible_user.pub') }}" - - name: HP - admin: true - keys: - - "{{ lookup('file', 'keys/operator.pub') }}" grafana_prometheus_sources: - 10.19.4.136 diff --git a/ansible/host_vars/gpu-01.lab.alexpires.me.yml b/ansible/host_vars/gpu-01.lab.alexpires.me.yml index d94e843..69a9b09 100644 --- a/ansible/host_vars/gpu-01.lab.alexpires.me.yml +++ b/ansible/host_vars/gpu-01.lab.alexpires.me.yml @@ -107,3 +107,35 @@ certbot_domains: - "gpu-01.lab.alexpires.me" certbot_cockpit_domain: "gpu-01.lab.alexpires.me" +ollama_models: + # Thinking models + - gpt-oss:latest + - gemma4:e4b + - qwen3.5:9b + - qwen3.5:4b + # Non-thinking models + - ministral-3:3b + - ministral-3:8b + - ministral-3:14b + - granite4.1:3b + - granite4.1:8b + - deepseek-r1:latest + - qwen2.5-coder:3b + # Embedding models + - embeddinggemma:latest + - nomic-embed-text:latest + # Whisper models + - dimavz/whisper-tiny:latest +ollama_model_files: + - file: "ollama/gpt-oss-latest-32k.modelfile" + name: "gpt-oss-32k:latest" + - file: "ollama/gemma4-e4b-32k.modelfile" + name: "gemma4-e4b-32k:latest" + - file: "ollama/qwen3.5-9b-32k.modelfile" + name: "qwen3.5-9b-32k:latest" + - file: "ollama/qwen3.5-9b-64k.modelfile" + name: "qwen3.5-9b-64k:latest" + - file: "ollama/qwen3.5-4b-32k.modelfile" + name: "qwen3.5-4b-32k:latest" + - file: "ollama/qwen3.5-4b-64k.modelfile" + name: "qwen3.5-4b-64k:latest" diff --git a/ansible/playbook_podman_ollama_rocm.yml b/ansible/playbook_podman_ollama_rocm.yml index beb7c7e..be17d60 100644 --- a/ansible/playbook_podman_ollama_rocm.yml +++ b/ansible/playbook_podman_ollama_rocm.yml @@ -7,15 +7,17 @@ podman_user_home: "{{ ollama_home | default('/home/ollama') }}" podman_user_folders: - data + __ollama_models__: "{{ ollama_models | default([]) }}" + __ollama_model_files__: "{{ ollama_model_files | default([]) }}" pods: - name: ollama-rocm build: dockerfile: "{{ lookup('file', 'dockerfiles/Dockerfile.ollama.rocm') }}" env: - OLLAMA_CONTEXT_LENGTH: 32000 OLLAMA_NUM_THREADS: 10 OLLAMA_NUM_PARALLEL: 2 OLLAMA_FLASH_ATTENTION: 1 + OLLAMA_KV_CACHE_TYPE: q8_0 OLLAMA_KV_CACHE: "true" HSA_OVERRIDE_GFX_VERSION: "11.0.0" ports: @@ -30,3 +32,80 @@ - name: Setup Pods ansible.builtin.include_tasks: file: tasks/podman.yml + + - name: Enable sudo access to podman user (temporary) + become: true + ansible.builtin.lineinfile: + path: "/etc/sudoers.d/ansible_{{ podman_user }}_tmp" + create: true + mode: "0440" + line: "{{ ansible_user_id }} ALL=({{ podman_user }}) NOPASSWD: ALL" + validate: "visudo -cf %s" + + - name: Create staging directory for model files + become: true + ansible.builtin.file: + path: "{{ ollama_home }}/data/.modelfiles" + state: directory + owner: "{{ podman_user }}" + group: "{{ podman_group }}" + mode: "0755" + when: __ollama_model_files__ | length > 0 + + - name: Copy model files to remote host staging directory + become: true + ansible.builtin.copy: + src: "{{ item.file }}" + dest: "{{ ollama_home }}/data/.modelfiles/{{ item.file | basename }}" + owner: "{{ podman_user }}" + group: "{{ podman_group }}" + mode: "0644" + loop: "{{ __ollama_model_files__ }}" + when: __ollama_model_files__ | length > 0 + + - name: Check existing ollama models + become: true + become_user: "{{ podman_user }}" + containers.podman.podman_container_exec: + name: ollama-rocm + command: "ollama ls" + register: ollama_existing_models + changed_when: false + + - name: Set fact for existing model names + ansible.builtin.set_fact: + __ollama_existing_model_names__: >- + {{ ollama_existing_models.stdout_lines | default([]) + | reject('match', '^\\s*$|^(NAME|\\-+)') + | map('regex_replace', '^\\s*(\\S+)\\s.*', '\\1') + | list }} + when: ollama_existing_models.stdout is defined and ollama_existing_models.stdout | length > 0 + changed_when: false + + - name: Pull ollama models that do not exist + become: true + become_user: "{{ podman_user }}" + containers.podman.podman_container_exec: + name: ollama-rocm + command: "ollama pull {{ item }}" + loop: "{{ __ollama_models__ }}" + when: + - __ollama_models__ | length > 0 + - item not in (__ollama_existing_model_names__ | default([])) + + - name: Create ollama models that do not exist + become: true + become_user: "{{ podman_user }}" + containers.podman.podman_container_exec: + name: ollama-rocm + command: "ollama create {{ item.name }} -f /home/worker/.ollama/.modelfiles/{{ item.file | basename }}" + loop: "{{ __ollama_model_files__ }}" + when: + - __ollama_model_files__ | length > 0 + - item.name not in (__ollama_existing_model_names__ | default([])) + + - name: Remove temporary sudo access + become: true + ansible.builtin.file: + path: "/etc/sudoers.d/ansible_{{ podman_user }}_tmp" + state: absent diff --git a/ansible/playbook_podman_qwen3.6.yml b/ansible/playbook_podman_qwen3.6.yml new file mode 100644 index 0000000..ca175a4 --- /dev/null +++ b/ansible/playbook_podman_qwen3.6.yml @@ -0,0 +1,52 @@ +- name: Setup Qwen3.6 - Coder (FIM - LLAMA.cpp) + hosts: qwen3_6_host + vars: + podman_user: "{{ qwen3_Z6_user | default('qwen36') }}" + podman_group: "{{ qwen3_6_group | default('qwen36') }}" + podman_extra_groups: "users,video" + podman_user_home: "{{ qwen3_6_home | default('/home/qwen36') }}" + llama_model: "{{ qwen3_6_model | default('bartowski/Qwen_Qwen3.6-35B-A3B-GGUF:IQ4_XS') }}" + __qwen3_6_context_length__: "{{ qwen3_6_context_length | default(65537) }}" + podman_user_folders: + - models + - .cache + - .cache/llama.cpp + pods: + - name: qwen3.6 + repo: + image: ghcr.io/ggml-org/llama.cpp:server-cuda + ports: + - "0.0.0.0:{{ qwen3_6_port | default(8012) }}:8080/tcp" + command: + - "-hf" + - "{{ llama_model }}" + - "-ngl" + - "40" + - "-ncmoe" + - "32" + - "-c" + - "{{ __qwen3_6_context_length__ }}" + - "-np" + - "1" + - "-ub" + - "512" + - "-b" + - "1024" + - "-fa" + - "on" + - "--no-mmap" + - "-ctk" + - "iq4_nl" + - "-ctv" + - "iq4_nl" + - "-cram" + - "{{ __qwen3_6_context_length__ }}" + volumes: + - "{{ podman_user_home }}/models:/models:Z" + - "{{ podman_user_home }}/.cache:/app/.cache:Z" + device: + - "nvidia.com/gpu=all" + tasks: + - name: Setup Pods + ansible.builtin.include_tasks: + file: tasks/podman.yml diff --git a/ansible/sectool.env b/ansible/sectool.env index 7b22aec..05dc74f 100644 --- a/ansible/sectool.env +++ b/ansible/sectool.env @@ -15,4 +15,4 @@ CLOUDNS_AUTH_ID=$CLOUDNS_AUTH_ID CLOUDNS_PASSWORD=$CLOUDNS_PASSWORD WIN_PROVISION_PASSWORD=$WIN_PROVISION_PASSWORD WIN_GAME_PASSWORD=$WIN_GAME_PASSWORD -WIN_HPZ440_PASSWORD=$WIN_HPZ440_PASSWORD \ No newline at end of file +WIN_CI_01_PASSWORD=$WIN_CI_01_PASSWORD \ No newline at end of file diff --git a/ansible/tasks/windows_prometheus.yml b/ansible/tasks/windows_prometheus.yml index beb02c4..88c6f30 100644 --- a/ansible/tasks/windows_prometheus.yml +++ b/ansible/tasks/windows_prometheus.yml @@ -36,12 +36,12 @@ dest: "{{ item.dest }}" force: false loop: - - { url: "{{ __prometheus_zip_url }}", dest: "{{ __prometheus_zip_path }}" } - - { url: "{{ __nssm_url }}", dest: "{{ __nssm_zip_path }}" } - - { - url: "{{ __windows_exporter_msi_url }}", - dest: "{{ __windows_exporter_msi_path }}", - } + - url: "{{ __prometheus_zip_url }}" + dest: "{{ __prometheus_zip_path }}" + - url: "{{ __nssm_url }}" + dest: "{{ __nssm_zip_path }}" + - url: "{{ __windows_exporter_msi_url }}" + dest: "{{ __windows_exporter_msi_path }}" - name: Extract downloaded archives community.windows.win_unzip: @@ -50,12 +50,11 @@ delete_archive: false creates: "{{ item.creates | default(omit) }}" loop: - - { - src: "{{ __prometheus_zip_path }}", - dest: "{{ prometheus_install_dir }}", - creates: "{{ __prometheus_home }}\\prometheus.exe", - } - - { src: "{{ __nssm_zip_path }}", dest: "{{ nssm_dir }}" } + - src: "{{ __prometheus_zip_path }}" + dest: "{{ prometheus_install_dir }}" + creates: "{{ __prometheus_home }}\\prometheus.exe" + - src: "{{ __nssm_zip_path }}" + dest: "{{ nssm_dir }}" - name: Verify Prometheus binary is present ansible.windows.win_stat: diff --git a/ansible/tasks/windows_ssh.yml b/ansible/tasks/windows_ssh.yml index 8f2409a..4b28afe 100644 --- a/ansible/tasks/windows_ssh.yml +++ b/ansible/tasks/windows_ssh.yml @@ -7,21 +7,100 @@ # - name: bob # keys: # - "ssh-rsa AAAA..." -- name: Ensure OpenSSH Server capability is installed (if available) +- name: Setting defaults for Windows OpenSSH Server installation + ansible.builtin.set_fact: + __ssh_default_ciphers__: + - chacha20-poly1305@openssh.com + - aes256-gcm@openssh.com + - aes256-ctr + __ssh_default_hostkey_algorithms__: + - ssh-ed25519-cert-v01@openssh.com + - ssh-rsa-cert-v01@openssh.com + - ssh-ed25519 + - ssh-rsa + - ecdsa-sha2-nistp521-cert-v01@openssh.com + - ecdsa-sha2-nistp384-cert-v01@openssh.com + - ecdsa-sha2-nistp256-cert-v01@openssh.com + - ecdsa-sha2-nistp521 + - ecdsa-sha2-nistp384 + - ecdsa-sha2-nistp256 + - sk-ecdsa-sha2-nistp256@openssh.com + __ssh_default_kex_algorithms__: + - mlkem768x25519-sha256 + - sntrup761x25519-sha512@openssh.com + - curve25519-sha256 + - curve25519-sha256@libssh.org + - ecdh-sha2-nistp521 + - ecdh-sha2-nistp384 + - ecdh-sha2-nistp256 + - diffie-hellman-group-exchange-sha256 + __ssh_default_macs__: + - hmac-sha2-512-etm@openssh.com + - hmac-sha2-256-etm@openssh.com + - hmac-sha2-512 + - hmac-sha2-256 + __ssh_default_auth_pubkey_accepted_algorithms__: + - ecdsa-sha2-nistp521 + - ecdsa-sha2-nistp384 + - ecdsa-sha2-nistp256 + - ssh-ed25519 + - rsa-sha2-512 + - rsa-sha2-256 + +- name: Override defaults with any provided variables + ansible.builtin.set_fact: + __ssh_users__: "{{ ssh_users | default([]) }}" + __ssh_ciphers__: "{{ ssh_ciphers | default(__ssh_default_ciphers__) }}" + __ssh_hostkey_algorithms__: "{{ ssh_hostkey_algorithms | default(__ssh_default_hostkey_algorithms__) }}" + __ssh_kex_algorithms__: "{{ ssh_kex_algorithms | default(__ssh_default_kex_algorithms__) }}" + __ssh_macs__: "{{ ssh_macs | default(__ssh_default_macs__) }}" + __ssh_max_auth_tries__: "{{ ssh_max_auth_tries | default(10) }}" + __ssh_max_sessions__: "{{ ssh_max_sessions | default(3) }}" + __ssh_pubkey_accepted_algorithms__: "{{ ssh_pubkey_accepted_algorithms | default(__ssh_default_auth_pubkey_accepted_algorithms__) }}" + +- name: Get OpenSSH Server capability state ansible.windows.win_shell: | - $cap = Get-WindowsCapability -Online | Where-Object { $_.Name -like 'OpenSSH.Server*' } - if ($cap -and $cap.State -ne 'Installed') { - Add-WindowsCapability -Online -Name $cap.Name - Write-Output 'installed' + $cap = Get-WindowsCapability -Online | Where-Object { $_.Name -like 'OpenSSH.Server*' } | Select-Object -First 1 + if ($cap) { + $cap.State.ToString().Trim() } else { - Write-Output 'present' + 'NotPresent' } args: executable: powershell.exe - register: openssh_install - changed_when: "'installed' in openssh_install.stdout" + register: openssh_capability_state + changed_when: false -- name: Ensure host keys exist (ssh-keygen -A) +- name: Install OpenSSH Server capability when not installed + ansible.windows.win_shell: | + $cap = Get-WindowsCapability -Online | Where-Object { $_.Name -like 'OpenSSH.Server*' } | Select-Object -First 1 + if (-not $cap) { + throw 'OpenSSH.Server capability not found on this host.' + } + + Add-WindowsCapability -Online -Name $cap.Name + args: + executable: powershell.exe + register: openssh_install + when: openssh_capability_state.stdout | trim != 'Installed' + +- name: Check whether OpenSSH host keys already exist + ansible.windows.win_shell: | + $keys = Get-ChildItem -Path 'C:\ProgramData\ssh' -Filter 'ssh_host_*_key' -File -ErrorAction SilentlyContinue + if ($keys -and $keys.Count -gt 0) { + Write-Output 'present' + } else { + Write-Output 'missing' + } + args: + executable: powershell.exe + register: ssh_hostkeys_state + changed_when: false + become: true + become_method: ansible.builtin.runas + become_user: "NT AUTHORITY\\SYSTEM" + +- name: Generate OpenSSH host keys when missing (ssh-keygen -A) ansible.windows.win_shell: | $exe = 'C:\Windows\System32\OpenSSH\ssh-keygen.exe' if (Test-Path $exe) { @@ -33,25 +112,96 @@ } register: ssh_keygen changed_when: "'hostkeys-generated' in ssh_keygen.stdout" + when: ssh_hostkeys_state.stdout | trim == 'missing' + become: true + become_method: ansible.builtin.runas + become_user: "NT AUTHORITY\\SYSTEM" -- name: Restrict private host key file permissions (icacls) and remove provision user access +- name: Ensure OpenSSH private host keys are owned by SYSTEM ansible.windows.win_shell: | - $current = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name + $changed = $false $keys = Get-ChildItem -Path 'C:\ProgramData\ssh' -Filter 'ssh_host_*_key' -File -ErrorAction SilentlyContinue if ($keys -and $keys.Count -gt 0) { foreach ($k in $keys) { + $before = (Get-Acl -Path $k.FullName).Owner + icacls.exe $k.FullName /setowner "*S-1-5-18" /C | Out-Null + $after = (Get-Acl -Path $k.FullName).Owner + if ($before -ne $after) { + $changed = $true + } + } + if ($changed) { + Write-Output 'hostkey-owner-changed' + } else { + Write-Output 'hostkey-owner-unchanged' + } + } else { + Write-Output 'no-keys' + } + register: ssh_hostkey_owner + changed_when: "'hostkey-owner-changed' in ssh_hostkey_owner.stdout" + become: true + become_method: ansible.builtin.runas + become_user: "NT AUTHORITY\\SYSTEM" + +- name: Restrict private host key file permissions (icacls) and remove provision user access + ansible.windows.win_shell: | + $changed = $false + $removePrincipal = "{{ ansible_user | default('') }}" + $keys = Get-ChildItem -Path 'C:\ProgramData\ssh' -Filter 'ssh_host_*_key' -File -ErrorAction SilentlyContinue + if ($keys -and $keys.Count -gt 0) { + foreach ($k in $keys) { + $before = (Get-Acl -Path $k.FullName).Sddl icacls.exe $k.FullName /inheritance:r icacls.exe $k.FullName /grant:r "S-1-5-18:(F)" - icacls.exe $k.FullName /remove $current 2>$null + if ($removePrincipal -and $removePrincipal -notmatch '^(?i:NT AUTHORITY\\SYSTEM|SYSTEM)$') { + icacls.exe $k.FullName /remove "$removePrincipal" 2>$null + } + $after = (Get-Acl -Path $k.FullName).Sddl + if ($before -ne $after) { + $changed = $true + } + } + if ($changed) { + Write-Output 'restricted-changed' + } else { + Write-Output 'restricted-unchanged' } - Write-Output "restricted: removed access for $current" } else { Write-Output 'no-keys' } args: executable: powershell.exe register: restrict_hostkey_perms - changed_when: "'restricted' in restrict_hostkey_perms.stdout" + changed_when: "'restricted-changed' in restrict_hostkey_perms.stdout" + become: true + become_method: ansible.builtin.runas + become_user: "NT AUTHORITY\\SYSTEM" + +- name: Restrict C:\ProgramData\ssh folder write permissions + ansible.windows.win_shell: | + $p = 'C:\ProgramData\ssh' + if (Test-Path $p) { + $before = (Get-Acl -Path $p).Sddl + icacls.exe $p /inheritance:r | Out-Null + icacls.exe $p /grant:r "S-1-5-18:(OI)(CI)(F)" "S-1-5-32-544:(OI)(CI)(F)" | Out-Null + icacls.exe $p /remove "S-1-5-32-545" 2>$null + $after = (Get-Acl -Path $p).Sddl + if ($before -ne $after) { + Write-Output 'ssh-dir-acl-changed' + } else { + Write-Output 'ssh-dir-acl-unchanged' + } + } else { + Write-Output 'no-dir' + } + args: + executable: powershell.exe + register: ssh_dir_acl + changed_when: "'ssh-dir-acl-changed' in ssh_dir_acl.stdout" + become: true + become_method: ansible.builtin.runas + become_user: "NT AUTHORITY\\SYSTEM" - name: Ensure sshd service is enabled and running ansible.windows.win_service: @@ -69,23 +219,14 @@ protocol: TCP profile: "Domain,Private" -- name: Ensure PubkeyAuthentication is enabled in sshd_config - community.windows.win_lineinfile: - path: 'C:\ProgramData\ssh\sshd_config' - regexp: "^#?PubkeyAuthentication" - line: "PubkeyAuthentication yes" - create: true - backup: true - register: sshd_pubkey_changed - -- name: Ensure AuthorizedKeysFile is set to .ssh/authorized_keys - community.windows.win_lineinfile: - path: 'C:\ProgramData\ssh\sshd_config' - regexp: "^#?AuthorizedKeysFile" - line: "AuthorizedKeysFile .ssh/authorized_keys" - create: true - backup: true - register: sshd_authkeys_changed +- name: Ensure SSH banner file is present + ansible.windows.win_copy: + src: "windows/issue.net" + dest: 'C:\ProgramData\ssh\issue.net' + force: true + become: true + become_method: ansible.builtin.runas + become_user: "NT AUTHORITY\\SYSTEM" - name: Detect available PowerShell for ssh subsystem ansible.windows.win_shell: | @@ -96,42 +237,58 @@ register: ssh_subsystem_path changed_when: false -- name: Configure sshd Subsystem to allow PowerShell sessions - community.windows.win_lineinfile: - path: 'C:\ProgramData\ssh\sshd_config' - regexp: '^Subsystem\s+powershell' - line: 'Subsystem powershell "{{ ssh_subsystem_path.stdout | trim }}" -sshs -NoLogo -NoProfile' - create: true - backup: true - register: sshd_subsystem_changed - when: ssh_subsystem_path.stdout != '' +- name: Set ssh subsystem executable path fact + ansible.builtin.set_fact: + __ssh_subsystem_path__: "{{ ssh_subsystem_path.stdout | trim }}" -- name: Restart sshd if config changed +- name: Detect supported SSH key exchange algorithms + ansible.windows.win_shell: | + $ssh = 'C:\Windows\System32\OpenSSH\ssh.exe' + if (Test-Path $ssh) { + & $ssh -Q kex + } else { + Write-Output '' + } + args: + executable: powershell.exe + register: ssh_supported_kex + changed_when: false + +- name: Render sshd_config from template + ansible.windows.win_template: + src: windows_sshd_config.j2 + dest: 'C:\ProgramData\ssh\sshd_config' + register: sshd_config_changed + become: true + become_method: ansible.builtin.runas + become_user: "NT AUTHORITY\\SYSTEM" + +- name: Restart sshd if config changed # noqa no-handler ansible.windows.win_service: name: sshd state: restarted - when: sshd_pubkey_changed.changed or sshd_authkeys_changed.changed or sshd_subsystem_changed.changed + when: sshd_config_changed.changed - name: Ensure users for SSH keys exist (no-op if already present) ansible.windows.win_user: name: "{{ item.name }}" state: present - loop: "{{ ssh_users | default([]) }}" + loop: "{{ __ssh_users__ | default([]) }}" loop_control: label: "{{ item.name }}" become: true - become_method: runas + become_method: ansible.builtin.runas become_user: "NT AUTHORITY\\SYSTEM" - name: Ensure user .ssh directory exists ansible.windows.win_file: path: "C:\\Users\\{{ item.name }}\\.ssh" state: directory - loop: "{{ ssh_users | default([]) }}" + loop: "{{ __ssh_users__ | default([]) }}" loop_control: label: "{{ item.name }}" become: true - become_method: runas + become_method: ansible.builtin.runas become_user: "NT AUTHORITY\\SYSTEM" - name: Deploy authorized_keys for each user @@ -139,11 +296,11 @@ dest: "C:\\Users\\{{ item.name }}\\.ssh\\authorized_keys" content: "{{ (item['keys'] | default([])) | join('\r\n') }}\r\n" force: true - loop: "{{ ssh_users | default([]) }}" + loop: "{{ __ssh_users__ | default([]) }}" loop_control: label: "{{ item.name }}" become: true - become_method: runas + become_method: ansible.builtin.runas become_user: "NT AUTHORITY\\SYSTEM" - name: Set ACL on .ssh directory to allow only the user and Administradores (directory) @@ -155,11 +312,11 @@ inheritance: false propagate: false state: present - loop: "{{ ssh_users | default([]) }}" + loop: "{{ __ssh_users__ | default([]) }}" loop_control: label: "{{ item.name }}" become: true - become_method: runas + become_method: ansible.builtin.runas become_user: "NT AUTHORITY\\SYSTEM" - name: Set ACL on authorized_keys file to allow only the user and Administradores (file) @@ -171,43 +328,46 @@ inheritance: false propagate: false state: present - loop: "{{ ssh_users | default([]) }}" + loop: "{{ __ssh_users__ | default([]) }}" loop_control: label: "{{ item.name }}" become: true - become_method: runas + become_method: ansible.builtin.runas become_user: "NT AUTHORITY\\SYSTEM" - name: Create administrators_authorized_keys from admin users' keys (if any) ansible.windows.win_copy: dest: 'C:\ProgramData\ssh\administrators_authorized_keys' - content: "{{ (ssh_users | default([]) | selectattr('admin','equalto',true) | map(attribute='keys') | list | flatten | default([])) | join('\r\n') }}\r\n" + content: >- + {{ (__ssh_users__ | default([]) | selectattr('admin', 'equalto', true) + | map(attribute='keys') | list | flatten | default([])) | join('\r\n') }}\r\n force: true - when: (ssh_users | default([]) | selectattr('admin','equalto',true) | list) | length > 0 + when: (__ssh_users__ | default([]) | selectattr('admin','equalto',true) | list) | length > 0 become: true - become_method: runas + become_method: ansible.builtin.runas become_user: "NT AUTHORITY\\SYSTEM" -- name: Tighten ACLs for administrators_authorized_keys (icacls) +- name: Restrict administrators_authorized_keys permissions ansible.windows.win_shell: | - $f = 'C:\ProgramData\ssh\administrators_authorized_keys' - $current = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name - if (Test-Path $f) { - icacls.exe $f /inheritance:r - icacls.exe $f /grant:r "S-1-5-18:(F)" "S-1-5-32-544:(F)" - icacls.exe $f /remove $current 2>$null - Write-Output 'restricted' + $path = 'C:\ProgramData\ssh\administrators_authorized_keys' + if (Test-Path $path) { + $before = (Get-Acl -Path $path).Sddl + icacls.exe $path /inheritance:r | Out-Null + icacls.exe $path /grant:r "S-1-5-18:(F)" "S-1-5-32-544:(F)" | Out-Null + $after = (Get-Acl -Path $path).Sddl + if ($before -ne $after) { + Write-Output 'admin-auth-keys-acl-changed' + } else { + Write-Output 'admin-auth-keys-acl-unchanged' + } } else { - Write-Output 'no-file' + Write-Output 'no-admin-auth-keys' } args: executable: powershell.exe register: restrict_admin_auth_perms - changed_when: "'restricted' in restrict_admin_auth_perms.stdout" + changed_when: "'admin-auth-keys-acl-changed' in restrict_admin_auth_perms.stdout" + when: (__ssh_users__ | default([]) | selectattr('admin','equalto',true) | list) | length > 0 become: true - become_method: runas + become_method: ansible.builtin.runas become_user: "NT AUTHORITY\\SYSTEM" - -- name: Debug administrators_authorized_keys ACL change - ansible.builtin.debug: - var: restrict_admin_auth_perms.stdout diff --git a/ansible/templates/windows_sshd_config.j2 b/ansible/templates/windows_sshd_config.j2 new file mode 100644 index 0000000..2515641 --- /dev/null +++ b/ansible/templates/windows_sshd_config.j2 @@ -0,0 +1,49 @@ +# Ansible managed +AcceptEnv LANG LC_* +AllowAgentForwarding no +AllowGroups provision users +AllowTcpForwarding yes +AuthenticationMethods any +Banner __PROGRAMDATA__/ssh/issue.net +ChallengeResponseAuthentication no +Ciphers {{ __ssh_ciphers__ | join(',') }} +ClientAliveCountMax 1 +ClientAliveInterval 200 +Compression no +GSSAPIAuthentication no +HostbasedAuthentication no +HostKeyAlgorithms {{ __ssh_hostkey_algorithms__ | join(',') }} +AuthorizedKeysFile .ssh/authorized_keys +PubkeyAcceptedAlgorithms {{ __ssh_pubkey_accepted_algorithms__ | join(',') }} +IgnoreUserKnownHosts yes +{% set __ssh_supported_kex__ = ssh_supported_kex.stdout_lines | default([]) | map('trim') | reject('equalto', '') | list %} +{% set __ssh_effective_kex__ = (__ssh_kex_algorithms__ | intersect(__ssh_supported_kex__)) if (__ssh_supported_kex__ | length > 0) else __ssh_kex_algorithms__ %} +{% if __ssh_effective_kex__ | length > 0 %} +KexAlgorithms {{ __ssh_effective_kex__ | join(',') }} +{% endif %} +LoginGraceTime 20 +LogLevel VERBOSE +Macs {{ __ssh_macs__ | join(',') }} +MaxAuthTries {{ __ssh_max_auth_tries__ }} +MaxSessions {{ __ssh_max_sessions__ }} +MaxStartups 10:30:60 +PasswordAuthentication no +PermitEmptyPasswords no +PermitRootLogin no +PermitUserEnvironment no +Port 22 +PrintLastLog yes +PrintMotd no +RekeyLimit 512M 1h +RequiredRSASize 2048 +StrictModes yes +TCPKeepAlive no +UseDNS no +X11Forwarding no +Subsystem sftp internal-sftp +{% if __ssh_subsystem_path__ | default('') | length > 0 %} +Subsystem powershell "{{ __ssh_subsystem_path__ }}" -sshs -NoLogo -NoProfile +{% endif %} +Match Group administrators + AuthorizedKeysFile __PROGRAMDATA__/ssh/administrators_authorized_keys + diff --git a/bin/parse_ollama_models.py b/bin/parse_ollama_models.py new file mode 100755 index 0000000..4accc14 --- /dev/null +++ b/bin/parse_ollama_models.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +""" +Utility to parse ollama ls output and extract model names. + +Replicates the Jinja2 filter logic from: + ollama_existing_models.stdout_lines + | reject('match', '^\\s*$|^(NAME|\\-+)') + | map('regex_replace', '^\\s*(\\S+)\\s.*', '\\1') + | list + +Usage: + python parse_ollama_models.py < input.txt + python parse_ollama_models.py models_output.txt + echo "model_name ..." | python parse_ollama_models.py +""" + +import re +import sys + + +def parse_ollama_models(lines): + """Parse ollama ls output lines and return list of model names.""" + results = [] + for line in lines: + # Skip empty lines and header rows (NAME, ---) + if not line.strip() or re.match(r'^(NAME|\-+)$', line.strip()): + continue + # Extract the model name (first non-whitespace token) + match = re.match(r'^\s*(\S+)\s.*', line) + if match: + results.append(match.group(1)) + return results + + +def main(): + # Read input from file argument, stdin, or skip + if len(sys.argv) > 1: + with open(sys.argv[1], 'r') as f: + lines = f.read().splitlines() + else: + lines = sys.stdin.read().splitlines() + + models = parse_ollama_models(lines) + + # Output: one model per line + for model in models: + print(model) + + # Also write to a JSON file if a second arg is given + if len(sys.argv) > 2: + import json + with open(sys.argv[2], 'w') as f: + json.dump(models, f, indent=2) + + +if __name__ == '__main__': + main() diff --git a/inventory/hosts b/inventory/hosts index 00bfcae..85a4d9e 100644 --- a/inventory/hosts +++ b/inventory/hosts @@ -107,18 +107,21 @@ gpu-01.lab.alexpires.me gpu-01.lab.alexpires.me [windows] -hpz440.lab.alexpires.me +ci-01.lab.alexpires.me [prometheus_linux] prod-01.alexpires.me dev-01.lab.alexpires.me [windows_prometheus] -game-01.lab.alexpires.me -hpz440.lab.alexpires.me +ci-01.lab.alexpires.me [game_station] steambox.local [llamacpp] gpu-01.lab.alexpires.me + +[qwen3_6_host] +gpu-01.lab.alexpires.me + diff --git a/terraform/apps/dev-01/config.tf b/terraform/apps/dev-01/config.tf index 8b6f58c..ce2d10f 100644 --- a/terraform/apps/dev-01/config.tf +++ b/terraform/apps/dev-01/config.tf @@ -116,6 +116,11 @@ locals { name = "search" type = "CNAME" content = "dev-01.${local.lab_domain}." + }, + { + name = "ci-01" + type = "A" + content = "10.19.4.207" } ] } diff --git a/terraform/apps/prod-01/config.tf b/terraform/apps/prod-01/config.tf index e1df8a2..e9b994d 100644 --- a/terraform/apps/prod-01/config.tf +++ b/terraform/apps/prod-01/config.tf @@ -139,12 +139,14 @@ locals { proxy_services = { "open-webui" = { - http_port = 8080 - https_port = 8443 - tls = true - resolver = "10.19.4.136" - fqdn = ["ai.alexpires.me"] - upstream = "https://open-webui.lab.alexpires.me" + http_port = 8080 + https_port = 8443 + tls = true + resolver = "10.19.4.136" + fqdn = ["ai.alexpires.me"] + read_timeout = "300s" + send_timeout = "300s" + upstream = "https://open-webui.lab.alexpires.me" locations = [ { path = "/ws/socket.io/" diff --git a/terraform/modules/apps/open-webui/ingress.tf b/terraform/modules/apps/open-webui/ingress.tf index bfd99d8..cad93ac 100644 --- a/terraform/modules/apps/open-webui/ingress.tf +++ b/terraform/modules/apps/open-webui/ingress.tf @@ -4,10 +4,12 @@ resource "kubernetes_ingress_v1" "openwebui" { namespace = kubernetes_namespace.openwebui.metadata[0].name annotations = { - "kubernetes.io/ingress.class" = "public" - "cert-manager.io/cluster-issuer" = "letsencrypt-prod" - "kubernetes.io/tls-acme" = "true" - "nginx.ingress.kubernetes.io/ssl-redirect" = "true" + "kubernetes.io/ingress.class" = "public" + "cert-manager.io/cluster-issuer" = "letsencrypt-prod" + "kubernetes.io/tls-acme" = "true" + "nginx.ingress.kubernetes.io/ssl-redirect" = "true" + "nginx.ingress.kubernetes.io/proxy-read-timeout" = "600s" + "nginx.ingress.kubernetes.io/proxy-send-timeout" = "600s" } } spec { diff --git a/terraform/modules/apps/searxng/config.tf b/terraform/modules/apps/searxng/config.tf index 5719da8..0e761f4 100644 --- a/terraform/modules/apps/searxng/config.tf +++ b/terraform/modules/apps/searxng/config.tf @@ -3,4 +3,7 @@ locals { app_version = var.tag app_fqdn = var.fqdn + + searxng_config = templatefile("${path.module}/resources/settings.yml.tpl", {}) + searxng_config_checksum = md5(local.searxng_config) } diff --git a/terraform/modules/apps/searxng/main.tf b/terraform/modules/apps/searxng/main.tf index 1998fad..8dd0ba1 100644 --- a/terraform/modules/apps/searxng/main.tf +++ b/terraform/modules/apps/searxng/main.tf @@ -26,6 +26,17 @@ resource "kubernetes_config_map" "searxng_config" { } } +resource "kubernetes_config_map" "searxng_config_file" { + metadata { + name = "searxng-config-file" + namespace = kubernetes_namespace.searxng.metadata[0].name + } + + data = { + "settings.yml" = local.searxng_config + } +} + resource "kubernetes_secret" "searxng_secrets" { metadata { name = "searxng-keys" @@ -64,6 +75,9 @@ resource "kubernetes_deployment" "searxng" { labels = { app = "searxng" } + annotations = { + "checksum/config" = local.searxng_config_checksum + } } spec { @@ -145,6 +159,12 @@ resource "kubernetes_deployment" "searxng" { name = "searxng-tmp" mount_path = "/tmp" } + + volume_mount { + name = "searxng-config-file" + mount_path = "/etc/searxng/settings.yml" + sub_path = "settings.yml" + } } volume { @@ -158,6 +178,13 @@ resource "kubernetes_deployment" "searxng" { name = "searxng-tmp" empty_dir {} } + + volume { + name = "searxng-config-file" + config_map { + name = kubernetes_config_map.searxng_config_file.metadata[0].name + } + } } } } diff --git a/terraform/modules/apps/searxng/resources/settings.yml.tpl b/terraform/modules/apps/searxng/resources/settings.yml.tpl new file mode 100644 index 0000000..a3a62fb --- /dev/null +++ b/terraform/modules/apps/searxng/resources/settings.yml.tpl @@ -0,0 +1,5 @@ +use_default_settings: true +search: + formats: + - html + - json diff --git a/terraform/modules/utils/proxy/config.tf b/terraform/modules/utils/proxy/config.tf index efbbcb0..1ba60fd 100644 --- a/terraform/modules/utils/proxy/config.tf +++ b/terraform/modules/utils/proxy/config.tf @@ -35,6 +35,9 @@ locals { tls = v.tls fqdn = length(v.fqdn) > 0 ? join(" ", v.fqdn) : k auth = length([for l in v.locations : true if l.private]) > 0 + send_timeout = v.send_timeout + read_timeout = v.read_timeout + connect_timeout = v.connect_timeout locations = [for l in v.locations : { path = l.path headers = merge(l.headers, coalesce(v.default_headers, local.default_headers)) diff --git a/terraform/modules/utils/proxy/resources/nginx/conf.d/default.conf.tpl b/terraform/modules/utils/proxy/resources/nginx/conf.d/default.conf.tpl index 7feb84f..102172d 100644 --- a/terraform/modules/utils/proxy/resources/nginx/conf.d/default.conf.tpl +++ b/terraform/modules/utils/proxy/resources/nginx/conf.d/default.conf.tpl @@ -44,6 +44,9 @@ location = /auth { rewrite ${l.rewrite}; %{ endif ~} proxy_pass $upstream; + proxy_connect_timeout ${connect_timeout}; + proxy_read_timeout ${read_timeout}; + proxy_send_timeout ${send_timeout}; proxy_http_version 1.1; %{ for k,v in l.headers ~} proxy_set_header ${k} ${v}; diff --git a/terraform/modules/utils/proxy/variables.tf b/terraform/modules/utils/proxy/variables.tf index c4d34f3..8c98633 100644 --- a/terraform/modules/utils/proxy/variables.tf +++ b/terraform/modules/utils/proxy/variables.tf @@ -41,6 +41,9 @@ variable "services" { ingress = optional(bool, true) service_http_port = optional(number, 80) service_https_port = optional(number, 443) + read_timeout = optional(string, "60s") + send_timeout = optional(string, "60s") + connect_timeout = optional(string, "250ms") locations = optional(list(object({ path = string, headers = optional(map(string), {}),