feat: Add ROCm support and update Ollama configurations

- Created ansible playbooks for setting up AMD ROCm container environment.
- Added ROCm and AMD Graphics repositories in ansible tasks.
- Installed necessary ROCm packages including rocm-dkms and developer tools.
- Updated Dockerfile for Ollama to use version 0.13.4 and created a new Dockerfile for ROCm support.
- Enhanced game station scripts for better performance and added MangoHud configuration.
- Modified systemd service files to allow CPU core allocation.
- Updated playbooks for Ollama and Qwen2 to include new configurations and environment variables.
- Added Wake-on-LAN systemd service configuration.
- Updated Terraform configurations to include secret management for the web UI.
- Refactored Kubernetes deployment for open-webui to include secret handling and improved volume management.

ansible/amd-rocm-container.yml

@@ -0,0 +1,31 @@
+- name: Add ROCm repository
+  become: true
+  ansible.builtin.dnf_repository:
+    name: rocm
+    description: ROCm Repository
+    baseurl: https://repo.radeon.com/rocm/el9/latest/main/
+    gpgcheck: yes
+    gpgkey: https://repo.radeon.com/rocm/rocm.gpg.key
+    enabled: yes
+
+- name: Add AMD Graphics repository
+  become: true
+  ansible.builtin.dnf_repository:
+    name: rocmgraphics
+    description: AMD Graphics Repository
+    baseurl: https://repo.radeon.com/graphics/latest/el/9.6/main/x86_64/
+    gpgcheck: yes
+    gpgkey: https://repo.radeon.com/rocm/rocm.gpg.key
+    enabled: yes
+
+- name: Install rocm-dkms and dependencies
+  become: true
+  ansible.builtin.package:
+    name:
+      - rocm
+      - rocm-developer-tools
+      - hipblas-devel
+      - hip-devel
+      - rocwmma-devel
+      - rocm-opencl-devel
+    state: present

ansible/files/dockerfiles/Dockerfile.ollama

@@ -1,4 +1,4 @@
-FROM docker.io/ollama/ollama:0.9.6
+FROM docker.io/ollama/ollama:0.13.4
 
 ARG UID=1000
 ARG GID=1000

ansible/files/dockerfiles/Dockerfile.ollama.rocm

@@ -0,0 +1,12 @@
+FROM docker.io/ollama/ollama:0.13.4-rocm
+
+ARG UID=1000
+ARG GID=1000
+RUN groupadd --system --gid ${GID} worker && \
+    useradd --system --gid ${GID} --uid ${UID} --home /home/worker worker && \
+    mkdir -p /home/worker && chown worker:worker /home/worker
+
+WORKDIR /home/worker
+USER worker
+ENTRYPOINT ["/bin/ollama"]
+CMD ["serve"]

ansible/files/game_station/bin/system/gamescope-session

@@ -10,13 +10,14 @@ CONF_FILE="$HOME/.config/gamescope/gamescope.conf"
 RESOLUTION=${RESOLUTION:-1920x1080}
 WIDTH="${RESOLUTION%x*}"
 HEIGHT="${RESOLUTION#*x}"
+FRAMERATE=${FRAMERATE:-60}
 
 export __GL_GSYNC_ALLOWED=1
 export __GL_VRR_ALLOWED=1
 export __GL_SHADER_DISK_CACHE=1
 export __GL_SYNC_TO_VBLANK=0
-export LIBVA_DRIVER_NAME=nvidia
 export __GLX_VENDOR_LIBRARY_NAME=nvidia
+export LIBVA_DRIVER_NAME=nvidia
 export NVD_BACKEND=direct
 
 GAMESCOPE_BIN="$(command -v gamescope || true)"
@@ -45,6 +46,15 @@ if [[ -z $tmpdir || -z ${XDG_RUNTIME_DIR+x} ]]; then
 	exit 0
 fi
 
+# Setup MangoHud config file in temp directory
+export MANGOHUD_CONFIGFILE="${tmpdir:+$tmpdir/mangohud.config}"
+
+# Initially write no_display to our config file
+# so we don't get mangoapp showing up before Steam initializes
+# on OOBE and stuff.
+mkdir -p "$(dirname "$MANGOHUD_CONFIGFILE")"
+echo "no_display" > "$MANGOHUD_CONFIGFILE"
+
 # Chromium (and therefore Steam) ignore XCursor and use on the GTK config
 kwriteconfig6 --file gtk-3.0/settings.ini  --group Settings --key gtk-cursor-theme-name steam
 

ansible/files/game_station/bin/system/steam-launcher

@@ -21,7 +21,7 @@ export VKD3D_SWAPCHAIN_LATENCY_FRAMES=3
 # Let's try this across the board to see if it breaks anything
 # Helps performance in HZD, Cyberpunk, at least
 # Expose 8 physical cores, instead of 4c/8t
-export WINE_CPU_TOPOLOGY=8:0,1,2,3,4,5,6,7
+# export WINE_CPU_TOPOLOGY=8:0,1,2,3,4,5,6,7
 
 # To expose vram info from radv's patch we're including
 export WINEDLLOVERRIDES=dxgi=n

ansible/files/game_station/systemd/user/session.slice

@@ -13,3 +13,4 @@ Documentation=man:systemd.special(7)
 
 [Slice]
 CPUWeight=100
+# AllowedCPUs={{ general_cores }}

ansible/files/game_station/systemd/user/steam.slice

@@ -13,3 +13,4 @@ Documentation=man:systemd.special(7)
 
 [Slice]
 CPUWeight=100
+# AllowedCPUs={{ general_cores }}

ansible/files/game_station/systemd/user/steamos.service

@@ -6,3 +6,4 @@ After=graphical-session.target
 ExecStart=/usr/local/bin/steamos
 EnvironmentFile=%t/steamos-environment
 Restart=on-failure
+# AllowedCPUs={{ general_cores }}

ansible/files/game_station/systemd/user/sunshine.slice

@@ -13,3 +13,4 @@ Documentation=man:systemd.special(7)
 
 [Slice]
 CPUWeight=100
+# AllowedCPUs={{ sunshine_cores }}

ansible/files/scripts/auto_suspend.sh

@@ -22,7 +22,7 @@ if ! command -v nvidia-smi &> /dev/null; then
 else
     # Check 3: Is GPU idle?
     GPU_UTIL=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits | head -n 1)
-    GPU_THRESHOLD=10
+    GPU_THRESHOLD=0
 
     if (( GPU_UTIL > GPU_THRESHOLD )); then
         echo "GPU is busy (utilization=${GPU_UTIL}%). Skipping suspend."
@@ -37,5 +37,25 @@ else
     fi
 fi
 
+if | command -v amd-smi &> /dev/null; then
+    # Check 5: Is AMD GPU idle?
+    AMD_GPU_UTIL=$(amd-smi monitor --csv | head -n 2 | tail -n 1 | cut -d ',' -f 8)
+    AMD_GPU_THRESHOLD=0
+
+    if (( AMD_GPU_UTIL > AMD_GPU_THRESHOLD )); then
+        echo "AMD GPU is busy (utilization=${AMD_GPU_UTIL}%). Skipping suspend."
+        exit 0
+    fi
+
+    if amd-smi process --csv | head -n 2 | tail -n 1 | grep -q "No running processes detected"; then
+        echo "No processes running on AMD GPU."
+    else
+        echo "Processes are running on AMD GPU. Skipping suspend."
+        exit 0
+    fi
+else
+    echo "amd-smi not found, skipping AMD GPU check"
+fi
+
 echo "System idle. Suspending..."
 shutdown now

ansible/host_vars/gpu-01.lab.alexpires.me.yml

@@ -1,7 +1,7 @@
 hostname: gpu-01.lab.alexpires.me
 ansible_user: "{{ lookup('env', 'ANSIBLE_USER') }}"
 
-network_interface: enp3s0
+network_interface: enp7s0
 
 # Users
 login_users:
@@ -42,8 +42,11 @@ fw_allowed_ports:
   - { rule: "allow", port: "11434", proto: "tcp", from: "10.19.4.0/24" } # Ollama (local network)
   - { rule: "allow", port: "11434", proto: "tcp", from: "10.5.5.5/32" } # Ollama (Laptop)
   - { rule: "allow", port: "8880", proto: "tcp", from: "10.19.4.0/24" } # Kokoro-FastAPI (local network)
+  - { rule: "allow", port: "8880", proto: "tcp", from: "10.5.5.5/32" } # Kokoro-FastAPI (Laptop)
   - { rule: "allow", port: "8188", proto: "tcp", from: "10.19.4.0/24" } # ComfyUI (local network)
   - { rule: "allow", port: "8999", proto: "tcp", from: "10.19.4.0/24" } # ComfyUI - SimpleHttpServer (local network)
+  - { rule: "allow", port: "8012", proto: "tcp", from: "10.19.4.0/24" } # Qwen2 (local network)
+  - { rule: "allow", port: "8012", proto: "tcp", from: "10.5.5.5/32" } # Qwen2 (Laptop)
 ufw_outgoing_traffic:
   - 22 # SSH
   - 53 # DNS

ansible/playbook_amd_rocm.yml

@@ -0,0 +1,7 @@
+- name: Setup AMD ROCm Container Environment
+  hosts: amd
+
+  tasks:
+    - name: Setup AMD ROCm Container Environment
+      ansible.builtin.include_tasks:
+        file: tasks/amd-rocm-container.yml

ansible/playbook_game_station.yml

@@ -9,6 +9,9 @@
     systemd_user_dir: "/home/{{ user_name }}/.config/systemd/user"
     systemd_system_dir: "/etc/systemd/system"
     grub_cfg: "/etc/default/grub"
+    system_cores: "0,8"
+    general_cores: "1-7,9-15"
+    sunshine_cores: "1-7,9-15"
     screen_resolution: "{{ lookup('env', 'GAME_RESOLUTION') | d('1920x1080', true) }}"
     screen_width: "{{ screen_resolution.split('x')[0] }}"
     screen_height: "{{ screen_resolution.split('x')[1] }}"
@@ -24,6 +27,8 @@
       - fbdev=1
       - mitigations=off
       - threadirqs
+      # - nohz_full=1-7,9-15
+      # - rcu_nocbs=1-7,9-15
       - drm.edid_firmware=DP-1:edid/aoc28e850.bin
       - video=DP-1:e
       - acpi_enforce_resources=lax
@@ -58,6 +63,142 @@
         - lizardbyte/stable
         - atim/heroic-games-launcher
 
+    # # --- CachyOS Kernel (conditional) ---
+    # - name: Detect supported x86_64 ISA levels
+    #   ansible.builtin.shell:
+    #     cmd: |
+    #       set -euo pipefail;
+    #       /lib64/ld-linux-x86-64.so.2 --help | grep -F "(supported, searched)"
+    #   register: glibc_loader_help
+    #   changed_when: false
+    #   failed_when: false
+
+    # - name: Determine CPU support for CachyOS kernel
+    #   ansible.builtin.set_fact:
+    #     has_x86_64_v3: "{{ ('x86-64-v3' in glibc_loader_help.stdout) or ('x86_64_v3' in glibc_loader_help.stdout) }}"
+    #     has_x86_64_v2: "{{ ('x86-64-v2' in glibc_loader_help.stdout) or ('x86_64_v2' in glibc_loader_help.stdout) }}"
+
+    # - name: Debug CPU ISA detection
+    #   ansible.builtin.debug:
+    #     msg: "CPU supports x86_64_v3: {{ has_x86_64_v3 }}, x86_64_v2: {{ has_x86_64_v2 }}"
+
+    # - name: Enable CachyOS COPR repository (kernel)
+    #   become: true
+    #   community.general.copr:
+    #     name: bieszczaders/kernel-cachyos
+    #     state: enabled
+    #   when: has_x86_64_v3 | bool or has_x86_64_v2 | bool
+
+    # - name: Enable CachyOS COPR repository (addons)
+    #   become: true
+    #   community.general.copr:
+    #     name: bieszczaders/kernel-cachyos-addons
+    #     state: enabled
+    #   when: has_x86_64_v3 | bool or has_x86_64_v2 | bool
+
+    # - name: Install CachyOS kernel for x86_64_v3 CPUs
+    #   become: true
+    #   ansible.builtin.dnf:
+    #     name:
+    #       - pesign
+    #       - openssl
+    #       - kernel-devel
+    #       - mokutil
+    #       - keyutils
+    #       - kernel-cachyos-lts
+    #       - kernel-cachyos-lts-devel
+    #       - uksmd
+    #     state: present
+    #   when: has_x86_64_v3 | bool
+
+    # - name: Ensures user is on /etc/pesign/users
+    #   become: true
+    #   ansible.builtin.lineinfile:
+    #     path: /etc/pesign/users
+    #     line: "{{ ansible_user }}"
+    #     state: present
+    #   when: has_x86_64_v3 | bool
+    #   notify: Run pesign
+
+    # - name: Run /usr/libexec/pesign/pesign-authorize
+    #   become: true
+    #   ansible.builtin.command: /usr/libexec/pesign/pesign-authorize
+    #   when: has_x86_64_v3 | bool
+    #   changed_when: false
+
+    # - name: Generate certificate for CachyOS kernel signing
+    #   become: true
+    #   ansible.builtin.shell: |
+    #     openssl req -new -x509 -newkey rsa:2048 -keyout "key.pem" \
+    #             -outform DER -out "cert.der" -nodes -days 36500 \
+    #             -subj "/CN=CachyOS Secure Boot/" && \
+    #     openssl pkcs12 -export -out key.p12 -inkey key.pem -in cert.der
+    #   when: has_x86_64_v3 | bool
+    #   changed_when: false
+    #   args:
+    #     executable: /bin/bash
+    #     creates:
+    #       - key.pem
+    #       - cert.der
+    #       - key.p12
+
+    # - name: Inform user to enroll MOK key
+    #   ansible.builtin.debug:
+    #     msg: |
+    #       Please enroll the MOK key for kernel module signing on next reboot.
+    #       Run:
+    #       sudo certutil -A -i cert.der -n "CachyOS Secure Boot" -d /etc/pki/pesign/ -t "Pu,Pu,Pu" && \
+    #       sudo pk12util -i key.p12 -d /etc/pki/pesign
+    #       sudo mokutil --import cert.der' if not prompted automatically.
+    #   when: has_x86_64_v3 | bool
+
+    # - name: Create /etc/kernel/postinst.d directory
+    #   become: true
+    #   ansible.builtin.file:
+    #     path: /etc/kernel/postinst.d
+    #     state: directory
+    #     owner: root
+    #     group: root
+    #     mode: "0755"
+
+    # - name: Enable Automatically signing kernel updates
+    #   become: true
+    #   ansible.builtin.copy:
+    #     dest: /etc/kernel/postinst.d/00-signing
+    #     owner: root
+    #     group: root
+    #     mode: "0755"
+    #     content: |
+    #       #!/bin/sh
+    #       set -e
+    #       KERNEL_IMAGE="$2"
+    #       MOK_KEY_NICKNAME="CachyOS Secure Boot"
+    #       if [ "$#" -ne "2" ] ; then
+    #         echo "Wrong count of command line arguments. This is not meant to be called directly." >&2
+    #         exit 1
+    #       fi
+    #       if [ ! -x "$(command -v pesign)" ] ; then
+    #         echo "pesign not executable. Bailing." >&2
+    #         exit 1
+    #       fi
+    #       if [ ! -w "$KERNEL_IMAGE" ] ; then
+    #         echo "Kernel image $KERNEL_IMAGE is not writable." >&2
+    #         exit 1
+    #       fi
+    #       echo "Signing $KERNEL_IMAGE..."
+    #       pesign --certificate "$MOK_KEY_NICKNAME" --in "$KERNEL_IMAGE" --sign --out "$KERNEL_IMAGE.signed"
+    #       mv "$KERNEL_IMAGE.signed" "$KERNEL_IMAGE"
+
+    # - name: Install CachyOS LTS kernel for x86_64_v2-only CPUs
+    #   become: true
+    #   ansible.builtin.dnf:
+    #     name:
+    #       - kernel-cachyos-lts
+    #       - kernel-cachyos-lts-devel-matched
+    #     state: present
+    #   when: (not has_x86_64_v3 | bool) and (has_x86_64_v2 | bool)
+
+    # --- Core system configuration ---
     - name: Ensure core gaming and streaming packages are installed
       become: true
       ansible.builtin.dnf:

ansible/playbook_podman_kokoro.yml

@@ -11,11 +11,20 @@
     pods:
       - name: kokoro
         repo:
-          image: ghcr.io/remsky/kokoro-fastapi-gpu
+          image: ghcr.io/remsky/kokoro-fastapi-cpu
+        env:
+          PYTHONPATH: /app:/app/api
+          # ONNX Optimization Settings for vectorized operations
+          ONNX_NUM_THREADS: 6  # Maximize core usage for vectorized ops
+          ONNX_INTER_OP_THREADS: 4  # Higher inter-op for parallel matrix operations
+          ONNX_EXECUTION_MODE: parallel
+          ONNX_OPTIMIZATION_LEVEL: all
+          ONNX_MEMORY_PATTERN: true
+          ONNX_ARENA_EXTEND_STRATEGY: kNextPowerOfTwo
         ports:
           - "0.0.0.0:{{ kokoro_port | default(8880) }}:8880/tcp"
-        device:
+        # device:
-          - "nvidia.com/gpu=all"
+        #   - "nvidia.com/gpu=all"
 
   tasks:
     - name: Setup Pods

ansible/playbook_podman_ollama.yml

@@ -3,7 +3,7 @@
   vars:
     podman_user: "{{ ollama_user | default('ollama') }}"
     podman_group: "{{ ollama_group | default('ollama') }}"
-    podman_extra_groups: "users"
+    podman_extra_groups: "users,video"
     podman_user_home: "{{ ollama_home | default('/home/ollama') }}"
     podman_user_folders:
       - data
@@ -11,6 +11,11 @@
       - name: ollama
         build:
           dockerfile: "{{ lookup('file', 'dockerfiles/Dockerfile.ollama') }}"
+        env:
+          OLLAMA_CONTEXT_SIZE: 65536
+          OLLAMA_NUM_THREADS: 6
+          OLLAMA_NUM_PARALLEL: 1
+          OLLAMA_KV_CACHE: "true"
         ports:
           - "0.0.0.0:{{ ollama_port | default(11434) }}:11434/tcp"
         volumes:

ansible/playbook_podman_ollama_rocm.yml

@@ -0,0 +1,30 @@
+- name: Setup ollama
+  hosts: ollama
+  vars:
+    podman_user: "{{ ollama_user | default('ollama') }}"
+    podman_group: "{{ ollama_group | default('ollama') }}"
+    podman_extra_groups: "users,video"
+    podman_user_home: "{{ ollama_home | default('/home/ollama') }}"
+    podman_user_folders:
+      - data
+    pods:
+      - name: ollama-rocm
+        build:
+          dockerfile: "{{ lookup('file', 'dockerfiles/Dockerfile.ollama.rocm') }}"
+        env:
+          OLLAMA_CONTEXT_SIZE: 65536
+          OLLAMA_NUM_THREADS: 6
+          OLLAMA_NUM_PARALLEL: 1
+          OLLAMA_KV_CACHE: "true"
+        ports:
+          - "0.0.0.0:{{ ollama_port | default(11434) }}:11434/tcp"
+        volumes:
+          - "{{ ollama_home }}/data:/home/worker/.ollama:Z"
+        device:
+          - "/dev/kfd:/dev/kfd:rw"
+          - "/dev/dri:/dev/dri:rw"
+
+  tasks:
+    - name: Setup Pods
+      ansible.builtin.include_tasks:
+        file: tasks/podman.yml

ansible/playbook_podman_qwen2.yml

@@ -0,0 +1,37 @@
+- name: Setup Qwen2.5 - Coder (FIM - LLAMA.cpp)
+  hosts: qwen2_host
+  vars:
+    podman_user: "{{ qwen2_user | default('qwen2') }}"
+    podman_group: "{{ qwen2_group | default('qwen2') }}"
+    podman_extra_groups: "users,video"
+    podman_user_home: "{{ qwen2_home | default('/home/qwen2') }}"
+    llama_model: "{{ qwen2_model | default('ggml-org/Qwen2.5-Coder-1.5B-Q8_0-GGUF:Q8_0') }}"
+    podman_user_folders:
+      - models
+      - .cache
+      - .cache/llama.cpp
+    pods:
+      - name: qwen2
+        repo:
+          image: docker.io/rocm/llama.cpp:llama.cpp-b6652.amd0_rocm7.0.0_ubuntu24.04_server
+        env:
+          LLAMA_ARG_THREADS: 6
+          LLAMA_ARG_PARALLEL: 1
+          LLAMA_ARG_CACHE: "true"
+          LLAMA_ARG_CTX_SIZE: 8192
+        ports:
+          - "0.0.0.0:{{ qwen2_port | default(8012) }}:8012/tcp"
+        command:
+          - "-hf"
+          - "{{ llama_model }}"
+          - "--fim-qwen-1.5b-default"
+        volumes:
+          - "{{ podman_user_home }}/models:/models:Z"
+          - "{{ podman_user_home }}/.cache:/app/.cache:Z"
+        device:
+          - "/dev/kfd:/dev/kfd:rw"
+          - "/dev/dri:/dev/dri:rw"
+  tasks:
+    - name: Setup Pods
+      ansible.builtin.include_tasks:
+        file: tasks/podman.yml

ansible/tasks/amd-rocm-container.yml

@@ -0,0 +1,33 @@
+- name: Add ROCm repository
+  become: true
+  ansible.builtin.yum_repository:
+    name: rocm
+    description: ROCm Repository
+    baseurl: https://repo.radeon.com/rocm/el9/latest/main/
+    gpgcheck: true
+    gpgkey: https://repo.radeon.com/rocm/rocm.gpg.key
+    enabled: true
+    state: present
+
+- name: Add AMD Graphics repository
+  become: true
+  ansible.builtin.yum_repository:
+    name: rocmgraphics
+    description: AMD Graphics Repository
+    baseurl: https://repo.radeon.com/graphics/latest/el/9.6/main/x86_64/
+    gpgkey: https://repo.radeon.com/rocm/rocm.gpg.key
+    gpgcheck: true
+    enabled: true
+    state: present
+
+- name: Install rocm-dkms and dependencies
+  become: true
+  ansible.builtin.package:
+    name:
+      - rocm
+      - rocm-developer-tools
+      - hipblas-devel
+      - hip-devel
+      - rocwmma-devel
+      - rocm-opencl-devel
+    state: present

ansible/tasks/podman.yml

@@ -112,6 +112,7 @@
         network: "{{ podman_network_name | default(omit) }}"
         ports: "{{ item.ports | default(omit) }}"
         volumes: "{{ item.volumes | default(omit) }}"
+        command: "{{ item.command | default(omit) }}"
         cmd_args:
           - "--userns=keep-id"
           - "--security-opt=label=disable"

ansible/tasks/wol.yml

@@ -23,12 +23,29 @@
     group: root
     mode: "0644"
 
+- name: Create systemd WOL config
+  become: true
+  ansible.builtin.copy:
+    dest: "/etc/systemd/system/wol.service"
+    content: |
+      [Unit]
+      Description=Enable Wake-on-LAN
+      After=network-online.target
+      [Service]
+      Type=oneshot
+      ExecStart=/sbin/ethtool --change {{ network_interface }} wol g
+      [Install]
+      WantedBy=network-online.target
+    owner: root
+    group: root
+    mode: "0644"
+
 - name: Enable and restart systemd-networkd
   become: true
   ansible.builtin.systemd:
-    name: systemd-networkd
+    name: wol.service
+    state: started
     enabled: true
-    state: restarted
 
 - name: Confirm WOL setting applied
   become: true

inventory/hosts

@@ -72,6 +72,9 @@ gpu-01.lab.alexpires.me
 [ollama]
 gpu-01.lab.alexpires.me
 
+[qwen2_host]
+gpu-01.lab.alexpires.me
+
 [kokoro]
 gpu-01.lab.alexpires.me
 
@@ -90,6 +93,9 @@ vh-01.lab.alexpires.me
 [nvidia]
 gpu-01.lab.alexpires.me
 
+[amd]
+gpu-01.lab.alexpires.me
+
 [wol]
 gpu-01.lab.alexpires.me
 

scripts/gamescope-steam-session

@@ -1,59 +0,0 @@
-#!/bin/bash
-echo "Starting sunshine service..."
-
-if [ -f "$HOME/.config/gamescope/gamescope.conf" ]; then
-    source "$HOME/.config/gamescope/gamescope.conf"
-fi
-OUTPUT_CONNECTOR=${OUTPUT_CONNECTOR:-HDMI-A-1}
-FRAMERATE=${FRAMERATE:-120}
-RESOLUTION=${RESOLUTION:-1920x1080}
-HDR_ENABLED=${HDR_ENABLED:-${HDR:-1}}
-
-echo "Building gamescope command..."
-
-# Choose backend based on environment
-BACKEND_ARGS=""
-DRM_ARGS=""
-IN_WAYLAND=0
-if [ -n "${WAYLAND_DISPLAY:-}" ] || [ "${XDG_SESSION_TYPE:-}" = "wayland" ]; then
-  IN_WAYLAND=1
-  BACKEND_ARGS="--backend wayland"
-fi
-
-# HDR only relevant for DRM path generally
-HDR_FLAG=""
-if [[ "$HDR_ENABLED" == "1" || "$HDR_ENABLED" == "true" || "$HDR_ENABLED" == "yes" ]]; then
-    HDR_FLAG="--hdr-enabled"
-    echo "   HDR flag added: --hdr-enabled"
-else
-    echo "   HDR disabled, no HDR flag added"
-fi
-
-WIDTH="${RESOLUTION%x*}"
-HEIGHT="${RESOLUTION#*x}"
-
-if [ "$IN_WAYLAND" -eq 0 ]; then
-  DRM_ARGS="-O \"$OUTPUT_CONNECTOR\" -r \"$FRAMERATE\" $HDR_FLAG"
-fi
-
-echo ""
-echo "Final Configuration Summary:"
-echo "   Backend: $([ "$IN_WAYLAND" -eq 1 ] && echo Wayland || echo DRM/KMS)"
-echo "   Output Connector: ${OUTPUT_CONNECTOR}"
-echo "   Framerate: ${FRAMERATE}Hz"
-echo "   Resolution: ${RESOLUTION}"
-echo "   HDR: $([ -n "$HDR_FLAG" ] && echo "enabled" || echo "disabled")"
-echo "   MangoHUD: enabled"
-echo "   Steam Mode: SteamOS3"
-
-echo ""
-echo "Launching gamescope session..."
-echo "Full command:"
-echo "   gamescope $BACKEND_ARGS $DRM_ARGS --mangoapp -w $WIDTH -h $HEIGHT -e -- /usr/bin/steamos"
-echo ""
-echo "========================================="
-
-LOGDIR="${HOME}/.local/logs"
-mkdir -p "$LOGDIR"
-echo "[$(date --iso-8601=seconds)] Launching (backend: $([ "$IN_WAYLAND" -eq 1 ] && echo wayland || echo drm)): gamescope $BACKEND_ARGS $DRM_ARGS --mangoapp -w $WIDTH -h $HEIGHT -e -- /usr/bin/steamos" >>"$LOGDIR/gamescope-launch.log"
-exec bash -lc "gamescope $BACKEND_ARGS $DRM_ARGS --mangoapp -w $WIDTH -h $HEIGHT -e -- /usr/bin/steamos"

terraform/apps/dev-01/apps.tf

@@ -4,7 +4,9 @@ module "open-webui" {
   persistent_folder = local.persistent_folder
   fqdn              = local.open_webui_fqdn
   ollama_proxy      = local.ollama_proxy
+  secret_key        = var.webui_secret_key
   tag               = "0.6-slim"
+  log_level         = local.open_webui_log_level
 }
 
 module "samba" {

terraform/apps/dev-01/config.tf

@@ -4,9 +4,12 @@ locals {
   persistent_folder = "/srv"
 
   lab_domain      = "lab.alexpires.me"
-  open_webui_fqdn = "open-webui.${local.lab_domain}"
   internal_issuer = "internal-ca"
 
+  # open-webui
+  open_webui_fqdn      = "open-webui.${local.lab_domain}"
+  open_webui_log_level = "DEBUG"
+
   # samba
   samba_shares = [
     {

terraform/apps/dev-01/secrets.tf

@@ -21,3 +21,9 @@ variable "telegram_bot_token" {
   type        = string
   sensitive   = true
 }
+
+variable "webui_secret_key" {
+  description = "The secret key for WEBUI"
+  type        = string
+  sensitive   = true
+}

terraform/apps/dev-01/sectool.env

@@ -5,3 +5,4 @@ TF_VAR_nextcloud_db_root_password=$MYSQL_ROOT_PASSWORD
 TF_VAR_nextcloud_admin_password=$NEXTCLOUD_ADMIN_PASSWORD
 TF_VAR_scaleway_project_id=$SCALEWAY_PROJECT_ID
 TF_VAR_telegram_bot_token=$TELEGRAM_BOT_TOKEN
+TF_VAR_webui_secret_key=$WEBUI_SECRET_KEY

terraform/modules/apps/open-webui/config.tf

@@ -1,5 +1,6 @@
 locals {
   persistent_folder = var.persistent_folder
-  openwebui_data    = "${local.persistent_folder}/open-webui"
+  openwebui_data    = "${local.persistent_folder}/open-webui.data"
+  openwebui_app     = "${local.persistent_folder}/open-webui.app"
 }
 

terraform/modules/apps/open-webui/variables.tf

@@ -34,3 +34,24 @@ variable "ollama_proxy" {
   }
 
 }
+
+variable "secret_key" {
+  description = "The secret key for WEBUI"
+  type        = string
+}
+
+variable "jwt_expires_in" {
+  description = "Always set a reasonable expiration time in production environments (e.g., 3600s, 1h, 7d etc.) to limit the lifespan of authentication tokens."
+  default     = "1h"
+  type        = string
+}
+
+variable "log_level" {
+  description = "The global log level for the application"
+  type        = string
+  default     = "INFO"
+  validation {
+    condition     = contains(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], var.log_level)
+    error_message = "log_level must be one of: DEBUG, INFO, WARNING, ERROR, CRITICAL"
+  }
+}

terraform/modules/apps/open-webui/webui.tf

@@ -8,6 +8,20 @@ resource "kubernetes_service_account" "openwebui" {
   automount_service_account_token = false
 }
 
+
+resource "kubernetes_secret" "openwebui" {
+  metadata {
+    name      = "openwebui-secret"
+    namespace = kubernetes_namespace.openwebui.metadata[0].name
+  }
+
+  data = {
+    "WEBUI_SECRET_KEY" = base64encode(var.secret_key)
+  }
+
+  type = "Opaque"
+
+}
 resource "kubernetes_deployment" "openwebui" {
   metadata {
     name      = "openwebui"
@@ -41,12 +55,39 @@ resource "kubernetes_deployment" "openwebui" {
         service_account_name            = kubernetes_service_account.openwebui.metadata[0].name
         automount_service_account_token = false
 
+        # We need an init container rsync the /app/backend from the original image to the hostPath volume, so that we don't overwrite user data on restarts
+        # also fix permissions
+        init_container {
+          name    = "startup"
+          image   = "ghcr.io/open-webui/open-webui:${var.tag}"
+          command = ["sh", "-c", "apt-get update && apt-get install -y --no-install-recommends rsync && rm -rf /target_app/* && rsync -avz --delete /app/backend/* /target_app/ --exclude data && chown -R 1000:1000 /target_app /target_data"]
+          security_context {
+            run_as_user  = 0
+            run_as_group = 0
+          }
+
+          volume_mount {
+            name       = "openwebui-app"
+            mount_path = "/target_app"
+          }
+
+          volume_mount {
+            name       = "openwebui-data"
+            mount_path = "/target_data"
+          }
+        }
+
+
         container {
           name              = "app"
           image             = "ghcr.io/open-webui/open-webui:${var.tag}"
+          image_pull_policy = "Always"
 
           security_context {
             allow_privilege_escalation = false
+            run_as_non_root            = true
+            run_as_user                = 1000
+            run_as_group               = 1000
           }
 
           port {
@@ -54,6 +95,27 @@ resource "kubernetes_deployment" "openwebui" {
             container_port = 8080
           }
 
+          env {
+            name = "WEBUI_SECRET_KEY"
+            value_from {
+              secret_key_ref {
+                name = kubernetes_secret.openwebui.metadata[0].name
+                key  = "WEBUI_SECRET_KEY"
+              }
+            }
+          }
+
+          env {
+            name  = "JWT_EXPIRES_IN"
+            value = var.jwt_expires_in
+          }
+
+          env {
+            name  = "GLOBAL_LOG_LEVEL"
+            value = var.log_level
+          }
+
+
           readiness_probe {
             tcp_socket {
               port = 8080
@@ -63,10 +125,23 @@ resource "kubernetes_deployment" "openwebui" {
             failure_threshold     = 10
           }
 
+          volume_mount {
+            name       = "openwebui-app"
+            mount_path = "/app/backend"
+          }
+
           volume_mount {
             name       = "openwebui-data"
             mount_path = "/app/backend/data"
           }
+
+        }
+
+        volume {
+          name = "openwebui-app"
+          host_path {
+            path = local.openwebui_app
+          }
         }
 
         volume {

terraform/modules/utils/cert-checker/main.tf

@@ -9,6 +9,7 @@ resource "kubernetes_service_account" "cert_checker" {
 resource "kubernetes_role" "cert_checker" {
   metadata {
     name      = "cert-checker-role"
+    namespace = var.namespace
   }
   rule {
     api_groups = ["apps"]