Enhance infrastructure and application configurations

- Updated AGENTS.md to include new directories for execution plans and implemented feature reports.
- Modified Ansible host variables for dev-02 and gpu-01 to change API endpoints and add new configurations for Scaleway.
- Adjusted firewall rules in gpu-01 to allow HTTPS traffic instead of the previous Ollama port.
- Added OAuth2 proxy configurations to gpu-01 for enhanced security.
- Removed deprecated open-webui module from Terraform app configurations.
- Updated Terraform configurations to reflect changes in local variables and removed unused secrets.
- Created a new sectool.json file for CloudNS integration with Bitwarden.
- Enhanced SELinux configurations for nginx to allow connections to any port and added oauth2-proxy port.

.gitea/actions/setup-ansible-env/action.yml

@@ -0,0 +1,99 @@
+name: Setup Ansible Environment
+description: Prepare the environment for A13labs CI/CD pipelines
+inputs:
+  access-token:
+    description: "Bitwarden access token"
+    required: true
+  project-id:
+    description: "Bitwarden project ID"
+    required: true
+  organization-id:
+    description: "Bitwarden organization ID"
+    required: true
+  ansible-user:
+    description: "The user to use for Ansible"
+    required: true
+  ssh-auth-socket:
+    description: "The SSH_AUTH_SOCK environment variable"
+    default: "/tmp/ssh_agent.sock"
+  host-group:
+    description: "The host group to setup the Ansible environment"
+    default: "public"
+outputs:
+  ssh-auth-sock:
+    description: "The SSH_AUTH_SOCK environment variable"
+    value: ${{ steps.start-ssh-agent.outputs.ssh-auth-sock }}
+  hosts:
+    description: "The list of hosts in the host group"
+    value: ${{ steps.read-hosts.outputs.hosts }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install required python packages
+      shell: bash
+      run: |
+        python3 -m pip install -r requirements/ansible.txt
+
+    - name: Prepare SSH key
+      shell: bash
+      env:
+        BW_ACCESS_TOKEN: ${{ inputs.access-token }}
+        BW_PROJECT_ID: ${{ inputs.project-id }}
+        BW_ORGANIZATION_ID: ${{ inputs.organization-id }}
+      run: |
+        sectool ssh unlock ansible
+        mkdir -p ~/.ssh
+        chmod 700 ~/.ssh
+        cp ssh-keys/ansible/id_ecdsa ~/.ssh/id_ecdsa
+        chmod 600 ~/.ssh/id_ecdsa
+
+    - name: Read hosts from inventory
+      id: read-hosts
+      shell: bash
+      env:
+        HOST_GROUP: ${{ inputs.host-group }}
+      run: |
+        ansible-inventory -i inventory/hosts --list --yaml --limit $HOST_GROUP | grep -oE '([a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}' | sort -u > /tmp/ansible_hosts
+        echo "hosts=$(cat /tmp/ansible_hosts)" >> $GITEA_OUTPUT
+
+    - name: Read SSH host keys from remote hosts
+      shell: bash
+      run: |
+        mkdir -p ~/.ssh
+        for HOST in $(cat /tmp/ansible_hosts); do
+          ssh-keyscan "$HOST" >> ~/.ssh/known_hosts
+        done
+        chmod 600 ~/.ssh/known_hosts
+
+    - name: Enable SSH host key algorithms
+      shell: bash
+      run: |
+        echo "Host *" >> ~/.ssh/config
+        echo "  HostKeyAlgorithms +sk-ecdsa-sha2-nistp256@openssh.com, sk-ssh-ed25519@openssh.com"
+        chmod 600 ~/.ssh/config
+
+    - name: Start SSH agent and add SSH key
+      id: start-ssh-agent
+      shell: bash
+      env:
+        BW_ACCESS_TOKEN: ${{ inputs.access-token }}
+        BW_PROJECT_ID: ${{ inputs.project-id }}
+        BW_ORGANIZATION_ID: ${{ inputs.organization-id }}
+        SSH_AUTH_SOCK: ${{ inputs.ssh-auth-socket }}
+      run: |
+        ssh-agent -a $SSH_AUTH_SOCK > /dev/null
+        echo 'sectool vault get SSH_PASSWORD' > ~/.ssh_askpass && chmod +x ~/.ssh_askpass
+        DISPLAY=None SSH_ASKPASS=~/.ssh_askpass ssh-add ~/.ssh/id_ecdsa < /dev/null
+        rm ~/.ssh_askpass
+        echo "ssh-auth-sock=$SSH_AUTH_SOCK" >> $GITEA_OUTPUT
+
+    - name: Check SSH Connection to remote hosts
+      shell: bash
+      env:
+        ANSIBLE_USER: ${{ inputs.ansible-user }}
+        SSH_AUTH_SOCK: ${{ inputs.ssh-auth-socket }}
+      run: |
+        for HOST in $(cat /tmp/ansible_hosts); do
+          ssh "$ANSIBLE_USER@$HOST" echo "SSH connection to "$HOST" successful"
+        done

.gitea/actions/setup-env/action.yml

@@ -0,0 +1,24 @@
+name: Setup Environment
+description: Prepare the environment for A13labs CI/CD pipelines
+outputs:
+  changed_files:
+    description: The list of changed files in the current pull request
+    value: ${{ steps.get_changed_files.outputs.changed_files }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Get changed files
+      id: get_changed_files
+      shell: bash
+      run: |
+        CHANGED_FILES=$(git diff --name-only ${{ gitea.event.before }} ${{ gitea.sha }})
+        echo "changed_files=$CHANGED_FILES" >> $GITEA_OUTPUT
+
+    - name: Setup Sectool
+      uses: ./.gitea/actions/setup-sectool
+
+    - name: Install Python packages
+      shell: bash
+      run: |
+        python3 -m pip install --upgrade pip

.gitea/actions/setup-k8s-env/action.yml

@@ -0,0 +1,54 @@
+name: Setup K8S Environment
+description: Prepare the environment for A13labs CI/CD pipelines
+inputs:
+  access-token:
+    description: "Bitwarden access token"
+    required: true
+  project-id:
+    description: "Bitwarden project ID"
+    required: true
+  organization-id:
+    description: "Bitwarden organization ID"
+    required: true
+  ansible-user:
+    description: "The user to use for Ansible"
+    required: true
+  k8s-target:
+    description: "The target host to deploy the K8S applications"
+    default: "microk8s"
+  ssh-auth-sock:
+    description: "The SSH_AUTH_SOCK environment variable"
+    default: "/tmp/ssh_agent.sock"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Copy Kube Config from remote host
+      shell: bash
+      env:
+        BW_ACCESS_TOKEN: ${{ inputs.access-token }}
+        BW_PROJECT_ID: ${{ inputs.project-id }}
+        BW_ORGANIZATION_ID: ${{ inputs.organization-id }}
+        ANSIBLE_USER: ${{ inputs.ansible-user }}
+        SSH_AUTH_SOCK: ${{ inputs.ssh-auth-sock }}
+        K8S_TARGET: ${{ inputs.k8s-target }}
+      run: |
+        mkdir -p ~/.kube
+        ansible -i inventory/hosts -u $ANSIBLE_USER -m fetch -a "src=/home/$ANSIBLE_USER/.kube/config dest=~/.kube/config flat=yes" $K8S_TARGET
+
+    - name: Forward K8S API Server port
+      shell: bash
+      env:
+        BW_ACCESS_TOKEN: ${{ inputs.access-token }}
+        BW_PROJECT_ID: ${{ inputs.project-id }}
+        BW_ORGANIZATION_ID: ${{ inputs.organization-id }}
+        ANSIBLE_USER: ${{ inputs.ansible-user }}
+        SSH_AUTH_SOCK: ${{ inputs.ssh-auth-sock }}
+        K8S_TARGET: ${{ inputs.k8s-target }}
+      run: |
+        ANSIBLE_HOST=$(ansible -i inventory/hosts -u $ANSIBLE_USER -m shell -a "hostname" $K8S_TARGET | grep -oE '([a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}' | head -1)
+        IP_ADDR=$(ansible -i inventory/hosts -u $ANSIBLE_USER -m shell -a "curl -s ifconfig.me" $K8S_TARGET | grep -oE '([0-9]+\.){3}[0-9]+')
+
+        ssh -f -N -L 16443:localhost:16443 "$ANSIBLE_USER@$ANSIBLE_HOST"
+
+        sed -i "s/server: https:\/\/$IP_ADDR:16443/server: https:\/\/localhost:16443/g" ~/.kube/config

.gitea/actions/setup-opentofu-env/action.yml

@@ -0,0 +1,19 @@
+name: Setup OpenTofu Environment
+description: Prepare the environment for A13labs CI/CD pipelines
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup OpenTofu
+      shell: bash
+      run: |
+        TOFU_VERSION="1.9.1"
+        curl -sL "https://github.com/opentofu/opentofu/releases/download/v${TOFU_VERSION}/tofu_${TOFU_VERSION}_linux_amd64.zip" -o /tmp/tofu.zip
+        unzip -o /tmp/tofu.zip -d /usr/local/bin
+        rm /tmp/tofu.zip
+        tofu version
+
+    - name: Install required python packages
+      shell: bash
+      run: |
+        python3 -m pip install -r requirements/opentofu.txt

.gitea/actions/setup-sectool/action.yml

@@ -0,0 +1,16 @@
+name: Setup Sectool
+description: Download and install the sectool binary from GitHub releases
+
+runs:
+  using: "composite"
+  steps:
+    - name: Download sectool binary
+      shell: bash
+      run: |
+        LATEST=$(curl -s https://api.github.com/repos/a13labs/sectool/releases/latest)
+        ASSET_URL=$(echo "$LATEST" | jq -r '.assets[] | select(.name | test("linux.*x64")) | .browser_download_url')
+        curl -sL "$ASSET_URL" -o /tmp/sectool.zip
+        unzip -o /tmp/sectool.zip -d /usr/local/bin
+        rm /tmp/sectool.zip
+        chmod +x /usr/local/bin/sectool
+        sectool version

.gitea/workflows/ansible-validate-on-pr.yml

@@ -0,0 +1,37 @@
+name: Ansible Validate On Pull Request
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "ansible/**"
+    types:
+      - opened
+      - synchronize
+      - reopened
+jobs:
+  ansible-validate:
+    runs-on: cicd-base
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup environment
+        uses: ./.gitea/actions/setup-env
+
+      - name: Install Python packages
+        run: |
+          python3 -m pip install -r requirements/ansible.txt
+
+      - name: Ansible Lint
+        run: |
+          cd ansible
+          ansible-lint .
+
+      - name: Ansible Playbook Syntax Check
+        run: |
+          cd ansible
+          ansible-playbook --syntax-check playbook_*.yml

.gitea/workflows/aws-validate-on-pr.yml

@@ -0,0 +1,39 @@
+name: Terraform AWS IAC Validate on Pull Request
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "terraform/iac/aws/**"
+    types:
+      - opened
+      - synchronize
+      - reopened
+jobs:
+  iac-validate:
+    runs-on: cicd-base
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup environment
+        uses: ./.gitea/actions/setup-env
+
+      - name: Setup OpenTofu environment
+        uses: ./.gitea/actions/setup-opentofu-env
+
+      - name: OpenTofu Init and Validate
+        env:
+          BW_PROJECT_ID: ${{ secrets.BW_PROJECT_ID }}
+          BW_ORGANIZATION_ID: ${{ secrets.BW_ORGANIZATION_ID }}
+          BW_ACCESS_TOKEN: ${{ secrets.BW_ACCESS_TOKEN }}
+        run: |
+          cd terraform/iac/aws
+          sectool -f ../../sectool.json exec tofu init
+          sectool -f ../../sectool.json exec tofu validate
+          sectool -f ../../sectool.json exec tofu fmt -check
+          sectool -f ../../sectool.json exec tofu plan

.gitea/workflows/cloudns-validate-on-pr.yml

@@ -0,0 +1,39 @@
+name: Terraform ClouDNS IAC Validate on Pull Request
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "terraform/iac/cloudns/**"
+    types:
+      - opened
+      - synchronize
+      - reopened
+jobs:
+  iac-validate:
+    runs-on: cicd-base
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup environment
+        uses: ./.gitea/actions/setup-env
+
+      - name: Setup OpenTofu environment
+        uses: ./.gitea/actions/setup-opentofu-env
+
+      - name: OpenTofu Init and Validate
+        env:
+          BW_PROJECT_ID: ${{ secrets.BW_PROJECT_ID }}
+          BW_ORGANIZATION_ID: ${{ secrets.BW_ORGANIZATION_ID }}
+          BW_ACCESS_TOKEN: ${{ secrets.BW_ACCESS_TOKEN }}
+        run: |
+          cd terraform/iac/cloudns
+          sectool -f ../../sectool.json exec tofu init
+          sectool -f ../../sectool.json exec tofu validate
+          sectool -f ../../sectool.json exec tofu fmt -check
+          sectool -f ../../sectool.json exec tofu plan

.gitea/workflows/contabo-validate-on-pr.yml

@@ -0,0 +1,39 @@
+name: Terraform Contabo IAC Validate on Pull Request
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "terraform/iac/contabo/**"
+    types:
+      - opened
+      - synchronize
+      - reopened
+jobs:
+  iac-validate:
+    runs-on: cicd-base
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup environment
+        uses: ./.gitea/actions/setup-env
+
+      - name: Setup OpenTofu environment
+        uses: ./.gitea/actions/setup-opentofu-env
+
+      - name: OpenTofu Init and Validate
+        env:
+          BW_PROJECT_ID: ${{ secrets.BW_PROJECT_ID }}
+          BW_ORGANIZATION_ID: ${{ secrets.BW_ORGANIZATION_ID }}
+          BW_ACCESS_TOKEN: ${{ secrets.BW_ACCESS_TOKEN }}
+        run: |
+          cd terraform/iac/contabo
+          sectool -f ../../sectool.json exec tofu init
+          sectool -f ../../sectool.json exec tofu validate
+          sectool -f ../../sectool.json exec tofu fmt -check
+          sectool -f ../../sectool.json exec tofu plan

.gitea/workflows/k8s-apps-validate.yml

@@ -0,0 +1,63 @@
+name: OpenTofu K8s Apps Validate on Pull Request
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "terraform/apps/**"
+    types:
+      - opened
+      - synchronize
+      - reopened
+jobs:
+  iac-validate:
+    runs-on: cicd-base
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup environment
+        uses: ./.gitea/actions/setup-env
+
+      - name: Setup Ansible Environment
+        id: setup_ansible_env
+        uses: ./.gitea/actions/setup-ansible-env
+        with:
+          project-id: ${{ secrets.BW_PROJECT_ID }}
+          organization-id: ${{ secrets.BW_ORGANIZATION_ID }}
+          access-token: ${{ secrets.BW_ACCESS_TOKEN }}
+          ansible-user: ${{ secrets.ANSIBLE_USER }}
+          host-group: microk8s
+
+      - name: Setup OpenTofu Environment
+        uses: ./.gitea/actions/setup-opentofu-env
+
+      - name: Setup Kubernetes Environment
+        uses: ./.gitea/actions/setup-k8s-env
+        with:
+          project-id: ${{ secrets.BW_PROJECT_ID }}
+          organization-id: ${{ secrets.BW_ORGANIZATION_ID }}
+          access-token: ${{ secrets.BW_ACCESS_TOKEN }}
+          ansible-user: ${{ secrets.ANSIBLE_USER }}
+
+      - name: OpenTofu Init and Validate
+        env:
+          BW_PROJECT_ID: ${{ secrets.BW_PROJECT_ID }}
+          BW_ORGANIZATION_ID: ${{ secrets.BW_ORGANIZATION_ID }}
+          BW_ACCESS_TOKEN: ${{ secrets.BW_ACCESS_TOKEN }}
+        run: |
+          cd terraform/apps/dev-01
+          sectool -f ../../sectool.json exec tofu init
+          sectool -f ../../sectool.json exec tofu validate
+          sectool -f ../../sectool.json exec tofu fmt -check
+          sectool -f ../../sectool.json exec tofu plan
+
+          cd terraform/apps/prod-01
+          sectool -f ../../sectool.json exec tofu init
+          sectool -f ../../sectool.json exec tofu validate
+          sectool -f ../../sectool.json exec tofu fmt -check
+          sectool -f ../../sectool.json exec tofu plan

.gitea/workflows/scaleway-validate-on-pr.yml

@@ -0,0 +1,39 @@
+name: Terraform Scaleway IAC Validate on Pull Request
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "terraform/iac/scaleway/**"
+    types:
+      - opened
+      - synchronize
+      - reopened
+jobs:
+  iac-validate:
+    runs-on: cicd-base
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup environment
+        uses: ./.gitea/actions/setup-env
+
+      - name: Setup OpenTofu environment
+        uses: ./.gitea/actions/setup-opentofu-env
+
+      - name: OpenTofu Init and Validate
+        env:
+          BW_PROJECT_ID: ${{ secrets.BW_PROJECT_ID }}
+          BW_ORGANIZATION_ID: ${{ secrets.BW_ORGANIZATION_ID }}
+          BW_ACCESS_TOKEN: ${{ secrets.BW_ACCESS_TOKEN }}
+        run: |
+          cd terraform/iac/scaleway
+          sectool -f ../../sectool.json exec tofu init
+          sectool -f ../../sectool.json exec tofu validate
+          sectool -f ../../sectool.json exec tofu fmt -check
+          sectool -f ../../sectool.json exec tofu plan

.github/copilot-instructions.md

@@ -0,0 +1,130 @@
+# Copilot Instructions for a13labs.infra
+
+This repository manages infrastructure mostly with Ansible and OpenTofu/Terraform.
+These instructions define how Terraform app modules should be developed, deployed, and debugged.
+
+## Scope and Structure
+
+- App roots:
+  - terraform/apps/dev-01
+  - terraform/apps/prod-01
+- Reusable modules:
+  - terraform/modules/apps
+  - terraform/modules/db
+  - terraform/modules/utils
+- New app modules should be created under terraform/modules/apps/<module-name>.
+- Use terraform/modules/apps/template as the base shape when creating a new app module.
+
+## Cluster References
+
+Always cover both clusters in planning and reviews:
+
+- dev-01
+  - App root: terraform/apps/dev-01
+  - Kubernetes context: microk8s-dev-01
+- pro-01
+  - App root: terraform/apps/prod-01
+  - Kubernetes context: microk8s-prod-01
+
+Note: The production folder name is prod-01, while discussions may refer to it as pro-01.
+
+## Module Development Standards
+
+When adding a new app module in terraform/modules/apps/<name>:
+
+1. Create the standard files:
+   - provider.tf
+   - variables.tf
+   - config.tf
+   - main.tf
+   - ingress.tf (if exposed over HTTP/HTTPS)
+2. Provider constraints should match repository conventions:
+   - required_version ~>1.8
+   - kubernetes provider version 2.36.0
+3. Keep names consistent across namespace, service account, deployment labels, and service selectors.
+4. Use locals in config.tf for derived paths and normalized values.
+5. Treat secrets as Terraform inputs (sensitive variables), not hardcoded literals.
+6. Prefer explicit image tags instead of latest for predictable rollouts.
+
+## Security and Hardening Defaults
+
+For Kubernetes deployments, prefer hardened defaults unless an app requires exceptions:
+
+- Disable service account token automount at both SA and pod level.
+- Use pod security context with non-root UID/GID and RuntimeDefault seccomp.
+- Use container security context:
+  - allow_privilege_escalation = false
+  - privileged = false
+  - drop all capabilities
+  - read_only_root_filesystem = true
+- Provide writable mounts only where required (for example cache path and /tmp via emptyDir).
+
+## Integrating a Module into an App Root
+
+For each cluster root (dev-01 and pro-01):
+
+1. Register module in apps.tf.
+2. Add needed locals in config.tf (FQDN, path, issuer, feature toggles).
+3. Add DNS record entries when the app is externally reachable.
+4. Add sensitive input variables in secrets.tf.
+5. Map TF_VAR entries in sectool.env.
+
+## Secrets Pattern (sectool + TF_VAR)
+
+Secret flow is:
+
+1. Declare sensitive variables in secrets.tf.
+2. Map TF_VAR_<name> in terraform/apps/<cluster>/sectool.env.
+3. Keep value source in vault/environment, not in repository.
+4. Run OpenTofu through sectool so variables are injected.
+
+Example workflow from an app root:
+
+- sectool -f ../../../sectool.json exec tofu init
+- sectool -f ../../../sectool.json exec tofu validate
+- sectool -f ../../../sectool.json exec tofu plan
+- sectool -f ../../../sectool.json exec tofu apply -auto-approve
+
+## OpenTofu and Deployment Workflow
+
+Preferred command flow per cluster root:
+
+1. Format and validate first:
+   - tofu fmt
+   - sectool -f ../../../sectool.json exec tofu validate
+2. Plan before apply:
+   - sectool -f ../../../sectool.json exec tofu plan
+3. Apply:
+   - sectool -f ../../../sectool.json exec tofu apply -auto-approve
+
+Targeted apply can be used for emergency recovery, then follow with a full plan to detect drift.
+
+## Runtime Debugging Playbook (kubectl)
+
+When a deployment fails (CrashLoopBackOff):
+
+1. Inspect pod state and events:
+   - kubectl -n <ns> get pods -o wide
+   - kubectl -n <ns> describe pod <pod>
+2. Check logs:
+   - kubectl -n <ns> logs <pod> --all-containers=true --tail=200
+   - kubectl -n <ns> logs <pod> --previous --all-containers=true --tail=200
+3. Validate rollout:
+   - kubectl -n <ns> rollout status deployment/<name> --timeout=120s
+4. If needed, test temporary env override with kubectl set env, verify recovery, then persist in Terraform module.
+
+## Known SearXNG Behavior in This Repo
+
+- SearXNG may fail on startup with IPv6 bind on hosts without IPv6 socket support.
+- Set GRANIAN_HOST to 0.0.0.0 in module-managed configuration.
+- Keep image tag pinned in app roots to avoid unexpected upstream behavior changes.
+
+## Review Checklist for Copilot Changes
+
+Before proposing Terraform changes:
+
+1. Confirm changes cover both dev-01 and pro-01 impact.
+2. Confirm secrets are wired via sensitive variables + sectool mapping.
+3. Confirm image tags are pinned.
+4. Confirm deployment hardening defaults are retained.
+5. Confirm commands use sectool + tofu in examples and runbooks.

.github/workflows/ansible-apply-hosts-changes.yml

@@ -1,53 +0,0 @@
-name: Ansible Apply because of host_vars change
-on:
-  push:
-    branches:
-      - main
-    paths:
-      - ansible/host_vars/*.alexpires.me.yml
-      - ansible/host_vars/*.a13labs.me.yml
-      - ansible/host_vars/*.a13labs.pt.yml
-jobs:
-  ansible-apply:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Setup environment
-        id: setup_env
-        uses: ./.github/actions/setup-env
-
-      - name: Setup Ansible environment
-        uses: ./.github/actions/setup-ansible-env
-        with:
-          project-id: ${{ secrets.BW_PROJECT_ID }}
-          organization-id: ${{ secrets.BW_ORGANIZATION_ID }}
-          access-token: ${{ secrets.BW_ACCESS_TOKEN }}
-          ansible-user: ${{ secrets.ANSIBLE_USER }}
-          trusted-hosts: ${{ secrets.TRUSTED_HOSTS }}
-
-      - name: Apply the playbook that triggered the workflow
-        env:
-          BW_PROJECT_ID: ${{ secrets.BW_PROJECT_ID }}
-          BW_ORGANIZATION_ID: ${{ secrets.BW_ORGANIZATION_ID }}
-          BW_ACCESS_TOKEN: ${{ secrets.BW_ACCESS_TOKEN }}
-          ANSIBLE_USER: ${{ secrets.ANSIBLE_USER }}
-          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-          CHANGED_FILES: ${{ steps.setup_env.outputs.CHANGED_FILES }}
-        run: |
-          cd ansible
-          sectool -f ../sectool.json exec ansible-playbook -u $ANSIBLE_USER playbook_fail2ban.yml
-          sectool -f ../sectool.json exec ansible-playbook -u $ANSIBLE_USER playbook_geoip.yml
-          sectool -f ../sectool.json exec ansible-playbook -u $ANSIBLE_USER playbook_duo_security.yml
-          sectool -f ../sectool.json exec ansible-playbook -u $ANSIBLE_USER playbook_encryption.yml
-          sectool -f ../sectool.json exec ansible-playbook -u $ANSIBLE_USER playbook_microk8s.yml
-          sectool -f ../sectool.json exec ansible-playbook -u $ANSIBLE_USER playbook_gitea.yml
-          sectool -f ../sectool.json exec ansible-playbook -u $ANSIBLE_USER playbook_nextcloud.yml
-          sectool -f ../sectool.json exec ansible-playbook -u $ANSIBLE_USER playbook_websites.yml
-          sectool -f ../sectool.json exec ansible-playbook -u $ANSIBLE_USER playbook_m3uproxy.yml
-          sectool -f ../sectool.json exec ansible-playbook -u $ANSIBLE_USER playbook_monitoring.yml
-          sectool -f ../sectool.json exec ansible-playbook -u $ANSIBLE_USER playbook_rustdesk.yml

.github/workflows/ansible-apply-playbook.yml

@@ -1,45 +0,0 @@
-name: Ansible Apply Playbook that Triggered the Workflow
-on:
-  push:
-    branches:
-      - main
-    paths:
-      - ansible/playbook_*.yml
-      - '!ansible/playbook_websites.yml'
-      - '!ansible/playbook_hardening*.yml'
-jobs:
-  ansible-apply:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Setup environment
-        id: setup_env
-        uses: ./.github/actions/setup-env
-
-      - name: Setup Ansible environment
-        uses: ./.github/actions/setup-ansible-env
-        with:
-          project-id: ${{ secrets.BW_PROJECT_ID }}
-          organization-id: ${{ secrets.BW_ORGANIZATION_ID }}
-          access-token: ${{ secrets.BW_ACCESS_TOKEN }}
-          ansible-user: ${{ secrets.ANSIBLE_USER }}
-          trusted-hosts: ${{ secrets.TRUSTED_HOSTS }}
-
-      - name: Apply the playbook that triggered the workflow
-        env:
-          BW_PROJECT_ID: ${{ secrets.BW_PROJECT_ID }}
-          BW_ORGANIZATION_ID: ${{ secrets.BW_ORGANIZATION_ID }}
-          BW_ACCESS_TOKEN: ${{ secrets.BW_ACCESS_TOKEN }}
-          ANSIBLE_USER: ${{ secrets.ANSIBLE_USER }}
-          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-          CHANGED_FILES: ${{ steps.setup_env.outputs.CHANGED_FILES }}
-        run: |
-          cd ansible
-          for PLAYBOOK in $(echo $CHANGED_FILES | grep -oP 'playbook_\K\w+'); do
-            sectool -f ../sectool.json exec ansible-playbook -u $ANSIBLE_USER playbook_$PLAYBOOK.yml
-          done

.github/workflows/ansible-validate-playbook-on-pr.yml.disabled

@@ -29,7 +29,6 @@ jobs:
           organization-id: ${{ secrets.BW_ORGANIZATION_ID }}
           access-token: ${{ secrets.BW_ACCESS_TOKEN }}
           ansible-user: ${{ secrets.ANSIBLE_USER }}
-          trusted-hosts: ${{ secrets.TRUSTED_HOSTS }}
 
       - name: Apply the playbook that triggered the workflow
         env:

.github/workflows/deploy-websites.yaml.disabled

@@ -37,7 +37,6 @@ jobs:
           organization-id: ${{ secrets.BW_ORGANIZATION_ID }}
           access-token: ${{ secrets.BW_ACCESS_TOKEN }}
           ansible-user: ${{ secrets.ANSIBLE_USER }}
-          trusted-hosts: ${{ secrets.TRUSTED_HOSTS }}
 
       - name: Run hugo_build command
         run: bin/hugo_build

.github/workflows/terraform-apps-contabo-apply.yml.disabled

@@ -1,13 +1,11 @@
 name: OpenTofu K8S Apps apply
 
 on:
-  schedule:
-    - cron: "0 2 * * 0"
   push:
     branches:
       - main
     paths:
-      - "terraform/k8sapps/**"
+      - "terraform/apps/contabo/**"
 jobs:
   iac-apply:
     runs-on: ubuntu-latest
@@ -37,6 +35,7 @@ jobs:
       - name: Setup Kubernetes Environment
         uses: ./.github/actions/setup-k8s-env
         with:
+          host: prod-01.alexpires.me
           project-id: ${{ secrets.BW_PROJECT_ID }}
           organization-id: ${{ secrets.BW_ORGANIZATION_ID }}
           access-token: ${{ secrets.BW_ACCESS_TOKEN }}

.gitignore

@@ -43,3 +43,4 @@ drafts/*
 *.auto.tfvars
 dev/*
 .ansible
+*.x.c

.opencode/plans/ci-orchestrator.md

@@ -0,0 +1,233 @@
+# Gitea → MicroK8s Webhook Orchestrator → Scaleway Containers (CI Runner System)
+
+A lightweight CI orchestration system where a self-hosted Gitea instance triggers a Go-based webhook service running inside MicroK8s. The service decides whether to run jobs locally or launch ephemeral CI runners in Scaleway Serverless Containers.
+
+---
+
+## 1. High-Level Architecture
+
+```text
+                    On-Premises (MicroK8s Cluster)
+
+┌──────────────────────────────────────────────────────────────┐
+│                        Gitea                                 │
+│                                                              │
+│  Repo push / PR event                                        │
+└───────────────┬──────────────────────────────────────────────┘
+                │ Webhook (internal cluster network)
+                ▼
+┌──────────────────────────────────────────────────────────────┐
+│              ci-orchestrator (Go service)                   │
+│                                                              │
+│  - Validates webhook signature (HMAC)                       │
+│  - Parses event payload                                     │
+│  - Applies routing rules                                    │
+│  - Decides runner target (local vs cloud)                   │
+└───────────────┬──────────────────────────────────────────────┘
+                │ HTTPS API call (authenticated)
+                ▼
+──────────────────────── Internet ───────────────────────────────
+                │
+                ▼
+┌──────────────────────────────────────────────────────────────┐
+│     Scaleway Serverless Containers API                     │
+│                                                              │
+│  - Starts ephemeral container                              │
+│  - Runs Gitea Runner                                       │
+│  - Executes CI job                                         │
+│  - Shuts down after completion                             │
+└──────────────────────────────────────────────────────────────┘
+```
+
+---
+
+## 2. Current State
+
+| Component | Location | Status |
+|---|---|---|
+| **Gitea** | `prod-01.alexpires.me` (K8s) | Deployed via Terraform module, TLS, ingress, S3 storage, MariaDB |
+| **MicroK8s** | `prod-01`, `dev-01` | nginx ingress, cert-manager, multiple apps running |
+| **Scaleway** | `fr-par` | Registry + S3 buckets + IAM keys exist, no Serverless Containers yet |
+| **Gitea Runners** | `ci-01.lab.alexpires.me` | Windows Podman + native runners already deployed |
+| **DNS** | CoreDNS in `dev-01` | `ci-01 → 10.19.4.207` already resolved |
+
+---
+
+## 3. Design Decisions
+
+1. **Orchestrator cluster:** `prod-01` for low latency to Gitea
+2. **Routing:** Cloud runners only trigger when `cloud` label is present
+3. **Runner image:** `gitea/act_runner` (official)
+4. **Webhook registration:** Terraform-managed via `go-gitea/gitea` provider
+
+---
+
+## 4. Implementation Plan
+
+### Phase 1: Scaleway IaC (`terraform/iac/scaleway/`)
+
+**New Terraform resources:**
+- `scaleway_container_namespace` — CI runners namespace
+- `scaleway_container` — the `gitea/act_runner` container (ephemeral, privacy=private)
+- `scaleway_iam_policy` — `ContainersInvoke` permission for the CI orchestrator app
+- `scaleway_container_domain` (optional) — custom domain if desired
+
+**IAM:**
+- New app `ci_runner_app` with policy granting `ContainersInvoke`
+- API key for the Go service to invoke containers
+
+**Existing reuse:**
+- `data.terraform_remote_state.scaleway` in `prod-01` for outputs
+- Existing `scaleway_registry_namespace` for pushing the runner image
+
+### Phase 2: Go Service (`ci-orchestrator/`)
+
+```
+ci-orchestrator/
+├── go.mod
+├── main.go                      # Entry: HTTP server
+├── Dockerfile                   # Multi-stage, distroless
+├── .dockerignore
+└── internal/
+    ├── webhook/
+    │   ├── gitea.go             # Parse Gitea push/PR payloads
+    │   └── hmac.go              # HMAC-SHA256 validation
+    ├── router/
+    │   ├── router.go            # Check if job has "cloud" label
+    │   └── rules.go             # Config-based routing rules
+    └── runner/
+        └── scaleway.go          # Invoke SCW Serverless Container
+```
+
+**Routing logic:**
+- Parse webhook → extract repo, branch, commit, labels (from PR body or commit message convention)
+- If `cloud` label → invoke Scaleway container
+- If no label → ignore (existing runners on `ci-01` handle it)
+
+**Endpoints:**
+- `POST /webhook/gitea` — main webhook receiver
+- `GET /health` — health check
+
+### Phase 3: Runner Container Image
+
+```dockerfile
+FROM gitea/act_runner:latest
+ENV GITEA_RUNNER_IGNORE_RUNNING_JOBS_IN_SCRIPT=true
+# Entrypoint configured via Terraform env vars
+```
+
+Push to: `rg.fr-par.scw.cloud/<ci-namespace>/act-runner:<tag>`
+
+### Phase 4: Terraform K8s Deployment (`terraform/modules/apps/ci-orchestrator/`)
+
+Deployed in `prod-01` namespace:
+
+| Resource | Purpose |
+|---|---|
+| `kubernetes_namespace` | `ci-orchestrator` |
+| `kubernetes_deployment` | Go service (hardened security context like other modules) |
+| `kubernetes_service` | ClusterIP, port 8080 |
+| `kubernetes_ingress_v1` | Ingress via nginx, path `/webhook/gitea` |
+| `kubernetes_config_map` | Routing rules config |
+| `kubernetes_secret` | HMAC key, Scaleway API credentials, Gitea URL |
+
+### Phase 5: Gitea Webhook Registration
+
+```hcl
+provider "gitea" {
+  address = "https://code.alexpires.me"
+  username = "<user>"
+  token    = "<token>"
+}
+
+resource "gitea_repository_webhook" "ci_orchestrator" {
+  username        = "<org-or-user>"
+  name            = "<repo-name>"
+  type            = "gitea"
+  url             = "https://code.alexpires.me/webhook/gitea"
+  secret          = var.webhook_secret
+  content_type    = "json"
+  branch_filter   = "*"
+  events          = ["push", "pull_request"]
+  active          = true
+}
+```
+
+Can be per-repo or a single wildcard. The Go service routes based on labels.
+
+### Phase 6: CI Flow
+
+```
+push/PR to Gitea
+       │
+       ▼
+Webhook → ci-orchestrator (prod-01)
+       │
+       ├─ no cloud label → ignore (local runners handle it)
+       │
+       └─ cloud label → invoke Scaleway container
+                              │
+                              ▼
+                    gitea/act_runner starts
+                    Registers with Gitea
+                    Receives and executes job
+                    Exits → container shuts down (cost: per-second)
+```
+
+---
+
+## 5. Provider Compatibility
+
+### `go-gitea/gitea` (v0.7.0)
+- `gitea_repository_webhook` resource — creates and manages webhooks
+- Supports: `url`, `secret`, `content_type`, `events`, `branch_filter`, `active`, `type`
+
+### `scaleway/scaleway` (v2.77.1)
+- `scaleway_container` — Serverless Container resource
+  - `privacy` can be set to `private` (requires IAM auth)
+  - `min_scale = 0`, `max_scale = 1` for ephemeral execution
+  - Supports `command`, `args`, `environment_variables`, `secret_environment_variables`
+- `scaleway_container_namespace` — namespace for containers
+- `scaleway_iam_policy` — IAM policies for container invocation
+
+---
+
+## 7. Knowns (Updated)
+
+### Gitea
+- URL: `https://code.alexpires.me`
+- Admin user: `alexandre.pires`
+- API token: `${GITEA_ADMIN_TOKEN}` (via sectool)
+- Provider config: `gitea_credentials` in `terraform/iac/scaleway/` or `prod-01/`
+
+### Webhook registration
+- Target: all existing repos + new ones
+- Method: one-time Ansible script to register webhooks on all repos, then run once per new repo
+- Webhook target URL: `https://code.alexpires.me/webhook/gitea` (points to Go service ingress)
+
+### Runner resources
+- CPU: 2 vCPU
+- Memory: 4Gi
+- Image: `gitea/act_runner:latest` (or pinned version)
+
+### Repository
+- Moved to Gitea: `https://code.alexpires.me/alexpires.me/a13labs.infra`
+- Origin remote updated to `ssh://git@code.alexpires.me:22/alexpires.me/a13labs.infra.git`
+- All branches and tags pushed
+
+### CI repo workflow
+- `ci-01` runner builds Terraform images using Gitea Actions
+- Images pushed to Scaleway Registry
+- Terraform state managed via S3 on Scaleway
+
+---
+
+## 8. Open Decisions (to be addressed during implementation)
+
+1. **Routing labels** — How will the Go service detect "cloud" intent? Options:
+   - PR body labels (`[cloud]` in title/body)
+   - Commit message convention (`[cloud]` prefix)
+   - Branch naming convention
+   - Separate "cloud" branch filter per repo
+2. **Go service ingress path** — Current plan: `/webhook/gitea`. Need to ensure it doesn't conflict with Gitea's own routes.
+3. **Scaling strategy** — `max_scale=1` per invocation, but multiple concurrent jobs? May need pool or queue.

.opencode/plans/gateway-api-migration.md

@@ -0,0 +1,321 @@
+# Gateway API Migration & Cloudflare Integration
+
+## Overview
+
+Migrate prod-01 and dev-01 clusters from the deprecated nginx-ingress controller (retired March 2026) to NGINX Gateway Fabric via the Gateway API, improve cross-cluster WireGuard connectivity, and add Cloudflare Tunnel for edge protection.
+
+## Current Architecture
+
+```
+Internet → Contabo Public IP → OPNsense → prod-01
+                              ↓
+                     nginx-ingress plugin (MicroK8s)
+                              ↓
+                         MetalLB (10.2.0.40-49)
+                              ↓
+               ┌───────────────┼─────────────────┐
+               ↓               ↓                  ↓
+        [proxy pod]      [app pods]        [TCP streams]
+      (nginx + WG)  (gitea, m3uproxy, etc)  (hostPort)
+          ↓
+    WireGuard → aristotle (Contabo VPN)
+          ↓
+      dev-01 backends (open-webui, nextcloud)
+```
+
+**Key facts:**
+- 14 `kubernetes_ingress_v1` resources across both clusters
+- All use `ingress.class: public` + nginx-specific annotations
+- Proxy module on prod-01 creates a double-proxy (Ingress → proxy pod → backends)
+- TCP/UDP streams use nginx with hostPorts
+- DNS: ClouDNS, no Cloudflare involvement
+- cert-manager with DNS-01 (dev-01) and HTTP-01 (prod-01) for Let's Encrypt
+- **Both clusters are single-node** — no need for MetalLB, NodePort is sufficient
+
+## Decisions Made
+
+| Decision | Choice | Rationale |
+|---|---|---|
+| Gateway API implementation | **NGINX Gateway Fabric** | Keeps nginx data plane, minimal behavioral changes, conformant v1.4.1 |
+| Cross-cluster connectivity | **Improve WireGuard** | Keep through aristotle, allowed_ips to dev-01 only (10.19.4.136/32), remove gpu-01 |
+| Ollama removal | **Remove** | gpu-01 runs llamacpp (not ollama), prod-01 ollama proxy is broken, dev-01 DNS CNAME stale |
+| Cloudflare scope | **Tunnels only** | Get DDoS+WAF, keep DNS with ClouDNS, least disruptive |
+| Proxy module | **All through proxy** | Gateway routes everything to proxy Service, simpler Terraform |
+| Website module | **Direct Gateway route** | Keep separate pods, no proxy overhead for static sites |
+| TLS termination | **At Cloudflare edge** | Simpler cert management, Cloudflare provides origin cert |
+
+## Target Architecture
+
+```
+Internet → Cloudflare Tunnel → Cloudflare Edge (TLS termination)
+                              ↓
+                     cloudflared pods (DaemonSet)
+                              ↓
+                     Gateway (NGINX Gateway Fabric, NodePort Service)
+                     HTTP listener :80 (from cloudflared)
+                              ↓
+               ┌───────────────┼─────────────────┐
+               ↓               ↓                  ↓
+        [proxy Service]  [app Services]    [TCP/UDP Routes]
+               ↓               ↓                  ↓
+      [proxy pod + WG]   [direct pods]      [direct pods]
+          (nginx)   (website, etc.)       (rustdesk, etc.)
+               ↓
+         Cross-cluster → dev-01 backends
+         (open-webui, nextcloud)
+```
+
+---
+
+## Phase 1: Gateway API Migration
+
+### 1A. Install NGINX Gateway Fabric (Ansible)
+
+**New file: `ansible/roles/microk8s/tasks/gateway-api.yml`**
+
+```
+- Install `snap` package manager (if needed)
+- Install helm via snap
+- Add nginx-stable helm repo
+- Install Gateway API CRDs (standard channel + experimental for TCPRoute/UDPRoute)
+- Install NGINX Gateway Fabric via Helm (experimental features enabled)
+- Create GatewayClass named `public`
+- Create Gateway Service with type = NodePort (single-node, no MetalLB needed)
+```
+
+**Modify: `ansible/roles/microk8s/tasks/main.yml`**
+- Include `gateway-api.yml` AFTER existing setup (runs alongside nginx-ingress initially)
+
+**Modify: `ansible/roles/microk8s/defaults/main.yml`**
+- Add variables: `gateway_fabric_version`, `gateway_fabric_experimental_enabled`
+- Remove MetalLB-related vars (`microk8s_lb_provider`, `microk8s_lb_ips`, `microk8s_metallb_version`)
+
+### 1B. Create Gateway Module (Terraform)
+
+**New module: `terraform/modules/utils/gateway/`**
+- `main.tf`: Gateway resource with HTTP (:80) listener, `ServiceType: NodePort`
+- `variables.tf`: TLS secret names, listener config
+- `outputs.tf`: Gateway service name, NodePort port numbers
+
+**Deploy in both clusters** (`prod-01/apps.tf`, `dev-01/apps.tf`)
+
+### 1C. Migrate Terraform Modules (Ingress → Gateway API Resources)
+
+#### Proxy module (`terraform/modules/utils/proxy/main.tf`)
+
+- **Remove**: `kubernetes_ingress_v1.ingress` resource (lines 160-203)
+- **Remove**: `host_port` from stream container ports in Deployment
+- **Remove**: ollama proxy service definition from prod-01 config.tf
+- **Add**: HTTPRoute for cross-cluster services (open-webui, nextcloud)
+- **Add**: TCPRoute resources for RustDesk (21115, 21116, 21117 TCP) and Prometheus (9090)
+- **Add**: UDPRoute resource for RustDesk (21116 UDP)
+- The proxy pod's nginx stream block can be removed (Gateway handles L4 routing)
+- **Keep**: WireGuard sidecar, auth sidecar, internal nginx reverse proxy config
+
+#### Website module (`terraform/modules/apps/nginx/main.tf`)
+
+- **Remove**: `kubernetes_ingress_v1.website_ingress` (lines 103-145)
+- **Add**: HTTPRoute per website with backendRef → website Service
+- Gateway handles TLS termination, website pods receive plain HTTP
+- Website pods can drop internal TLS (simplification)
+
+#### Gitea module (`terraform/modules/apps/gitea/ingress.tf`)
+
+- **Remove**: `kubernetes_ingress_v1.gitea_ingress`
+- **Add**: HTTPRoute with backendRef → Gitea Service
+- Use `BackendTLSPolicy` if Gitea serves HTTPS internally
+
+#### M3UProxy module (`terraform/modules/apps/m3uproxy/main.tf`)
+
+- **Remove**: `kubernetes_ingress_v1.m3uproxy_ingress_http` (lines 241-274)
+- **Add**: HTTPRoute with backendRef → m3uproxy Service
+
+#### dev-01 modules
+
+| Module | Current | Change |
+|---|---|---|
+| `open-webui/ingress.tf` | kubernetes_ingress_v1 | HTTPRoute |
+| `grafana/grafana.tf` | kubernetes_ingress_v1 | HTTPRoute |
+| `mstream/ingress.tf` | kubernetes_ingress_v1 | HTTPRoute |
+| `searxng/ingress.tf` | kubernetes_ingress_v1 | HTTPRoute |
+| `vaultwarden/ingress.tf` | kubernetes_ingress_v1 | HTTPRoute |
+| `terraform-mcp/ingress.tf` | kubernetes_ingress_v1 | HTTPRoute |
+| `nextcloud/ingress.tf` | kubernetes_ingress_v1 | HTTPRoute |
+| `proxy/main.tf` | kubernetes_ingress_v1 | HTTPRoute (opencode, vscode) |
+
+#### Annotation → Gateway API Mapping
+
+| Current nginx annotation | Gateway API equivalent |
+|---|---|
+| `nginx.org/ssl-redirect: true` | HTTPRoute with `RequestRedirect` filter |
+| `nginx.org/redirect-to-https: true` | HTTPRoute on HTTP listener with redirect |
+| `nginx.ingress.kubernetes.io/backend-protocol: HTTPS` | `BackendTLSPolicy` resource |
+| `nginx.ingress.kubernetes.io/proxy-body-size` | `UpstreamSettings` policy (NGF custom) |
+| `nginx.ingress.kubernetes.io/proxy-request-buffering` | `ProxySettings` policy (NGF custom) |
+| `nginx.ingress.kubernetes.io/proxy-buffer-size` | `UpstreamSettings` policy |
+| `cert-manager.io/cluster-issuer` | cert-manager annotation on Gateway |
+
+### 1D. cert-manager Integration
+
+Use **approach B**: Keep existing Certificate resources, reference secrets in Gateway TLS config.
+
+- Each module's `kubernetes_manifest` Certificate resource stays unchanged
+- Gateway HTTPS listener references cert secrets via `certificateRefs`
+- This avoids alpha cert-manager Gateway integration
+
+### 1E. Migration Execution Order
+
+1. Apply new Ansible role (installs Gateway Fabric alongside nginx-ingress)
+2. Deploy Gateway resource to both clusters
+3. Migrate modules on **dev-01 first** (lower risk):
+   - Deploy HTTPRoutes for each module, one at a time
+   - Verify each service works
+   - Remove old Ingress resources
+4. Migrate modules on **prod-01**:
+   - Same process, one module at a time
+5. Disable nginx-ingress plugin on both clusters
+6. Clean up IngressClass, old LoadBalancer Service
+
+---
+
+## Phase 2: Cross-Cluster Connectivity
+
+### 2A. Improve WireGuard
+
+**Modify: `terraform/apps/prod-01/config.tf`** (WireGuard config):
+- Change `allowed_ips` from `[10.19.4.106/32, 10.19.4.136/32]` → `["10.19.4.136/32"]` (dev-01 only)
+- Remove gpu-01 (10.19.4.106) - ollama proxy being removed, no need to reach gpu-01
+- `persistent_keepalive` already set to 25
+
+**Modify: `terraform/modules/utils/proxy/resources/wireguard/wireguard.conf.tpl`**:
+- Template `PersistentKeepalive` from config variable
+- Template `AllowedIPs` from config list
+
+**Also clean up**:
+- Remove `"ollama"` entry from `proxy_services` in prod-01 config.tf
+- Remove `ollama` CNAME from dev-01 DNS zones in dev-01 config.tf
+
+**Modify: `terraform/modules/utils/proxy/main.tf`** (WireGuard container):
+- Improve liveness probe: `wg show` instead of `wg show | grep -q transfer`
+- Add readiness probe
+- Add resource limits (memory cap)
+
+### 2B. DNS Resolution
+
+No changes needed. The proxy's per-location resolver in `default.conf.tpl` already handles DNS correctly for cross-cluster backends.
+
+---
+
+## Phase 3: Cloudflare Tunnels
+
+### 3A. Cloudflare Setup
+
+- Create/verify Cloudflare account with `alexpires.me` domain
+- Keep DNS with ClouDNS (Cloudflare acts as reverse proxy only)
+- Create Cloudflare Tunnel for prod-01
+- Configure route rules: `https://*.alexpires.me` → Gateway Service
+- Set up origin rules (TLS terminates at Cloudflare edge → HTTP to cloudflared)
+
+### 3B. Deploy cloudflared
+
+**New module: `terraform/modules/utils/cloudflare-tunnel/`**
+- DaemonSet of `cloudflare/cloudflared` pods
+- ConfigMap with tunnel credentials and route definitions
+- Each public FQDN mapped to `http://<gateway-service>.<namespace>:80`
+
+### 3C. Gateway Configuration Changes
+
+- Gateway uses HTTP (:80) NodePort listener for Cloudflare Tunnel traffic
+- Cloudflare Tunnel connects to `http://<node_ip>:<nodeport>`
+- No HTTPS listener needed since all traffic comes through Cloudflare Tunnel
+
+### 3D. Firewall Changes
+
+- Close UFW ports 80/443 (cloudflared uses ephemeral outbound ports)
+- Keep SSH + WireGuard ports open
+
+### 3E. Security Benefits
+
+With Cloudflare in front:
+- No open ports on prod-01 (80, 443, etc. closed)
+- DDoS protection (automatic with Cloudflare)
+- WAF rules (OWASP Top 10, custom rules)
+- Rate limiting on public endpoints
+- Bot management
+
+---
+
+## Phase 4: Cleanup
+
+- Remove nginx-ingress plugin from MicroK8s (`microk8s.disable ingress`)
+- Remove unused IngressClass resources
+- Remove existing MetalLB (delete metallb-system namespace, IPAddressPool, L2Advertisement)
+- Delete orphaned `tasks/loadbalancer.yml` and `tasks/loadbalancer/metallb.yaml`
+- Remove MetalLB vars from `defaults/main.yml`
+- Update documentation
+
+---
+
+## File Change Summary
+
+### New Files
+| File | Purpose |
+|---|---|
+| `ansible/roles/microk8s/tasks/gateway-api.yml` | NGINX Gateway Fabric install |
+| `terraform/modules/utils/gateway/main.tf` | Gateway + ReferenceGrant resources |
+| `terraform/modules/utils/gateway/variables.tf` | Gateway configuration |
+| `terraform/modules/utils/gateway/outputs.tf` | Gateway outputs |
+| `terraform/modules/utils/cloudflare-tunnel/main.tf` | cloudflared DaemonSet |
+| `terraform/modules/utils/cloudflare-tunnel/variables.tf` | Tunnel config |
+
+### Deleted Files
+| File | Reason |
+|---|---|
+| `ansible/roles/microk8s/tasks/loadbalancer.yml` | Not wired into main.yml, was for nginx-ingress LB |
+| `ansible/roles/microk8s/tasks/loadbalancer/metallb.yaml` | MetalLB no longer needed (single-node, NodePort) |
+
+### Modified Files
+| File | Change |
+|---|---|
+| `ansible/roles/microk8s/tasks/main.yml` | Include gateway-api.yml |
+| `ansible/roles/microk8s/defaults/main.yml` | Add Gateway Fabric defaults, remove MetalLB vars |
+| `terraform/modules/utils/proxy/main.tf` | Ingress → HTTPRoute + TCPRoute + UDPRoute |
+| `terraform/modules/utils/proxy/variables.tf` | Update for Gateway API |
+| `terraform/modules/utils/proxy/resources/wireguard/wireguard.conf.tpl` | Add PersistentKeepalive |
+| `terraform/modules/apps/nginx/main.tf` | Ingress → HTTPRoute |
+| `terraform/modules/apps/gitea/ingress.tf` | Ingress → HTTPRoute |
+| `terraform/modules/apps/m3uproxy/main.tf` | Ingress → HTTPRoute |
+| `terraform/modules/apps/open-webui/ingress.tf` | Ingress → HTTPRoute |
+| `terraform/modules/apps/grafana/grafana.tf` | Ingress → HTTPRoute |
+| `terraform/modules/apps/mstream/ingress.tf` | Ingress → HTTPRoute |
+| `terraform/modules/apps/searxng/ingress.tf` | Ingress → HTTPRoute |
+| `terraform/modules/apps/vaultwarden/ingress.tf` | Ingress → HTTPRoute |
+| `terraform/modules/apps/terraform-mcp/ingress.tf` | Ingress → HTTPRoute |
+| `terraform/modules/apps/nextcloud/ingress.tf` | Ingress → HTTPRoute |
+| `terraform/modules/apps/ssh_locker_web/` | Ingress → HTTPRoute |
+| `terraform/apps/prod-01/apps.tf` | Add gateway module |
+| `terraform/apps/prod-01/config.tf` | WireGuard (remove gpu-01), remove ollama proxy |
+| `terraform/apps/dev-01/config.tf` | Remove ollama DNS CNAME |
+| `terraform/apps/dev-01/apps.tf` | Add gateway module |
+
+---
+
+## Risk Assessment
+
+| Risk | Mitigation |
+|---|---|
+| TCPRoute/UDPRoute experimental in NGF | Start with HTTPRoute only, keep proxy streams temporarily |
+| Cloudflare Tunnel adds latency | Monitor, acceptable for reverse proxy traffic |
+| cert-manager Gateway integration alpha | Use approach B (separate Certificate resources) |
+| Downtime during migration | Run both controllers in parallel, switch via DNS TTL |
+| Removing gpu-01 from WireGuard allowed_ips | Verify no active proxy depends on gpu-01 reachability |
+| Removing existing MetalLB on prod-01 | Do after Cloudflare Tunnel is fully operational |
+
+## Timeline Estimate
+
+| Phase | Duration | Dependencies |
+|---|---|---|
+| Phase 1 (Gateway API) | 2-3 weeks | None |
+| Phase 2 (WireGuard) | 1-2 weeks | Can run parallel with Phase 1 |
+| Phase 3 (Cloudflare) | 1-2 weeks | After Phase 1 |
+| Phase 4 (Cleanup) | 1 week | After Phase 3 |

.opencode/plans/gitea-actions-migration.md

@@ -0,0 +1,170 @@
+# Gitea Actions Migration Plan — Validate-Only Phase
+
+## Overview
+
+Migrate validate-only workflows from `.github/workflows/` (all `.disabled`) to `.gitea/workflows/` on the `alexpires.me/a13labs.infra` repo. Runs on `cicd-base` runner label. Terraform removed entirely — OpenTofu only.
+
+## Status: IMPLEMENTED
+
+All files created, secrets configured as placeholders. Ready for push and testing.
+
+## Scope
+
+### Workflows In Scope (6)
+
+| Workflow | Trigger | Path Filter | What It Does |
+|---|---|---|---|
+| `ansible-validate-on-pr` | PR (opened/synchronize/reopened) → `main` | `ansible/**` | ansible-lint + syntax-check |
+| `aws-validate-on-pr` | PR → `main` | `terraform/iac/aws/**` | tofu init, validate, fmt -check, plan |
+| `cloudns-validate-on-pr` | PR → `main` | `terraform/iac/cloudns/**` | tofu init, validate, fmt -check, plan |
+| `contabo-validate-on-pr` | PR → `main` | `terraform/iac/contabo/**` | tofu init, validate, fmt -check, plan |
+| `scaleway-validate-on-pr` | PR → `main` | `terraform/iac/scaleway/**` | tofu init, validate, fmt -check, plan |
+| `k8s-apps-validate` | PR → `main` | `terraform/apps/**` | ansible+k8s env + tofu validate dev-01 & prod-01 |
+
+### Workflows Deferred (8)
+
+All apply workflows, `deploy-websites`, `actions-validate`, `ansible-validate-playbook-on-pr` — out of scope.
+
+## Directory Layout
+
+```
+.gitea/
+├── actions/
+│   ├── setup-sectool/
+│   │   └── action.yml
+│   ├── setup-env/
+│   │   └── action.yml
+│   ├── setup-ansible-env/
+│   │   └── action.yml
+│   ├── setup-k8s-env/
+│   │   └── action.yml
+│   ├── setup-opentofu-env/
+│   │   └── action.yml
+│   └── setup-scaleway-registry/
+│       └── action.yml
+└── workflows/
+    ├── ansible-validate-on-pr.yml
+    ├── aws-validate-on-pr.yml
+    ├── build-and-push-images.yml
+    ├── cloudns-validate-on-pr.yml
+    ├── contabo-validate-on-pr.yml
+    ├── k8s-apps-validate.yml
+    ├── scaleway-validate-on-pr.yml
+    └── weekly-image-rebuild.yml
+```
+
+## Action Migration Details
+
+### `setup-sectool` (New)
+
+Replaces `uses: a13labs/setup-sectool@v1` (external GitHub action). Local composite action that downloads the sectool binary from `a13labs/sectool` GitHub releases.
+
+- Fetches latest release from `https://api.github.com/repos/a13labs/sectool/releases/latest`
+- Downloads Linux amd64 binary (tar.gz)
+- Installs to `/usr/local/bin/sectool`
+- Runs `sectool version` to verify
+
+### `setup-env` (Converted)
+
+Changes from `.github/actions/setup-env/action.yml`:
+- `outputs.changed_files` → uses `$GITEA_OUTPUT` instead of `$GITHUB_OUTPUT`
+- `git diff --name-only ${{ github.event.before }} ${{ github.sha }}` → `${{ gitea.event.before }} ${{ gitea.sha }}`
+- `uses: a13labs/setup-sectool@v1` → `uses: ./.gitea/actions/setup-sectool`
+
+### `setup-ansible-env` (Converted)
+
+Changes from `.github/actions/setup-ansible-env/action.yml`:
+- `outputs.ssh-auth-sock` and `outputs.hosts` → use `$GITEA_OUTPUT`
+- No `github.event` references — minimal conversion needed
+
+### `setup-opentofu-env` (Unchanged)
+
+No changes needed. `opentofu/setup-opentofu@v1` stays as-is (Gitea pulls from GitHub).
+
+### `setup-k8s-env` (Unchanged)
+
+Carried over as-is. No GitHub-specific variables in this action.
+
+### `setup-terraform-env` (Not Migrated)
+
+Dropped entirely. Terraform is no longer used in this project.
+
+### `setup-scaleway-registry` (New)
+
+Local composite action for image build workflows. Creates `~/.docker/config.json` with registry auth and configures buildah storage/network settings for rootless operation.
+
+- Inputs: `registry-endpoint`, `registry-username`, `registry-password`
+- Used by: `build-and-push-images.yml`, `weekly-image-rebuild.yml`
+
+## New Workflows (Image Build & Push)
+
+### `build-and-push-images.yml` (New)
+
+Builds and pushes runner profile Docker images to Scaleway registry.
+
+| Trigger | Tags | Scanning |
+|---|---|---|
+| Push to main (dockerfiles changed) | `{sha}` + `latest` | Yes (Trivy, warn only) |
+| Push tag (`v*`) | `{tag}` + `{sha}` + `latest` | Yes (Trivy, warn only) |
+| Pull request | `pr-{number}` | No |
+
+Jobs: `check-paths` → `auth` → `build-base` → `build-images` (matrix: 6 images)
+
+### `weekly-image-rebuild.yml` (New)
+
+Scheduled weekly (Monday 6AM). Rebuilds all images against updated upstream bases, tags only `:latest`, full Trivy scanning.
+
+See `plans/image-build-push.md` for full details.
+
+## Path Corrections
+
+GitHub workflows referenced stale paths from a previous repo layout:
+
+| Source (GitHub) | Actual | Notes |
+|---|---|---|
+| `terraform/aws-iac/` | `terraform/iac/aws/` | Restructured |
+| `terraform/cloudns-iac/` | `terraform/iac/cloudns/` | Restructured |
+| `terraform/contabo-iac/` | `terraform/iac/contabo/` | Restructured |
+| `terraform/scaleway-iac/` | `terraform/iac/scaleway/` | Restructured |
+| `terraform/k8sapps/` | `terraform/apps/dev-01/` + `terraform/apps/prod-01/` | Renamed + split |
+
+## Gitea Syntax Conversions
+
+| GitHub | Gitea |
+|---|---|
+| `$GITHUB_OUTPUT` | `$GITEA_OUTPUT` |
+| `${{ github.event.before }}` | `${{ gitea.event.before }}` |
+| `${{ github.sha }}` | `${{ gitea.sha }}` |
+| `runs-on: ubuntu-latest` | `runs-on: cicd-base` |
+| `uses: ./.github/actions/...` | `uses: ./.gitea/actions/...` |
+
+## Repo Secrets
+
+| Secret | Used By | Purpose | Status |
+|---|---|---|---|
+| `BW_PROJECT_ID` | All workflows | Bitwarden project ID | **Configured** |
+| `BW_ORGANIZATION_ID` | All workflows | Bitwarden org ID | **Configured** |
+| `BW_ACCESS_TOKEN` | All workflows | Bitwarden API token | **Configured** |
+| `ANSIBLE_USER` | Ansible + K8s workflows | SSH user for remote hosts | **Configured** |
+| `SCALEWAY_REGISTRY_ENDPOINT` | Image build workflows | Scaleway registry endpoint | **Needs setup** |
+| `SCALEWAY_CI_ACCESS_KEY` | Image build workflows | CI IAM API access key | **Needs setup** |
+| `SCALEWAY_CI_SECRET_KEY` | Image build workflows | CI IAM API secret key | **Needs setup** |
+
+## Phase B: Scaleway IaC Changes
+
+IaC changes in `terraform/iac/scaleway/` add CI IAM application and outputs. See `plans/image-build-push.md` for details.
+
+- `iam.tf` — added `ci_app` IAM application, policy, and API key
+- `outputs.tf` — added `registry_endpoint`, `ci_app_api_access_key`, `ci_app_api_secret_key`
+- `sectool.env` — no changes needed (CI app resources are hardcoded)
+
+## Next Steps
+
+1. **Push** all changes to `alexpires.me/a13labs.infra` on Gitea (note: `registry.tf` changed `is_public` from `false` to `true` on `a13labs` namespace)
+2. **SSH to ci-01** — rebuild `cicd-base` image locally, update Gitea runner container — DONE
+3. **Add Bitwarden secrets** — `SCALEWAY_CI_ACCESS_KEY` and `SCALEWAY_CI_SECRET_KEY` (populated after IaC apply)
+4. **Apply Scaleway IaC** — `sectool exec tofu apply` in `terraform/iac/scaleway/`
+5. **Extract IaC outputs** — `registry_endpoint`, `ci_app_api_access_key`, `ci_app_api_secret_key`
+6. **Configure Gitea secrets** — `SCALEWAY_REGISTRY_ENDPOINT`, `SCALEWAY_CI_ACCESS_KEY`, `SCALEWAY_CI_SECRET_KEY`
+7. **Test validate workflows** — PR touching `ansible/` or `terraform/iac/` paths
+8. **Test build workflow** — PR touching `ansible/files/dockerfiles/linux-runners/`

.opencode/plans/image-build-push.md

@@ -0,0 +1,155 @@
+# Image Build & Push to Scaleway Registry
+
+## Goal
+Build runner profile container images from `ansible/files/dockerfiles/linux-runners/`, scan for vulnerabilities, and push to the Scaleway container registry (`a13labs` namespace).
+
+## Decisions
+
+| Decision | Choice | Rationale |
+|---|---|---|
+| Registry namespace | `a13labs` (existing) | Clean project branding |
+| Tagging strategy | Git tag + SHA + latest | Full traceability and history |
+| Initial scope | Runner profiles only (7 → 4) | Consolidated image-build, infra-tools, security-scan into cicd-base |
+| Trivy scan severity | Warn only | Review manually, never block builds |
+| Update consumers | Not this iteration | Build+push pipeline first, consumers later |
+| Path filtering | In-workflow check | Gitea paths filter unreliable with tags |
+| PR scanning | Disabled | Keep PR builds fast |
+
+## Image Inventory & Dependencies
+
+```
+docker.io/gitea/runner:latest
+        ↓
+  gitea-runner-base  (build stage 1, pushed first)
+        ↓
+  ┌─────────────────────────────┐
+  │ cicd-base, cpp-build, go-build │ (build stage 2, all parallel)
+  └─────────────────────────────┘
+```
+
+| Image | Dockerfile | Base |
+|---|---|---|
+| gitea-runner-base | `Dockerfile.gitea-runner-base` | `docker.io/gitea/runner:latest` |
+| cicd-base | `Dockerfile.cicd-base` | `local/gitea-runner-base:latest` |
+| cpp-build | `Dockerfile.cpp-build` | `local/gitea-runner-base:latest` |
+| go-build | `Dockerfile.go-build` | `local/gitea-runner-base:latest` |
+
+> **Note:** `image-build`, `infra-tools`, and `security-scan` runners were decommissioned and their tooling consolidated into `cicd-base`. All workflows now run on `cicd-base` or the language-specific runners (`cpp-build`, `go-build`).
+
+## Registry Naming
+
+Images pushed as `fr-par-2.registry.scaleway.com/a13labs/{image-name}:{tag}`
+
+## Phases
+
+### Phase A: Prepare CI Runner (DONE)
+
+**Step 1 — Add build tools to `Dockerfile.cicd-base`**
+
+DONE. Added the following packages to the existing cicd-base image:
+- Build packages: `buildah`, `fuse-overlayfs`, `podman`, `shadow-uidmap`, `skopeo`, `slirp4netns`, `iptables` (from `Dockerfile.image-build`)
+- Storage config: `storage.conf` (vfs driver) and `containers.conf` (from `Dockerfile.image-build`)
+- Trivy binary download (from `Dockerfile.security-scan`, v0.72.0)
+
+**Step 2 — Register updated cicd-base on Gitea runner** (DONE)
+
+cicd-base image rebuilt on `ci-01` and Gitea runner container registration updated.
+
+### Phase B: Scaleway IaC Changes (DONE - code changes, manual apply needed)
+
+**Step 3 — Add CI IAM app** (`terraform/iac/scaleway/iam.tf`) - DONE
+- New `scaleway_iam_application` "ci_app" with `RegistryFullAccess` permission set
+- New `scaleway_iam_policy` "registry_full_access" granting ci_app registry access
+- New `scaleway_iam_api_key` "ci_app_api" for ci_app credentials
+
+**Step 4 — Add registry outputs** (`terraform/iac/scaleway/outputs.tf`) - DONE
+- `registry_endpoint` — from `scaleway_registry_namespace.a13labs.endpoint`
+- `ci_app_api_access_key`, `ci_app_api_secret_key` — sensitive
+
+**Step 5 — No `sectool.env` changes needed** - DONE
+The CI app resources are hardcoded, no TF_VAR mappings required.
+
+**Step 6 — Apply and extract secrets** (MANUAL)
+- Add `SCALEWAY_CI_ACCESS_KEY` and `SCALEWAY_CI_SECRET_KEY` to Bitwarden
+- Run `sectool exec tofu apply` in `terraform/iac/scaleway/`
+- Add `SCALEWAY_REGISTRY_ENDPOINT`, `SCALEWAY_CI_ACCESS_KEY`, `SCALEWAY_CI_SECRET_KEY` as Gitea repo secrets
+
+### Phase C: Gitea Actions & Workflow (DONE)
+
+**Step 7 — Create `.gitea/actions/setup-scaleway-registry/action.yml`** - DONE
+Composite action that:
+- Creates `~/.docker/config.json` with Scaleway registry auth
+- Configures buildah storage and network settings for rootless operation
+- Inputs: `registry-endpoint`, `registry-username`, `registry-password`
+
+**Step 8 — Create `.gitea/workflows/build-and-push-images.yml`** - DONE
+
+Jobs:
+- `check-paths`: Determines if dockerfiles changed (handles push, PR, tag events)
+- `auth`: Sets up registry authentication (gates on path check)
+- `build-base`: Builds and pushes `gitea-runner-base`
+- `build-images`: Matrix build of 6 dependent images
+
+Tagging strategy:
+- Push to main: `{sha}` + `latest`
+- Push tag (v*): `{tag}` + `{sha}` + `latest`
+- PR: `pr-{number}` (no scanning)
+
+**Step 9 — Create `.gitea/workflows/weekly-image-rebuild.yml`** - DONE
+Scheduled weekly (`cron: "0 6 * * 1"` — Monday 6AM):
+- Same build jobs but only tag `:latest`
+- Uses `--pull-always` to force rebuild against updated upstream base images
+- Full Trivy scan (warn only, artifacts)
+
+## Build Order & Dependency Resolution
+
+Each dependent Dockerfile says `FROM local/gitea-runner-base:latest`. The workflow replaces `local/gitea-runner-base` with the full registry URL in a temp copy before building, so buildah pulls the freshly pushed base image.
+
+## Manual Setup Checklist
+
+Before the workflows can run successfully, complete these steps:
+
+- [ ] Push code changes to repo (note: `registry.tf` changed `is_public` from `false` to `true` on `a13labs` namespace)
+- [x] SSH to `ci-01`, rebuild `cicd-base` image locally, update Gitea runner container — DONE
+- [ ] Add `SCALEWAY_CI_ACCESS_KEY` and `SCALEWAY_CI_SECRET_KEY` to Bitwarden (leave blank initially, they'll be populated after IaC apply)
+- [ ] Apply Scaleway IaC: `sectool exec tofu apply` in `terraform/iac/scaleway/`
+- [ ] From IaC outputs, extract `registry_endpoint`, `ci_app_api_access_key`, `ci_app_api_secret_key`
+- [ ] Add as Gitea repo secrets: `SCALEWAY_REGISTRY_ENDPOINT`, `SCALEWAY_CI_ACCESS_KEY`, `SCALEWAY_CI_SECRET_KEY`
+- [ ] Push a test commit touching `ansible/files/dockerfiles/linux-runners/` to trigger workflow
+- [ ] Verify workflow runs and images appear in Scaleway registry
+
+## Gitea Secrets Required
+
+| Secret | Source | Used by |
+|---|---|---|
+| `SCALEWAY_REGISTRY_ENDPOINT` | IaC output `registry_endpoint` | build-and-push, weekly-rebuild |
+| `SCALEWAY_CI_ACCESS_KEY` | IaC output `ci_app_api_access_key` | build-and-push, weekly-rebuild |
+| `SCALEWAY_CI_SECRET_KEY` | IaC output `ci_app_api_secret_key` | build-and-push, weekly-rebuild |
+
+## File Summary
+
+| Action | File | Status |
+|---|---|---|
+| Modify | `ansible/files/dockerfiles/linux-runners/Dockerfile.cicd-base` | DONE |
+| Modify | `terraform/iac/scaleway/iam.tf` | DONE |
+| Modify | `terraform/iac/scaleway/outputs.tf` | DONE |
+| Modify | `terraform/iac/scaleway/sectool.env` | DONE |
+| Create | `.gitea/actions/setup-scaleway-registry/action.yml` | DONE |
+| Create | `.gitea/workflows/build-and-push-images.yml` | DONE |
+| Create | `.gitea/workflows/weekly-image-rebuild.yml` | DONE |
+
+## Post-Iteration (NOT now)
+
+- Update runner Dockerfiles to use registry base URLs
+- Update Ansible playbooks to `pull` from registry instead of `build`
+- Add PR close job to clean up `pr-{number}` tags from registry
+- Expand to AI service Dockerfiles (comfyui, ollama, llamacpp variants)
+
+## Consolidation
+
+`image-build`, `infra-tools`, and `security-scan` runner profiles have been decommissioned. Their tooling was already absorbed into `cicd-base`:
+- **image-build** tools (buildah, podman, storage config) → merged into `Dockerfile.cicd-base`
+- **security-scan** tools (trivy) → merged into `Dockerfile.cicd-base`
+- **infra-tools** tools (ansible, pip packages) → installed at workflow runtime via `requirements/ansible.txt`
+
+Unused Dockerfiles removed: `Dockerfile.image-build`, `Dockerfile.infra-tools`, `Dockerfile.security-scan`

.opencode/plans/llama_server_template_command.md

@@ -0,0 +1,271 @@
+# Plan: Refactor llama_server role — template-based container command
+
+## Current State
+
+The `llama_server` role deploys `llama.cpp` server via Podman (EL) or NSSB (Windows) in router mode.
+
+**Existing structure:**
+
+| Component | Location |
+|---|---|
+| `defaults/main.yml` | User/group, image, tag, port, bind, models list, `argv_extra`, exporter config, Windows config |
+| `tasks/el/main.yml` | User/group creation → podman image pull → **container created inline** with hardcoded flags → systemd wrapper → HF model sync → exporter |
+| `tasks/windows/main.yml` | Binary download → NSSM install with same hardcoded flags → HF sync → firewall → exporter |
+| `templates/podman.service.j2` | Thin systemd wrapper: `podman start/stop llamacpp` |
+| `templates/llama_exporter.service.j2` | Exporter systemd service |
+| `vars/main.yml` | Internal non-overridable vars |
+| `files/scripts/` | `llama_hf_sync.py` (Linux + Windows), `llama_exporter.py`, tests |
+
+**Current container command** (`tasks/el/main.yml:142-152`):
+
+```yaml
+command: >-
+  {{
+    [
+      '--metrics',
+      '--models-preset',
+      '/models/.router/models.ini',
+      '--models-max',
+      (llama_server_models_max | string)
+    ]
+    + (llama_server_argv_extra | default([]))
+  }}
+```
+
+**Current Windows command** (`tasks/windows/main.yml:220-238`):
+
+```yaml
+argv: >-
+  {{
+    [
+      'nssm.exe',
+      'install',
+      'llama-server',
+      (llama_server_windows_install_dir ~ '\\' ~ llama_server_windows_bin_name),
+      '--port',
+      (llama_server_port | default(11434) | string),
+      '--host',
+      (llama_server_host | default('127.0.0.1') | string),
+      '--metrics',
+      '--models-preset',
+      (__llama_preset_file | string),
+      '--models-max',
+      (llama_server_models_max | string)
+    ]
+    + (llama_server_argv_extra | default([]))
+  }}
+```
+
+**Problem:** Both commands are hardcoded inline. The only extensibility is `llama_server_argv_extra` (append-only list), which has no validation, no conditional logic, and no self-documentation.
+
+## Goals
+
+- Replace hardcoded command construction with Jinja2 templates for both EL and Windows paths.
+- Add role variables for all major llama.cpp server flags with sensible defaults and conditional rendering.
+- Maintain backward compatibility: `llama_server_argv_extra` still works as a suffix.
+- Enable future configuration of MCP servers, SSL/TLS, API keys, chat templates, and performance tuning without manual playbook edits.
+
+## Changes Required
+
+### 1. Add new role variables to `defaults/main.yml`
+
+Add a new section after the existing defaults:
+
+```yaml
+# --- Server configuration ---
+llama_server_bind: "0.0.0.0"
+llama_server_host: "127.0.0.1"
+llama_server_port: 11434
+
+# SSL/TLS
+llama_server_ssl_enabled: false
+llama_server_ssl_cert: ""
+llama_server_ssl_key: ""
+
+# Authentication
+llama_server_api_key: ""
+
+# Chat
+llama_server_chat_template: ""
+
+# Performance
+llama_server_ctx_size: 4096
+llama_server_n_gpu_layers: 35
+llama_server_batch_size: 512
+llama_server_ubatch_size: 512
+llama_server_threads: 0
+llama_server_threads_batch: 0
+
+# Features
+llama_server_embeddings: false
+llama_server_cors: false
+llama_server_log_timestamps: false
+llama_server_log_level: "warn"
+llama_server_log_print: false
+llama_server_slot_save_context: false
+
+# Web UI
+llama_server_webui: ""
+llama_server_webui_author: ""
+
+# MCP
+llama_server_mcp_servers: []
+
+# KV cache quantization
+llama_server_cache_type_q4: false
+
+# Speculative decoding
+llama_server_speculative_model: ""
+llama_server_draft: 0
+
+# RoPE scaling
+llama_server_rope_freq_scale: 0.0
+
+# GPU offload
+llama_server_tensor_split: 0
+llama_server_split_mode: "layer"
+llama_server_main_gpu: 0
+```
+
+### 2. Create EL container command template
+
+**File:** `templates/llama_server_command_el.j2`
+
+```jinja2
+{{
+  [
+    '--metrics'
+  ]
+  + (llama_server_models_preset_global_path | default('/models/.router/models.ini') | ternary(['--models-preset', llama_server_models_preset_global_path], []))
+  + (llama_server_models_max | default(1) | string | ternary(['--models-max', llama_server_models_max | string], []))
+  + (llama_server_host | default('127.0.0.1') | ternary(['--host', llama_server_host], []))
+  + (llama_server_ssl_enabled | default(false) | bool | ternary(
+      ['--ssl', '--ssl-cert', llama_server_ssl_cert, '--ssl-key', llama_server_ssl_key] if llama_server_ssl_cert and llama_server_ssl_key else [], []))
+  + (llama_server_api_key | default('') | ternary(['--api-key', llama_server_api_key], []))
+  + (llama_server_chat_template | default('') | ternary(['--chat-template', llama_server_chat_template], []))
+  + (llama_server_ctx_size | default(4096) | ternary(['--ctx-size', llama_server_ctx_size | string], []))
+  + (llama_server_n_gpu_layers | default(35) | ternary(['--n-gpu-layers', llama_server_n_gpu_layers | string], []))
+  + (llama_server_batch_size | default(512) | ternary(['--batch-size', llama_server_batch_size | string], []))
+  + (llama_server_ubatch_size | default(512) | ternary(['--ubatch-size', llama_server_ubatch_size | string], []))
+  + (llama_server_threads | default(0) | ternary(['--threads', llama_server_threads | string], []))
+  + (llama_server_threads_batch | default(0) | ternary(['--threads-batch', llama_server_threads_batch | string], []))
+  + (llama_server_embeddings | default(false) | bool | ternary(['--embedding'], []))
+  + (llama_server_cors | default(false) | bool | ternary(['--cors'], []))
+  + (llama_server_log_timestamps | default(false) | bool | ternary(['--log-timestamps'], []))
+  + (llama_server_log_level | default('warn') | ternary(['--log-level', llama_server_log_level], []))
+  + (llama_server_log_print | default(false) | bool | ternary(['--log-print'], []))
+  + (llama_server_slot_save_context | default(false) | bool | ternary(['--slot-save-context'], []))
+  + (llama_server_webui | default('') | ternary(['--webui-name', llama_server_webui], []))
+  + (llama_server_webui_author | default('') | ternary(['--webui-author', llama_server_webui_author], []))
+  + (llama_server_mcp_servers | default([]) | length | ternary(['--mcp-servers', (llama_server_mcp_servers | to_json)], []))
+  + (llama_server_cache_type_q4 | default(false) | bool | ternary(['--cache-type-q4'], []))
+  + (llama_server_speculative_model | default('') | ternary(['--speculative-model', llama_server_speculative_model], []))
+  + (llama_server_draft | default(0) | ternary(['--draft', llama_server_draft | string], []))
+  + (llama_server_rope_freq_scale | default(0.0) | ternary(['--rope-freq-scale', llama_server_rope_freq_scale | string], []))
+  + (llama_server_tensor_split | default(0) | ternary(['--tensor-split', llama_server_tensor_split | string], []))
+  + (llama_server_split_mode | default('layer') | ternary(['--split-mode', llama_server_split_mode], []))
+  + (llama_server_main_gpu | default(0) | ternary(['--main-gpu', llama_server_main_gpu | string], []))
+  + (llama_server_argv_extra | default([]))
+}}
+```
+
+**Note:** The above uses a Jinja2 expression that produces a JSON array string. The `ternary()` calls handle conditional inclusion. The `llama_server_mcp_servers` serializes to JSON for the `--mcp-servers` flag.
+
+### 3. Create Windows service command template
+
+**File:** `templates/llama_server_command_windows.j2`
+
+Same logic as EL but output as a flat list suitable for NSSM `nssm.exe install` argv (each flag and value as separate list elements).
+
+### 4. Refactor `tasks/el/main.yml`
+
+Replace the hardcoded command block (lines 142-152) in the `podman_container` task:
+
+```yaml
+- name: Set fact llama_server_el_command
+  ansible.builtin.set_fact:
+    llama_server_el_command: "{{ lookup('template', 'templates/llama_server_command_el.j2') }}"
+
+- name: Create pods
+  containers.podman.podman_container:
+    name: "llamacpp"
+    image: "{{ llama_server_image }}:{{ llama_server_tag }}"
+    state: started
+    device: "{{ llama_server_devices | default(omit) }}"
+    env: "{{ llama_server_env | default(omit) }}"
+    ports: "{{ llama_server_bind }}:{{ llama_server_port }}:8080/tcp"
+    volumes:
+      - "{{ llama_server_home }}/models:/models:Z"
+      - "{{ llama_server_home }}/.cache:/app/.cache:Z"
+    command: "{{ llama_server_el_command }}"
+```
+
+### 5. Refactor `tasks/windows/main.yml`
+
+Replace the hardcoded `nssm.exe install` argv block (lines 220-238):
+
+```yaml
+- name: Set fact llama_server_windows_argv
+  ansible.builtin.set_fact:
+    llama_server_windows_argv: "{{ lookup('template', 'templates/llama_server_command_windows.j2') }}"
+
+- name: Install llama-server service via NSSM
+  ansible.windows.win_command:
+    argv: "{{ llama_server_windows_argv }}"
+  args:
+    chdir: "{{ llama_server_windows_install_dir }}"
+  register: __llama_nssm_install
+  changed_when: __llama_nssm_install.rc == 0
+```
+
+### 6. Create MCP servers preset file template (optional, for future)
+
+**File:** `templates/mcp_servers.json.j2`
+
+```json
+{{ llama_server_mcp_servers | to_nice_json }}
+```
+
+Used when MCP servers need to be written to a file and passed via `--mcp-servers` flag pointing to a path.
+
+### 7. Add tasks to write SSL/TLS and API key files (conditional)
+
+When `llama_server_ssl_cert` or `llama_server_api_key` are set, ensure the referenced files exist on the remote host. This may require:
+- A task to create the API key file if provided as a variable (not file path)
+- A task to validate SSL cert/key paths exist
+
+## Files to modify
+
+| File | Change |
+|------|--------|
+| `ansible/roles/llama_server/defaults/main.yml` | Add new role variables section |
+| `ansible/roles/llama_server/templates/llama_server_command_el.j2` | **New** — EL container command template |
+| `ansible/roles/llama_server/templates/llama_server_command_windows.j2` | **New** — Windows NSSM command template |
+| `ansible/roles/llama_server/templates/mcp_servers.json.j2` | **New** — MCP servers JSON preset (optional) |
+| `ansible/roles/llama_server/tasks/el/main.yml` | Replace hardcoded command with `lookup('template')` + `set_fact` |
+| `ansible/roles/llama_server/tasks/windows/main.yml` | Replace hardcoded nssm argv with `lookup('template')` + `set_fact` |
+| `ansible/roles/llama_server/tasks/el/llama_exporter.yml` | No change |
+| `ansible/roles/llama_server/files/scripts/` | No change |
+
+## Rollback plan
+
+- The `llama_server_argv_extra` variable continues to work as a suffix in both templates.
+- If template rendering fails, revert `tasks/el/main.yml` and `tasks/windows/main.yml` to the hardcoded command blocks.
+- Delete the new template files.
+- No data loss — templates only affect command construction, not state.
+
+## Testing
+
+1. **Baseline test:** Run role with existing defaults — should produce identical command to current hardcoded version.
+2. **Variable test:** Set individual variables (e.g., `llama_server_ctx_size: 8192`, `llama_server_embeddings: true`) and verify they appear in the rendered command.
+3. **Conditional test:** Set `llama_server_ssl_enabled: true` with cert/key paths — verify `--ssl`, `--ssl-cert`, `--ssl-key` flags are included.
+4. **MCP test:** Set `llama_server_mcp_servers` to a list of dicts — verify JSON serialization in `--mcp-servers` flag.
+5. **Backward compat test:** Use `llama_server_argv_extra` alongside new variables — verify both are present.
+6. **Windows test:** Same scenarios on Windows target with NSSM.
+
+## Open questions
+
+1. **Should `--models-preset` and `--models-max` remain in the template or stay hardcoded?** They're core to router mode and unlikely to change. Keeping them in the template makes the command self-contained but adds complexity.
+2. **For `mcp_servers`** — does llama.cpp accept the flag as `--mcp-servers '{"name":"..."}'` (JSON string) or `--mcp-servers /path/to/config.json` (file path)? This affects whether we need a separate file template.
+3. **Should defaults be opinionated** (e.g., `llama_server_ctx_size: 4096`) or minimal (only required flags)? Opinionated defaults reduce host var boilerplate but may not suit all use cases.
+4. **Should we add assertions** for mutually exclusive flags (e.g., `--split-mode` values)?

.opencode/plans/llamacpp-nginx-proxy.md

@@ -0,0 +1,234 @@
+# Plan: Replace OpenWebUI with llama.cpp behind nginx proxy
+
+## Current State
+
+- **llama.cpp** runs on `gpu-01.lab.alexpires.me` via Podman on port `11434` (bound to `127.0.0.1` with API key)
+- **nginx proxy** role runs on `dev-02.lab.alexpires.me` and `gpu-01.lab.alexpires.me`
+- **OpenWebUI** has been decommissioned (removed from K8s, DNS, inventory)
+- **DNS** for `lab.alexpires.me` is managed via Terraform (`terraform/apps/dev-01/config.tf` zones)
+- **oauth2-proxy** template uses OIDC provider (`code.alexpires.me`), `set_xauthrequest = true`
+
+## Goals
+
+- Deploy llama.cpp behind nginx proxy at `llm.lab.alexpires.me` ✅
+- API paths (`/api/*`) accessible without OAuth2 (token auth enforced via `--api-key`) ✅
+- All other paths (chat UI, etc.) protected with OAuth2 via oauth2-proxy ✅
+- Issue SSL certificate for `llm.lab.alexpires.me` via CloudNS DNS ✅
+- OpenWebUI decommissioned ✅
+
+## Important Constraint
+
+> **Resolved.** Phase 2 completed: llama.cpp now bound to `127.0.0.1` with API key. OpenWebUI decommissioned.
+
+## Changes Required
+
+> All changes have been implemented. See execution order below for completed steps.
+
+### 1. Add DNS record for llm.lab.alexpires.me ✅
+
+**File:** `terraform/apps/dev-01/config.tf`
+
+Add A record to the `lab.alexpires.me` zone:
+```hcl
+{
+  name    = "llm"
+  type    = "A"
+  content = "10.19.4.106"
+}
+```
+
+Run `tofu plan/apply` on `terraform/apps/dev-01/` before Phase 1 so DNS resolves.
+
+### 2. Add gpu-01 to `[nginx]` inventory group ✅
+
+### 3. Update nginx-server.conf.j2 — per-site `no_auth` conditional ✅
+
+### 4. Configure gpu-01 host vars for nginx proxy ✅
+
+### 5. Add firewall rules for HTTPS ✅
+
+### 6. Update llama_server role — support configurable bind address ✅
+
+### 7. Two-phase transition ✅
+
+**Phase 1 — Deploy nginx proxy (llama.cpp still on 0.0.0.0, no API key)**
+
+- Keep `llama_server_bind: "0.0.0.0"` (current behavior, default)
+- Do NOT set API key
+- nginx proxies to `127.0.0.1:11434` (container is bound to `0.0.0.0:11434`, reachable from localhost)
+- OpenWebUI continues to work (still connects to `gpu-01:11434` directly)
+- Test: `llm.lab.alexpires.me/` requires OAuth2 login, `llm.lab.alexpires.me/api/*` is open
+
+**Phase 2 — Switch to localhost binding + enable API key (after validation)**
+
+- Change `llama_server_bind: "127.0.0.1"` in host vars
+- Add `--api-key` to `llama_server_argv_extra` in host vars:
+  ```yaml
+  llama_server_argv_extra:
+    - "--api-key"
+    - "{{ lookup('env', 'LLAMA_SERVER_API_KEY') }}"
+    # ... existing args ...
+  ```
+- OpenWebUI will break (can no longer reach llama.cpp on `gpu-01:11434`) — decommission OpenWebUI
+- Verify all consumers migrate to `llm.lab.alexpires.me/api/*` with `Authorization: Bearer <key>`
+- Remove old firewall rules for port 11434 (no longer externally reachable)
+
+## Execution Order
+
+### Phase 1 — Deploy nginx proxy (llama.cpp unchanged, OpenWebUI still works)
+
+> **Status: COMPLETE** — All steps executed and verified.
+
+1. **Update Terraform DNS** — add `llm` A record in `terraform/apps/dev-01/config.tf`, run `tofu apply` ✅
+2. **Update inventory** — add gpu-01 to `[nginx]` group, clean up duplicate `[certbot]` group ✅
+3. **Update nginx-server.conf.j2** — add `no_auth` per-site conditional (template stays generic) ✅
+4. **Update llama_server role** — add `llama_server_bind` variable (default `0.0.0.0`) ✅
+5. **Update host vars** — add nginx sites (with `extra_locations` for `/api/`), OAuth2 creds, firewall rules ✅
+6. **Run nginx proxy playbook** — installs nginx + oauth2-proxy + certbot + deploys config ✅
+7. **Fix SELinux** — add `nginx_connect_any` boolean + port 4180 to `http_port_t` (manually applied, added to role for persistence) ✅
+8. **Verify chat UI** — `llm.lab.alexpires.me/` returns oauth2-proxy login page (200) ✅
+9. **Verify API** — `llm.lab.alexpires.me/api/v1/models` returns 200 with models list ✅
+10. **Verify OpenWebUI still works** — llama.cpp still bound to `0.0.0.0:11434` ✅
+
+1. **Update Terraform DNS** — add `llm` A record in `terraform/apps/dev-01/config.tf`, run `tofu apply`
+2. **Update inventory** — add gpu-01 to `[nginx]` group, clean up duplicate `[certbot]` group
+3. **Update nginx-server.conf.j2** — add `no_auth` per-site conditional (template stays generic)
+4. **Update llama_server role** — add `llama_server_bind` variable (default `0.0.0.0`)
+5. **Update host vars** — add nginx sites (with `extra_locations` for `/api/`), OAuth2 creds, firewall rules
+6. **Run nginx proxy playbook** — installs nginx + oauth2-proxy + certbot + deploys config (all in one run)
+7. **Verify chat UI** — test `llm.lab.alexpires.me/` with OAuth2 login
+8. **Verify API** — test `llm.lab.alexpires.me/api/*` (no auth yet, same as current direct access)
+9. **Verify OpenWebUI still works** — confirm dev-01 still connects to gpu-01:11434
+
+### Phase 2 — Switch to localhost + API key (after validation, decommission OpenWebUI)
+
+> **Status: COMPLETE** — All steps executed and verified.
+
+10. **Update host vars** — change `llama_server_bind: "127.0.0.1"`, add `--api-key` to `llama_server_argv_extra` ✅
+11. **Update firewall** — remove port 11434 rules from `fw_allowed_ports` ✅
+12. **Run llama_server playbook** — restart container with new binding + API key ✅
+13. **Run hardening playbook** — `playbook_hardening_cis.yml --limit gpu-01.lab.alexpires.me -t ufw` to apply firewall changes ✅
+14. **Verify API** — `llm.lab.alexpires.me/api/v1/chat/completions` returns 401 without key, 200 with key ✅
+15. **Decommission OpenWebUI** — removed from `terraform/apps/dev-01/apps.tf` ✅
+16. **Remove OpenWebUI DNS records** — removed `open-webui` CNAME from `terraform/apps/dev-01/config.tf` zones ✅
+17. **Remove OpenWebUI locals** — removed `open_webui_fqdn`, `open_webui_log_level`, `llm_proxy` from config.tf ✅
+18. **Remove `webui_secret_key` variable** — removed from `terraform/apps/dev-01/secrets.tf` ✅
+19. **Remove `[openwebui]` group** — cleaned up `inventory/hosts` ✅
+20. **Run `tofu apply`** on `terraform/apps/dev-01/` — destroyed 8 OpenWebUI K8s resources (namespace, deployment, service, ingress, secret, service accounts) ✅
+
+### Phase 2 — Switch to localhost + API key (after validation, decommission OpenWebUI)
+
+10. **Update host vars** — change `llama_server_bind: "127.0.0.1"`, add `--api-key` to `llama_server_argv_extra`
+11. **Update firewall** — remove port 11434 rules from `fw_allowed_ports`
+12. **Run llama_server playbook** — restart container with new binding + API key
+13. **Run hardening playbook** — `playbook_hardening_cis.yml --limit gpu-01.lab.alexpires.me -t ufw` to apply firewall changes
+14. **Verify API** — test `llm.lab.alexpires.me/api/*` requires API key
+15. **Decommission OpenWebUI** — remove Terraform module from `terraform/apps/dev-01/apps.tf` and `config.tf`
+16. **Remove OpenWebUI DNS records** — clean up `open-webui` CNAME in `terraform/apps/dev-01/config.tf` zones
+17. **Remove `[openwebui]` group** — clean up `inventory/hosts`
+18. **Run `tofu apply`** on `terraform/apps/dev-01/` — destroys OpenWebUI K8s resources
+
+## Playbooks to run
+
+### Phase 1
+
+> **COMPLETE** — All steps executed and verified.
+
+```sh
+cd ansible
+sectool exec ansible-playbook -u $ANSIBLE_USER playbook_nginx_proxy.yml    # nginx + oauth2 + certbot (all-in-one)
+```
+
+### Phase 2
+
+> **COMPLETE** — All steps executed and verified.
+
+```sh
+cd ansible
+sectool exec ansible-playbook -u $ANSIBLE_USER playbook_llama_server.yml                # container binding + API key
+sectool exec ansible-playbook -u $ANSIBLE_USER playbook_hardening_cis.yml -t ufw        # remove port 11434 rules
+```
+
+Then in terraform:
+```sh
+cd terraform/apps/dev-01
+sectool exec tofu plan    # review OpenWebUI destruction
+sectool exec tofu apply --auto-approve   # apply
+```
+
+## Files to modify
+
+| File | Phase | Change |
+|------|-------|--------|
+| `terraform/apps/dev-01/config.tf` | 1 | Add `llm` A record to zones |
+| `inventory/hosts` | 1 | Add gpu-01 to `[nginx]` group, clean up duplicate `[certbot]` |
+| `ansible/roles/nginx_proxy/templates/nginx-server.conf.j2` | 1 | Add `no_auth` per-site conditional (3 blocks) |
+| `ansible/roles/llama_server/defaults/main.yml` | 1 | Add `llama_server_bind` variable (default `0.0.0.0`) |
+| `ansible/roles/llama_server/tasks/el/main.yml` | 1 | Use `llama_server_bind` in podman port mapping |
+| `ansible/roles/nginx_proxy/tasks/nginx.yml` | 1 | Add SELinux tasks (`nginx_connect_any` boolean + oauth2-proxy port to `http_port_t`) |
+| `ansible/host_vars/gpu-01.lab.alexpires.me.yml` | 1 | Add nginx sites (with `extra_locations` for `/api/`), OAuth2 creds, firewall rules |
+| `ansible/host_vars/gpu-01.lab.alexpires.me.yml` | 2 | Change `llama_server_bind` to `127.0.0.1`, add `--api-key` to `llama_server_argv_extra`, remove port 11434 fw rules ✅ |
+| `terraform/apps/dev-01/apps.tf` | 2 | Remove `module "open-webui"` block ✅ |
+| `terraform/apps/dev-01/config.tf` | 2 | Remove `open_webui_fqdn`, `open_webui_log_level`, `llm_proxy` locals; remove `open-webui` DNS record from zones ✅ |
+| `terraform/apps/dev-01/secrets.tf` | 2 | Remove `webui_secret_key` variable ✅ |
+| `inventory/hosts` | 2 | Remove `[openwebui]` group ✅ |
+
+## Rollback plan
+
+### Phase 1 rollback (nginx proxy broken)
+
+- Remove gpu-01 from `[nginx]` group in `inventory/hosts`
+- Re-run hardening playbook on gpu-01 to clean up nginx/oauth2-proxy packages
+- Revert `nginx-server.conf.j2` template changes
+- llama.cpp continues running on `0.0.0.0:11434` — no impact
+- OpenWebUI continues to work
+
+## Issues Encountered & Resolved
+
+### 1. nginx `copy` module missing `remote_src: true`
+**File:** `ansible/roles/nginx_proxy/tasks/oauth2_proxy.yml:16`
+The `copy` module was trying to copy from the controller instead of the remote host. Added `remote_src: true`.
+
+### 2. SELinux blocking nginx → oauth2-proxy / llama.cpp connections
+**Symptom:** `connect() to 127.0.0.1:4180 failed (13: Permission denied)` in nginx error log
+**Fix:** 
+- Set SELinux boolean `nginx_connect_any` to `true` (persistent)
+- Added port 4180 to `http_port_t` via `semanage port`
+- Added both as Ansible tasks in `nginx.yml` for persistence
+
+### 3. Template `no_auth` attribute error
+**Symptom:** `object of type 'dict' has no attribute 'no_auth'`
+**Fix:** Changed `{% if not item.no_auth %}` to `{% if not (item.no_auth | default(false) | bool) %}` to handle missing attribute on dicts.
+
+### 4. Playbook failed on model query after container restart (Phase 2)
+**Symptom:** `urlopen error [Errno 111] Connection refused` when querying `http://127.0.0.1:11434/models?reload=1`
+**Cause:** The playbook's "Query llama router model list" task ran before the container was fully started. The container did start correctly with `127.0.0.1:11434` binding and API key active.
+**Impact:** Non-blocking — the container was running correctly; only the model list query task failed.
+
+## Known Warnings (non-blocking)
+
+- `types_hash_bucket_size: 64` — nginx warning, non-blocking
+- oauth2-proxy PKCE warning (`--code-challenge-method`) — non-blocking, S256 is auto-negotiated
+
+## Rollback plan
+
+### Phase 1 rollback (nginx proxy broken)
+
+- Revert `llama_server_bind` to `0.0.0.0` in host vars
+- Remove `--api-key` from `llama_server_argv_extra`
+- Re-add port 11434 firewall rules
+- Re-run `playbook_llama_server.yml` + `playbook_hardening_cis.yml -t ufw` — container restarts with old settings
+- OpenWebUI still works (llama.cpp is back on `0.0.0.0` without API key)
+- Fix issues and re-attempt Phase 2
+
+### Phase 2 rollback (after OpenWebUI decommission)
+
+- Restore `llama_server_bind: "0.0.0.0"` in host vars
+- Remove `--api-key` from `llama_server_argv_extra`
+- Re-add port 11434 firewall rules
+- Restore `module "open-webui"` in `terraform/apps/dev-01/apps.tf`
+- Restore `open_webui_fqdn`, `open_webui_log_level`, `llm_proxy` locals in `config.tf`
+- Restore `open-webui` DNS CNAME record in `config.tf` zones
+- Restore `webui_secret_key` variable in `secrets.tf`
+- Re-add `[openwebui]` group in `inventory/hosts`
+- Run `tofu apply` to recreate OpenWebUI K8s resources

.opencode/plans/tls-oauth2-sso-dev-02.md

@@ -0,0 +1,605 @@
+# TLS + OAuth2 SSO for dev-02
+
+## Goal
+
+Add HTTPS, TLS termination, and OAuth2/OIDC SSO to dev-02 services (code-server + opencode),
+exposed directly on the internal network for VPN/office access.
+
+## Architecture
+
+```
+                        ┌──────────────────────────────────────────┐
+                        │          dev-02.lab.alexpires.me         │
+                        │  (Fedora bare metal)                     │
+                        │                                          │
+                        │  ┌──────┐  auth_request   ┌───────────┐ │
+                        │  │nginx │─────────────────►│oauth2-    │ │
+                        │  │:443 │◄──────────────────│proxy:4180 │ │
+                        │  │TLS  │                   │(session)  │ │
+                        │  │SSO  │                   │           │ │
+                        │  └──┬───┘                   └─────┬─────┘ │
+                        │     │ proxy_pass (trusts auth)      │      │
+                        │     │                               │OIDC  │
+                        │  ┌──▼──────┐                  ┌──────▼────┐ │
+                        │  │ide:3000 │                  │Gitea IdP  │ │
+                        │  │(127.0.0.1)                 │code.      │ │
+                        │  │code-   │                  │alexpires. │ │
+                        │  │server  │                  │me (pub)   │ │
+                        │  └────────┘                  └───────────┘ │
+                        │  ┌────────┐                                │
+                        │  │agent:4096│                               │
+                        │  │(127.0.0.1)                               │
+                        │  │opencode │                                │
+                        │  └────────┘                                │
+                        └──────────────────────────────────────────┘
+                               │              │
+                    HTTPS 443  │              │ OIDC
+                    (internal) │              │
+                                ▼              ▼
+                        ┌──────────────────────────────────────────┐
+                        │  Browser (VPN/office)          │  code.alexpires.me (public)
+                        │  ide.lab.alexpires.me:443 ──────────►  Gitea IdP
+                        │  agent.lab.alexpires.me:443 ──────────►
+                        └──────────────────────────────────────────┘
+```
+
+## SSO Flow
+
+1. User visits `https://ide.lab.alexpires.me:443` (direct, requires VPN/office)
+2. nginx detects no valid session cookie → `auth_request` to oauth2-proxy
+3. oauth2-proxy returns 401 → nginx redirects browser to Gitea login
+4. Browser → `code.alexpires.me` (Gitea, public, always reachable)
+5. User logs in → Gitea redirects back to `https://ide.lab.alexpires.me/oauth2/callback`
+6. oauth2-proxy exchanges code → access_token → creates session cookie
+7. Cookie scoped to `*.lab.alexpires.me` → SSO across lab subdomains
+8. Subsequent requests: nginx trusts cookie → proxies to backend (no re-login)
+
+**Access constraint:** SSO only works when the user can reach `lab.alexpires.me` (VPN or office network).
+This is inherent to the architecture — the OAuth2 callback URL (`ide.lab.alexpires.me/oauth2/callback`)
+must be reachable by the browser after Gitea redirects back. All `lab.alexpires.me` services
+follow this same constraint.
+
+## Certificate Strategy
+
+- **SAN cert** for `ide.lab.alexpires.me` and `agent.lab.alexpires.me`
+- Obtained via certbot DNS-01 challenge (cloudns plugin)
+- Auto-renewal via cron with `--deploy-hook "systemctl reload nginx"`
+- No port 80/443 exposure needed (DNS-01 challenge doesn't need HTTP)
+- Certbot stores certs in `/etc/letsencrypt/live/` (symlinked from `/etc/letsencrypt/archive/`)
+- nginx references:
+  - `ssl_certificate /etc/letsencrypt/live/ide.lab.alexpires.me/fullchain.pem;`
+  - `ssl_certificate_key /etc/letsencrypt/live/ide.lab.alexpires.me/privkey.pem;`
+
+## Service Binding
+
+| Service | Before | After |
+|---------|--------|-------|
+| code-server | `0.0.0.0:3000` | `127.0.0.1:3000` |
+| opencode | `0.0.0.0:4096` | `127.0.0.1:4096` |
+| oauth2-proxy | N/A | `127.0.0.1:4180` |
+| nginx | N/A | `0.0.0.0:443` (only public port) |
+
+## Pre-requisites
+
+Before applying, register an OAuth2 app in Gitea:
+- **Name:** `lab-services`
+- **Redirect URI:** `https://ide.lab.alexpires.me/oauth2/callback`
+- **Scopes:** `openid`, `profile`, `email`
+- **Result:** `client_id` + `client_secret` → stored in `sectool` / env vars
+
+Generate a cookie secret:
+```sh
+openssl rand -base64 32
+```
+Store via `sectool` as `OAUTH2_PROXY_COOKIE_SECRET`, `OAUTH2_PROXY_CLIENT_ID`, `OAUTH2_PROXY_CLIENT_SECRET`.
+
+## Changes Required
+
+### 1. New Role: `nginx_proxy`
+
+```
+ansible/roles/nginx_proxy/
+├── defaults/main.yml           # certbot, nginx, oauth2 config, upstreams
+├── tasks/
+│   ├── main.yml                # Entry point (includes certbot, nginx, oauth2_proxy)
+│   ├── certbot.yml             # Install certbot, DNS credentials, obtain cert, cron renewal
+│   ├── nginx.yml               # Install nginx, deploy config, systemd, SELinux context
+│   └── oauth2_proxy.yml        # Download binary, config, systemd
+├── templates/
+│   ├── nginx.conf.j2           # Main nginx config (worker processes, logging, etc.)
+│   ├── nginx-server.conf.j2    # Server blocks (SAN domains, TLS, auth_request, proxy)
+│   └── oauth2-proxy.cfg.j2     # OAuth2/OIDC config for Gitea
+└── files/
+    └── .gitkeep
+```
+
+#### `defaults/main.yml`
+
+```yaml
+nginx_proxy_certbot_enabled: true
+nginx_proxy_certbot_email: "c.alexandre.pires@alexpires.me"
+nginx_proxy_certbot_domains:
+  - "ide.lab.alexpires.me"
+  - "agent.lab.alexpires.me"
+nginx_proxy_certbot_cron_minute: "0"
+nginx_proxy_certbot_cron_hour: "3"
+nginx_proxy_certbot_credential_file: "/etc/letsencrypt/cloudns.ini"
+nginx_proxy_certbot_credential_mode: "0600"
+nginx_proxy_certbot_nginx_reload: true
+
+nginx_proxy_nginx_enabled: true
+nginx_proxy_nginx_bind: "0.0.0.0"
+nginx_proxy_nginx_port: 443
+nginx_proxy_nginx_worker_connections: 1024
+nginx_proxy_nginx_log_format: "main"
+nginx_proxy_nginx_access_log: "/var/log/nginx/access.log"
+nginx_proxy_nginx_error_log: "/var/log/nginx/error.log warn"
+nginx_proxy_nginx_ssl_protocols: "TLSv1.2 TLSv1.3"
+nginx_proxy_nginx_ssl_ciphers: "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384"
+nginx_proxy_nginx_ssl_prefer_server_ciphers: "on"
+nginx_proxy_nginx_hsts_max_age: "31536000"
+nginx_proxy_nginx_hsts_include_subdomains: true
+
+nginx_proxy_oauth2_proxy_enabled: true
+nginx_proxy_oauth2_proxy_bind: "127.0.0.1"
+nginx_proxy_oauth2_proxy_port: 4180
+nginx_proxy_oauth2_proxy_version: "2.13.0"
+nginx_proxy_oauth2_proxy_cookie_domain: ".lab.alexpires.me"
+nginx_proxy_oauth2_proxy_cookie_name: "_alexpires_oauth2_proxy"
+nginx_proxy_oauth2_proxy_cookie_secret: "{{ lookup('env', 'OAUTH2_PROXY_COOKIE_SECRET', default='') }}"
+nginx_proxy_oauth2_proxy_client_id: "{{ lookup('env', 'OAUTH2_PROXY_CLIENT_ID', default='') }}"
+nginx_proxy_oauth2_proxy_client_secret: "{{ lookup('env', 'OAUTH2_PROXY_CLIENT_SECRET', default='') }}"
+nginx_proxy_oauth2_proxy_redirect_url: "https://ide.lab.alexpires.me/oauth2/callback"
+nginx_proxy_oauth2_proxy_provider: "gitea"
+nginx_proxy_oauth2_proxy_provider_url: "https://code.alexpires.me"
+nginx_proxy_oauth2_proxy_scopes: "openid profile email"
+nginx_proxy_oauth2_proxy_email_domains: []
+nginx_proxy_oauth2_proxy_upstream: "static://200"
+nginx_proxy_oauth2_proxy_skip_jwt: false
+nginx_proxy_oauth2_proxy_set_basic_auth: false
+nginx_proxy_oauth2_proxy_set_xauthrequest: true
+nginx_proxy_oauth2_proxy_ssl_insecure_skip_verify: false
+nginx_proxy_oauth2_proxy_login_button: "Sign in with Gitea"
+nginx_proxy_oauth2_proxy_proxy_prefix: "/oauth2"
+nginx_proxy_oauth2_proxy_silent_login: false
+nginx_proxy_oauth2_proxy_skip_provider_whoami: false
+nginx_proxy_oauth2_proxy_cookie_refresh: "0s"
+nginx_proxy_oauth2_proxy_cookie_expire: "168h"
+nginx_proxy_oauth2_proxy_session_storage_type: "memory"
+nginx_proxy_oauth2_proxy_request_logging: true
+nginx_proxy_oauth2_proxy_log_file: "/var/log/oauth2-proxy.log"
+nginx_proxy_oauth2_proxy_log_level: "info"
+
+nginx_proxy_upstreams:
+  ide:
+    host: "127.0.0.1"
+    port: 3000
+  agent:
+    host: "127.0.0.1"
+    port: 4096
+
+nginx_proxy_cloudns_auth_id: "{{ lookup('env', 'CLOUDNS_AUTH_ID') }}"
+nginx_proxy_cloudns_auth_password: "{{ lookup('env', 'CLOUDNS_PASSWORD') }}"
+nginx_proxy_cloudns_nameserver: "109.201.133.111"
+```
+
+#### `tasks/certbot.yml`
+
+- Install certbot + certbot-dns-cloudns via pip
+- Create CloudDNS credentials file at `/etc/letsencrypt/cloudns.ini`:
+  ```ini
+  dns_cloudns_auth_id = {{ nginx_proxy_cloudns_auth_id }}
+  dns_cloudns_auth_password = {{ nginx_proxy_cloudns_auth_password }}
+  ```
+- Permissions: `0600`, owner: `root:root`
+- Obtain SAN cert:
+  ```sh
+  certbot certonly --dns-cloudns \
+    -d ide.lab.alexpires.me \
+    -d agent.lab.alexpires.me \
+    --non-interactive \
+    --agree-tos \
+    -m {{ nginx_proxy_certbot_email }}
+  ```
+- Create cron job for renewal (daily at 3:00 AM):
+  ```sh
+  certbot renew --deploy-hook "systemctl reload nginx"
+  ```
+- Certs stored in `/etc/letsencrypt/live/ide.lab.alexpires.me/`
+
+#### `tasks/nginx.yml`
+
+- Install nginx package
+- Render `nginx.conf` (global):
+  ```nginx
+  user nginx;
+  worker_processes auto;
+  error_log {{ nginx_proxy_nginx_error_log }};
+  pid /run/nginx.pid;
+
+  events {
+      worker_connections {{ nginx_proxy_nginx_worker_connections }};
+  }
+
+  http {
+      log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+      access_log {{ nginx_proxy_nginx_access_log }} main;
+
+      sendfile on;
+      tcp_nopush on;
+      tcp_nodelay on;
+      keepalive_timeout 65;
+      types_hash_max_size 2048;
+
+      include /etc/nginx/mime.types;
+      default_type application/octet-stream;
+
+      # WebSocket upgrade mapping
+      map $http_upgrade $connection_upgrade {
+          default upgrade;
+          ''      close;
+      }
+
+      include /etc/nginx/conf.d/*.conf;
+  }
+  ```
+- Render `nginx-server.conf` (server blocks):
+  ```nginx
+  # IDE (code-server)
+  server {
+      listen 443 ssl http2;
+      server_name ide.lab.alexpires.me;
+
+      ssl_certificate /etc/letsencrypt/live/ide.lab.alexpires.me/fullchain.pem;
+      ssl_certificate_key /etc/letsencrypt/live/ide.lab.alexpires.me/privkey.pem;
+      ssl_protocols {{ nginx_proxy_nginx_ssl_protocols }};
+      ssl_ciphers {{ nginx_proxy_nginx_ssl_ciphers }};
+      ssl_prefer_server_ciphers {{ nginx_proxy_nginx_ssl_prefer_server_ciphers }};
+
+      add_header Strict-Transport-Security "max-age={{ nginx_proxy_nginx_hsts_max_age }}; {{ nginx_proxy_nginx_hsts_include_subdomains ? 'includeSubDomains' : '' }}" always;
+
+      # OAuth2 auth_request
+      location = /oauth2/ {
+          proxy_pass http://127.0.0.1:{{ nginx_proxy_oauth2_proxy_port }};
+          proxy_set_header Host $host;
+          proxy_set_header X-Real-IP $remote_addr;
+          proxy_set_header X-Scheme $scheme;
+          proxy_set_header X-Auth-Request-Redirect $request_uri;
+      }
+
+      location = /oauth2/callback {
+          proxy_pass http://127.0.0.1:{{ nginx_proxy_oauth2_proxy_port }}/oauth2/callback;
+          proxy_set_header Host $host;
+          proxy_set_header X-Real-IP $remote_addr;
+          proxy_set_header X-Scheme $scheme;
+          proxy_set_header X-Auth-Request-Redirect $request_uri;
+      }
+
+      location /oauth2/ {
+          proxy_pass http://127.0.0.1:{{ nginx_proxy_oauth2_proxy_port }};
+          proxy_set_header Host $host;
+          proxy_set_header X-Real-IP $remote_addr;
+          proxy_set_header X-Scheme $scheme;
+      }
+
+      location / {
+          auth_request /oauth2/;
+          auth_request_set $auth_status $upstream_status;
+          auth_request_set $auth_cookie $upstream_http_set_cookie;
+
+          add_header Set-Cookie $auth_cookie;
+
+          proxy_pass http://127.0.0.1:{{ nginx_proxy_upstreams.ide.port }};
+          proxy_set_header Host $host;
+          proxy_set_header X-Real-IP $remote_addr;
+          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header X-Forwarded-Proto $scheme;
+          proxy_set_header X-Forwarded-Host $host;
+          proxy_set_header X-Forwarded-Port $server_port;
+
+          # WebSocket support
+          proxy_http_version 1.1;
+          proxy_set_header Upgrade $http_upgrade;
+          proxy_set_header Connection $connection_upgrade;
+
+          # Timeouts for code-server
+          proxy_read_timeout 3600s;
+          proxy_send_timeout 3600s;
+      }
+  }
+
+  # Agent (opencode)
+  server {
+      listen 443 ssl http2;
+      server_name agent.lab.alexpires.me;
+
+      ssl_certificate /etc/letsencrypt/live/ide.lab.alexpires.me/fullchain.pem;
+      ssl_certificate_key /etc/letsencrypt/live/ide.lab.alexpires.me/privkey.pem;
+      ssl_protocols {{ nginx_proxy_nginx_ssl_protocols }};
+      ssl_ciphers {{ nginx_proxy_nginx_ssl_ciphers }};
+      ssl_prefer_server_ciphers {{ nginx_proxy_nginx_ssl_prefer_server_ciphers }};
+
+      add_header Strict-Transport-Security "max-age={{ nginx_proxy_nginx_hsts_max_age }}; {{ nginx_proxy_nginx_hsts_include_subdomains ? 'includeSubDomains' : '' }}" always;
+
+      # OAuth2 auth_request
+      location = /oauth2/ {
+          proxy_pass http://127.0.0.1:{{ nginx_proxy_oauth2_proxy_port }};
+          proxy_set_header Host $host;
+          proxy_set_header X-Real-IP $remote_addr;
+          proxy_set_header X-Scheme $scheme;
+          proxy_set_header X-Auth-Request-Redirect $request_uri;
+      }
+
+      location = /oauth2/callback {
+          proxy_pass http://127.0.0.1:{{ nginx_proxy_oauth2_proxy_port }}/oauth2/callback;
+          proxy_set_header Host $host;
+          proxy_set_header X-Real-IP $remote_addr;
+          proxy_set_header X-Scheme $scheme;
+          proxy_set_header X-Auth-Request-Redirect $request_uri;
+      }
+
+      location /oauth2/ {
+          proxy_pass http://127.0.0.1:{{ nginx_proxy_oauth2_proxy_port }};
+          proxy_set_header Host $host;
+          proxy_set_header X-Real-IP $remote_addr;
+          proxy_set_header X-Scheme $scheme;
+      }
+
+      location / {
+          auth_request /oauth2/;
+          auth_request_set $auth_status $upstream_status;
+          auth_request_set $auth_cookie $upstream_http_set_cookie;
+
+          add_header Set-Cookie $auth_cookie;
+
+          proxy_pass http://127.0.0.1:{{ nginx_proxy_upstreams.agent.port }};
+          proxy_set_header Host $host;
+          proxy_set_header X-Real-IP $remote_addr;
+          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header X-Forwarded-Proto $scheme;
+          proxy_set_header X-Forwarded-Host $host;
+          proxy_set_header X-Forwarded-Port $server_port;
+
+          # WebSocket support
+          proxy_http_version 1.1;
+          proxy_set_header Upgrade $http_upgrade;
+          proxy_set_header Connection "upgrade";
+
+          # Timeouts for opencode
+          proxy_read_timeout 3600s;
+          proxy_send_timeout 3600s;
+      }
+  }
+  ```
+- Enable + start nginx service
+- Set SELinux context: `semanage fcontext -a -t httpd_cert_t "/etc/letsencrypt(/.*)?"` + `restorecon -Rv /etc/letsencrypt`
+
+#### `tasks/oauth2_proxy.yml`
+
+- Download latest release from GitHub (Go binary for linux/amd64):
+  ```sh
+  wget -qO /tmp/oauth2-proxy.tar.gz "https://github.com/oauth2-proxy/oauth2-proxy/releases/download/v{{ nginx_proxy_oauth2_proxy_version }}/oauth2-proxy-v{{ nginx_proxy_oauth2_proxy_version }}.linux-amd64.tar.gz"
+  tar xzf /tmp/oauth2-proxy.tar.gz -C /tmp/
+  mv /tmp/oauth2-proxy-v{{ nginx_proxy_oauth2_proxy_version }}.linux-amd64/oauth2-proxy /usr/local/bin/
+  ```
+- Render config file at `/etc/oauth2-proxy/oauth2-proxy.cfg`:
+  ```ini
+  provider = "{{ nginx_proxy_oauth2_proxy_provider }}"
+  provider_url = "{{ nginx_proxy_oauth2_proxy_provider_url }}"
+  client_id = "{{ nginx_proxy_oauth2_proxy_client_id }}"
+  client_secret = "{{ nginx_proxy_oauth2_proxy_client_secret }}"
+  cookie_domain = "{{ nginx_proxy_oauth2_proxy_cookie_domain }}"
+  cookie_name = "{{ nginx_proxy_oauth2_proxy_cookie_name }}"
+  cookie_secret = "{{ nginx_proxy_oauth2_proxy_cookie_secret }}"
+  cookie_expire = "{{ nginx_proxy_oauth2_proxy_cookie_expire | default('168h') }}"
+  cookie_refresh = "{{ nginx_proxy_oauth2_proxy_cookie_refresh | default('0s') }}"
+  redirect_url = "{{ nginx_proxy_oauth2_proxy_redirect_url }}"
+  login_url = "{{ nginx_proxy_oauth2_proxy_provider_url }}/login/oauth/authorize"
+  redeem_url = "{{ nginx_proxy_oauth2_proxy_provider_url }}/login/oauth/access_token"
+  openid_connectissuerurl = "{{ nginx_proxy_oauth2_proxy_provider_url }}"
+  scopes = "{{ nginx_proxy_oauth2_proxy_scopes }}"
+  email_domains = {{ nginx_proxy_oauth2_proxy_email_domains | to_json }}
+  upstream = "{{ nginx_proxy_oauth2_proxy_upstream }}"
+  skip_jwt_bearer_tokens = {{ nginx_proxy_oauth2_proxy_skip_jwt | lower }}
+  set_basic_auth = {{ nginx_proxy_oauth2_proxy_set_basic_auth | lower }}
+  set_xauthrequest = {{ nginx_proxy_oauth2_proxy_set_xauthrequest | lower }}
+  ssl_insecure_skip_verify = {{ nginx_proxy_oauth2_proxy_ssl_insecure_skip_verify | lower }}
+  proxy_prefix = "{{ nginx_proxy_oauth2_proxy_proxy_prefix }}"
+  silent_login = {{ nginx_proxy_oauth2_proxy_silent_login | lower }}
+  skip_provider_whoami = {{ nginx_proxy_oauth2_proxy_skip_provider_whoami | lower }}
+  listen = "{{ nginx_proxy_oauth2_proxy_bind }}:{{ nginx_proxy_oauth2_proxy_port }}"
+  {% if nginx_proxy_oauth2_proxy_request_logging | default(true) %}
+  logfile_filename = "{{ nginx_proxy_oauth2_proxy_log_file | default('/var/log/oauth2-proxy.log') }}"
+  {% endif %}
+  {% if nginx_proxy_oauth2_proxy_log_level | default('info') %}
+  log_level = "{{ nginx_proxy_oauth2_proxy_log_level | default('info') }}"
+  {% endif %}
+  cookie_secure = false
+  {% if nginx_proxy_oauth2_proxy_login_button | default('Sign in with Gitea') %}
+  custom_login_message = "Sign in with your Gitea account"
+  {% endif %}
+  ```
+- Create systemd service at `/etc/systemd/system/oauth2-proxy.service`:
+  ```ini
+  [Unit]
+  Description=oauth2-proxy for {{ nginx_proxy_oauth2_proxy_cookie_domain }}
+  After=network.target
+
+  [Service]
+  Type=simple
+  User=nginx
+  ExecStart=/usr/local/bin/oauth2-proxy --config /etc/oauth2-proxy/oauth2-proxy.cfg
+  Restart=always
+  RestartSec=5
+
+  [Install]
+  WantedBy=multi-user.target
+  ```
+- Enable + start service
+- Set config file permissions: `0640`, owner `root:nginx` (contains `client_secret`)
+- Create log directory with proper permissions: `mkdir -p /var/log && chown nginx:nginx /var/log/oauth2-proxy.log`
+
+### Notes
+
+- **opencode without credentials** — Verify opencode starts and functions correctly when `OPENCODE_SERVER_USERNAME`/`OPENCODE_SERVER_PASSWORD` are not set. If opencode requires them to start, we may need to keep them but they will be ignored since nginx/oauth2-proxy handles auth.
+
+### 2. Update `linux_dev_station` Role
+
+#### `defaults/main.yml`
+
+- Change `linux_dev_station_code_server_host` default to `127.0.0.1`
+- Change `linux_dev_station_opencode_server_host` default to `127.0.0.1`
+
+#### `templates/code-server.service.j2`
+
+- Change `--bind-addr` to `127.0.0.1:3000`
+- **Remove** `Environment="PASSWORD={{ linux_dev_station_code_server_password }}"` — SSO handles auth, no password needed
+
+#### `templates/opencode.service.j2`
+
+- Change `--hostname` to `127.0.0.1`
+- **Remove** `Environment="OPENCODE_SERVER_USERNAME={{ linux_dev_station_opencode_server_username }}"` — SSO handles auth
+- **Remove** `Environment="OPENCODE_SERVER_PASSWORD={{ linux_dev_station_opencode_server_password }}"` — SSO handles auth
+
+### 3. Update dev-01 DNS & Remove Proxy Config
+
+#### `terraform/apps/dev-01/config.tf`
+
+**Add DNS records** for `ide` and `agent` pointing to dev-02 (`10.19.4.210`):
+
+```hcl
+{
+  name    = "ide"
+  type    = "A"
+  content = "10.19.4.210"
+},
+{
+  name    = "agent"
+  type    = "A"
+  content = "10.19.4.210"
+},
+```
+
+**Remove** the `opencode` and `code-server` entries from `proxy_services` (no longer needed, dev-02 handles its own TLS).
+
+**Remove** the `opencode` and `vscode` CNAME records from the `zones` list:
+```hcl
+// Remove these:
+{
+  name    = "opencode"
+  type    = "CNAME"
+  content = "dev-01.${local.lab_domain}."
+},
+{
+  name    = "vscode"
+  type    = "CNAME"
+  content = "dev-01.${local.lab_domain}."
+},
+```
+
+### 4. Update Host Vars & Inventory
+
+#### `ansible/host_vars/dev-02.lab.alexpires.me.yml`
+
+Add:
+
+```yaml
+# Certbot
+nginx_proxy_certbot_enabled: true
+nginx_proxy_certbot_email: "c.alexandre.pires@alexpires.me"
+nginx_proxy_certbot_domains:
+  - "ide.lab.alexpires.me"
+  - "agent.lab.alexpires.me"
+nginx_proxy_certbot_credential_file: "/etc/letsencrypt/cloudns.ini"
+nginx_proxy_cloudns_auth_id: "{{ lookup('env', 'CLOUDNS_AUTH_ID') }}"
+nginx_proxy_cloudns_auth_password: "{{ lookup('env', 'CLOUDNS_PASSWORD') }}"
+nginx_proxy_cloudns_nameserver: "109.201.133.111"
+
+# OAuth2 / Gitea
+nginx_proxy_oauth2_proxy_cookie_secret: "{{ lookup('env', 'OAUTH2_PROXY_COOKIE_SECRET') }}"
+nginx_proxy_oauth2_proxy_client_id: "{{ lookup('env', 'OAUTH2_PROXY_CLIENT_ID') }}"
+nginx_proxy_oauth2_proxy_client_secret: "{{ lookup('env', 'OAUTH2_PROXY_CLIENT_SECRET') }}"
+```
+
+#### `inventory/hosts`
+
+Add `dev-02.lab.alexpires.me` to `[certbot]` group:
+```ini
+[certbot]
+dev-02.lab.alexpires.me
+```
+
+### 5. Gitea OAuth2 Application
+
+Register one OAuth2 app in Gitea:
+- **Name:** `lab-services`
+- **Redirect URI:** `https://ide.lab.alexpires.me/oauth2/callback`
+- **Scopes:** `openid`, `profile`, `email`
+- **Result:** `client_id` + `client_secret` → stored in `sectool` / env vars
+
+### 6. Firewall Changes (handled by existing firewall role)
+
+| Port | Before | After |
+|------|--------|-------|
+| 22 | `10.19.4.0/24` + VPN | Same |
+| 3000 | `10.19.4.0/24` | **CLOSED** |
+| 4096 | `10.19.4.0/24` | **CLOSED** |
+| 443 | — | `10.19.4.0/24` + VPN |
+
+The `linux_dev_station` role already manages firewall rules. Update `fw_allowed_ports` in host_vars:
+```yaml
+fw_allowed_ports:
+  - { rule: "allow", port: "22", proto: "tcp", from: "10.19.4.0/24" }
+  - { rule: "allow", port: "22", proto: "tcp", from: "10.5.5.5/32" }
+  - { rule: "allow", port: "443", proto: "tcp", from: "10.19.4.0/24" }
+  - { rule: "allow", port: "443", proto: "tcp", from: "10.5.5.5/32" }
+```
+
+## SELinux Considerations
+
+- Fedora ships with SELinux enforcing by default
+- nginx needs permission to read certs: `semanage fcontext -a -t httpd_cert_t "/etc/letsencrypt(/.*)?"` + `restorecon -Rv /etc/letsencrypt`
+- oauth2-proxy runs as `nginx` user — ensure systemd service file sets `User=nginx`
+- nginx needs outbound HTTP/HTTPS to Gitea and CloudDNS API — these are allowed by default for the `httpd_t` domain
+
+## Playbook Structure
+
+```yaml
+# playbook_linux_dev_station.yml
+- hosts: linux_dev
+  roles:
+    - linux_dev_station
+
+# playbook_nginx_proxy.yml
+- hosts: nginx
+  roles:
+    - nginx_proxy
+```
+
+The `nginx_proxy` role is a standalone, generic role — not tied to `linux_dev_station`.
+
+## Apply Order
+
+1. Register Gitea OAuth2 app → store creds in `sectool`
+2. Run `playbook_nginx_proxy.yml` on dev-02 (Ansible)
+3. Update `fw_allowed_ports` in dev-02 host_vars (close 3000/4096, add 443)
+4. Update dev-01 DNS + remove proxy config (Terraform)
+5. Verify: `https://ide.lab.alexpires.me` → SSO via Gitea
+6. Verify: `https://agent.lab.alexpires.me` → SSO via Gitea
+
+## Files to Create
+
+| File | Action |
+|------|--------|
+| `ansible/roles/nginx_proxy/` | **NEW** role |
+| `ansible/playbook_nginx_proxy.yml` | **NEW** standalone playbook (`hosts: certbot`) |
+| `ansible/roles/linux_dev_station/defaults/main.yml` | Update bind to `127.0.0.1` |
+| `ansible/roles/linux_dev_station/templates/code-server.service.j2` | `--bind-addr 127.0.0.1:3000` |
+| `ansible/roles/linux_dev_station/templates/opencode.service.j2` | `--hostname 127.0.0.1` |
+| `ansible/host_vars/dev-02.lab.alexpires.me.yml` | Add certbot + oauth2 vars |
+| `inventory/hosts` | Add dev-02 to `[certbot]` |
+| `terraform/apps/dev-01/config.tf` | Add DNS records for ide/agent, remove proxy config, remove CNAMEs |
+| `implemented/linux_dev_station.md` | Update report |

.opencode/skills/ansible-develop/SKILL.md

@@ -0,0 +1,486 @@
+---
+name: ansible-develop
+description: >
+  Use when developing, writing, or debugging Ansible playbooks and roles in this
+  repository. Covers playbook and role conventions, inventory group targeting,
+  molecule-podman testing, variable scoping (defaults/vars/host_vars), Jinja2
+  templates, tag-based execution, and known patterns such as the podman shared
+  tasks helper and dual Debian/RedHat OS-family blocks.
+---
+
+# Ansible Playbook & Role Skill
+
+## Purpose
+
+Use this skill when creating new playbooks or roles in `ansible/`. It covers
+project conventions for structure, testing, variable management, secrets, and
+execution patterns used across all existing automation.
+
+---
+
+## Repository Layout
+
+```
+ansible/
+  ansible.cfg              # inventory = ../inventory/hosts
+  requirements.yml         # collections: ansible.posix, community.general,
+                           # containers.podman, community.libvirt, ...
+  .yamllint.yml            # 120-char line limit, extended default rules
+  playbook_*.yml           # top-level entry points (run from repo root)
+  tasks/                   # shared task fragments (e.g. tasks/podman.yml)
+  roles/
+    <name>/
+      defaults/main.yml    # public defaults (overridable by callers)
+      vars/main.yml        # internal values (non-overridable)
+      tasks/main.yml       # entry point; include_ or block per sub-feature
+      handlers/main.yml    # notify handlers (services, reloads)
+      templates/           # Jinja2 files (.j2 extension)
+      files/               # static files copied as-is
+      meta/main.yml        # galaxy metadata, min_ansible_version, dependencies
+      molecule/default/    # tests: converge.yml, prepare.yml, verify.yml, molecule.yml
+      tests/inventory      # small static inventory for role tests
+      tests/test.yml       # molecule verifier
+      .yamllint.yml        # (optional) role-level yamllint overrides
+      README.md            # role documentation
+inventory/
+  hosts                    # host groups (ini-format inventory)
+  host_vars/               # per-host YAML variable files
+```
+
+Playbooks live in `ansible/` and are run from the repository root. The
+inventory path is relative: `../inventory/hosts` (as declared in
+`ansible/ansible.cfg`).
+
+---
+
+## Inventory & Group Targeting
+
+The inventory at `inventory/hosts` uses **ini-format groups**. Playbooks target
+groups via `hosts: <groupname>`. Group names follow kebab-case convention.
+
+Existing groups:
+
+| Group             | Hosts                                          |
+|-------------------|------------------------------------------------|
+| `all`             | prod-01, dev-01, vh-01, gpu-01                 |
+| `hardening`       | prod-01, dev-01, vh-01, gpu-01                 |
+| `ubuntu`          | prod-01, dev-01                                  |
+| `fedora`          | vh-01, gpu-01                                    |
+| `microk8s`        | prod-01, dev-01                                  |
+| `monitoring`      | prod-01                                          |
+| `duo`             | prod-01, dev-01, vh-01, gpu-01                 |
+| `virtualization`  | vh-01                                          |
+| `ollama`          | gpu-01                                         |
+| `comfyui`         | gpu-01                                         |
+| `microk8s`        | prod-01, dev-01                                |
+| `certbot`         | vh-01, gpu-01                                  |
+| `nvidia`          | gpu-01                                         |
+| `amd`             | gpu-01                                         |
+| `public`          | prod-01                                        |
+| `private`         | dev-01, vh-01, gpu-01                          |
+| `windows`         | ci-01                                          |
+
+When adding a new playbook, **add a new group** in `inventory/hosts` and target
+it from the playbook.
+
+---
+
+## Playbook Patterns
+
+Two patterns are used. Choose the one that fits the task.
+
+### Pattern 1 — Role-based (preferred for reusable logic)
+
+Use when the task is encapsulated in a role or may be reused across playbooks.
+
+```yaml
+---
+- name: My feature setup
+  hosts: mygroup
+  gather_facts: true
+
+  roles:
+    - role: "my_role"
+      tags:
+        - roles
+        - roles::my_role
+```
+
+### Pattern 2 — Inline tasks
+
+Use for single-shot playbooks that configure one specific host with custom logic
+(e.g., virt_host, encryption).
+
+```yaml
+---
+- name: My inline setup
+  hosts: mygroup
+  become: true
+  gather_facts: true
+
+  tasks:
+    - name: Ensure the system is Debian
+      ansible.builtin.assert:
+        that:
+          - ansible_os_family == 'Debian'
+        fail_msg: "This playbook only supports Debian hosts."
+
+    - name: Install required packages
+      become: true
+      ansible.builtin.apt:
+        name:
+          - package-one
+          - package-two
+        state: present
+        update_cache: true
+```
+
+### Shared task fragments
+
+Complex operations (e.g. podman pod/container lifecycle) go into `tasks/` and
+are included via `ansible.builtin.include_tasks`. See `tasks/podman.yml` for
+the pattern: required-var checks, OS-family `set_fact`, `become_user` blocks,
+and loop-based resource creation.
+
+---
+
+## Role Development
+
+### Minimal role directory structure
+
+```
+roles/my_role/
+  defaults/main.yml    # public defaults
+  tasks/main.yml       # entry point
+  handlers/main.yml    # notify handlers
+  templates/           # Jinja2 files
+  meta/main.yml        # galaxy metadata
+  molecule/default/    # tests
+```
+
+### defaults/main.yml
+
+Declare **all** public variables here with safe defaults. This is how callers
+override behavior.
+
+```yaml
+---
+my_feature_enabled: true
+my_feature_port: 8080
+my_feature_users: []
+```
+
+Use `lookup('env', 'VAR_NAME')` for secrets with empty default:
+
+```yaml
+my_api_key: "{{ lookup('env', 'MY_API_KEY') | default('') }}"
+```
+
+### tasks/main.yml
+
+Use `block` + `when` for OS-family conditional logic:
+
+```yaml
+---
+- name: Install packages (Debian)
+  when: ansible_os_family == 'Debian'
+  block:
+    - name: Install deps
+      become: true
+      ansible.builtin.apt:
+        name: "{{ item }}"
+        state: present
+        update_cache: true
+      loop:
+        - package-one
+
+- name: Install packages (RedHat)
+  when: ansible_os_family == 'RedHat'
+  block:
+    - name: Install deps
+      become: true
+      ansible.builtin.dnf:
+        name: "{{ item }}"
+        state: present
+      loop:
+        - package-one
+```
+
+Use `include_tasks` for sub-features, tagged separately:
+
+```yaml
+- name: Include volume provisioning
+  ansible.builtin.include_tasks: volume.yml
+  tags:
+    - roles::my_role::volume
+
+- name: Include KMS setup
+  ansible.builtin.include_tasks: kms.yml
+  tags:
+    - roles::my_role::kms
+```
+
+### meta/main.yml
+
+Always declare `galaxy_info` with namespace `a13labs`, author
+`Alexandre Pires`, and company `A13Labs`:
+
+```yaml
+---
+galaxy_info:
+  author: Alexandre Pires
+  description: Role to do something
+  company: A13Labs
+  role_name: my_role
+  namespace: a13labs
+  license: MIT
+  min_ansible_version: "2.1"
+```
+
+### Tag conventions
+
+Use nested tags following the pattern `roles::<role_name>::<feature>`. Top-level
+playbooks use `roles` and `roles::<role_name>`.
+
+---
+
+## Molecule Testing (Podman)
+
+Every role should have a `molecule/default/` test scenario using the Podman
+driver.
+
+### molecule.yml
+
+```yaml
+---
+driver:
+  name: podman
+platforms:
+  - name: instance
+    image: ubuntu:latest
+    pre_build_image: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    privileged: true
+provisioner:
+  name: ansible
+  env:
+    MY_ROLE_VAR: "test_value"
+  playbooks:
+    converge: converge.yml
+    prepare: prepare.yml
+verifier:
+  name: ansible
+```
+
+### prepare.yml
+
+Must install python3 and sudo using raw (facts are unavailable in containers):
+
+```yaml
+---
+- name: Prepare
+  hosts: all
+  gather_facts: false
+  tasks:
+    - name: Update cache
+      ansible.builtin.raw: apt update
+
+    - name: Install required packages
+      ansible.builtin.raw: apt install -y python3 sudo
+```
+
+### converge.yml
+
+```yaml
+---
+- name: Converge
+  hosts: all
+  gather_facts: true
+  tasks:
+    - name: Include my_role
+      ansible.builtin.include_role:
+        name: "my_role"
+```
+
+### Testing
+
+Run from the role directory:
+
+```bash
+cd ansible/roles/my_role
+molecule test
+```
+
+This runs: prepare -> converge -> verify -> destroy.
+
+**Important**: Most containers lack systemd. Guard service-related tasks:
+`when: ansible_service_mgr == "systemd"`.
+
+---
+
+## Secrets & Environment Variables
+
+Secrets are injected via `sectool` using `sectool.json` at the repository root.
+The `ANSIBLE_USER` env var must be set before opening opencode.
+
+Secrets in roles use `lookup('env', 'VAR_NAME')` — `sectool` injects them
+automatically when playbooks are run through `sectool exec`.
+
+Pattern in role defaults:
+
+```yaml
+api_key: "{{ lookup('env', 'MY_API_KEY') | default('') }}"
+```
+
+Pattern in host_vars (for host-level secrets):
+
+```yaml
+cloudns_auth_id: "{{ lookup('env', 'CLOUDNS_AUTH_ID') }}"
+cloudns_auth_password: "{{ lookup('env', 'CLOUDNS_PASSWORD') }}"
+```
+
+Never hardcode secrets. Always use `lookup('env', ...)` with a `| default('')`
+fallback.
+
+---
+
+## Execution
+
+All Ansible commands must be run from the **`ansible/` directory** using
+`sectool` to inject secrets:
+
+```bash
+cd ansible/
+
+# Role-based playbook
+sectool -f ../sectool.json exec ansible-playbook -u $ANSIBLE_USER playbook_myfeature.yml
+
+# Tag-based partial run
+sectool -f ../sectool.json exec ansible-playbook -u $ANSIBLE_USER playbook_myfeature.yml --tags roles::my_role::volume
+
+# Limit to specific group
+sectool -f ../sectool.json exec ansible-playbook -u $ANSIBLE_USER playbook_myfeature.yml --limit mygroup
+
+# Role tests (molecule runs independently, no sectool wrapper needed)
+cd ansible/roles/my_role && molecule test
+```
+
+`$ANSIBLE_USER` must be set in the shell before opening opencode.
+
+---
+
+## Linting
+
+### yamllint
+
+```bash
+cd ansible
+yamllint -c .yamllint.yml .
+```
+
+Rules: 120-char line width (warning), forbid implicit/explicit octal, extended
+default ruleset.
+
+### ansible-lint
+
+If available, run from the `ansible/` directory:
+
+```bash
+cd ansible
+ansible-lint
+```
+
+---
+
+## Known Patterns
+
+### Dual OS-family support
+
+Always support both Debian and RedHat families when packages differ:
+
+```yaml
+- name: Set binary path (Debian)
+  when: ansible_os_family == 'Debian'
+  ansible.builtin.set_fact:
+    podman_binary: /usr/bin/podman
+
+- name: Set binary path (RedHat)
+  when: ansible_os_family == 'RedHat'
+  ansible.builtin.set_fact:
+    podman_binary: /usr/sbin/podman
+```
+
+### become_user blocks
+
+When a task needs to run as a different user (e.g. podman operations), use a
+`become: true` + `become_user` block:
+
+```yaml
+- name: Run tasks as podman user
+  become: true
+  become_user: "{{ podman_user }}"
+  block:
+    - name: Build images
+      containers.podman.podman_image:
+        ...
+```
+
+### systemd service generation
+
+Use `ansible.builtin.template` to generate `.service` files from `.j2`
+templates, then `daemon_reload` + `systemd` enable:
+
+```yaml
+- name: Create systemd service file
+  become: true
+  ansible.builtin.template:
+    src: myservice.service.j2
+    dest: "/etc/systemd/system/my-service.service"
+    mode: "0644"
+  notify: Reload daemon
+
+handlers:
+  - name: Reload daemon
+    changed_when: false
+    ansible.builtin.systemd:
+      daemon_reload: true
+```
+
+### nmcli bridge creation
+
+Network bridge configuration uses `nmcli connection add` with `async` +
+`wait_for` to handle reconnection after interface changes:
+
+```yaml
+- name: Create bridge
+  ansible.builtin.command: nmcli connection add ...
+  register: bridge_created
+  async: 10
+  poll: 0
+
+- name: Wait for reconnection
+  ansible.builtin.wait_for:
+    host: "{{ bridge_ip }}"
+    port: 22
+    timeout: 30
+  delegate_to: localhost
+```
+
+---
+
+## Review Checklist
+
+Before proposing a new playbook or role:
+
+- [ ] Playbook targets a defined group in `inventory/hosts` (add one if needed).
+- [ ] Role has `defaults/main.yml` with all public variables + safe defaults.
+- [ ] Secrets use `lookup('env', 'VAR') | default('')`, never hardcoded.
+- [ ] Dual OS-family support when package names differ.
+- [ ] Tasks tagged with `roles::<role_name>::<feature>`.
+- [ ] Molecule `molecule/default/` scenario with prepare + converge.
+- [ ] `molecule.yml` uses Podman driver with `privileged: true` + cgroup mount.
+- [ ] Line length within 120 characters.
+- [ ] `yamllint -c ansible/.yamllint.yml ansible/` passes.
+- [ ] Service tasks guarded by `when: ansible_service_mgr == "systemd"` in
+      molecule containers.

.opencode/skills/terraform-develop-app/SKILL.md

@@ -0,0 +1,301 @@
+---
+name: terraform-develop-app
+description: >
+  Use when developing, deploying, or debugging Terraform app modules in this
+  repository. Covers module scaffolding, secrets injection with sectool and
+  TF_VAR, OpenTofu command workflow, Kubernetes runtime debugging with kubectl,
+  hardening defaults for pod and container security contexts, and known quirks
+  such as the SearXNG IPv6 Granian bind issue. Also covers integrating modules
+  into dev-01 and prod-01 app roots including DNS records, sectool.env mapping,
+  and targeted apply recovery patterns.
+---
+
+# Terraform App Module Skill
+
+## Purpose
+
+Use this skill when developing, deploying, or debugging Terraform app modules in this repository. It covers module authoring conventions, secrets injection with sectool, OpenTofu command workflow, Kubernetes runtime debugging, and known behavioral quirks discovered in production.
+
+---
+
+## Repository Layout
+
+```
+terraform/
+  apps/
+    dev-01/          # dev cluster root
+    prod-01/         # prod cluster root
+  modules/
+    apps/            # reusable app modules
+    db/              # database modules
+    utils/           # utility modules
+```
+
+App roots are the deployment entry points. Reusable logic lives in modules and is consumed by app roots.
+
+Use `terraform/modules/apps/template` as the baseline shape when creating a new module.
+
+---
+
+## Cluster References
+
+| Name   | Folder                   | Kubernetes context   |
+|--------|--------------------------|----------------------|
+| dev-01 | terraform/apps/dev-01    | microk8s-dev-01      |
+| pro-01 | terraform/apps/prod-01   | microk8s-prod-01     |
+
+When planning or reviewing changes, always consider both clusters. The production folder is named `prod-01`; discussions may call it `pro-01`.
+
+---
+
+## Creating a New App Module
+
+Create `terraform/modules/apps/<name>/` with these files:
+
+### provider.tf
+
+```hcl
+terraform {
+  required_version = "~>1.8"
+  required_providers {
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "2.36.0"
+    }
+  }
+}
+```
+
+### variables.tf
+
+Declare at minimum:
+
+```hcl
+variable "persistent_folder" {
+  description = "Path for persistent data on the host"
+  type        = string
+}
+
+variable "fqdn" {
+  description = "FQDN for the service"
+  type        = string
+}
+
+variable "tag" {
+  description = "Image tag to deploy"
+  type        = string
+  default     = "x.y.z"   # always pin, never use latest
+}
+```
+
+Add sensitive variables at module level only when the module itself needs to inject them into a Kubernetes secret. If the secret is passed from the app root, declare it sensitive there.
+
+### config.tf
+
+Use locals to normalize inputs. Do not append the app name to `persistent_folder` if the caller already passes an app-specific path:
+
+```hcl
+locals {
+  app_folder  = var.persistent_folder
+  app_version = var.tag
+  app_fqdn    = var.fqdn
+}
+```
+
+### main.tf resource order
+
+1. `kubernetes_namespace`
+2. `kubernetes_service_account` — with `automount_service_account_token = false`
+3. `kubernetes_config_map` (non-sensitive config)
+4. `kubernetes_secret` (sensitive data)
+5. `kubernetes_deployment`
+6. `kubernetes_service`
+
+### ingress.tf (when HTTP/HTTPS exposed)
+
+```hcl
+resource "kubernetes_ingress_v1" "<name>" {
+  metadata {
+    annotations = {
+      "kubernetes.io/ingress.class"              = "public"
+      "cert-manager.io/cluster-issuer"           = var.issuer
+      "kubernetes.io/tls-acme"                   = "true"
+      "nginx.ingress.kubernetes.io/ssl-redirect" = "true"
+    }
+  }
+  ...
+}
+```
+
+Accept `issuer` as a variable (default `"letsencrypt-prod"`) so the same module works for both clusters and internal CA scenarios.
+
+---
+
+## Hardening Defaults
+
+Apply these to every new deployment unless there is a documented exception:
+
+```hcl
+spec {
+  automount_service_account_token = false
+
+  security_context {
+    run_as_non_root = true
+    run_as_user     = <uid>
+    run_as_group    = <gid>
+    fs_group        = <gid>
+    seccomp_profile {
+      type = "RuntimeDefault"
+    }
+  }
+
+  container {
+    security_context {
+      allow_privilege_escalation = false
+      privileged                 = false
+      read_only_root_filesystem  = true
+      capabilities {
+        drop = ["ALL"]
+      }
+    }
+
+    # provide writable scratch via emptyDir, not root filesystem
+    volume_mount {
+      name       = "<name>-tmp"
+      mount_path = "/tmp"
+    }
+  }
+
+  volume {
+    name = "<name>-tmp"
+    empty_dir {}
+  }
+}
+```
+
+---
+
+## Integrating a Module into an App Root
+
+Do this for each cluster root:
+
+1. Add a `module` block in `apps.tf`:
+   ```hcl
+   module "myapp" {
+     source            = "../../modules/apps/myapp"
+     persistent_folder = local.myapp_folder
+     fqdn              = local.myapp_fqdn
+     tag               = "x.y.z"
+     secret_key        = var.myapp_secret_key
+   }
+   ```
+
+2. Add locals in `config.tf`:
+   ```hcl
+   myapp_folder = "/mnt/md0/myapp"
+   myapp_fqdn   = "myapp.${local.lab_domain}"
+   ```
+
+3. Add a DNS CNAME record inside `local.zones[0].records`:
+   ```hcl
+   { name = "myapp", type = "CNAME", content = "dev-01.${local.lab_domain}." }
+   ```
+
+4. Add sensitive variable in `secrets.tf`:
+   ```hcl
+   variable "myapp_secret_key" {
+     description = "Secret key for MyApp"
+     type        = string
+     sensitive   = true
+   }
+   ```
+
+5. Map the env variable in `sectool.env`:
+   ```
+   TF_VAR_myapp_secret_key=$MYAPP_SECRET_KEY
+   ```
+
+---
+
+## Secrets Pattern (sectool + TF_VAR)
+
+Secrets are sourced from Bitwarden via `sectool` using `sectool.json` at the repository root.
+
+Flow:
+1. Store secret value in Bitwarden under the project referenced in `sectool.json`.
+2. Declare the variable in `secrets.tf` with `sensitive = true`.
+3. Map `TF_VAR_<name>=$ENV_VAR_NAME` in `terraform/apps/<cluster>/sectool.env`.
+4. Run all OpenTofu commands via `sectool exec` so variables are injected automatically.
+
+---
+
+## OpenTofu Workflow
+
+Run from the relevant app root (e.g., `terraform/apps/dev-01`):
+
+```bash
+# Initialise
+sectool -f ../../../sectool.json exec tofu init
+
+# Format and validate
+tofu fmt
+sectool -f ../../../sectool.json exec tofu validate
+
+# Plan
+sectool -f ../../../sectool.json exec tofu plan
+
+# Apply
+sectool -f ../../../sectool.json exec tofu apply -auto-approve
+
+# Targeted apply (emergency recovery only — always follow with a full plan)
+sectool -f ../../../sectool.json exec tofu apply -auto-approve -target=module.<name>
+```
+
+---
+
+## Runtime Debugging Playbook
+
+When a pod is in CrashLoopBackOff:
+
+```bash
+# 1. Inspect state and events
+kubectl -n <ns> get pods -o wide
+kubectl -n <ns> describe pod <pod>
+
+# 2. Check current and previous logs
+kubectl -n <ns> logs <pod> --all-containers=true --tail=200
+kubectl -n <ns> logs <pod> --previous --all-containers=true --tail=200
+
+# 3. Validate rollout after a fix
+kubectl -n <ns> rollout status deployment/<name> --timeout=120s
+
+# 4. Test a runtime env override before persisting in Terraform
+kubectl -n <ns> set env deployment/<name> KEY=value
+# verify recovery, then add KEY to the module ConfigMap and re-apply
+```
+
+---
+
+## Known Quirks
+
+### SearXNG — IPv6 bind failure
+
+The `searxng/searxng` image defaults to `GRANIAN_HOST=::` (IPv6). On nodes without IPv6 socket support this causes an immediate crash:
+
+```
+RuntimeError: Address family not supported by protocol (os error 97)
+```
+
+Fix: set `GRANIAN_HOST = "0.0.0.0"` in the module ConfigMap.
+
+---
+
+## Review Checklist
+
+Before proposing or applying any Terraform change:
+
+- [ ] Changes scoped to both dev-01 and prod-01 where relevant.
+- [ ] Sensitive inputs declared in `secrets.tf` + mapped in `sectool.env`.
+- [ ] Image tag is pinned (not `latest`).
+- [ ] Hardening defaults present (non-root, drop-all capabilities, read-only root fs).
+- [ ] Commands in examples use `sectool -f ../../../sectool.json exec tofu`.

.opencode/skills/terraform-update-app-version/SKILL.md

@@ -0,0 +1,115 @@
+---
+name: terraform-update-app-version
+description: >
+  Use when updating a container image version for an app managed by Terraform in
+  this repository. Covers how to find where the tag is set, update dev-01 and
+  prod-01 app roots safely, run sectool-wrapped OpenTofu validation and plan,
+  apply changes, and verify rollout with kubectl.
+---
+
+# Terraform Update App Version Skill
+
+## Purpose
+
+Use this skill when you need to update the deployed container image version for an app module.
+
+This repository usually sets the image tag in the app root module call (`terraform/apps/<cluster>/apps.tf`) and passes it to the module variable `tag`, then to `local.app_version` inside the module.
+
+Always use pinned tags. Do not use `latest`.
+
+---
+
+## Scope
+
+Update both clusters unless the request explicitly says otherwise:
+
+- dev-01: `terraform/apps/dev-01`
+- pro-01 folder: `terraform/apps/prod-01`
+
+Note: production is often called "pro-01" in discussion, but the folder name is `prod-01`.
+
+---
+
+## Where to Change the Version
+
+Common location:
+
+- `terraform/apps/<cluster>/apps.tf`
+
+Pattern:
+
+```hcl
+module "<app_name>" {
+  source = "../../modules/apps/<app_name>"
+  ...
+  tag    = "<image-tag>"
+}
+```
+
+Module wiring pattern:
+
+- `terraform/modules/apps/<app_name>/variables.tf` defines `variable "tag"`
+- `terraform/modules/apps/<app_name>/config.tf` typically sets `local.app_version = var.tag`
+- `terraform/modules/apps/<app_name>/main.tf` uses `${local.app_version}` in image reference
+
+If tag is not in the app root, check the module defaults and root overrides before editing.
+
+---
+
+## Update Workflow
+
+Run from each app root directory (`terraform/apps/dev-01` and `terraform/apps/prod-01`) as needed.
+
+1. Edit `apps.tf` and set the new pinned tag.
+2. Validate configuration:
+
+```bash
+tofu fmt
+sectool -f ../../../sectool.json exec tofu validate
+```
+
+3. Review plan:
+
+```bash
+sectool -f ../../../sectool.json exec tofu plan
+```
+
+4. Apply:
+
+```bash
+sectool -f ../../../sectool.json exec tofu apply -auto-approve
+```
+
+5. Verify rollout:
+
+```bash
+kubectl --context microk8s-<cluster> -n <namespace> rollout status deployment/<deployment> --timeout=120s
+kubectl --context microk8s-<cluster> -n <namespace> get pods -o wide
+kubectl --context microk8s-<cluster> -n <namespace> logs deployment/<deployment> --all-containers=true --tail=200
+```
+
+---
+
+## Emergency Recovery
+
+If immediate recovery is required, a targeted apply can be used temporarily:
+
+```bash
+sectool -f ../../../sectool.json exec tofu apply -auto-approve -target=module.<app_name>
+```
+
+Afterward, always run a full plan to detect drift:
+
+```bash
+sectool -f ../../../sectool.json exec tofu plan
+```
+
+---
+
+## Review Checklist
+
+- [ ] New image tag is pinned and explicit.
+- [ ] Both cluster roots were considered (dev-01 and prod-01).
+- [ ] Commands were run with `sectool -f ../../../sectool.json exec tofu`.
+- [ ] Rollout and pod health were verified with kubectl.
+- [ ] Any runtime hotfix used during debugging was persisted back into Terraform.

AGENTS.md

@@ -0,0 +1,147 @@
+# AGENTS.md - A13Labs Infrastructure
+
+## Repo Structure
+
+- **`ansible/`** — Playbooks, roles, inventory. Runs against bare-metal servers.
+- **`.opencode/plans/`** — Plans under execution
+- **`implemented/`** — Reports of implemented features
+- **`terraform/`** — IaC (OpenTofu for K8s app deployments + cloud provisioning).
+- **`inventory/hosts`** — Ansible inventory. All hosts and groups.
+- **`requirements/`** — pip requirements files: `ansible.txt`, `dev.txt`, `opentofu.txt`.
+- **`sectool`** — Secret manager (Bitwarden, configured in `sectool.json`). Both root and `ansible/` have their own `sectool.json` + `sectool.env`.
+
+## Key Facts
+
+- **No `Makefile`, no `justfile`, no pre-commit config.** Commands are run directly.
+- **All GitHub workflows are `.disabled`** — CI is not active. Verification is local-only.
+- **`ansible/ansible.cfg`** points to `../inventory/hosts` (relative to `ansible/` dir). Always run playbooks from the `ansible/` directory, or use `-i` to override.
+- **YAML linting:** `ansible-lint` config at `ansible/.ansible-lint`. `.yamllint.yml` at `ansible/.yamllint.yml` (120 char line warning).
+- **No `.pre-commit-config.yaml` exists** — linting is manual or CI-only.
+
+## Ansible
+
+### Required Environment vars
+
+```sh
+export ANSIBLE_USER=provision 
+export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES # if on a MAC
+```
+
+### Running playbooks
+
+**Note:** ```sectool``` just need to be used when secrets must be injected to the environment, if the command does not requires secrets injection do not use ```sectool``` 
+
+```
+sh
+cd ansible
+sectool exec ansible-playbook -u $ANSIBLE_USER  playbook_<name>.yml
+```
+
+### Role testing (Molecule + Podman - Linux Only)
+```sh
+cd ansible/roles/<role_name>
+molecule test          # create -> converge -> verify -> destroy
+molecule converge       # run converge only
+molecule destroy        # clean up
+```
+
+**Molecule gotchas:**
+- Always run `molecule` commands **from the role directory**, not from repo root.
+- Molecule containers use `podman` driver. No systemd by default — guard service tasks with `when: ansible_service_mgr == "systemd"`.
+- `molecule/default/prepare.yml` must install python3 and sudo via `raw:` before Ansible can run Python modules.
+- `provisioner.env.ANSIBLE_ROLES_PATH` should point up to the parent (e.g., `../../../`).
+
+### Ansible-lint
+```sh
+ansible-lint  # runs from repo root or ansible/
+```
+Excluded paths: `.git/`, `.github/`, `tests/`. Warns on: `line-length`, `yaml[line-length]`, `var-naming[no-role-prefix]`.
+
+## Terraform / OpenTofu
+
+### App deployments (K8s)
+- `terraform/apps/dev-01/` — dev Kubernetes cluster (microk8s)
+- `terraform/apps/prod-01/` — prod Kubernetes cluster (microk8s)
+- Uses Kubernetes + TLS providers. `provider.tf` sets `insecure = true`.
+
+### Cloud IaC
+- `terraform/iac/aws/` — AWS infrastructure
+- `terraform/iac/contabo/` — Contabo VPS
+- `terraform/iac/cloudns/` — DNS (cloudns)
+- `terraform/iac/scaleway/` — Scaleway infrastructure
+
+### Running Terraform / OpenTofu
+
+Each terraform directory has its own `sectool.env`. Run `sectool` to load secrets before `tofu init/plan/apply`:
+To validate plane (mandatory before apply)
+```sh
+sectool exec tofu plan
+```
+Apply:
+```sh
+sectool exec tofu apply --auto-approve
+```
+
+## Secrets
+
+To list secrets:
+```sh
+sectool vault list
+```
+To set a secret:
+```sh
+sectool vault set <secret name> <secret value>
+```
+To set a secret:
+```sh
+sectool vault get <secret name> <secret value>
+```
+
+
+### Module structure
+- `terraform/modules/apps/` — application modules (Nextcloud, etc.)
+- `terraform/modules/db/` — database modules (MariaDB with backups)
+- `terraform/modules/utils/` — utilities (exporters, cert-checker, prometheus, trivy, etc.)
+
+## Scripts (`scripts/`)
+
+One-off utility scripts — not library code:
+- `install-opnsense.sh` — boots OPNsense on a VM via virt-install (one-time router setup)
+- `chronograf.sh` — `kubectl port-forward` wrapper for Chronograf pod
+- `proxy-pt.sh` — `kubectl port-forward` wrapper for m3uproxy pod
+- `steamos` — launches SteamOS gaming session with sunshine
+- `cpu_power_monitor.py` — RAPL power monitoring (requires root)
+
+## Environment Setup
+
+```sh
+# Ansible + dev tools
+pip install -r requirements/ansible.txt -r requirements/dev.txt
+
+# Terraform/OpenTofu deps
+pip install -r requirements/opentofu.txt
+```
+
+## Contabo CLI
+
+```sh
+sectool exec cntb get instances --oauth2-clientid="$CONTABO_CLIENT_ID" ...
+```
+
+## Windows hosts
+
+- Use `ConfigureRemotingForAnsible.ps1` in `scripts/` to enable WinRM on Windows targets.
+- On MSYS2: install packages listed in `Readme.MD` (go, python, cryptography, paramiko, rpds-py, etc.).
+- Python venv: `python -m venv .venv --system-site-packages`
+
+## Style Conventions
+
+- YAML: 120 char line length (warning threshold)
+- Ansible roles follow galaxy conventions: `defaults/`, `tasks/`, `meta/`, `files/`, `templates/`
+- `tasks/main.yml` is the entrypoint for every role
+- Host vars: `ansible/host_vars/<hostname>.yml`
+
+
+## Troubleshooting and planning
+
+- When troubleshooting and planning use deepwiki whenever possible to get documentation or information, deepwiki provides information about projects hosted on github to agents.

Readme.MD

@@ -27,3 +27,126 @@ Setup:
 ```
 sectool exec cntb get instances --oauth2-clientid="$CONTABO_CLIENT_ID" --oauth2-client-secret="$CONTABO_CLIENT_SECRET" --oauth2-user="$CONTABO_API_USER" --oauth2-password="$CONTABO_API_PASSWORD"
 ```
+
+# Windows
+
+## Create a Dedicated “Ansible” User (One-Time Setup) — Linux
+
+```
+# Create the provision user (Debian/Ubuntu)
+sudo useradd -m -s /bin/bash -G sudo provision
+sudo passwd provision
+
+# OR (RHEL/CentOS/Fedora)
+sudo useradd -m -s /bin/bash -G wheel provision
+sudo passwd provision
+```
+
+```
+# Allow passwordless sudo for the provision user (recommended for Ansible)
+echo 'provision ALL=(ALL) NOPASSWD:ALL' | sudo tee /etc/sudoers.d/provision
+sudo chmod 440 /etc/sudoers.d/provision
+
+# Or edit safely:
+sudo visudo -f /etc/sudoers.d/provision
+# and add:
+# provision ALL=(ALL) NOPASSWD:ALL
+```
+
+```
+# Install SSH authorized key for the provision user (recommended over passwords)
+sudo -u provision mkdir -p /home/provision/.ssh
+sudo -u provision chmod 700 /home/provision/.ssh
+echo "<your_public_key_here>" | sudo -u provision tee /home/provision/.ssh/authorized_keys
+sudo -u provision chmod 600 /home/provision/.ssh/authorized_keys
+```
+
+Notes:
+- Replace <your_public_key_here> with the actual public key.
+- Use a strong password if you must set one; prefer SSH keys.
+- For tighter security, restrict the sudoers entry to only the commands Ansible requires instead of ALL.
+- Validate sudoers syntax with visudo to avoid lockout.
+
+## Configure windows to allow Ansible to connect via WinRM
+
+Copy the script located in scripts "ConfigureRemotingForAnsible.ps1" and run it on the target machine.
+
+```
+Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass
+.\ConfigureRemotingForAnsible.ps1
+```
+
+# Using MSYS2
+
+Upgrade your system:
+
+```
+pacman -Syu
+```
+
+Install the following packages:
+
+```
+mingw-w64-ucrt-x86_64-go
+mingw-w64-ucrt-x86_64-make
+mingw-w64-ucrt-x86_64-python
+ucrt-w64-ucrt-x86_64-gcc
+mingw-w64-ucrt-x86_64-python-cryptography
+mingw-w64-ucrt-x86_64-python-paramiko
+mingw-w64-ucrt-x86_64-python-rpds-py
+python-lxml
+mingw-w64-ucrt-x86_64-python-ruamel-yaml
+mingw-w64-ucrt-x86_64-python-sspilib
+unzip
+```
+
+Create a python environment use:
+
+```
+python -m venv .venv  --system-site-packages 
+```
+
+# macOS Dev Station
+
+Use the repository hybrid flow: bootstrap once with shell script, then manage day-2 changes with Ansible.
+
+## 1) Bootstrap prerequisites on the Mac
+
+```
+./scripts/setup-dev-station.sh
+```
+
+This installs only prerequisites (Homebrew, base tools, Ansible) and avoids persistent shell side effects.
+
+## 2) Configure the Mac with Ansible
+
+1. Add or adjust the host in `inventory/hosts` under `[darwin_dev]`.
+2. Configure host vars in `ansible/host_vars/mac-01.lab.alexpires.me.yml`.
+3. Run:
+
+```
+cd ansible
+sectool exec ansible-playbook playbook_macos_dev_station.yml --limit darwin_dev
+```
+
+The macOS role configures tools, tmux, code-server and opencode as system launchd daemons running under a dedicated service user. This allows services to start at boot without requiring an interactive login.
+
+OpenCode server auth is injected from environment variables. Ensure `OPENCODE_SERVER_USERNAME` and `OPENCODE_SERVER_PASSWORD` are available through `ansible/sectool.env` values before running the playbook.
+code-server password auth is also env-backed. Ensure `CODE_SERVER_PASSWORD` is available through `ansible/sectool.env` values before running the playbook.
+Hard mode (system daemons + dedicated service user) also requires sudo rights. Use passwordless sudo for your ansible user or set `ANSIBLE_BECOME_PASSWORD` through `ansible/sectool.env` values before running the playbook.
+
+## 3) Optional DNS entry in dev-01 CoreDNS
+
+From `terraform/apps/dev-01`, you can inject an optional DNS record for the Mac dev station:
+
+```
+export TF_VAR_mac_dev_station_dns_name="mac-mini"
+export TF_VAR_mac_dev_station_dns_type="A"
+export TF_VAR_mac_dev_station_dns_target="10.19.4.250"
+
+sectool -f ../../../sectool.json exec tofu validate
+sectool -f ../../../sectool.json exec tofu plan
+sectool -f ../../../sectool.json exec tofu apply -auto-approve
+```
+
+Use `CNAME` + hostname target instead of `A` when you want an alias.

ansible/ansible.cfg

@@ -1,5 +1,6 @@
 [defaults]
 inventory = ../inventory/hosts
+allow_world_readable_tmpfiles = True
 
 [ssh_connection]
 ssh_args = -o ControlMaster=auto -o ControlPersist=60s

ansible/apply.sh

@@ -1,55 +0,0 @@
-#!/bin/bash
-
-# check if sectool is available
-command -v sectool >/dev/null 2>&1 || { echo >&2 "sectool is not installed.  Aborting."; exit 1; }
-
-# check if ansible is available
-command -v ansible >/dev/null 2>&1 || { echo >&2 "ansible is not installed.  Aborting."; exit 1; }
-
-# check if ansible-playbook is available
-command -v ansible-playbook >/dev/null 2>&1 || { echo >&2 "ansible-playbook is not installed.  Aborting."; exit 1; }
-
-function help {
-    echo "Usage: $0 -d <playbook>"
-    echo "Options:"
-    echo "  -d, --dry-run: Run ansible-playbook in dry run mode (don't apply changes)"
-    exit 1
-}
-
-if [ -z $ANSIBLE_USER ]; then
-    echo "ANSIBLE_USER environment variable not set"
-    exit 1
-fi
-
-# parse arguments
-while [ "$1" != "" ]; do
-    case $1 in
-        -d | --dry-run )        shift
-                                DRY_RUN="-DC"
-                                ;;
-        -h | --help )           help
-                                ;;
-        * )                     PLAYBOOK=$1
-                                break
-                                ;;
-    esac
-done
-
-if [ -z "$PLAYBOOK" ]; then
-    echo "Playbook not specified"
-    help
-else
-    if [ ! -f "$PLAYBOOK" ]; then
-        echo "Playbook $PLAYBOOK not found"
-        help
-    fi
-fi
-
-if [ -z "$DRY_RUN" ]; then
-    echo "Applying playbook $PLAYBOOK"
-else
-    echo "Dry run playbook $PLAYBOOK"
-fi
-
-# execute ansible-playbook
-sectool exec ansible-playbook -u $ANSIBLE_USER $DRY_RUN $PLAYBOOK

ansible/files/configs/sshd_config.d/gitea.conf

@@ -1,3 +1,5 @@
 Match User git
+  Banner none
+  PermitTTY no
   AuthorizedKeysCommandUser git
-  AuthorizedKeysCommand /usr/local/bin/gitea-auth -u %u -t %t -k %k
+  AuthorizedKeysCommand /usr/local/bin/k8s_gitea_auth -r gitea gitea-app -u %u -t %t -k %k

ansible/files/dockerfiles/Dockerfile.comfyui

@@ -0,0 +1,35 @@
+# Use NVIDIA's PyTorch image with CUDA support
+FROM docker.io/pytorch/pytorch:2.1.0-cuda11.8-cudnn8-runtime
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+# Install system dependencies
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get install -y tzdata git ffmpeg libgl1 libglib2.0-0 && \
+    rm -rf /var/lib/apt/lists/*
+
+ARG UID=1000
+ARG GID=1000
+RUN groupadd --system --gid ${GID} worker && \
+    adduser --system --gid ${GID} --uid ${UID} --home /home/worker worker
+
+WORKDIR /home/worker
+USER worker
+
+# Clone ComfyUI
+RUN git clone https://github.com/comfyanonymous/ComfyUI.git .
+
+# Install Python dependencies
+RUN pip install --upgrade pip && \
+    pip install -r requirements.txt && \
+    rm -rf /home/worker/.cache/pip
+
+# Expose port
+EXPOSE 8188
+
+# Run ComfyUI
+CMD ["python", "main.py", "--listen", "0.0.0.0", "--port", "8188"]
+

ansible/files/dockerfiles/Dockerfile.llamacpp.cuda

@@ -0,0 +1,11 @@
+FROM ghcr.io/ggml-org/llama.cpp:server-cuda
+
+ARG UID=1000
+ARG GID=1000
+RUN groupadd --system --gid ${GID} worker && \
+    useradd --system --gid ${GID} --uid ${UID} --home /home/worker worker && \
+    mkdir -p /home/worker && chown worker:worker /home/worker
+
+WORKDIR /home/worker
+USER worker
+ENTRYPOINT ["/app/llama-server"]

ansible/files/dockerfiles/Dockerfile.llamacpp.rocm

@@ -0,0 +1,14 @@
+FROM ghcr.io/ggml-org/llama.cpp:server-rocm
+
+ARG UID=1000
+ARG GID=1000
+RUN groupadd --system --gid ${GID} worker && \
+    useradd --system --gid ${GID} --uid ${UID} --home /home/worker worker && \
+    mkdir -p /home/worker && chown worker:worker /home/worker && \
+    cp /app/llama-server /home/worker/ && \
+    chmod +x /home/worker/llama-server && \
+    chown worker:worker /home/worker/llama-server
+
+WORKDIR /home/worker
+USER worker
+ENTRYPOINT ["/home/worker/llama-server"]

ansible/files/dockerfiles/Dockerfile.ollama

@@ -0,0 +1,12 @@
+FROM docker.io/ollama/ollama:latest
+
+ARG UID=1000
+ARG GID=1000
+RUN groupadd --system --gid ${GID} worker && \
+    useradd --system --gid ${GID} --uid ${UID} --home /home/worker worker && \
+    mkdir -p /home/worker && chown worker:worker /home/worker
+
+WORKDIR /home/worker
+USER worker
+ENTRYPOINT ["/bin/ollama"]
+CMD ["serve"]

ansible/files/dockerfiles/Dockerfile.ollama.rocm

@@ -0,0 +1,12 @@
+FROM docker.io/ollama/ollama:rocm
+
+ARG UID=1000
+ARG GID=1000
+RUN groupadd --system --gid ${GID} worker && \
+    useradd --system --gid ${GID} --uid ${UID} --home /home/worker worker && \
+    mkdir -p /home/worker && chown worker:worker /home/worker
+
+WORKDIR /home/worker
+USER worker
+ENTRYPOINT ["/bin/ollama"]
+CMD ["serve"]

ansible/files/dockerfiles/linux-runners/Dockerfile.cicd-base

@@ -0,0 +1,31 @@
+FROM local/gitea-runner-base:latest
+
+# Base image sets USER runner; switch back to root for package installs
+USER root
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	buildah \
+	fuse-overlayfs \
+	iptables \
+	podman \
+	skopeo \
+	slirp4netns \
+	uidmap \
+	&& rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /etc/containers /var/lib/containers /run/containers && \
+	printf '%s\n' '[storage]' 'driver = "vfs"' > /etc/containers/storage.conf && \
+	printf '%s\n' '[engine]' > /etc/containers/containers.conf && \
+	chown -R 1000:1000 /var/lib/containers
+
+ENV BUILDAH_ISOLATION=chroot
+ENV STORAGE_DRIVER=vfs
+
+ARG TRIVY_VERSION=0.72.0
+RUN curl -fsSL -o /tmp/trivy.tar.gz \
+	"https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" && \
+	tar -xzf /tmp/trivy.tar.gz -C /tmp && \
+	install -m 0755 /tmp/trivy /usr/local/bin/trivy && \
+	rm -f /tmp/trivy.tar.gz /tmp/trivy
+
+# Default to non-root execution
+USER runner

ansible/files/dockerfiles/linux-runners/Dockerfile.cpp-build

@@ -0,0 +1,11 @@
+FROM local/gitea-runner-base:latest
+
+# Base image sets USER runner; switch back to root for package installs
+USER root
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	build-essential \
+	cmake \
+	&& rm -rf /var/lib/apt/lists/*
+
+# Default to non-root execution
+USER runner

ansible/files/dockerfiles/linux-runners/Dockerfile.gitea-runner-base

@@ -0,0 +1,38 @@
+# Stage 1: Extract the gitea-runner binary from the official Alpine-based image
+FROM docker.io/gitea/runner:latest AS runner
+
+# Stage 2: Debian-based image with native glibc support
+FROM debian:bookworm-slim
+
+COPY --from=runner /usr/local/bin/gitea-runner /usr/local/bin/gitea-runner
+COPY --from=runner /usr/local/bin/run.sh /usr/local/bin/run.sh
+RUN chmod +x /usr/local/bin/run.sh
+
+ENTRYPOINT ["/usr/local/bin/run.sh"]
+
+# Create dedicated runner user (UID 1000) for non-root container execution
+RUN groupadd -g 1000 runner && \
+    useradd -u 1000 -g 1000 -m -d /home/runner -s /bin/bash runner
+
+# Install runner tooling
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash \
+    ca-certificates \
+    curl \
+    git \
+    jq \
+    make \
+    nodejs \
+    openssh-client \
+    python3 \
+    python3-pip \
+    rsync \
+    unzip \
+    zip \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set work directory
+ENV RUNNER_WORKDIR=/home/runner
+
+# Ensure container runs as non-root by default
+USER runner

ansible/files/dockerfiles/linux-runners/Dockerfile.go-build

@@ -0,0 +1,10 @@
+FROM local/gitea-runner-base:latest
+
+# Base image sets USER runner; switch back to root for package installs
+USER root
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	golang-go \
+	&& rm -rf /var/lib/apt/lists/*
+
+# Default to non-root execution
+USER runner

ansible/files/dockerfiles/windows-runners/Dockerfile.windows-base

@@ -0,0 +1,22 @@
+FROM mcr.microsoft.com/windows/servercore:ltsc2022
+
+SHELL ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command"]
+
+# Base image for Windows Gitea runners with Git tools and gitea runner.
+ARG ACT_RUNNER_VERSION=2.0.0
+RUN Set-StrictMode -Version Latest; \
+    $ErrorActionPreference = 'Stop'; \
+    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; \
+    iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')); \
+  choco config set commandExecutionTimeoutSeconds 14400; \
+  choco feature disable -n showDownloadProgress; \
+    choco feature enable -n allowGlobalConfirmation; \
+  choco install git --yes --no-progress --limit-output; \
+    New-Item -ItemType Directory -Path C:\gitea-runner -Force | Out-Null; \
+    $runnerUrl = 'https://gitea.com/gitea/runner/releases/download/v{0}/gitea-runner-{0}-windows-amd64.exe' -f $env:ACT_RUNNER_VERSION; \
+    Invoke-WebRequest -UseBasicParsing -Uri $runnerUrl -OutFile C:\gitea-runner\act_runner.exe
+
+COPY ["start-runner.ps1", "C:/gitea-runner/start-runner.ps1"]
+
+ENV PATH="C:\ProgramData\chocolatey\bin;C:\Program Files\Git\cmd;C:\Program Files\Git\bin;C:\Program Files\Git\usr\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Windows\System32\OpenSSH\;C:\Users\ContainerAdministrator\AppData\Local\Microsoft\WindowsApps"
+ENTRYPOINT ["C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", "C:\\gitea-runner\\start-runner.ps1"]

ansible/files/dockerfiles/windows-runners/Dockerfile.windows-cpp

@@ -0,0 +1,8 @@
+FROM local/gitea-windows-runner:windows-base
+
+SHELL ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command"]
+
+# C/C++ profile with cmake and ninja.
+RUN choco feature enable -n allowGlobalConfirmation; \
+    choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System'; \
+    choco install ninja

ansible/files/dockerfiles/windows-runners/Dockerfile.windows-dotnet

@@ -0,0 +1,7 @@
+FROM local/gitea-windows-runner:windows-base
+
+SHELL ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command"]
+
+# .NET profile for building and testing managed workloads.
+RUN choco feature enable -n allowGlobalConfirmation; \
+    choco install dotnet-sdk --version=8.0.204

ansible/files/dockerfiles/windows-runners/Dockerfile.windows-go

@@ -0,0 +1,7 @@
+FROM local/gitea-windows-runner:windows-base
+
+SHELL ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command"]
+
+# Go profile for Windows builds.
+RUN choco feature enable -n allowGlobalConfirmation; \
+    choco install golang --version=1.22.4

ansible/files/dockerfiles/windows-runners/Dockerfile.windows-image-build

@@ -0,0 +1,9 @@
+FROM local/gitea-windows-runner:windows-msys
+
+SHELL ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command"]
+
+# Dedicated profile for image builds. Docker daemon access is provided at runtime
+# through the host named pipe mount configured in host_vars.
+RUN Set-StrictMode -Version Latest; \
+	$ErrorActionPreference = 'Stop'; \
+	choco install docker-cli --yes --no-progress --limit-output

ansible/files/dockerfiles/windows-runners/Dockerfile.windows-msys

@@ -0,0 +1,7 @@
+FROM local/gitea-windows-runner:windows-base
+
+SHELL ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command"]
+
+# MSYS-focused profile with common GNU toolchain extras.
+RUN choco feature enable -n allowGlobalConfirmation; \
+    choco install make msys2 unzip --yes --no-progress --limit-output

ansible/files/dockerfiles/windows-runners/start-runner.ps1

@@ -0,0 +1,17 @@
+$ErrorActionPreference = "Stop"
+$runner = "C:\gitea-runner\act_runner.exe"
+$config = if ($env:CONFIG_FILE) { $env:CONFIG_FILE } else { "C:\runner-data\config.yaml" }
+$runnerFile = if ($env:GITEA_RUNNER_FILE) { $env:GITEA_RUNNER_FILE } else { "C:\runner-data\.runner" }
+
+if (-not (Test-Path -Path $runnerFile) -and $env:GITEA_INSTANCE_URL -and $env:GITEA_RUNNER_REGISTRATION_TOKEN) {
+  $labels = if ($env:GITEA_RUNNER_LABELS) { $env:GITEA_RUNNER_LABELS } else { "windows:host" }
+  $name = if ($env:GITEA_RUNNER_NAME) { $env:GITEA_RUNNER_NAME } else { $env:COMPUTERNAME }
+
+  & $runner register --no-interactive --instance $env:GITEA_INSTANCE_URL --token $env:GITEA_RUNNER_REGISTRATION_TOKEN --name $name --labels $labels --config $config
+  if ($LASTEXITCODE -ne 0) {
+    exit $LASTEXITCODE
+  }
+}
+
+& $runner daemon --config $config
+exit $LASTEXITCODE

ansible/files/game_station/bin/system/gamescope-session

@@ -0,0 +1,157 @@
+#!/bin/bash
+set -euo pipefail
+LOG_FILE="$HOME/.local/logs/session.log"
+
+log() { echo "[$(date --iso-8601=seconds)] $*" | tee -a "$LOG_FILE"; }
+
+CONF_FILE="$HOME/.config/gamescope/gamescope.conf"
+[ -f "$CONF_FILE" ] && source "$CONF_FILE"
+
+RESOLUTION=${RESOLUTION:-1920x1080}
+WIDTH="${RESOLUTION%x*}"
+HEIGHT="${RESOLUTION#*x}"
+FRAMERATE=${FRAMERATE:-60}
+
+export __GL_GSYNC_ALLOWED=1
+export __GL_VRR_ALLOWED=1
+export __GL_SHADER_DISK_CACHE=1
+export __GL_SYNC_TO_VBLANK=0
+export __GLX_VENDOR_LIBRARY_NAME=nvidia
+export LIBVA_DRIVER_NAME=nvidia
+export NVD_BACKEND=direct
+
+GAMESCOPE_BIN="$(command -v gamescope || true)"
+if [ -z "$GAMESCOPE_BIN" ]; then
+	log "ERROR: gamescope not found in PATH"
+	exit 127
+fi
+
+HDR_FLAG=""
+if [[ "$HDR_ENABLED" == "1" || "$HDR_ENABLED" == "true" || "$HDR_ENABLED" == "yes" ]]; then
+    HDR_FLAG="--hdr-enabled"
+    log "HDR flag added: --hdr-enabled"
+else
+    log "HDR disabled, no HDR flag added"
+fi
+
+# Create run directory file for startup and stats sockets
+#   shellcheck disable=SC2030 # (broken warning)
+tmpdir="$([[ -n ${XDG_RUNTIME_DIR+x} ]] && mktemp -p "$XDG_RUNTIME_DIR" -d -t gamescope.XXXXXXX)"
+socket="${tmpdir:+$tmpdir/startup.socket}"
+stats="${tmpdir:+$tmpdir/stats.pipe}"
+# Fail early if we don't have a proper runtime directory setup
+#   shellcheck disable=SC2031 # (broken warning)
+if [[ -z $tmpdir || -z ${XDG_RUNTIME_DIR+x} ]]; then
+	echo >&2 "!! Failed to find run directory in which to create stats session sockets (is \$XDG_RUNTIME_DIR set?)"
+	exit 0
+fi
+
+# Setup MangoHud config file in temp directory
+export MANGOHUD_CONFIGFILE="${tmpdir:+$tmpdir/mangohud.config}"
+
+# Initially write no_display to our config file
+# so we don't get mangoapp showing up before Steam initializes
+# on OOBE and stuff.
+mkdir -p "$(dirname "$MANGOHUD_CONFIGFILE")"
+echo "no_display" > "$MANGOHUD_CONFIGFILE"
+
+# Chromium (and therefore Steam) ignore XCursor and use on the GTK config
+kwriteconfig6 --file gtk-3.0/settings.ini  --group Settings --key gtk-cursor-theme-name steam
+
+# Workaround for Steam login issue while Steam client change propagates out of Beta
+touch ~/.steam/root/config/SteamAppData.vdf || true
+
+# Increase open file limit for gamescope session
+ulimit -n 524288
+
+# 1048576 = 1M - passing it like that omits the 'M' suffix - xargs removes whitespace
+free_disk_space_megs=$(df ~/ --output=avail -B1048576 | sed -n '2 p' | xargs)
+minimum_free_disk_space_needed_megs=500
+
+if [[ "$free_disk_space_megs" -lt "$minimum_free_disk_space_needed_megs" ]]; then
+	echo >&2 "gamescope-session: not enough disk space to proceed, trying to find game to delete"
+
+	find ~/.local/share/Steam/steamapps/common/ -mindepth 1 -maxdepth 1 -type d -printf "%T@ %p\0" | sort -n -z | while IFS= read -r -d $'\0' line; do
+		timestamp=${line%% *}
+                game_folder=${line#* }
+
+		[[ -d $game_folder ]]  || continue
+
+		acf=$(grep -F -- "$(basename -- "$game_folder")" ~/.local/share/Steam/steamapps/*.acf | grep \"installdir\" | cut -d: -f1)
+		[[ -e "$acf" ]] || continue
+
+		echo >&2 "gamescope-session: deleting $(basename "$game_folder")"
+		appid=$(basename "$acf" | cut -d_ -f2 | cut -d. -f1)
+
+		# TODO leave a note for Steam to display UI to explain what happened, if this logic stays
+		# intentionally leave compatdata; could be unclouded save files there
+		rm -rf --one-file-system -- "$game_folder" "$acf" ~/.local/share/Steam/steamapps/shadercache/"$appid"
+
+		free_disk_space_megs=$(df ~/ --output=avail -B1048576 | sed -n '2 p' | xargs)
+		[[ "$free_disk_space_megs" -lt "$minimum_free_disk_space_needed_megs" ]] || break
+	done
+fi
+
+export GAMESCOPE_STATS="$stats"
+mkfifo -- "$stats"
+mkfifo -- "$socket"
+
+# Attempt to claim global session if we're the first one running (e.g. /run/1000/gamescope)
+linkname="gamescope-stats"
+#   shellcheck disable=SC2031 # (broken warning)
+sessionlink="${XDG_RUNTIME_DIR:+$XDG_RUNTIME_DIR/}${linkname}" # Account for XDG_RUNTIME_DIR="" (notfragileatall)
+lockfile="$sessionlink".lck
+exec 9>"$lockfile" # Keep as an fd such that the lock lasts as long as the session if it is taken
+if flock -n 9 && rm -f "$sessionlink" && ln -sf "$tmpdir" "$sessionlink"; then
+	# Took the lock.  Don't blow up if those commands fail, though.
+	echo >&2 "Claimed global gamescope stats session at \"$sessionlink\""
+else
+	echo >&2 "!! Failed to claim global gamescope stats session"
+fi
+
+read_gamescope_env() {
+	log "Waiting for gamescope to provide DISPLAY and GAMESCOPE_WAYLAND_DISPLAY via socket..."
+	if read -r -t 3 response_x_display response_wl_display <> "$socket"; then
+		log "Received gamescope environment:"
+		log "  DISPLAY=$response_x_display"
+		log "  GAMESCOPE_WAYLAND_DISPLAY=$response_wl_display"
+		export DISPLAY="$response_x_display"
+		export GAMESCOPE_WAYLAND_DISPLAY="$response_wl_display"
+
+		# Sync environment variables to systemd, then notify we are ready to launch subsequent services
+		log "Storing gamescope environment for systemd..."
+		env > $XDG_RUNTIME_DIR/gamescope-environment
+		systemd-notify --ready
+	fi
+}
+
+# Spawned in parallel to read values from gamescope
+(read_gamescope_env &)
+
+log "Starting Gamescope session..."
+
+# Build gamescope argument array for safe quoting and readability
+GAMESCOPE_ARGS=(
+	--generate-drm-mode fixed
+	--fade-out-duration 200
+	--cursor-scale-height 720
+	-f
+	-W "$WIDTH" -H "$HEIGHT"
+	-w "$WIDTH" -h "$HEIGHT"
+	--rt --mangoapp -e -R "$socket" -T "$stats"
+)
+
+# Append HDR flag if enabled
+if [ -n "${HDR_FLAG:-}" ]; then
+    GAMESCOPE_ARGS+=("$HDR_FLAG")
+fi
+
+export XDG_CURRENT_DESKTOP=gamescope
+export XDG_SESSION_TYPE=wayland
+export DESKTOP_SESSION=gamescope
+
+log "Launching Gamescope: $GAMESCOPE_BIN ${GAMESCOPE_ARGS[*]} -- $*"
+sudo setcap 'CAP_SYS_NICE=eip' "$GAMESCOPE_BIN"
+
+exec "$GAMESCOPE_BIN" "${GAMESCOPE_ARGS[@]}" >>"$LOG_FILE" 2>&1
+

ansible/files/game_station/bin/system/jupiter-biosupdate

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -euo pipefail
+LOGFILE="$HOME/.local/logs/jupiter-biosupdate.log"
+
+log() { echo "[$(date --iso-8601=seconds)] $*" | tee -a "$LOGFILE"; }
+
+log "Starting Jupiter BIOS update..."
+log "Arguments passed: ${@}"
+echo "Not applicable for this OS"

ansible/files/game_station/bin/system/launch-gamescope

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -euo pipefail
+LOGFILE="$HOME/.local/logs/gamescope.log"
+export PATH="$HOME/.local/bin:$PATH"
+
+log() { echo "[$(date --iso-8601=seconds)] $*" | tee -a "$LOGFILE"; }
+
+# SDDM sets this to wayland but apps using Gamescope must use x11
+export XDG_SESSION_TYPE=x11
+
+# Update the enviroment with DESKTOP_SESSION and all XDG variables
+log "Updating systemd user environment with XDG variables"
+dbus-update-activation-environment --systemd DESKTOP_SESSION `env | grep ^XDG_ | cut -d = -f 1`
+
+# This makes it so that xdg-desktop-portal doesn't find any portal implementations and doesn't start them and makes
+# them crash/exit because the dbus env has no DISPLAY. In turn this causes dbus calls to the portal which don't rely
+# on implementations to hang (such as SDL talking to the real time portal)
+# Plasma resets this variable when it starts
+systemctl --user set-environment XDG_DESKTOP_PORTAL_DIR=""
+
+# Remove these as they prevent gamescope-session from starting correctly
+systemctl --user unset-environment DISPLAY XAUTHORITY
+
+# If this shell script is killed then stop gamescope-session
+trap 'systemctl --user stop gamescope-session.target' HUP INT TERM
+
+# Start gamescope-session and wait
+log "Starting Gamescope Session"
+systemctl --user --wait start gamescope-session.target &
+wait
+log "Gamescope Session Ended - Performing Final Cleanup"
+
+# The 'wait' above blocks until gamescope-session.target stops
+# We want to wait until *everything* has finished. We know systemd will have
+# queued a stop job on graphical-session-pre aleady
+# by also queuing up a job we can block until that completes
+systemctl --user stop graphical-session-pre.target
+log "Gamescope Session Ended - Cleanup Complete"

ansible/files/game_station/bin/system/launch-kde

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+set -euo pipefail
+LOGFILE="$HOME/.local/logs/desktop.log"
+export PATH="$HOME/.local/bin:$PATH"
+
+log() { echo "[$(date --iso-8601=seconds)] $*" | tee -a "$LOGFILE"; }
+
+DESKTOP_BIN="$(command -v plasmashell || true)"
+if [ -z "$DESKTOP_BIN" ]; then
+  log "ERROR: plasmashell not found in PATH"
+  exit 127
+fi
+
+export XDG_CURRENT_DESKTOP=plasma
+export XDG_CURRENT_DESKTOP=KDE
+
+export QT_QPA_PLATFORM=wayland
+export GDK_BACKEND=wayland
+export CLUTTER_BACKEND=wayland
+export SDL_VIDEODRIVER=wayland
+export MOZ_ENABLE_WAYLAND=1
+export KDEDIRS=/usr
+export KDE_FULL_SESSION=true
+export DESKTOP_SESSION=plasma
+
+unset DISPLAY
+
+mkdir -p $XDG_RUNTIME_DIR
+chmod 700 $XDG_RUNTIME_DIR
+
+log "Launching Plasma: $DESKTOP_BIN"
+systemd-run --user --scope --quiet \
+  --unit=plasma.scope \
+  --slice=desktop.slice \
+  "$DESKTOP_BIN" >>"$LOGFILE" 2>&1 &
+plasma_scope_pid=$!
+log "Plasma started with scope pid $plasma_scope_pid"
+
+{
+  SUNSHINE_BIN="$(command -v sunshine || true)"
+  if [ -z "$SUNSHINE_BIN" ]; then
+    log "ERROR: sunshine not found in PATH"
+  else
+    SUNSHINE_LOG_FILE="$HOME/.local/logs/sunshine.log"
+    log "Found Sunshine: $SUNSHINE_BIN, starting after 5s delay"
+    sleep 5
+    systemd-run --user --scope --quiet \
+    --unit=sunshine.scope \
+    --slice=sunshine.slice \
+    "$SUNSHINE_BIN" "$HOME/.config/sunshine/sunshine.conf"
+  fi
+} &
+
+wait "$plasma_scope_pid"
+status=$?
+
+# Stop Sunshine gracefully
+if systemctl --user is-active --quiet sunshine.scope; then
+  log "Stopping Sunshine"
+  systemctl --user stop sunshine.scope
+fi
+
+log "Plasma session ended."
+exit "$status"

ansible/files/game_station/bin/system/steam-launcher

@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+set -euo pipefail
+LOGFILE="$HOME/.local/logs/steam.log"
+export PATH="$HOME/.local/bin:$PATH"
+
+export SDL_VIDEO_MINIMIZE_ON_FOCUS_LOSS=0
+
+# There is no way to set a color space for an NV12
+# buffer in Wayland. And the color management protocol that is
+# meant to let this happen is missing the color range...
+# So just workaround this with an ENV var that Remote Play Together
+# and Gamescope will use for now.
+export GAMESCOPE_NV12_COLORSPACE=k_EStreamColorspace_BT601
+export STEAM_GAMESCOPE_HDR_SUPPORTED=1
+
+# Workaround older versions of vkd3d-proton setting this
+# too low (desc.BufferCount), resulting in symptoms that are potentially like
+# swapchain starvation.
+export VKD3D_SWAPCHAIN_LATENCY_FRAMES=3
+
+# Let's try this across the board to see if it breaks anything
+# Helps performance in HZD, Cyberpunk, at least
+# Expose 8 physical cores, instead of 4c/8t
+# export WINE_CPU_TOPOLOGY=8:0,1,2,3,4,5,6,7
+
+# To expose vram info from radv's patch we're including
+export WINEDLLOVERRIDES=dxgi=n
+
+# Disable automatic audio device switching in steam, now handled by wireplumber
+export STEAM_DISABLE_AUDIO_DEVICE_SWITCHING=1
+
+# Enable support for xwayland isolation per-game in Steam
+# export STEAM_MULTIPLE_XWAYLANDS=0
+
+# We have NIS support
+export STEAM_GAMESCOPE_NIS_SUPPORTED=1
+
+# Enable tearing controls in steam
+export STEAM_GAMESCOPE_TEARING_SUPPORTED=1
+
+# Enable VRR controls in steam
+export STEAM_GAMESCOPE_VRR_SUPPORTED=1
+
+# When set to 1, a toggle will show up in the steamui to control whether dynamic refresh rate is applied to the steamui
+export STEAM_GAMESCOPE_DYNAMIC_REFRESH_IN_STEAM_SUPPORTED=0
+
+# Don't wait for buffers to idle on the client side before sending them to gamescope
+export vk_xwayland_wait_ready=false
+
+# Scaling support
+export STEAM_GAMESCOPE_FANCY_SCALING_SUPPORT=1
+
+# Color management support
+export STEAM_GAMESCOPE_COLOR_MANAGED=1
+export STEAM_GAMESCOPE_VIRTUAL_WHITE=1
+
+# Temporary crutch until dummy plane interactions / etc are figured out
+# export GAMESCOPE_DISABLE_ASYNC_FLIPS=1
+
+# Use Vulkan for the Steam Overlay
+export STEAM_OVERLAY_RENDERER=vk
+
+# Set input method modules for Qt/GTK that will show the Steam keyboard
+export QT_IM_MODULE=steam
+export GTK_IM_MODULE=Steam
+
+export XCURSOR_THEME=steam
+export XCURSOR_SCALE=256
+
+export GAMESCOPE_WSI=0
+export STEAM_USE_PW_CAPTURE=0               # Deck prefers gamescope capture path
+export STEAM_DISABLE_GAMESCOPE_LAUNCH=0     # allow gamescope handling
+
+log() { echo "[$(date --iso-8601=seconds)] $*" | tee -a "$LOGFILE"; }
+
+mkdir -p $XDG_RUNTIME_DIR
+chmod 700 $XDG_RUNTIME_DIR
+
+if [ -r /proc/sys/user/max_user_namespaces ]; then
+  max_ns="$(cat /proc/sys/user/max_user_namespaces || echo 0)"
+  if [ "${max_ns:-0}" -eq 0 ]; then
+    log "ERROR: user.max_user_namespaces is 0. Enable it with: sudo sysctl -w user.max_user_namespaces=15000"
+    exit 1
+  fi
+fi
+
+if command -v bwrap >/dev/null 2>&1; then
+  bwrap_bin="$(command -v bwrap)"
+  if ! [ -u "$bwrap_bin" ]; then
+    # Not fatal if user namespaces are enabled, but warn for visibility
+    log "WARN: $bwrap_bin is not setuid root. If Steam still fails, run: sudo dnf reinstall bubblewrap && sudo chmod 4755 /usr/bin/bwrap"
+  fi
+else
+  log "WARN: bubblewrap (bwrap) not found. Steam may fail to start."
+fi
+
+if findmnt -no OPTIONS / | grep -qw nosuid; then
+  log "WARN: root filesystem is mounted nosuid. Setuid helpers like bubblewrap will not work."
+fi
+
+STEAM_BIN="$(command -v steam || true)"
+if [ -z "$STEAM_BIN" ]; then
+  log "ERROR: steam not found in PATH"
+  exit 127
+fi
+
+log "Updating steam: /opt/steamcmd/steamcmd.sh +quit"
+/opt/steamcmd/steamcmd.sh +quit >>"$LOGFILE" 2>&1 || {
+  log "ERROR: Steam update failed"
+  exit 1
+}
+
+log "Launching Steam"
+STEAM_ARGS=("-steamdeck" "-steamos3" "-gamepadui")
+systemd-run --user --scope --quiet \
+  --unit=steam.scope \
+  --slice=steam.slice \
+  "$STEAM_BIN" "${STEAM_ARGS[@]}" >>"$LOGFILE" 2>&1 &
+steam_scope_pid=$!
+log "Steam started with pid $steam_scope_pid"
+
+{
+  SUNSHINE_BIN="$(command -v sunshine || true)"
+  if [ -z "$SUNSHINE_BIN" ]; then
+    log "ERROR: sunshine not found in PATH"
+  else
+    SUNSHINE_LOG_FILE="$HOME/.local/logs/sunshine.log"
+    log "Found Sunshine: $SUNSHINE_BIN, starting after 5s delay"
+    sleep 5
+    systemd-run --user --scope --quiet \
+    --unit=sunshine.scope \
+    --slice=sunshine.slice \
+    "$SUNSHINE_BIN" "$HOME/.config/sunshine/sunshine.conf"
+  fi
+} &
+
+# Wait for Steam to exit
+wait "$steam_scope_pid"
+status=$?
+
+# Stop Sunshine gracefully
+if systemctl --user is-active --quiet sunshine.scope; then
+  log "Stopping Sunshine"
+  systemctl --user stop sunshine.scope
+fi
+
+log "Steam session ended."
+exit "$status"

ansible/files/game_station/bin/system/steam-short-session-tracker

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+short_session_tracker_file="/tmp/steamos-short-session-tracker"
+short_session_start_file="/tmp/steamos-short-session-start"
+short_session_duration=60
+short_session_count_before_reset=3
+
+do_repair() {
+  # This should really be part of the steam launcher as `steam --repair` or similar, but we presently only have
+  # `steam --reset` which is too heavy of a hammer.
+
+  echo >&2 "steam-short-session-tracker: Re-bootstrapping steam from OS copy."
+  # at this point, might as well make sure we won't relaunch on a sideloaded build
+  rm -f -- "$HOME/devkit-game/devkit-steam"
+  mkdir -p ~/.local/share/Steam
+  # remove some caches and stateful things known to cause Steam to fail to start if corrupt
+  rm -rf --one-file-system ~/.local/share/Steam/config/widevine
+  # cleanup the steam config dir, i.e. ~/.steam. We should try to preserve registry.vdf if possible
+  steam_config_dir="$HOME/.steam"
+  steam_config_backup_dir="$HOME/dot-steam.bak.$(date +%s)"
+  registry_vdf="$steam_config_dir/registry.vdf"
+  registry_backup_vdf="$steam_config_backup_dir/.steam/registry.vdf"
+  mv "$steam_config_dir" "$steam_config_backup_dir"
+  mkdir -p "$steam_config_dir"
+  cp -f "$registry_backup_vdf" "$registry_vdf" || true
+  # restore clean copy of binaries from RO partition
+  tar xf /usr/lib/steam/bootstraplinux_ubuntu12_32.tar.xz -C ~/.local/share/Steam
+  # rearm
+  rm "$short_session_tracker_file"
+}
+
+handle_started() {
+  short_session_count=$(< "$short_session_tracker_file" wc -l)
+  touch $short_session_start_file
+
+  if [[ "$short_session_count" -ge "$short_session_count_before_reset" ]]; then
+    echo >&2 "steam-short-session-tracker: Steam failed to start $short_session_count_before_reset times within $short_session_duration seconds"
+
+    if [ -f "$HOME/.config/inhibit-short-session-tracker" ]; then
+      echo >&2 "steam-short-session-tracker: Auto-repair disabled by config. Remove ~/.config/inhibit-short-session-tracker to re-enable"
+      return
+    fi
+
+    do_repair
+  fi
+}
+
+handle_stopped() {
+  seconds_since_started=$(($(date +%s) - $(date +%s -r "$short_session_start_file")))
+
+  if [[ "seconds_since_started" -lt "$short_session_duration" ]]; then
+    echo "frog" >> "$short_session_tracker_file"
+  else
+    rm "$short_session_tracker_file"
+  fi
+}
+
+case "$1" in
+  --track-started)
+    echo "steam-short-session-tracker: Tracking Steam start (unimplemented)"
+    # handle_started
+    ;;
+  --track-stopped)
+    echo "steam-short-session-tracker: Tracking Steam stop (unimplemented)"
+    # handle_stopped
+    ;;
+  --repair-now)
+    echo "steam-short-session-tracker: Performing immediate repair (unimplemented)"
+    # do_repair
+    ;;
+  *)
+    echo "Usage: $0 --track-started | --track-stopped | --repair-now"
+    exit 1
+    ;;
+esac
+
+exit 0

ansible/files/game_station/bin/system/steamos

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -euo pipefail
+LOG_FILE="$HOME/.local/logs/session.log"
+
+log() { echo "[$(date --iso-8601=seconds)] $*" | tee -a "$LOG_FILE"; }
+
+if [[ -f "$HOME/.desktop-session-plasma" ]]; then
+  log "Plasma session selected."
+  rm "$HOME/.desktop-session-plasma"
+  exec /usr/bin/startplasma-wayland
+else
+  log "Gamescope session selected."
+  exec /usr/local/bin/launch-gamescope 
+fi

ansible/files/game_station/bin/system/steamos-select-branch

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -euo pipefail
+LOGFILE="$HOME/.local/logs/steamos-select-branch.log"
+
+log() { echo "[$(date --iso-8601=seconds)] $*" | tee -a "$LOGFILE"; }
+
+if [[ $# -eq 1 ]]; then
+  case "$1" in
+    "-c")
+      log "Current branch requested"
+      echo "stable"
+      exit 0
+      ;;
+    "-l")
+      log "Listing available branches"
+      exit 0
+      ;;
+    "rel")
+      log "Legacy branch 'rel' requested"
+      exit 0
+      ;;
+    "stable" | "rc" | "beta" | "bc" | "preview" | "pc" | "main" | "staging")
+      log "Branch '$1' requested"      
+      exit 0
+      ;;
+  esac
+fi
+
+echo "Usage: steamos-select-branch <-c|-l|rel|rc|beta|bc|preview|pc|main>" 1>&2
+    

ansible/files/game_station/bin/system/steamos-session

@@ -0,0 +1,26 @@
+# #!/bin/bash
+set -euo pipefail
+LOG_FILE="$HOME/.local/logs/session.log"
+
+log() { echo "[$(date --iso-8601=seconds)] $*" | tee -a "$LOG_FILE"; }
+
+env > $XDG_RUNTIME_DIR/steamos-environment
+log "Starting SteamOS Session"
+while true; do
+    systemctl --user --wait start steamos.service
+    wait_status=$?
+    log "Graphical session service exited with status $wait_status"
+    if [ $wait_status -ne 0 ]; then
+        log "Graphical session service exited with error; ending loop."
+        break
+    fi
+    if [ -f "$HOME/.steamos-session-switch" ]; then
+        log "Graphical session service exited; switch file present -> restarting..."
+        rm "$HOME/.steamos-session-switch"
+        continue
+    fi
+    log "Graphical session service exited; switch file absent -> ending loop."
+    break
+done
+sleep 5
+log "SteamOS Session Ended"

ansible/files/game_station/bin/system/steamos-session-select

@@ -0,0 +1,39 @@
+#!/bin/bash
+set -euo pipefail
+LOGFILE="$HOME/.local/logs/session.log"
+
+log() { echo "[$(date --iso-8601=seconds)] $*" | tee -a "$LOGFILE"; }
+
+die() { echo >&2 "!! $*"; exit 1; }
+
+# File this script will modify
+CONF_FILE="/etc/sddm.conf.d/zz-steamos-autologin.conf"
+
+session="${1:-gamescope}"
+
+case "$session" in
+  plasma*)
+    log "Plasma session selected."
+    touch "$HOME/.desktop-session-plasma"
+  ;;
+  gamescope)
+    log "Gamescope session selected."
+    if [[ -f "$HOME/.desktop-session-plasma" ]]; then
+      rm "$HOME/.desktop-session-plasma"
+    fi
+  ;;
+  *)
+    log "!! Unrecognized session '$session'"
+    exit 1
+  ;;
+esac
+
+log "Signaling session switch..."
+touch "$HOME/.steamos-session-switch"
+
+# It is possible to get things remaining from the previous session if SDDM is sigkilled by root systemd
+# and our units are still running in the user systemd
+systemctl --user stop gamescope-session.target
+systemctl --user stop plasma-workspace.target
+systemctl --user reset-failed
+

ansible/files/game_station/bin/system/steamos-update

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -euo pipefail
+LOGFILE="$HOME/.local/logs/steamos-update.log"
+
+log() { echo "[$(date --iso-8601=seconds)] $*" | tee -a "$LOGFILE"; }
+
+log "Starting SteamOS update..."
+log "Arguments passed: ${@}"
+echo "Not applicable for this OS"
+# TODO: Implement SteamOS update logic for now inform no update available
+exit 7

ansible/files/game_station/bin/user/launch_genesis.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+LOGFILE="$HOME/.local/logs/retroarch-genesis.log"
+
+log() { echo "[$(date --iso-8601=seconds)] $*" | tee -a "$LOGFILE"; }
+RETROARCH_HOME="$HOME/.local/share/Steam/steamapps/common/RetroArch"
+if [ ! -d "$RETROARCH_HOME" ]; then
+  log "ERROR: RetroArch directory not found at $RETROARCH_HOME, install it via Steam"
+  exit 127
+fi
+
+RETROARCH_CORE=genesis_plus_gx_libretro.so
+STEAM_BIN=$(command -v steam || true)
+if [ ! -f "$STEAM_BIN" ] || [ ! -f "$RETROARCH_HOME/cores/$RETROARCH_CORE" ]; then
+  log "ERROR: steam or retroarch core not found, install Steam or RetroArch PPSSPP core via Steam"
+  exit 127
+fi
+
+log "Launching RetroArch: $STEAM_BIN -applaunch 1118310 -L \"$RETROARCH_HOME/cores/$RETROARCH_CORE\" \"${@}\""
+"$STEAM_BIN" -applaunch 1118310 -L "$RETROARCH_HOME/cores/$RETROARCH_CORE" "${@}" >>"$LOGFILE"

ansible/files/game_station/bin/user/launch_ppspp.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+LOGFILE="$HOME/.local/logs/retroarch-ppsspp.log"
+
+log() { echo "[$(date --iso-8601=seconds)] $*" | tee -a "$LOGFILE"; }
+RETROARCH_HOME="$HOME/.local/share/Steam/steamapps/common/RetroArch"
+if [ ! -d "$RETROARCH_HOME" ]; then
+  log "ERROR: RetroArch directory not found at $RETROARCH_HOME, install it via Steam"
+  exit 127
+fi
+
+RETROARCH_CORE=ppsspp_libretro.so
+STEAM_BIN=$(command -v steam || true)
+if [ ! -f "$STEAM_BIN" ] || [ ! -f "$RETROARCH_HOME/cores/$RETROARCH_CORE" ]; then
+  log "ERROR: steam or retroarch core not found, install Steam or RetroArch PPSSPP core via Steam"
+  exit 127
+fi
+
+log "Launching RetroArch: $STEAM_BIN -applaunch 1118310 -L \"$RETROARCH_HOME/cores/$RETROARCH_CORE\" \"${@}\""
+"$STEAM_BIN" -applaunch 1118310 -L "$RETROARCH_HOME/cores/$RETROARCH_CORE" "${@}" >>"$LOGFILE"

ansible/files/game_station/bin/user/launch_rpcs3.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -euo pipefail
+LOGFILE="$HOME/.local/logs/rpcs3.log"
+
+log() { echo "[$(date --iso-8601=seconds)] $*" | tee -a "$LOGFILE"; }
+
+RPCS_BIN="$(command -v rpcs3 || true)"
+if [ -z "$RPCS_BIN" ]; then
+  log "ERROR: rpcs3 not found in PATH"
+  exit 127
+fi
+
+log "Launching RPCS3: $RPCS_BIN \"${@}\""
+"$RPCS_BIN" --no-gui --fullscreen "${@}" >>"$LOGFILE"
+

ansible/files/game_station/bin/user/launch_snes.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+LOGFILE="$HOME/.local/logs/retroarch-snes.log"
+
+log() { echo "[$(date --iso-8601=seconds)] $*" | tee -a "$LOGFILE"; }
+RETROARCH_HOME="$HOME/.local/share/Steam/steamapps/common/RetroArch"
+if [ ! -d "$RETROARCH_HOME" ]; then
+  log "ERROR: RetroArch directory not found at $RETROARCH_HOME, install it via Steam"
+  exit 127
+fi
+
+RETROARCH_CORE=mesen-s_libretro.so
+STEAM_BIN=$(command -v steam || true)
+if [ ! -f "$STEAM_BIN" ] || [ ! -f "$RETROARCH_HOME/cores/$RETROARCH_CORE" ]; then
+  log "ERROR: steam or retroarch core not found, install Steam or RetroArch PPSSPP core via Steam"
+  exit 127
+fi
+
+log "Launching RetroArch: $STEAM_BIN -applaunch 1118310 -L \"$RETROARCH_HOME/cores/$RETROARCH_CORE\" \"${@}\""
+"$STEAM_BIN" -applaunch 1118310 -L "$RETROARCH_HOME/cores/$RETROARCH_CORE" "${@}" >>"$LOGFILE"

ansible/files/game_station/bin/user/update_roms.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+LOGFILE="$HOME/.local/logs/update_roms.log"
+
+log() { echo "[$(date --iso-8601=seconds)] $*" | tee -a "$LOGFILE"; }
+
+ROM_MANAGER_BIN="$(command -v srm || true)"
+if [ -z "$ROM_MANAGER_BIN" ]; then
+  log "ERROR: srm (Steam ROM Manager) not found in PATH"
+  exit 127
+fi
+log "Updating Steam ROM Manager"
+$ROM_MANAGER_BIN add >> "$LOGFILE" 2>&1 || {
+  log "ERROR: Steam ROM Manager update failed"
+  exit 1
+}
+log "Steam ROM Manager update completed successfully, restarting Steam service if running"
+if systemctl --user is-active --quiet steam.scope; then
+  log "Restarting Steam service to apply changes"
+  systemctl --user restart steam.scope
+  log "Steam service restarted"
+else
+  log "Steam service is not running, no need to restart"
+fi

ansible/files/game_station/firmware/aoc28e850.bin

@@ -0,0 +1,5 @@
+AP///////wAhVwkhutc0AQ8iAQOARid4AiiVp1VOoyYPUFQhCADRwIHAAQABAAEAAQABAAEAAjqA
+GHE4LUBYLEUAxI4hAAAeAAAA/QAYVR5kHgAKICAgICAgAAAA/ABIRCBUTyBVU0IKICAgAAAAEAAA
+AAAAAAAAAAAAAAAAAZYCAzBxThAfIiEgBBM+PTxfZAUUIwkHB4MBAABuAwwAEAAAPCAAgAECAwTl
+DmFgZmVqXgCgoKApUDAgJQCwEzIAABgZZACAo6AsULAQNRCwEzIAABgAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAArw==

ansible/files/game_station/gamescope/gamescope.conf

@@ -0,0 +1,5 @@
+OUTPUT_CONNECTOR={{ output_connector }}
+FRAMERATE={{ framerate }}
+RESOLUTION={{ screen_resolution }}
+HDR_ENABLED={{ hdr_enabled }}
+DRM_DEVICE={{ drm_device }}

ansible/files/game_station/sddm.conf/steamos.conf

@@ -0,0 +1,7 @@
+[General]
+DisplayServer=wayland
+
+[Autologin]
+Relogin=true
+Session=steamos.desktop
+User={{ user_name }}

ansible/files/game_station/shortcuts/alvr.desktop

@@ -0,0 +1,8 @@
+[Desktop Entry]
+Name=ALVR
+Comment=Linux Virtual Reality
+Exec=/opt/alvr/bin/alvr_dashboard
+Icon=alvr.png
+Terminal=false
+Type=Application
+Categories=Game;

ansible/files/game_station/shortcuts/steamos.desktop

@@ -0,0 +1,8 @@
+[Desktop Entry]
+Name=SteamOS
+Comment=Switch to SteamOS 
+Exec=/usr/local/bin/steamos-session-select
+Icon=steamos.png
+Terminal=false
+Type=Application
+Categories=Game;

ansible/files/game_station/srm/userConfigurations.json

@@ -0,0 +1,368 @@
+[
+    {
+        "parserType": "Glob",
+        "configTitle": "Sony PlayStation 3 - RPCS3 (Extracted ISO)",
+        "steamDirectory": "${steamdirglobal}",
+        "romDirectory": "/home/gameuser/.config/rpcs3/games",
+        "steamCategories": [
+            "PS3"
+        ],
+        "executableArgs": "\"${filePath}\"",
+        "executableModifier": "\"${exePath}\"",
+        "startInDirectory": "",
+        "titleModifier": "${fuzzyTitle}",
+        "steamInputEnabled": "1",
+        "imageProviders": [
+            "sgdb"
+        ],
+        "onlineImageQueries": [
+            "${fuzzyTitle}"
+        ],
+        "imagePool": "${fuzzyTitle}",
+        "drmProtect": false,
+        "userAccounts": {
+            "specifiedAccounts": [
+                "Global"
+            ]
+        },
+        "parserInputs": {
+            "glob": "${title}/PS3_GAME/USRDIR/@(eboot.bin|EBOOT.BIN)"
+        },
+        "executable": {
+            "path": "/home/gameuser/.local/bin/launch_rpcs3.sh",
+            "shortcutPassthrough": false,
+            "appendArgsToExecutable": true
+        },
+        "titleFromVariable": {
+            "limitToGroups": [
+                "PS3"
+            ],
+            "caseInsensitiveVariables": false,
+            "skipFileIfVariableWasNotFound": false
+        },
+        "fuzzyMatch": {
+            "replaceDiacritics": true,
+            "removeCharacters": true,
+            "removeBrackets": true
+        },
+        "controllers": {
+            "ps4": null,
+            "ps5": null,
+            "ps5_edge": null,
+            "xbox360": null,
+            "xboxone": null,
+            "xboxelite": null,
+            "switch_joycon_left": null,
+            "switch_joycon_right": null,
+            "switch_pro": null,
+            "neptune": null,
+            "steamcontroller_gordon": null
+        },
+        "imageProviderAPIs": {
+            "sgdb": {
+                "nsfw": false,
+                "humor": false,
+                "styles": [],
+                "stylesHero": [],
+                "stylesLogo": [],
+                "stylesIcon": [],
+                "imageMotionTypes": [
+                    "static"
+                ],
+                "sizes": [],
+                "sizesHero": [],
+                "sizesIcon": []
+            }
+        },
+        "defaultImage": {
+            "tall": "",
+            "long": "",
+            "hero": "",
+            "logo": "",
+            "icon": ""
+        },
+        "localImages": {
+            "tall": "",
+            "long": "",
+            "hero": "",
+            "logo": "",
+            "icon": ""
+        },
+        "parserId": "176098777083347867",
+        "disabled": false,
+        "version": 25
+    },
+    {
+        "parserType": "Glob",
+        "configTitle": "Sega Genesis/Mega Drive - Retroarch - Genesis Plus GX",
+        "steamDirectory": "${steamdirglobal}",
+        "romDirectory": "/home/gameuser/Emulation/genesis",
+        "steamCategories": [
+            "Genesis/Mega Drive"
+        ],
+        "executableArgs": "\"${filePath}\"",
+        "executableModifier": "\"${exePath}\"",
+        "startInDirectory": "",
+        "titleModifier": "${fuzzyTitle}",
+        "steamInputEnabled": "1",
+        "imageProviders": [
+            "sgdb"
+        ],
+        "onlineImageQueries": [
+            "${fuzzyTitle}"
+        ],
+        "imagePool": "${fuzzyTitle}",
+        "drmProtect": false,
+        "userAccounts": {
+            "specifiedAccounts": [
+                "Global"
+            ]
+        },
+        "parserInputs": {
+            "glob": "${title}@(.7z|.7Z|.gen|.GEN|.md|.MD|.smd|.SMD|.zip|.ZIP|.bin|.BIN)"
+        },
+        "executable": {
+            "path": "/home/gameuser/.local/bin/launch_genesis.sh",
+            "shortcutPassthrough": false,
+            "appendArgsToExecutable": true
+        },
+        "titleFromVariable": {
+            "limitToGroups": [],
+            "caseInsensitiveVariables": false,
+            "skipFileIfVariableWasNotFound": false
+        },
+        "fuzzyMatch": {
+            "replaceDiacritics": true,
+            "removeCharacters": true,
+            "removeBrackets": true
+        },
+        "controllers": {
+            "ps4": null,
+            "ps5": null,
+            "ps5_edge": null,
+            "xbox360": null,
+            "xboxone": null,
+            "xboxelite": null,
+            "switch_joycon_left": null,
+            "switch_joycon_right": null,
+            "switch_pro": null,
+            "neptune": null,
+            "steamcontroller_gordon": null
+        },
+        "imageProviderAPIs": {
+            "sgdb": {
+                "nsfw": false,
+                "humor": false,
+                "styles": [],
+                "stylesHero": [],
+                "stylesLogo": [],
+                "stylesIcon": [],
+                "imageMotionTypes": [
+                    "static"
+                ],
+                "sizes": [],
+                "sizesHero": [],
+                "sizesIcon": []
+            }
+        },
+        "defaultImage": {
+            "tall": "",
+            "long": "",
+            "hero": "",
+            "logo": "",
+            "icon": ""
+        },
+        "localImages": {
+            "tall": "",
+            "long": "",
+            "hero": "",
+            "logo": "",
+            "icon": ""
+        },
+        "parserId": "176029625316137679",
+        "disabled": false,
+        "version": 25
+    },
+    {
+        "parserType": "Glob",
+        "configTitle": "Sony PlayStation Portable - Retroarch - PPSSPP",
+        "steamDirectory": "${steamdirglobal}",
+        "romDirectory": "/home/gameuser/Emulation/psp",
+        "steamCategories": [
+            "PSP"
+        ],
+        "executableArgs": "\"${filePath}\"",
+        "executableModifier": "\"${exePath}\"",
+        "startInDirectory": "",
+        "titleModifier": "${fuzzyTitle}",
+        "steamInputEnabled": "1",
+        "imageProviders": [
+            "sgdb"
+        ],
+        "onlineImageQueries": [
+            "${fuzzyTitle}"
+        ],
+        "imagePool": "${fuzzyTitle}",
+        "drmProtect": false,
+        "userAccounts": {
+            "specifiedAccounts": [
+                "Global"
+            ]
+        },
+        "parserInputs": {
+            "glob": "${title}@(.7z|.7Z|.elf|.ELF|.cso|.CSO|.iso|.ISO|.pbp|.PBP|.prx|.PRX)"
+        },
+        "executable": {
+            "path": "/home/gameuser/.local/bin/launch_ppspp.sh",
+            "shortcutPassthrough": false,
+            "appendArgsToExecutable": true
+        },
+        "titleFromVariable": {
+            "limitToGroups": [],
+            "caseInsensitiveVariables": false,
+            "skipFileIfVariableWasNotFound": false
+        },
+        "fuzzyMatch": {
+            "replaceDiacritics": true,
+            "removeCharacters": true,
+            "removeBrackets": true
+        },
+        "controllers": {
+            "ps4": null,
+            "ps5": null,
+            "ps5_edge": null,
+            "xbox360": null,
+            "xboxone": null,
+            "xboxelite": null,
+            "switch_joycon_left": null,
+            "switch_joycon_right": null,
+            "switch_pro": null,
+            "neptune": null,
+            "steamcontroller_gordon": null
+        },
+        "imageProviderAPIs": {
+            "sgdb": {
+                "nsfw": false,
+                "humor": false,
+                "styles": [],
+                "stylesHero": [],
+                "stylesLogo": [],
+                "stylesIcon": [],
+                "imageMotionTypes": [
+                    "static"
+                ],
+                "sizes": [],
+                "sizesHero": [],
+                "sizesIcon": []
+            }
+        },
+        "defaultImage": {
+            "tall": "",
+            "long": "",
+            "hero": "",
+            "logo": "",
+            "icon": ""
+        },
+        "localImages": {
+            "tall": "",
+            "long": "",
+            "hero": "",
+            "logo": "",
+            "icon": ""
+        },
+        "parserId": "176029685701892152",
+        "disabled": false,
+        "version": 25
+    },
+    {
+        "parserType": "Glob",
+        "configTitle": "Nintendo SNES - Retroarch - Mesen-S",
+        "steamDirectory": "${steamdirglobal}",
+        "romDirectory": "/home/gameuser/Emulation/snes",
+        "steamCategories": [
+            "SNES"
+        ],
+        "executableArgs": "\"${filePath}\"",
+        "executableModifier": "\"${exePath}\"",
+        "startInDirectory": "",
+        "titleModifier": "${fuzzyTitle}",
+        "steamInputEnabled": "1",
+        "imageProviders": [
+            "sgdb"
+        ],
+        "onlineImageQueries": [
+            "${fuzzyTitle}"
+        ],
+        "imagePool": "${fuzzyTitle}",
+        "drmProtect": false,
+        "userAccounts": {
+            "specifiedAccounts": [
+                "Global"
+            ]
+        },
+        "parserInputs": {
+            "glob": "${title}@(.7z|.7Z|.bml|.BML|.sfc|.SFC|.smc|.SMC|.zip|.ZIP)"
+        },
+        "executable": {
+            "path": "/home/gameuser/.local/bin/launch_snes.sh",
+            "shortcutPassthrough": false,
+            "appendArgsToExecutable": true
+        },
+        "titleFromVariable": {
+            "limitToGroups": [],
+            "caseInsensitiveVariables": false,
+            "skipFileIfVariableWasNotFound": false
+        },
+        "fuzzyMatch": {
+            "replaceDiacritics": true,
+            "removeCharacters": true,
+            "removeBrackets": true
+        },
+        "controllers": {
+            "ps4": null,
+            "ps5": null,
+            "ps5_edge": null,
+            "xbox360": null,
+            "xboxone": null,
+            "xboxelite": null,
+            "switch_joycon_left": null,
+            "switch_joycon_right": null,
+            "switch_pro": null,
+            "neptune": null,
+            "steamcontroller_gordon": null
+        },
+        "imageProviderAPIs": {
+            "sgdb": {
+                "nsfw": false,
+                "humor": false,
+                "styles": [],
+                "stylesHero": [],
+                "stylesLogo": [],
+                "stylesIcon": [],
+                "imageMotionTypes": [
+                    "static"
+                ],
+                "sizes": [],
+                "sizesHero": [],
+                "sizesIcon": []
+            }
+        },
+        "defaultImage": {
+            "tall": "",
+            "long": "",
+            "hero": "",
+            "logo": "",
+            "icon": ""
+        },
+        "localImages": {
+            "tall": "",
+            "long": "",
+            "hero": "",
+            "logo": "",
+            "icon": ""
+        },
+        "parserId": "176029713768938935",
+        "disabled": false,
+        "version": 25
+    }
+]

ansible/files/game_station/sunshine/apps.json

@@ -0,0 +1,11 @@
+{
+    "env": {
+        "PATH": "$(PATH)"
+    },
+    "apps": [
+        {
+            "name": "SteamDeck",
+            "image-path": "steam.png"
+        }
+    ]
+}

ansible/files/game_station/sunshine/sunshine.conf

@@ -0,0 +1,2 @@
+system_tray=disable
+log_path = /home/{{ user_name }}/.local/logs/sunshine.log

ansible/files/game_station/systemd/user/gamescope-mangoapp.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=mangoapp
+After=graphical-session.target
+PartOf=graphical-session.target
+
+[Service]
+Type=exec
+ExecStart=/usr/bin/mangoapp
+Restart=always
+EnvironmentFile=%t/gamescope-environment
+TimeoutStopSec=5

ansible/files/game_station/systemd/user/gamescope-session.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=Gamescope Session
+Before=graphical-session.target
+PartOf=graphical-session.target
+Wants=graphical-session-pre.target
+After=graphical-session-pre.target
+RefuseManualStart=yes
+
+[Service]
+TimeoutStartSec=5
+TimeoutStopSec=10
+ExecStart=gamescope-session
+Type=notify
+NotifyAccess=all
+# Make Steam's srt-logger write to the journal with it's own prefixes
+Environment=SRT_LOG_TO_JOURNAL=1
+Slice=session.slice

ansible/files/game_station/systemd/user/gamescope-session.target

@@ -0,0 +1,13 @@
+[Unit]
+Description=Gamescope Session
+Requires=graphical-session.target
+BindsTo=graphical-session.target
+After=graphical-session.target
+
+Requires=gamescope-session.service
+BindsTo=gamescope-session.service
+
+Upholds=steam-launcher.service
+
+Wants=ibus-gamescope.service
+Wants=gamescope-mangoapp.service

ansible/files/game_station/systemd/user/ibus-gamescope.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Wrapper for Ibus
+PartOf=graphical-session.target
+After=graphical-session.target
+
+[Service]
+Type=dbus
+ExecStart=/usr/bin/ibus-daemon -r --panel=disable --emoji-extension=disable
+BusName=org.freedesktop.IBus
+EnvironmentFile=%t/gamescope-environment
+Slice=session.slice
+TimeoutStopSec=5

ansible/files/game_station/systemd/user/session.slice

@@ -0,0 +1,16 @@
+#  SPDX-License-Identifier: LGPL-2.1-or-later
+#
+#  This file is part of systemd.
+#
+#  systemd is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU Lesser General Public License as published by
+#  the Free Software Foundation; either version 2.1 of the License, or
+#  (at your option) any later version.
+
+[Unit]
+Description=User Core Session Slice
+Documentation=man:systemd.special(7)
+
+[Slice]
+CPUWeight=100
+# AllowedCPUs={{ general_cores }}

ansible/files/game_station/systemd/user/steam-launcher.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=Steam Launcher
+After=gamescope-session.target
+PartOf=gamescope-session.target
+
+[Service]
+ExecStart=steam-launcher
+
+# Track short sessions to trigger a repair after enough failures
+ExecStartPre=steam-short-session-tracker --track-started
+ExecStopPost=steam-short-session-tracker --track-stopped
+
+# To close properly we need to kill the child of the steam wrapper, as the wrapper itself does not forward signals
+ExecStop=/bin/bash -c 'kill -TERM $(pgrep -P $MAINPID || echo $MAINPID)'
+# Disable the term signal sent from systemd, as we we handle it manually above
+# But still want systemd to kill the entire cgroup on timeout
+KillSignal=SIGCONT
+KillMode=mixed
+TimeoutStopSec=60
+
+Type=exec
+EnvironmentFile=%t/gamescope-environment

ansible/files/game_station/systemd/user/steam.slice

@@ -0,0 +1,16 @@
+#  SPDX-License-Identifier: LGPL-2.1-or-later
+#
+#  This file is part of systemd.
+#
+#  systemd is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU Lesser General Public License as published by
+#  the Free Software Foundation; either version 2.1 of the License, or
+#  (at your option) any later version.
+
+[Unit]
+Description=User Core Session Slice
+Documentation=man:systemd.special(7)
+
+[Slice]
+CPUWeight=100
+# AllowedCPUs={{ general_cores }}

ansible/files/game_station/systemd/user/steamos.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=User graphical session service
+After=graphical-session.target
+
+[Service]
+ExecStart=/usr/local/bin/steamos
+EnvironmentFile=%t/steamos-environment
+Restart=on-failure
+# AllowedCPUs={{ general_cores }}

ansible/files/game_station/systemd/user/sunshine.slice

@@ -0,0 +1,16 @@
+#  SPDX-License-Identifier: LGPL-2.1-or-later
+#
+#  This file is part of systemd.
+#
+#  systemd is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU Lesser General Public License as published by
+#  the Free Software Foundation; either version 2.1 of the License, or
+#  (at your option) any later version.
+
+[Unit]
+Description=User Core Session Slice
+Documentation=man:systemd.special(7)
+
+[Slice]
+CPUWeight=100
+# AllowedCPUs={{ sunshine_cores }}

ansible/files/game_station/wayland-sessions/steamos.desktop

@@ -0,0 +1,8 @@
+[Desktop Entry]
+Encoding=UTF-8
+Name=SteamOS
+Comment=SteamOS graphical session
+Exec=/usr/local/bin/steamos-session
+Icon=steamicon.png
+Type=Application
+DesktopNames=gamescope

ansible/files/keys/localnetwork.pub

@@ -0,0 +1 @@
+ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAHHOPBR9p9kq5Cqzpe4cr3jHnweaYrHPXv5sXNzt+sCXP54uc5rWUBhxW2OQVvQzJ47dEVhEKi4WSC7LcuKS2G5AQDzWXNiasHvYIYQU3F/EknVCZnsiXYqXphYkJA6rJCNRnISZCIC1mocq6PB7J08ONdRFCvjfUBuVRT8BNGXNmQ/zQ==

ansible/files/ollama/gemma4-e4b-32k.modelfile

@@ -0,0 +1,2 @@
+FROM gemma4:e4b
+PARAMETER num_ctx 32359

ansible/files/ollama/gpt-oss-latest-32k.modelfile

@@ -0,0 +1,2 @@
+FROM gpt-oss:latest
+PARAMETER num_ctx 32359

ansible/files/ollama/qwen3.5-4b-32k.modelfile

@@ -0,0 +1,2 @@
+FROM qwen3.5:9b
+PARAMETER num_ctx 32359