diff --git a/ansible/files/scripts/auto_suspend.sh b/ansible/files/scripts/auto_suspend.sh new file mode 100644 index 0000000..6065302 --- /dev/null +++ b/ansible/files/scripts/auto_suspend.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +IDLE_TIMEOUT_MINUTES=20 + +# Check 1: Any logged-in users? +if who | grep -qv "tty"; then + echo "Active user sessions found. Skipping suspend." + exit 0 +fi + +# Check 2: CPU idle percentage +CPU_IDLE=$(mpstat 1 1 | tail -n 1 | awk '/:/ {print $12}' | sed 's/,/./g') +CPU_THRESHOLD=90 + +if (( $(echo "$CPU_IDLE < $CPU_THRESHOLD" | bc -l) )); then + echo "CPU too active. Skipping suspend." + exit 0 +fi + +if ! command -v nvidia-smi &> /dev/null; then + echo "nvidia-smi not found, skipping GPU check" +else + # Check 3: Is GPU idle? + GPU_UTIL=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits | head -n 1) + GPU_THRESHOLD=10 + + if (( GPU_UTIL > GPU_THRESHOLD )); then + echo "GPU is busy (utilization=${GPU_UTIL}%). Skipping suspend." + exit 0 + fi + + # Check 4: Is OLLAMA active? + OLLAMA_ACTIVE=$(nvidia-smi --query-compute-apps=name,used_memory --format=csv,noheader,nounits | grep ollama | wc -l) + if (( OLLAMA_ACTIVE > 0 )); then + echo "OLLMAMA is active. Skipping suspend." + exit 0 + fi +fi + +echo "System idle. Suspending..." +shutdown now diff --git a/ansible/host_vars/dev-01.lab.alexpires.me.yml b/ansible/host_vars/dev-01.lab.alexpires.me.yml index 240ed72..6b6cf2c 100644 --- a/ansible/host_vars/dev-01.lab.alexpires.me.yml +++ b/ansible/host_vars/dev-01.lab.alexpires.me.yml @@ -38,11 +38,13 @@ fw_allowed_ports: - { rule: "allow", port: "16443", proto: "tcp", from: "10.19.4.0/24" } # Kubernetes API (local network) - { rule: "allow", port: "139", proto: "tcp", from: "10.19.4.0/24" } # Samba (local network only) - { rule: "allow", port: "445", proto: "tcp", from: "10.19.4.0/24" } # Samba (local network only) + - { rule: "allow", port: "11434", proto: "tcp", from: "10.19.4.0/24" } # Ollama (local network only) - { rule: "allow", port: "443", proto: "tcp", from: "10.5.5.2/32" } # HTTPS (Contabo VPN) - { rule: "allow", port: "53", proto: "udp", from: "10.19.4.0/24" } # DNS (local network) - { rule: "allow", port: "53", proto: "udp", from: "10.5.5.2/32" } # DNS (Contabo VPN) ufw_outgoing_traffic: + - 9 # For Wake-on-LAN - 22 # SSH - 53 # DNS - 80 # HTTP diff --git a/ansible/playbook_auto_suspend.yml b/ansible/playbook_auto_suspend.yml new file mode 100644 index 0000000..4bdb8de --- /dev/null +++ b/ansible/playbook_auto_suspend.yml @@ -0,0 +1,7 @@ +- name: Setup Auto Suspend + hosts: auto_suspend + + tasks: + - name: Setup Auto Suspend + ansible.builtin.include_tasks: + file: tasks/auto_suspend.yml diff --git a/ansible/tasks/auto_suspend.yml b/ansible/tasks/auto_suspend.yml new file mode 100644 index 0000000..680a54f --- /dev/null +++ b/ansible/tasks/auto_suspend.yml @@ -0,0 +1,129 @@ +- name: Install dependencies + become: true + ansible.builtin.package: + name: "{{ item }}" + state: present + loop: + - sysstat + - bc + +- name: Enable sysstat service + become: true + ansible.builtin.systemd: + name: sysstat + enabled: true + state: started + +- name: Read existing GRUB_CMDLINE_LINUX + ansible.builtin.shell: set -o pipefail && grep '^GRUB_CMDLINE_LINUX=' /etc/default/grub | cut -d= -f2- | tr -d '"' + register: grub_line + changed_when: false + failed_when: grub_line.stdout == "" + +- name: Set fact for existing_cmdline + ansible.builtin.set_fact: + existing_cmdline: "{{ grub_line.stdout }}" + +- name: Set fedora grub file locations + ansible.builtin.set_fact: + grub_config_path_bios: /boot/grub2/grub.cfg + when: ansible_distribution == "Fedora" and ansible_os_family == "RedHat" + +- name: Show current GRUB_CMDLINE_LINUX + ansible.builtin.debug: + msg: "Current GRUB_CMDLINE_LINUX: {{ existing_cmdline }}" + +- name: Updated GRUB_CMDLINE_LINUX + ansible.builtin.debug: + msg: "Current GRUB_CMDLINE_LINUX: {{ existing_cmdline | regex_replace(' *mem_sleep_default=\\S+', '') }} mem_sleep_default=deep" + +- name: Ensure mem_sleep_default=deep is set in GRUB_CMDLINE_LINUX + become: true + vars: + grub_cmdline_linux_line: >- + GRUB_CMDLINE_LINUX="{{ existing_cmdline | regex_replace(' *mem_sleep_default=\\S+', '') }} mem_sleep_default=deep" + ansible.builtin.lineinfile: + path: /etc/default/grub + regexp: '^GRUB_CMDLINE_LINUX=' + line: "{{ grub_cmdline_linux_line }}" + +- name: Generate grub config for BIOS + become: true + changed_when: false + ansible.builtin.command: grub2-mkconfig -o {{ grub_config_path_bios }} + +- name: Create idle check and suspend script + become: true + ansible.builtin.copy: + dest: /usr/local/bin/auto_suspend_script.sh + mode: '0755' + owner: root + group: root + content: "{{ lookup('file', 'scripts/auto_suspend.sh') }}" + +- name: Create systemd service for auto suspend + become: true + ansible.builtin.copy: + dest: /etc/systemd/system/auto-suspend.service + content: | + [Unit] + Description=Check system idleness and suspend if idle + After=multi-user.target + + [Service] + Type=oneshot + ExecStart=/usr/local/bin/auto_suspend_script.sh + owner: root + group: root + mode: '0644' + +- name: Create sleep.conf.d folder if it doesn't exist + become: true + ansible.builtin.file: + path: /etc/systemd/sleep.conf.d + state: directory + owner: root + group: root + mode: '0755' + +- name: Create sleep.conf.d for deep sleep + become: true + ansible.builtin.copy: + dest: /etc/systemd/sleep.conf.d/mem-deep.conf + content: | + [Sleep] + MemorySleepMode=deep + owner: root + group: root + mode: '0644' + +- name: Create systemd timer for auto suspend + become: true + ansible.builtin.copy: + dest: /etc/systemd/system/auto-suspend.timer + content: | + [Unit] + Description=Run idle suspend check every 10 minutes + + [Timer] + OnBootSec=10min + OnUnitActiveSec=10min + Unit=auto-suspend.service + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + +- name: Reload systemd daemon + become: true + ansible.builtin.systemd: + daemon_reload: true + +- name: Enable and start auto-suspend timer + become: true + ansible.builtin.systemd: + name: auto-suspend.timer + enabled: true + state: started diff --git a/inventory/hosts b/inventory/hosts index c76baf6..79383a6 100644 --- a/inventory/hosts +++ b/inventory/hosts @@ -91,4 +91,7 @@ vh-01.lab.alexpires.me gpu-01.lab.alexpires.me [wol] +gpu-01.lab.alexpires.me + +[auto_suspend] gpu-01.lab.alexpires.me \ No newline at end of file diff --git a/terraform/apps/dev-01/apps.tf b/terraform/apps/dev-01/apps.tf index ee46fc1..cd728d7 100644 --- a/terraform/apps/dev-01/apps.tf +++ b/terraform/apps/dev-01/apps.tf @@ -3,6 +3,7 @@ module "open-webui" { persistent_folder = local.persistent_folder fqdn = local.open_webui_fqdn + ollama_proxy = local.ollama_proxy } module "samba" { diff --git a/terraform/apps/dev-01/config.tf b/terraform/apps/dev-01/config.tf index 65b4be0..3aea57b 100644 --- a/terraform/apps/dev-01/config.tf +++ b/terraform/apps/dev-01/config.tf @@ -45,7 +45,7 @@ locals { { name = "ollama" type = "CNAME" - content = "gpu-01.${local.lab_domain}." + content = "dev-01.${local.lab_domain}." }, { name = "kokoro" @@ -103,6 +103,14 @@ locals { } } + ollama_proxy = { + enabled = true + mac = "a0:8c:fd:e2:3c:fa" + ip = "10.19.4.106" + port = "11434" + scheme = "http" + } + scaleway_smtp_host = "smtp.tem.scw.cloud" nextcloud_fqdn = "cloud.alexpires.me" diff --git a/terraform/modules/apps/comfyui/config.tf b/terraform/modules/apps/comfyui/config.tf deleted file mode 100644 index bd119c6..0000000 --- a/terraform/modules/apps/comfyui/config.tf +++ /dev/null @@ -1,4 +0,0 @@ -locals { - comfyui_version = "frontend${var.tag}" -} - diff --git a/terraform/modules/apps/comfyui/main.tf b/terraform/modules/apps/comfyui/main.tf deleted file mode 100644 index e459a6e..0000000 --- a/terraform/modules/apps/comfyui/main.tf +++ /dev/null @@ -1,98 +0,0 @@ -resource "kubernetes_namespace" "comfyui" { - metadata { - name = "comfyui" - } -} - -resource "kubernetes_service_account" "comfyui" { - - metadata { - name = "comfyui-sa" - namespace = kubernetes_namespace.comfyui.metadata[0].name - } - - automount_service_account_token = false -} - -resource "kubernetes_deployment" "comfyui" { - metadata { - name = "comfyui" - namespace = kubernetes_namespace.comfyui.metadata[0].name - labels = { - comfyui = "comfyui" - } - } - - spec { - replicas = 1 - - selector { - match_labels = { - comfyui = "comfyui" - } - } - - strategy { - type = "RollingUpdate" - } - - template { - metadata { - labels = { - comfyui = "comfyui" - } - } - - spec { - service_account_name = kubernetes_service_account.comfyui.metadata[0].name - automount_service_account_token = false - - container { - name = "comfyui" - image = "a13labs/comfyui:${local.comfyui_version}" - - - security_context { - allow_privilege_escalation = false - } - - port { - name = "frontend" - container_port = 5173 - protocol = "TCP" - } - - env { - name = "DEV_SERVER_COMFYUI_URL" - value = coalesce(var.backend, "http://localhost:8188") - } - } - } - } - } -} - -resource "kubernetes_service" "comfyui" { - metadata { - name = "frontend" - namespace = kubernetes_namespace.comfyui.metadata[0].name - } - - spec { - selector = { - comfyui = "comfyui" - } - - port { - name = "frontend" - port = 5173 - target_port = 5173 - } - } - - lifecycle { - ignore_changes = [ - metadata[0].annotations - ] - } -} diff --git a/terraform/modules/apps/comfyui/provider.tf b/terraform/modules/apps/comfyui/provider.tf deleted file mode 100644 index 2eaa508..0000000 --- a/terraform/modules/apps/comfyui/provider.tf +++ /dev/null @@ -1,9 +0,0 @@ -terraform { - required_version = "~>1.8" - required_providers { - kubernetes = { - source = "hashicorp/kubernetes" - version = "2.36.0" - } - } -} diff --git a/terraform/modules/apps/comfyui/variables.tf b/terraform/modules/apps/comfyui/variables.tf deleted file mode 100644 index 3385ab3..0000000 --- a/terraform/modules/apps/comfyui/variables.tf +++ /dev/null @@ -1,11 +0,0 @@ -variable "tag" { - description = "The version of app to install" - type = string - default = "v0.0.1" -} - -variable "backend" { - description = "The backend to use for ComfyUI" - type = string - default = "" -} diff --git a/terraform/modules/apps/open-webui/ingress.tf b/terraform/modules/apps/open-webui/ingress.tf new file mode 100644 index 0000000..bfd99d8 --- /dev/null +++ b/terraform/modules/apps/open-webui/ingress.tf @@ -0,0 +1,34 @@ +resource "kubernetes_ingress_v1" "openwebui" { + metadata { + name = "ingress" + namespace = kubernetes_namespace.openwebui.metadata[0].name + + annotations = { + "kubernetes.io/ingress.class" = "public" + "cert-manager.io/cluster-issuer" = "letsencrypt-prod" + "kubernetes.io/tls-acme" = "true" + "nginx.ingress.kubernetes.io/ssl-redirect" = "true" + } + } + spec { + tls { + hosts = [var.fqdn] + secret_name = "openwebui-tls" + } + rule { + host = var.fqdn + http { + path { + backend { + service { + name = "http" + port { + number = 8080 + } + } + } + } + } + } + } +} diff --git a/terraform/modules/apps/open-webui/main.tf b/terraform/modules/apps/open-webui/main.tf index e426579..d6bc9c5 100644 --- a/terraform/modules/apps/open-webui/main.tf +++ b/terraform/modules/apps/open-webui/main.tf @@ -3,146 +3,3 @@ resource "kubernetes_namespace" "openwebui" { name = "openwebui" } } - -resource "kubernetes_service_account" "openwebui" { - - metadata { - name = "openwebui-sa" - namespace = kubernetes_namespace.openwebui.metadata[0].name - } - - automount_service_account_token = false -} -resource "kubernetes_deployment" "openwebui" { - metadata { - name = "openwebui" - namespace = kubernetes_namespace.openwebui.metadata[0].name - labels = { - openwebui = "openwebui" - } - } - - spec { - replicas = 1 - - selector { - match_labels = { - openwebui = "openwebui" - } - } - - strategy { - type = "RollingUpdate" - } - - template { - metadata { - labels = { - openwebui = "openwebui" - } - } - - spec { - service_account_name = kubernetes_service_account.openwebui.metadata[0].name - automount_service_account_token = false - - container { - name = "openwebui-container" - image = "ghcr.io/open-webui/open-webui:${var.tag}" - - security_context { - allow_privilege_escalation = false - } - - port { - name = "http" - container_port = 8080 - } - - readiness_probe { - tcp_socket { - port = 8080 - } - initial_delay_seconds = 5 - period_seconds = 10 - failure_threshold = 10 - } - - volume_mount { - name = "openwebui-data" - mount_path = "/app/backend/data" - } - } - - volume { - name = "openwebui-data" - host_path { - path = local.openwebui_data - } - } - } - } - } -} - -resource "kubernetes_service" "openwebui" { - metadata { - name = "http" - namespace = kubernetes_namespace.openwebui.metadata[0].name - } - - spec { - selector = { - openwebui = "openwebui" - } - - port { - name = "http" - port = 8080 - target_port = 8080 - } - - cluster_ip = "None" - } - - lifecycle { - ignore_changes = [ - metadata[0].annotations - ] - } -} - -resource "kubernetes_ingress_v1" "openwebui" { - metadata { - name = "ingress" - namespace = kubernetes_namespace.openwebui.metadata[0].name - - annotations = { - "kubernetes.io/ingress.class" = "public" - "cert-manager.io/cluster-issuer" = "letsencrypt-prod" - "kubernetes.io/tls-acme" = "true" - "nginx.ingress.kubernetes.io/ssl-redirect" = "true" - } - } - spec { - tls { - hosts = [var.fqdn] - secret_name = "openwebui-tls" - } - rule { - host = var.fqdn - http { - path { - backend { - service { - name = "http" - port { - number = 8080 - } - } - } - } - } - } - } -} diff --git a/terraform/modules/apps/open-webui/ollama_proxy.tf b/terraform/modules/apps/open-webui/ollama_proxy.tf new file mode 100644 index 0000000..0bc7b88 --- /dev/null +++ b/terraform/modules/apps/open-webui/ollama_proxy.tf @@ -0,0 +1,101 @@ +resource "kubernetes_service_account" "ollama_proxy" { + + count = var.ollama_proxy.enabled ? 1 : 0 + metadata { + name = "ollama-proxy-sa" + namespace = kubernetes_namespace.openwebui.metadata[0].name + } + + automount_service_account_token = false +} + +resource "kubernetes_deployment" "ollama_proxy" { + + count = var.ollama_proxy.enabled ? 1 : 0 + metadata { + name = "ollama-proxy" + namespace = kubernetes_namespace.openwebui.metadata[0].name + labels = { + app = "ollama-proxy" + } + } + + spec { + replicas = 1 + selector { + match_labels = { + app = "ollama-proxy" + } + } + + strategy { + type = "Recreate" + } + + template { + metadata { + labels = { + app = "ollama-proxy" + } + } + + spec { + service_account_name = kubernetes_service_account.openwebui.metadata[0].name + automount_service_account_token = false + + host_network = true + container { + name = "app" + image = "a13labs/wol_proxy:v0.4.0" + image_pull_policy = "Always" + + security_context { + capabilities { + add = ["NET_RAW"] + } + } + + port { + name = "http" + container_port = var.ollama_proxy.port + } + + env { + name = "UPSTREAM_MAC" + value = var.ollama_proxy.mac + } + + env { + name = "UPSTREAM_IP" + value = var.ollama_proxy.ip + } + + env { + name = "UPSTREAM_PORT" + value = var.ollama_proxy.port + } + + env { + name = "UPSTREAM_SCHEME" + value = var.ollama_proxy.scheme + } + + env { + name = "PROXY_PORT" + value = var.ollama_proxy.port + } + + readiness_probe { + tcp_socket { + port = 11434 + } + initial_delay_seconds = 5 + period_seconds = 10 + failure_threshold = 10 + } + } + } + } + } +} + diff --git a/terraform/modules/apps/open-webui/variables.tf b/terraform/modules/apps/open-webui/variables.tf index 00189d8..679387f 100644 --- a/terraform/modules/apps/open-webui/variables.tf +++ b/terraform/modules/apps/open-webui/variables.tf @@ -20,3 +20,17 @@ variable "issuer" { default = "internal-ca" } +variable "ollama_proxy" { + description = "Configuration for the Ollama proxy" + type = object({ + enabled = bool + mac = optional(string, "") + ip = optional(string, "") + port = optional(string, "11434") + scheme = optional(string, "http") + }) + default = { + enabled = false + } + +} diff --git a/terraform/modules/apps/open-webui/webui.tf b/terraform/modules/apps/open-webui/webui.tf new file mode 100644 index 0000000..6749423 --- /dev/null +++ b/terraform/modules/apps/open-webui/webui.tf @@ -0,0 +1,108 @@ +resource "kubernetes_service_account" "openwebui" { + + metadata { + name = "openwebui-sa" + namespace = kubernetes_namespace.openwebui.metadata[0].name + } + + automount_service_account_token = false +} + +resource "kubernetes_deployment" "openwebui" { + metadata { + name = "openwebui" + namespace = kubernetes_namespace.openwebui.metadata[0].name + labels = { + app = "openwebui" + } + } + + spec { + replicas = 1 + + selector { + match_labels = { + app = "openwebui" + } + } + + strategy { + type = "RollingUpdate" + } + + template { + metadata { + labels = { + app = "openwebui" + } + } + + spec { + service_account_name = kubernetes_service_account.openwebui.metadata[0].name + automount_service_account_token = false + + container { + name = "app" + image = "ghcr.io/open-webui/open-webui:${var.tag}" + + security_context { + allow_privilege_escalation = false + } + + port { + name = "http" + container_port = 8080 + } + + readiness_probe { + tcp_socket { + port = 8080 + } + initial_delay_seconds = 5 + period_seconds = 10 + failure_threshold = 10 + } + + volume_mount { + name = "openwebui-data" + mount_path = "/app/backend/data" + } + } + + volume { + name = "openwebui-data" + host_path { + path = local.openwebui_data + } + } + } + } + } +} + +resource "kubernetes_service" "openwebui" { + metadata { + name = "http" + namespace = kubernetes_namespace.openwebui.metadata[0].name + } + + spec { + selector = { + app = "openwebui" + } + + port { + name = "http" + port = 8080 + target_port = 8080 + } + + cluster_ip = "None" + } + + lifecycle { + ignore_changes = [ + metadata[0].annotations + ] + } +}