Added auto-suspend, added wol proxy for ollama

ansible/files/scripts/auto_suspend.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+IDLE_TIMEOUT_MINUTES=20
+
+# Check 1: Any logged-in users?
+if who | grep -qv "tty"; then
+    echo "Active user sessions found. Skipping suspend."
+    exit 0
+fi
+
+# Check 2: CPU idle percentage
+CPU_IDLE=$(mpstat 1 1 | tail -n 1 | awk '/:/ {print $12}' | sed 's/,/./g')
+CPU_THRESHOLD=90
+
+if (( $(echo "$CPU_IDLE < $CPU_THRESHOLD" | bc -l) )); then
+    echo "CPU too active. Skipping suspend."
+    exit 0
+fi
+
+if ! command -v nvidia-smi &> /dev/null; then
+  echo "nvidia-smi not found, skipping GPU check"
+else
+    # Check 3: Is GPU idle?
+    GPU_UTIL=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits | head -n 1)
+    GPU_THRESHOLD=10
+
+    if (( GPU_UTIL > GPU_THRESHOLD )); then
+        echo "GPU is busy (utilization=${GPU_UTIL}%). Skipping suspend."
+        exit 0
+    fi
+
+    # Check 4: Is OLLAMA active?
+    OLLAMA_ACTIVE=$(nvidia-smi --query-compute-apps=name,used_memory --format=csv,noheader,nounits | grep ollama | wc -l)
+    if (( OLLAMA_ACTIVE > 0 )); then
+        echo "OLLMAMA is active. Skipping suspend."
+        exit 0
+    fi
+fi
+
+echo "System idle. Suspending..."
+shutdown now

ansible/host_vars/dev-01.lab.alexpires.me.yml

@@ -38,11 +38,13 @@ fw_allowed_ports:
   - { rule: "allow", port: "16443", proto: "tcp", from: "10.19.4.0/24" } # Kubernetes API (local network)
   - { rule: "allow", port: "139", proto: "tcp", from: "10.19.4.0/24" } # Samba (local network only)
   - { rule: "allow", port: "445", proto: "tcp", from: "10.19.4.0/24" } # Samba (local network only)
+  - { rule: "allow", port: "11434", proto: "tcp", from: "10.19.4.0/24" } # Ollama (local network only)
   - { rule: "allow", port: "443", proto: "tcp", from: "10.5.5.2/32" } # HTTPS (Contabo VPN)
   - { rule: "allow", port: "53", proto: "udp", from: "10.19.4.0/24" } # DNS (local network)
   - { rule: "allow", port: "53", proto: "udp", from: "10.5.5.2/32" } # DNS (Contabo VPN)
 
 ufw_outgoing_traffic:
+  - 9 # For Wake-on-LAN
   - 22 # SSH
   - 53 # DNS
   - 80 # HTTP

ansible/playbook_auto_suspend.yml

@@ -0,0 +1,7 @@
+- name: Setup Auto Suspend
+  hosts: auto_suspend
+
+  tasks:
+    - name: Setup Auto Suspend
+      ansible.builtin.include_tasks:
+        file: tasks/auto_suspend.yml

ansible/tasks/auto_suspend.yml

@@ -0,0 +1,129 @@
+- name: Install dependencies
+  become: true
+  ansible.builtin.package:
+    name: "{{ item }}"
+    state: present
+  loop:
+    - sysstat
+    - bc
+
+- name: Enable sysstat service
+  become: true
+  ansible.builtin.systemd:
+    name: sysstat
+    enabled: true
+    state: started
+
+- name: Read existing GRUB_CMDLINE_LINUX
+  ansible.builtin.shell: set -o pipefail && grep '^GRUB_CMDLINE_LINUX=' /etc/default/grub | cut -d= -f2- | tr -d '"'
+  register: grub_line
+  changed_when: false
+  failed_when: grub_line.stdout == ""
+
+- name: Set fact for existing_cmdline
+  ansible.builtin.set_fact:
+    existing_cmdline: "{{ grub_line.stdout }}"
+
+- name: Set fedora grub file locations
+  ansible.builtin.set_fact:
+    grub_config_path_bios: /boot/grub2/grub.cfg
+  when: ansible_distribution == "Fedora" and ansible_os_family == "RedHat"
+
+- name: Show current GRUB_CMDLINE_LINUX
+  ansible.builtin.debug:
+    msg: "Current GRUB_CMDLINE_LINUX: {{ existing_cmdline }}"
+
+- name: Updated GRUB_CMDLINE_LINUX
+  ansible.builtin.debug:
+    msg: "Current GRUB_CMDLINE_LINUX: {{ existing_cmdline | regex_replace(' *mem_sleep_default=\\S+', '') }} mem_sleep_default=deep"
+
+- name: Ensure mem_sleep_default=deep is set in GRUB_CMDLINE_LINUX
+  become: true
+  vars:
+    grub_cmdline_linux_line: >-
+      GRUB_CMDLINE_LINUX="{{ existing_cmdline | regex_replace(' *mem_sleep_default=\\S+', '') }} mem_sleep_default=deep"
+  ansible.builtin.lineinfile:
+    path: /etc/default/grub
+    regexp: '^GRUB_CMDLINE_LINUX='
+    line: "{{ grub_cmdline_linux_line }}"
+
+- name: Generate grub config for BIOS
+  become: true
+  changed_when: false
+  ansible.builtin.command: grub2-mkconfig -o {{ grub_config_path_bios }}
+
+- name: Create idle check and suspend script
+  become: true
+  ansible.builtin.copy:
+    dest: /usr/local/bin/auto_suspend_script.sh
+    mode: '0755'
+    owner: root
+    group: root
+    content: "{{ lookup('file', 'scripts/auto_suspend.sh') }}"
+
+- name: Create systemd service for auto suspend
+  become: true
+  ansible.builtin.copy:
+    dest: /etc/systemd/system/auto-suspend.service
+    content: |
+      [Unit]
+      Description=Check system idleness and suspend if idle
+      After=multi-user.target
+
+      [Service]
+      Type=oneshot
+      ExecStart=/usr/local/bin/auto_suspend_script.sh
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Create sleep.conf.d folder if it doesn't exist
+  become: true
+  ansible.builtin.file:
+    path: /etc/systemd/sleep.conf.d
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+
+- name: Create sleep.conf.d for deep sleep
+  become: true
+  ansible.builtin.copy:
+    dest: /etc/systemd/sleep.conf.d/mem-deep.conf
+    content: |
+      [Sleep]
+      MemorySleepMode=deep
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Create systemd timer for auto suspend
+  become: true
+  ansible.builtin.copy:
+    dest: /etc/systemd/system/auto-suspend.timer
+    content: |
+      [Unit]
+      Description=Run idle suspend check every 10 minutes
+
+      [Timer]
+      OnBootSec=10min
+      OnUnitActiveSec=10min
+      Unit=auto-suspend.service
+
+      [Install]
+      WantedBy=timers.target
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Reload systemd daemon
+  become: true
+  ansible.builtin.systemd:
+    daemon_reload: true
+
+- name: Enable and start auto-suspend timer
+  become: true
+  ansible.builtin.systemd:
+    name: auto-suspend.timer
+    enabled: true
+    state: started

inventory/hosts

@@ -91,4 +91,7 @@ vh-01.lab.alexpires.me
 gpu-01.lab.alexpires.me
 
 [wol]
+gpu-01.lab.alexpires.me
+
+[auto_suspend]
 gpu-01.lab.alexpires.me

terraform/apps/dev-01/apps.tf

@@ -3,6 +3,7 @@ module "open-webui" {
 
   persistent_folder = local.persistent_folder
   fqdn              = local.open_webui_fqdn
+  ollama_proxy      = local.ollama_proxy
 }
 
 module "samba" {

terraform/apps/dev-01/config.tf

@@ -45,7 +45,7 @@ locals {
         {
           name    = "ollama"
           type    = "CNAME"
-          content = "gpu-01.${local.lab_domain}."
+          content = "dev-01.${local.lab_domain}."
         },
         {
           name    = "kokoro"
@@ -103,6 +103,14 @@ locals {
     }
   }
 
+  ollama_proxy = {
+    enabled = true
+    mac     = "a0:8c:fd:e2:3c:fa"
+    ip      = "10.19.4.106"
+    port    = "11434"
+    scheme  = "http"
+  }
+
   scaleway_smtp_host = "smtp.tem.scw.cloud"
 
   nextcloud_fqdn             = "cloud.alexpires.me"

terraform/modules/apps/comfyui/config.tf

@@ -1,4 +0,0 @@
-locals {
-  comfyui_version = "frontend${var.tag}"
-}
-

terraform/modules/apps/comfyui/main.tf

@@ -1,98 +0,0 @@
-resource "kubernetes_namespace" "comfyui" {
-  metadata {
-    name = "comfyui"
-  }
-}
-
-resource "kubernetes_service_account" "comfyui" {
-
-  metadata {
-    name      = "comfyui-sa"
-    namespace = kubernetes_namespace.comfyui.metadata[0].name
-  }
-
-  automount_service_account_token = false
-}
-
-resource "kubernetes_deployment" "comfyui" {
-  metadata {
-    name      = "comfyui"
-    namespace = kubernetes_namespace.comfyui.metadata[0].name
-    labels = {
-      comfyui = "comfyui"
-    }
-  }
-
-  spec {
-    replicas = 1
-
-    selector {
-      match_labels = {
-        comfyui = "comfyui"
-      }
-    }
-
-    strategy {
-      type = "RollingUpdate"
-    }
-
-    template {
-      metadata {
-        labels = {
-          comfyui = "comfyui"
-        }
-      }
-
-      spec {
-        service_account_name            = kubernetes_service_account.comfyui.metadata[0].name
-        automount_service_account_token = false
-
-        container {
-          name  = "comfyui"
-          image = "a13labs/comfyui:${local.comfyui_version}"
-
-
-          security_context {
-            allow_privilege_escalation = false
-          }
-
-          port {
-            name           = "frontend"
-            container_port = 5173
-            protocol       = "TCP"
-          }
-
-          env {
-            name  = "DEV_SERVER_COMFYUI_URL"
-            value = coalesce(var.backend, "http://localhost:8188")
-          }
-        }
-      }
-    }
-  }
-}
-
-resource "kubernetes_service" "comfyui" {
-  metadata {
-    name      = "frontend"
-    namespace = kubernetes_namespace.comfyui.metadata[0].name
-  }
-
-  spec {
-    selector = {
-      comfyui = "comfyui"
-    }
-
-    port {
-      name        = "frontend"
-      port        = 5173
-      target_port = 5173
-    }
-  }
-
-  lifecycle {
-    ignore_changes = [
-      metadata[0].annotations
-    ]
-  }
-}

terraform/modules/apps/comfyui/provider.tf

@@ -1,9 +0,0 @@
-terraform {
-  required_version = "~>1.8"
-  required_providers {
-    kubernetes = {
-      source  = "hashicorp/kubernetes"
-      version = "2.36.0"
-    }
-  }
-}

terraform/modules/apps/comfyui/variables.tf

@@ -1,11 +0,0 @@
-variable "tag" {
-  description = "The version of app to install"
-  type        = string
-  default     = "v0.0.1"
-}
-
-variable "backend" {
-  description = "The backend to use for ComfyUI"
-  type        = string
-  default     = ""
-}

terraform/modules/apps/open-webui/ingress.tf

@@ -0,0 +1,34 @@
+resource "kubernetes_ingress_v1" "openwebui" {
+  metadata {
+    name      = "ingress"
+    namespace = kubernetes_namespace.openwebui.metadata[0].name
+
+    annotations = {
+      "kubernetes.io/ingress.class"              = "public"
+      "cert-manager.io/cluster-issuer"           = "letsencrypt-prod"
+      "kubernetes.io/tls-acme"                   = "true"
+      "nginx.ingress.kubernetes.io/ssl-redirect" = "true"
+    }
+  }
+  spec {
+    tls {
+      hosts       = [var.fqdn]
+      secret_name = "openwebui-tls"
+    }
+    rule {
+      host = var.fqdn
+      http {
+        path {
+          backend {
+            service {
+              name = "http"
+              port {
+                number = 8080
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

terraform/modules/apps/open-webui/main.tf

@@ -3,146 +3,3 @@ resource "kubernetes_namespace" "openwebui" {
     name = "openwebui"
   }
 }
-
-resource "kubernetes_service_account" "openwebui" {
-
-  metadata {
-    name      = "openwebui-sa"
-    namespace = kubernetes_namespace.openwebui.metadata[0].name
-  }
-
-  automount_service_account_token = false
-}
-resource "kubernetes_deployment" "openwebui" {
-  metadata {
-    name      = "openwebui"
-    namespace = kubernetes_namespace.openwebui.metadata[0].name
-    labels = {
-      openwebui = "openwebui"
-    }
-  }
-
-  spec {
-    replicas = 1
-
-    selector {
-      match_labels = {
-        openwebui = "openwebui"
-      }
-    }
-
-    strategy {
-      type = "RollingUpdate"
-    }
-
-    template {
-      metadata {
-        labels = {
-          openwebui = "openwebui"
-        }
-      }
-
-      spec {
-        service_account_name            = kubernetes_service_account.openwebui.metadata[0].name
-        automount_service_account_token = false
-
-        container {
-          name  = "openwebui-container"
-          image = "ghcr.io/open-webui/open-webui:${var.tag}"
-
-          security_context {
-            allow_privilege_escalation = false
-          }
-
-          port {
-            name           = "http"
-            container_port = 8080
-          }
-
-          readiness_probe {
-            tcp_socket {
-              port = 8080
-            }
-            initial_delay_seconds = 5
-            period_seconds        = 10
-            failure_threshold     = 10
-          }
-
-          volume_mount {
-            name       = "openwebui-data"
-            mount_path = "/app/backend/data"
-          }
-        }
-
-        volume {
-          name = "openwebui-data"
-          host_path {
-            path = local.openwebui_data
-          }
-        }
-      }
-    }
-  }
-}
-
-resource "kubernetes_service" "openwebui" {
-  metadata {
-    name      = "http"
-    namespace = kubernetes_namespace.openwebui.metadata[0].name
-  }
-
-  spec {
-    selector = {
-      openwebui = "openwebui"
-    }
-
-    port {
-      name        = "http"
-      port        = 8080
-      target_port = 8080
-    }
-
-    cluster_ip = "None"
-  }
-
-  lifecycle {
-    ignore_changes = [
-      metadata[0].annotations
-    ]
-  }
-}
-
-resource "kubernetes_ingress_v1" "openwebui" {
-  metadata {
-    name      = "ingress"
-    namespace = kubernetes_namespace.openwebui.metadata[0].name
-
-    annotations = {
-      "kubernetes.io/ingress.class"              = "public"
-      "cert-manager.io/cluster-issuer"           = "letsencrypt-prod"
-      "kubernetes.io/tls-acme"                   = "true"
-      "nginx.ingress.kubernetes.io/ssl-redirect" = "true"
-    }
-  }
-  spec {
-    tls {
-      hosts       = [var.fqdn]
-      secret_name = "openwebui-tls"
-    }
-    rule {
-      host = var.fqdn
-      http {
-        path {
-          backend {
-            service {
-              name = "http"
-              port {
-                number = 8080
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-}

terraform/modules/apps/open-webui/ollama_proxy.tf

@@ -0,0 +1,101 @@
+resource "kubernetes_service_account" "ollama_proxy" {
+
+  count = var.ollama_proxy.enabled ? 1 : 0
+  metadata {
+    name      = "ollama-proxy-sa"
+    namespace = kubernetes_namespace.openwebui.metadata[0].name
+  }
+
+  automount_service_account_token = false
+}
+
+resource "kubernetes_deployment" "ollama_proxy" {
+
+  count = var.ollama_proxy.enabled ? 1 : 0
+  metadata {
+    name      = "ollama-proxy"
+    namespace = kubernetes_namespace.openwebui.metadata[0].name
+    labels = {
+      app = "ollama-proxy"
+    }
+  }
+
+  spec {
+    replicas = 1
+    selector {
+      match_labels = {
+        app = "ollama-proxy"
+      }
+    }
+
+    strategy {
+      type = "Recreate"
+    }
+
+    template {
+      metadata {
+        labels = {
+          app = "ollama-proxy"
+        }
+      }
+
+      spec {
+        service_account_name            = kubernetes_service_account.openwebui.metadata[0].name
+        automount_service_account_token = false
+
+        host_network = true
+        container {
+          name              = "app"
+          image             = "a13labs/wol_proxy:v0.4.0"
+          image_pull_policy = "Always"
+
+          security_context {
+            capabilities {
+              add = ["NET_RAW"]
+            }
+          }
+
+          port {
+            name           = "http"
+            container_port = var.ollama_proxy.port
+          }
+
+          env {
+            name  = "UPSTREAM_MAC"
+            value = var.ollama_proxy.mac
+          }
+
+          env {
+            name  = "UPSTREAM_IP"
+            value = var.ollama_proxy.ip
+          }
+
+          env {
+            name  = "UPSTREAM_PORT"
+            value = var.ollama_proxy.port
+          }
+
+          env {
+            name  = "UPSTREAM_SCHEME"
+            value = var.ollama_proxy.scheme
+          }
+
+          env {
+            name  = "PROXY_PORT"
+            value = var.ollama_proxy.port
+          }
+
+          readiness_probe {
+            tcp_socket {
+              port = 11434
+            }
+            initial_delay_seconds = 5
+            period_seconds        = 10
+            failure_threshold     = 10
+          }
+        }
+      }
+    }
+  }
+}
+

terraform/modules/apps/open-webui/variables.tf

@@ -20,3 +20,17 @@ variable "issuer" {
   default     = "internal-ca"
 }
 
+variable "ollama_proxy" {
+  description = "Configuration for the Ollama proxy"
+  type = object({
+    enabled = bool
+    mac     = optional(string, "")
+    ip      = optional(string, "")
+    port    = optional(string, "11434")
+    scheme  = optional(string, "http")
+  })
+  default = {
+    enabled = false
+  }
+
+}

terraform/modules/apps/open-webui/webui.tf

@@ -0,0 +1,108 @@
+resource "kubernetes_service_account" "openwebui" {
+
+  metadata {
+    name      = "openwebui-sa"
+    namespace = kubernetes_namespace.openwebui.metadata[0].name
+  }
+
+  automount_service_account_token = false
+}
+
+resource "kubernetes_deployment" "openwebui" {
+  metadata {
+    name      = "openwebui"
+    namespace = kubernetes_namespace.openwebui.metadata[0].name
+    labels = {
+      app = "openwebui"
+    }
+  }
+
+  spec {
+    replicas = 1
+
+    selector {
+      match_labels = {
+        app = "openwebui"
+      }
+    }
+
+    strategy {
+      type = "RollingUpdate"
+    }
+
+    template {
+      metadata {
+        labels = {
+          app = "openwebui"
+        }
+      }
+
+      spec {
+        service_account_name            = kubernetes_service_account.openwebui.metadata[0].name
+        automount_service_account_token = false
+
+        container {
+          name  = "app"
+          image = "ghcr.io/open-webui/open-webui:${var.tag}"
+
+          security_context {
+            allow_privilege_escalation = false
+          }
+
+          port {
+            name           = "http"
+            container_port = 8080
+          }
+
+          readiness_probe {
+            tcp_socket {
+              port = 8080
+            }
+            initial_delay_seconds = 5
+            period_seconds        = 10
+            failure_threshold     = 10
+          }
+
+          volume_mount {
+            name       = "openwebui-data"
+            mount_path = "/app/backend/data"
+          }
+        }
+
+        volume {
+          name = "openwebui-data"
+          host_path {
+            path = local.openwebui_data
+          }
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_service" "openwebui" {
+  metadata {
+    name      = "http"
+    namespace = kubernetes_namespace.openwebui.metadata[0].name
+  }
+
+  spec {
+    selector = {
+      app = "openwebui"
+    }
+
+    port {
+      name        = "http"
+      port        = 8080
+      target_port = 8080
+    }
+
+    cluster_ip = "None"
+  }
+
+  lifecycle {
+    ignore_changes = [
+      metadata[0].annotations
+    ]
+  }
+}