Add configuration for CoreDNS and update host variables for lab environment

ansible/host_vars/dev-01.lab.alexpires.yml

@@ -1,4 +1,4 @@
-hostname: dev-01.alexpires.local
+hostname: dev-01.lab.alexpires.me
 ansible_user: "{{ lookup('env', 'ANSIBLE_USER') }}"
 
 # Users
@@ -32,13 +32,15 @@ sshd_max_auth_tries: 10
 # Firewall ports
 ufw_enable: true
 fw_allowed_ports:
-  - { rule: "allow", port: "22", proto: "tcp", from: "10.5.5.5/32" } # Wireguard VPN (Laptop)
+  - { rule: "allow", port: "22", proto: "tcp", from: "10.5.5.5/32" } # SSH Access (VPN device)
-  - { rule: "allow", port: "22", proto: "tcp", from: "10.19.4.0/24" } # local network
+  - { rule: "allow", port: "22", proto: "tcp", from: "10.19.4.0/24" } # SSH Access (local network)
   - { rule: "allow", port: "16443", proto: "tcp", from: "10.5.5.5/32" } # Kubernetes API (Laptop)
   - { rule: "allow", port: "16443", proto: "tcp", from: "10.19.4.0/24" } # Kubernetes API (local network)
-  - { rule: "allow", port: "139", proto: "tcp", from: "10.19.4.0/24" } # Samba (local network)
+  - { rule: "allow", port: "139", proto: "tcp", from: "10.19.4.0/24" } # Samba (local network only)
-  - { rule: "allow", port: "445", proto: "tcp", from: "10.19.4.0/24" } # Samba (local network)
+  - { rule: "allow", port: "445", proto: "tcp", from: "10.19.4.0/24" } # Samba (local network only)
-  - { rule: "allow", port: "443", proto: "tcp", from: "10.5.5.2/32" } # Wireguard VPN (Kokoro-FastAPI)
+  - { rule: "allow", port: "443", proto: "tcp", from: "10.5.5.2/32" } # HTTPS (Contabo VPN)
+  - { rule: "allow", port: "53", proto: "udp", from: "10.19.4.0/24" } # DNS (local network)
+  - { rule: "allow", port: "53", proto: "udp", from: "10.5.5.2/32" } # DNS (Contabo VPN)
 
 ufw_outgoing_traffic:
   - 22 # SSH

ansible/host_vars/gpu-01.lab.alexpires.yml

@@ -1,4 +1,4 @@
-hostname: gpu-01.alexpires.local
+hostname: gpu-01.lab.alexpires.me
 ansible_user: "{{ lookup('env', 'ANSIBLE_USER') }}"
 
 # Users

ansible/host_vars/vh-01.lab.alexpires.yml

@@ -1,4 +1,4 @@
-hostname: vh-01.alexpires.local
+hostname: vh-01.lab.alexpires.me
 ansible_user: "{{ lookup('env', 'ANSIBLE_USER') }}"
 
 # Users

inventory/hosts

@@ -1,30 +1,30 @@
 [all]
 prod-01.alexpires.me
-dev-01.alexpires.dev
+dev-01.lab.alexpires.me
-vh-01.alexpires.dev
+vh-01.lab.alexpires.me
-gpu-01.alexpires.dev
+gpu-01.lab.alexpires.me
 
 [hardening]
 prod-01.alexpires.me
-dev-01.alexpires.dev
+dev-01.lab.alexpires.me
-vh-01.alexpires.dev
+vh-01.lab.alexpires.me
-gpu-01.alexpires.dev
+gpu-01.lab.alexpires.me
 
 [ubuntu]
 prod-01.alexpires.me
-dev-01.alexpires.dev
+dev-01.lab.alexpires.me
 
 [fedora]
-vh-01.alexpires.dev
+vh-01.lab.alexpires.me
-gpu-01.alexpires.dev
+gpu-01.lab.alexpires.me
 
 [public]
 prod-01.alexpires.me
 
 [private]
-dev-01.alexpires.dev
+dev-01.lab.alexpires.me
-vh-01.alexpires.dev
+vh-01.lab.alexpires.me
-gpu-01.alexpires.dev
+gpu-01.lab.alexpires.me
 
 [gitea]
 prod-01.alexpires.me
@@ -40,7 +40,7 @@ prod-01.alexpires.me
 
 [microk8s]
 prod-01.alexpires.me
-dev-01.alexpires.dev
+dev-01.lab.alexpires.me
 
 [iac]
 prod-01.alexpires.me
@@ -61,22 +61,22 @@ prod-01.alexpires.me
 prod-01.alexpires.me
 
 [virtualization]
-vh-01.alexpires.dev
+vh-01.lab.alexpires.me
 
 [duo]
 prod-01.alexpires.me
-dev-01.alexpires.dev
+dev-01.lab.alexpires.me
-vh-01.alexpires.dev
+vh-01.lab.alexpires.me
-gpu-01.alexpires.dev
+gpu-01.lab.alexpires.me
 
 [ollama]
-gpu-01.alexpires.dev
+gpu-01.lab.alexpires.me
 
 [kokoro]
-gpu-01.alexpires.dev
+gpu-01.lab.alexpires.me
 
 [openwebui]
-dev-01.alexpires.dev
+dev-01.lab.alexpires.me
 
 [mcp]
-dev-01.alexpires.dev
+dev-01.lab.alexpires.me

terraform/apps/dev-01/apps.tf

@@ -10,3 +10,10 @@ module "samba" {
 
   shared_folder = local.samba_shared_folder
 }
+
+module "dns" {
+  source = "../../modules/apps/coredns"
+
+  forward_dns_servers = local.dns_nameservers
+  zones               = local.zones
+}

terraform/apps/dev-01/config.tf

@@ -3,10 +3,54 @@ locals {
   # storage
   persistent_folder = "/srv"
 
-  primary_domain  = "alexpires.me"
+  lab_domain      = "lab.alexpires.me"
-  open_webui_fqdn = "ai.${local.primary_domain}"
+  open_webui_fqdn = "open-webui.${local.lab_domain}"
   internal_issuer = "internal-ca"
 
   # samba
   samba_shared_folder = "/mnt/usb"
+
+  # DNS
+  dns_nameservers = [
+    "193.110.81.0",
+    "185.253.5.0"
+  ]
+
+  zones = [
+    {
+      fqdn = "${local.lab_domain}"
+      records = [
+        {
+          name    = "gpu-01"
+          type    = "A"
+          content = "10.19.4.106"
+        },
+        {
+          name    = "dev-01"
+          type    = "A"
+          content = "10.19.4.136"
+        },
+        {
+          name    = "ollama"
+          type    = "CNAME"
+          content = "gpu-01.${local.lab_domain}."
+        },
+        {
+          name    = "kokoro"
+          type    = "CNAME"
+          content = "gpu-01.${local.lab_domain}."
+        },
+        {
+          name    = "open-webui"
+          type    = "CNAME"
+          content = "dev-01.${local.lab_domain}."
+        },
+        {
+          name    = "smb"
+          type    = "CNAME"
+          content = "dev-01.${local.lab_domain}."
+        },
+      ]
+    }
+  ]
 }

terraform/apps/prod-01/config.tf

@@ -97,16 +97,15 @@ locals {
       http_port  = 8080
       https_port = 8443
       tls        = true
+      resolver   = "10.19.4.136"
       fqdn       = ["ai.alexpires.me"]
-      upstream   = "https://10.19.4.136"
+      upstream   = "https://open-webui.lab.alexpires.me"
       locations = [
         {
           path = "/ws/socket.io/"
           headers = {
-            "Host"            = "$host"
+            "Upgrade"    = "$http_upgrade"
-            "X-Forwarded-For" = "$remote_addr"
+            "Connection" = "upgrade"
-            "Upgrade"         = "$http_upgrade"
-            "Connection"      = "upgrade"
           }
         },
         # {
@@ -117,10 +116,6 @@ locals {
         # },
         {
           path = "/"
-          headers = {
-            "Host"            = "$host"
-            "X-Forwarded-For" = "$remote_addr"
-          }
         },
       ]
     },

terraform/modules/apps/coredns/config.tf

@@ -0,0 +1,21 @@
+
+locals {
+  configfile = templatefile("${path.module}/resources/Corefile.tpl", {
+    zones               = var.zones
+    forward_dns_servers = join(" ", var.forward_dns_servers)
+  })
+  configfile_checksum = md5(local.configfile)
+  db_files = {
+    for z in var.zones : z.fqdn => templatefile("${path.module}/resources/db.zone.tpl", {
+      fqdn          = z.fqdn
+      domain_master = z.domain_master
+      timestamp     = formatdate("YYYYMMDD", timestamp())
+      refresh_rate  = z.refresh_rate
+      retry_rate    = z.retry_rate
+      expire_time   = z.expire_time
+      ttl           = z.ttl
+      records       = z.records
+    })
+  }
+  db_files_checksum = md5(jsonencode(local.db_files))
+}

terraform/modules/apps/coredns/main.tf

@@ -0,0 +1,195 @@
+resource "kubernetes_namespace" "dns" {
+  metadata {
+    name = "dns"
+  }
+}
+
+resource "kubernetes_service_account" "dns" {
+
+  metadata {
+    name      = "dns-sa"
+    namespace = kubernetes_namespace.dns.metadata[0].name
+  }
+
+  automount_service_account_token = false
+}
+
+resource "kubernetes_config_map" "dns" {
+  metadata {
+    name      = "dns-config"
+    namespace = kubernetes_namespace.dns.metadata[0].name
+  }
+
+  data = merge({
+    "Corefile" = local.configfile
+
+  }, local.db_files)
+}
+
+resource "kubernetes_deployment" "dns" {
+  metadata {
+    name      = "dns"
+    namespace = kubernetes_namespace.dns.metadata[0].name
+    labels = {
+      dns = "dns"
+    }
+  }
+
+  spec {
+    replicas = 1
+
+    selector {
+      match_labels = {
+        dns = "dns"
+      }
+    }
+
+    strategy {
+      type = "RollingUpdate"
+    }
+
+    template {
+      metadata {
+        labels = {
+          dns = "dns"
+        }
+        annotations = {
+          "checksum/corefile" = local.configfile_checksum
+          "checksum/db-files" = local.db_files_checksum
+        }
+      }
+
+      spec {
+        service_account_name            = kubernetes_service_account.dns.metadata[0].name
+        automount_service_account_token = false
+
+        container {
+          name              = "dns"
+          image             = "docker.io/coredns/coredns:${var.tag}"
+          image_pull_policy = "IfNotPresent"
+          args = [
+            "-conf",
+            "/etc/coredns/Corefile",
+          ]
+
+          liveness_probe {
+            http_get {
+              path   = "/health"
+              port   = 8080
+              scheme = "HTTP"
+            }
+            initial_delay_seconds = 60
+            period_seconds        = 10
+            success_threshold     = 1
+            timeout_seconds       = 5
+          }
+
+          readiness_probe {
+            http_get {
+              path   = "/ready"
+              port   = 8181
+              scheme = "HTTP"
+            }
+            period_seconds    = 10
+            success_threshold = 1
+            timeout_seconds   = 1
+          }
+
+          resources {
+            requests = {
+              memory = "100Mi"
+            }
+          }
+
+          security_context {
+            allow_privilege_escalation = false
+            capabilities {
+              drop = ["ALL"]
+              add  = ["NET_BIND_SERVICE"]
+            }
+          }
+
+          port {
+            name           = "dns-tcp"
+            container_port = 53
+            host_port      = 53
+            protocol       = "TCP"
+          }
+
+          port {
+            name           = "dns-udp"
+            container_port = 53
+            host_port      = 53
+            protocol       = "UDP"
+          }
+
+          port {
+            name           = "metrics"
+            container_port = 9153
+            protocol       = "TCP"
+          }
+
+          volume_mount {
+            name       = "config"
+            mount_path = "/etc/coredns"
+            read_only  = true
+          }
+        }
+
+        volume {
+          name = "config"
+          config_map {
+            name         = kubernetes_config_map.dns.metadata[0].name
+            default_mode = "0644"
+            items {
+              key  = "Corefile"
+              path = "Corefile"
+            }
+            dynamic "items" {
+              for_each = local.db_files
+              content {
+                key  = items.key
+                path = "db.${items.key}"
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_service" "dns" {
+  metadata {
+    name      = "dns-ports"
+    namespace = kubernetes_namespace.dns.metadata[0].name
+  }
+
+  spec {
+    selector = {
+      dns = "dns"
+    }
+
+    port {
+      name        = "dns-tcp"
+      protocol    = "TCP"
+      port        = 53
+      target_port = 53
+    }
+
+    port {
+      name        = "dns-udp"
+      protocol    = "UDP"
+      port        = 53
+      target_port = 53
+    }
+
+    type = "LoadBalancer"
+  }
+
+  lifecycle {
+    ignore_changes = [
+      metadata[0].annotations
+    ]
+  }
+}

terraform/modules/apps/coredns/provider.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = "~>1.8"
+  required_providers {
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "2.36.0"
+    }
+  }
+}

terraform/modules/apps/coredns/resources/Corefile.tpl

@@ -0,0 +1,24 @@
+.:53 {
+    errors
+    health {
+      lameduck 5s
+    }
+    ready
+    log . {
+      class error
+    }
+    prometheus :9153
+    forward . ${forward_dns_servers}
+    cache 30
+    loop
+    reload
+    loadbalance
+}
+
+%{ for z in zones ~}
+${z.fqdn}:53 {
+    file /etc/coredns/db.${z.fqdn}
+    log
+    errors
+}
+%{ endfor ~}

terraform/modules/apps/coredns/resources/db.zone.tpl

@@ -0,0 +1,13 @@
+$ORIGIN ${fqdn}.
+@         IN  SOA dns.${fqdn}. ${domain_master}.${fqdn}. (
+              ${timestamp} ; serial
+              ${refresh_rate}     ; refresh
+              ${retry_rate}     ; retry
+              ${expire_time}  ; expire
+              ${ttl} )   ; minimum
+
+          IN  NS  dns.${fqdn}.
+
+%{ for r in records ~}
+${r.name} IN ${r.type} ${r.content}
+%{ endfor ~}

terraform/modules/apps/coredns/variables.tf

@@ -0,0 +1,29 @@
+variable "tag" {
+  description = "The version of app to install"
+  type        = string
+  default     = "latest"
+}
+
+variable "forward_dns_servers" {
+  description = "Forward DNS servers"
+  type        = list(string)
+  default     = ["8.8.8.8", "8.8.4.4"]
+}
+
+variable "zones" {
+  description = "Zones to be managed by coredns"
+  type = list(object({
+    fqdn          = string
+    domain_master = optional(string, "admin")
+    refresh_rate  = optional(number, 7200)
+    retry_rate    = optional(number, 3600)
+    expire_time   = optional(number, 1209600)
+    ttl           = optional(number, 3600)
+    records = list(object({
+      name    = string
+      type    = string
+      content = string
+    }))
+  }))
+  default = []
+}

terraform/modules/apps/nginx-reverse-proxy/config.tf

@@ -27,6 +27,7 @@ locals {
       http_port  = v.http_port
       https_port = v.https_port
       upstream   = v.upstream
+      resolver   = v.resolver
       tls        = v.tls
       fqdn       = join(" ", v.fqdn)
       auth       = length([for l in v.locations : true if l.private]) > 0
@@ -35,6 +36,7 @@ locals {
         headers  = merge(l.headers, coalesce(v.default_headers, local.default_headers))
         private  = l.private
         upstream = l.upstream
+        resolver = l.resolver
         rewrite  = l.rewrite
       }]
       custom_snippet = v.custom_snippet

terraform/modules/apps/nginx-reverse-proxy/main.tf

@@ -310,6 +310,27 @@ resource "kubernetes_deployment" "reverse_proxy" {
             allow_privilege_escalation = false
           }
 
+          readiness_probe {
+            http_get {
+              path = "/health"
+              port = 8888
+            }
+            initial_delay_seconds = 5
+            period_seconds        = 10
+          }
+
+          liveness_probe {
+            http_get {
+              path = "/health"
+              port = 8888
+            }
+            initial_delay_seconds = 5
+            period_seconds        = 10
+            failure_threshold     = 10
+            timeout_seconds       = 5
+            success_threshold     = 1
+          }
+
           dynamic "port" {
             for_each = { for k, v in var.services : k => v.http_port }
             content {

terraform/modules/apps/nginx-reverse-proxy/resources/nginx/conf.d/default.conf.tpl

@@ -21,13 +21,21 @@ location = /auth {
 %{ endif }
 %{ for l in locations ~}
     location ${l.path} {
+%{ if l.resolver == "" ~}
+%{ if resolver != "" ~}
+        resolver ${resolver};
+%{ endif ~}
+%{ else ~}
+        resolver ${l.resolver};
+%{ endif ~}
 %{ if l.private ~}
-        auth_request /auth;
+        auth_request /auth<;
 %{ endif ~}
 %{ if l.rewrite != "" ~}
         rewrite ${l.rewrite};
 %{ endif ~}
-        proxy_pass %{ if l.upstream != "" ~}${l.upstream}%{ else ~}${upstream}%{ endif ~};
+        set $upstream "%{ if l.upstream != "" ~}${l.upstream}%{ else ~}${upstream}%{ endif ~}"; 
+        proxy_pass $upstream;
         proxy_http_version 1.1;
 %{ for k,v in l.headers ~}        
         proxy_set_header ${k} ${v};

terraform/modules/apps/nginx-reverse-proxy/resources/nginx/nginx.conf

@@ -3,7 +3,6 @@ worker_processes  auto;
 error_log  /var/log/nginx/error.log notice;
 pid        /tmp/nginx.pid;
 
-
 events {
     worker_connections  1024;
 }
@@ -32,5 +31,15 @@ http {
 
     keepalive_timeout  65;
 
+    server {
+        listen       8888;
+        server_name  _;
+        location /health {
+            access_log off;
+            return 200 "OK";
+        }
+    }
+
     include /etc/nginx/conf.d/*.conf;
-}
+}
+

terraform/modules/apps/nginx-reverse-proxy/variables.tf

@@ -31,6 +31,7 @@ variable "services" {
   type = map(object({
     fqdn            = list(string)
     upstream        = string
+    resolver        = optional(string, "")
     default_headers = optional(map(string), {}),
     http_port       = optional(number, 80)
     https_port      = optional(number, 443)
@@ -40,6 +41,7 @@ variable "services" {
       headers  = optional(map(string), {}),
       private  = optional(bool, false),
       upstream = optional(string, ""),
+      resolver = optional(string, "")
       rewrite  = optional(string, "")
     })), [])
     custom_snippet = optional(string, "")