diff --git a/ansible/host_vars/dev-01.alexpires.dev.yml b/ansible/host_vars/dev-01.lab.alexpires.yml similarity index 88% rename from ansible/host_vars/dev-01.alexpires.dev.yml rename to ansible/host_vars/dev-01.lab.alexpires.yml index 15febd0..3ad5886 100644 --- a/ansible/host_vars/dev-01.alexpires.dev.yml +++ b/ansible/host_vars/dev-01.lab.alexpires.yml @@ -1,4 +1,4 @@ -hostname: dev-01.alexpires.local +hostname: dev-01.lab.alexpires.me ansible_user: "{{ lookup('env', 'ANSIBLE_USER') }}" # Users @@ -32,13 +32,15 @@ sshd_max_auth_tries: 10 # Firewall ports ufw_enable: true fw_allowed_ports: - - { rule: "allow", port: "22", proto: "tcp", from: "10.5.5.5/32" } # Wireguard VPN (Laptop) - - { rule: "allow", port: "22", proto: "tcp", from: "10.19.4.0/24" } # local network + - { rule: "allow", port: "22", proto: "tcp", from: "10.5.5.5/32" } # SSH Access (VPN device) + - { rule: "allow", port: "22", proto: "tcp", from: "10.19.4.0/24" } # SSH Access (local network) - { rule: "allow", port: "16443", proto: "tcp", from: "10.5.5.5/32" } # Kubernetes API (Laptop) - { rule: "allow", port: "16443", proto: "tcp", from: "10.19.4.0/24" } # Kubernetes API (local network) - - { rule: "allow", port: "139", proto: "tcp", from: "10.19.4.0/24" } # Samba (local network) - - { rule: "allow", port: "445", proto: "tcp", from: "10.19.4.0/24" } # Samba (local network) - - { rule: "allow", port: "443", proto: "tcp", from: "10.5.5.2/32" } # Wireguard VPN (Kokoro-FastAPI) + - { rule: "allow", port: "139", proto: "tcp", from: "10.19.4.0/24" } # Samba (local network only) + - { rule: "allow", port: "445", proto: "tcp", from: "10.19.4.0/24" } # Samba (local network only) + - { rule: "allow", port: "443", proto: "tcp", from: "10.5.5.2/32" } # HTTPS (Contabo VPN) + - { rule: "allow", port: "53", proto: "udp", from: "10.19.4.0/24" } # DNS (local network) + - { rule: "allow", port: "53", proto: "udp", from: "10.5.5.2/32" } # DNS (Contabo VPN) ufw_outgoing_traffic: - 22 # SSH diff --git a/ansible/host_vars/gpu-01.alexpires.dev.yml b/ansible/host_vars/gpu-01.lab.alexpires.yml similarity index 98% rename from ansible/host_vars/gpu-01.alexpires.dev.yml rename to ansible/host_vars/gpu-01.lab.alexpires.yml index f2c7221..50a0dbf 100644 --- a/ansible/host_vars/gpu-01.alexpires.dev.yml +++ b/ansible/host_vars/gpu-01.lab.alexpires.yml @@ -1,4 +1,4 @@ -hostname: gpu-01.alexpires.local +hostname: gpu-01.lab.alexpires.me ansible_user: "{{ lookup('env', 'ANSIBLE_USER') }}" # Users diff --git a/ansible/host_vars/vh-01.alexpires.dev.yml b/ansible/host_vars/vh-01.lab.alexpires.yml similarity index 99% rename from ansible/host_vars/vh-01.alexpires.dev.yml rename to ansible/host_vars/vh-01.lab.alexpires.yml index 6f12207..5f5b526 100644 --- a/ansible/host_vars/vh-01.alexpires.dev.yml +++ b/ansible/host_vars/vh-01.lab.alexpires.yml @@ -1,4 +1,4 @@ -hostname: vh-01.alexpires.local +hostname: vh-01.lab.alexpires.me ansible_user: "{{ lookup('env', 'ANSIBLE_USER') }}" # Users diff --git a/inventory/hosts b/inventory/hosts index aa38574..1e41a42 100644 --- a/inventory/hosts +++ b/inventory/hosts @@ -1,30 +1,30 @@ [all] prod-01.alexpires.me -dev-01.alexpires.dev -vh-01.alexpires.dev -gpu-01.alexpires.dev +dev-01.lab.alexpires.me +vh-01.lab.alexpires.me +gpu-01.lab.alexpires.me [hardening] prod-01.alexpires.me -dev-01.alexpires.dev -vh-01.alexpires.dev -gpu-01.alexpires.dev +dev-01.lab.alexpires.me +vh-01.lab.alexpires.me +gpu-01.lab.alexpires.me [ubuntu] prod-01.alexpires.me -dev-01.alexpires.dev +dev-01.lab.alexpires.me [fedora] -vh-01.alexpires.dev -gpu-01.alexpires.dev +vh-01.lab.alexpires.me +gpu-01.lab.alexpires.me [public] prod-01.alexpires.me [private] -dev-01.alexpires.dev -vh-01.alexpires.dev -gpu-01.alexpires.dev +dev-01.lab.alexpires.me +vh-01.lab.alexpires.me +gpu-01.lab.alexpires.me [gitea] prod-01.alexpires.me @@ -40,7 +40,7 @@ prod-01.alexpires.me [microk8s] prod-01.alexpires.me -dev-01.alexpires.dev +dev-01.lab.alexpires.me [iac] prod-01.alexpires.me @@ -61,22 +61,22 @@ prod-01.alexpires.me prod-01.alexpires.me [virtualization] -vh-01.alexpires.dev +vh-01.lab.alexpires.me [duo] prod-01.alexpires.me -dev-01.alexpires.dev -vh-01.alexpires.dev -gpu-01.alexpires.dev +dev-01.lab.alexpires.me +vh-01.lab.alexpires.me +gpu-01.lab.alexpires.me [ollama] -gpu-01.alexpires.dev +gpu-01.lab.alexpires.me [kokoro] -gpu-01.alexpires.dev +gpu-01.lab.alexpires.me [openwebui] -dev-01.alexpires.dev +dev-01.lab.alexpires.me [mcp] -dev-01.alexpires.dev +dev-01.lab.alexpires.me diff --git a/terraform/apps/dev-01/apps.tf b/terraform/apps/dev-01/apps.tf index 29b1cd9..ced2f70 100644 --- a/terraform/apps/dev-01/apps.tf +++ b/terraform/apps/dev-01/apps.tf @@ -10,3 +10,10 @@ module "samba" { shared_folder = local.samba_shared_folder } + +module "dns" { + source = "../../modules/apps/coredns" + + forward_dns_servers = local.dns_nameservers + zones = local.zones +} diff --git a/terraform/apps/dev-01/config.tf b/terraform/apps/dev-01/config.tf index 7049e10..22eb6b7 100644 --- a/terraform/apps/dev-01/config.tf +++ b/terraform/apps/dev-01/config.tf @@ -3,10 +3,54 @@ locals { # storage persistent_folder = "/srv" - primary_domain = "alexpires.me" - open_webui_fqdn = "ai.${local.primary_domain}" + lab_domain = "lab.alexpires.me" + open_webui_fqdn = "open-webui.${local.lab_domain}" internal_issuer = "internal-ca" # samba samba_shared_folder = "/mnt/usb" + + # DNS + dns_nameservers = [ + "193.110.81.0", + "185.253.5.0" + ] + + zones = [ + { + fqdn = "${local.lab_domain}" + records = [ + { + name = "gpu-01" + type = "A" + content = "10.19.4.106" + }, + { + name = "dev-01" + type = "A" + content = "10.19.4.136" + }, + { + name = "ollama" + type = "CNAME" + content = "gpu-01.${local.lab_domain}." + }, + { + name = "kokoro" + type = "CNAME" + content = "gpu-01.${local.lab_domain}." + }, + { + name = "open-webui" + type = "CNAME" + content = "dev-01.${local.lab_domain}." + }, + { + name = "smb" + type = "CNAME" + content = "dev-01.${local.lab_domain}." + }, + ] + } + ] } diff --git a/terraform/apps/prod-01/config.tf b/terraform/apps/prod-01/config.tf index ea73cc8..d4a7a95 100644 --- a/terraform/apps/prod-01/config.tf +++ b/terraform/apps/prod-01/config.tf @@ -97,16 +97,15 @@ locals { http_port = 8080 https_port = 8443 tls = true + resolver = "10.19.4.136" fqdn = ["ai.alexpires.me"] - upstream = "https://10.19.4.136" + upstream = "https://open-webui.lab.alexpires.me" locations = [ { path = "/ws/socket.io/" headers = { - "Host" = "$host" - "X-Forwarded-For" = "$remote_addr" - "Upgrade" = "$http_upgrade" - "Connection" = "upgrade" + "Upgrade" = "$http_upgrade" + "Connection" = "upgrade" } }, # { @@ -117,10 +116,6 @@ locals { # }, { path = "/" - headers = { - "Host" = "$host" - "X-Forwarded-For" = "$remote_addr" - } }, ] }, diff --git a/terraform/modules/apps/coredns/config.tf b/terraform/modules/apps/coredns/config.tf new file mode 100644 index 0000000..12195f5 --- /dev/null +++ b/terraform/modules/apps/coredns/config.tf @@ -0,0 +1,21 @@ + +locals { + configfile = templatefile("${path.module}/resources/Corefile.tpl", { + zones = var.zones + forward_dns_servers = join(" ", var.forward_dns_servers) + }) + configfile_checksum = md5(local.configfile) + db_files = { + for z in var.zones : z.fqdn => templatefile("${path.module}/resources/db.zone.tpl", { + fqdn = z.fqdn + domain_master = z.domain_master + timestamp = formatdate("YYYYMMDD", timestamp()) + refresh_rate = z.refresh_rate + retry_rate = z.retry_rate + expire_time = z.expire_time + ttl = z.ttl + records = z.records + }) + } + db_files_checksum = md5(jsonencode(local.db_files)) +} diff --git a/terraform/modules/apps/coredns/main.tf b/terraform/modules/apps/coredns/main.tf new file mode 100644 index 0000000..745e11d --- /dev/null +++ b/terraform/modules/apps/coredns/main.tf @@ -0,0 +1,195 @@ +resource "kubernetes_namespace" "dns" { + metadata { + name = "dns" + } +} + +resource "kubernetes_service_account" "dns" { + + metadata { + name = "dns-sa" + namespace = kubernetes_namespace.dns.metadata[0].name + } + + automount_service_account_token = false +} + +resource "kubernetes_config_map" "dns" { + metadata { + name = "dns-config" + namespace = kubernetes_namespace.dns.metadata[0].name + } + + data = merge({ + "Corefile" = local.configfile + + }, local.db_files) +} + +resource "kubernetes_deployment" "dns" { + metadata { + name = "dns" + namespace = kubernetes_namespace.dns.metadata[0].name + labels = { + dns = "dns" + } + } + + spec { + replicas = 1 + + selector { + match_labels = { + dns = "dns" + } + } + + strategy { + type = "RollingUpdate" + } + + template { + metadata { + labels = { + dns = "dns" + } + annotations = { + "checksum/corefile" = local.configfile_checksum + "checksum/db-files" = local.db_files_checksum + } + } + + spec { + service_account_name = kubernetes_service_account.dns.metadata[0].name + automount_service_account_token = false + + container { + name = "dns" + image = "docker.io/coredns/coredns:${var.tag}" + image_pull_policy = "IfNotPresent" + args = [ + "-conf", + "/etc/coredns/Corefile", + ] + + liveness_probe { + http_get { + path = "/health" + port = 8080 + scheme = "HTTP" + } + initial_delay_seconds = 60 + period_seconds = 10 + success_threshold = 1 + timeout_seconds = 5 + } + + readiness_probe { + http_get { + path = "/ready" + port = 8181 + scheme = "HTTP" + } + period_seconds = 10 + success_threshold = 1 + timeout_seconds = 1 + } + + resources { + requests = { + memory = "100Mi" + } + } + + security_context { + allow_privilege_escalation = false + capabilities { + drop = ["ALL"] + add = ["NET_BIND_SERVICE"] + } + } + + port { + name = "dns-tcp" + container_port = 53 + host_port = 53 + protocol = "TCP" + } + + port { + name = "dns-udp" + container_port = 53 + host_port = 53 + protocol = "UDP" + } + + port { + name = "metrics" + container_port = 9153 + protocol = "TCP" + } + + volume_mount { + name = "config" + mount_path = "/etc/coredns" + read_only = true + } + } + + volume { + name = "config" + config_map { + name = kubernetes_config_map.dns.metadata[0].name + default_mode = "0644" + items { + key = "Corefile" + path = "Corefile" + } + dynamic "items" { + for_each = local.db_files + content { + key = items.key + path = "db.${items.key}" + } + } + } + } + } + } + } +} + +resource "kubernetes_service" "dns" { + metadata { + name = "dns-ports" + namespace = kubernetes_namespace.dns.metadata[0].name + } + + spec { + selector = { + dns = "dns" + } + + port { + name = "dns-tcp" + protocol = "TCP" + port = 53 + target_port = 53 + } + + port { + name = "dns-udp" + protocol = "UDP" + port = 53 + target_port = 53 + } + + type = "LoadBalancer" + } + + lifecycle { + ignore_changes = [ + metadata[0].annotations + ] + } +} diff --git a/terraform/modules/apps/coredns/provider.tf b/terraform/modules/apps/coredns/provider.tf new file mode 100644 index 0000000..2eaa508 --- /dev/null +++ b/terraform/modules/apps/coredns/provider.tf @@ -0,0 +1,9 @@ +terraform { + required_version = "~>1.8" + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.36.0" + } + } +} diff --git a/terraform/modules/apps/coredns/resources/Corefile.tpl b/terraform/modules/apps/coredns/resources/Corefile.tpl new file mode 100644 index 0000000..be170ae --- /dev/null +++ b/terraform/modules/apps/coredns/resources/Corefile.tpl @@ -0,0 +1,24 @@ +.:53 { + errors + health { + lameduck 5s + } + ready + log . { + class error + } + prometheus :9153 + forward . ${forward_dns_servers} + cache 30 + loop + reload + loadbalance +} + +%{ for z in zones ~} +${z.fqdn}:53 { + file /etc/coredns/db.${z.fqdn} + log + errors +} +%{ endfor ~} \ No newline at end of file diff --git a/terraform/modules/apps/coredns/resources/db.zone.tpl b/terraform/modules/apps/coredns/resources/db.zone.tpl new file mode 100644 index 0000000..994ee3f --- /dev/null +++ b/terraform/modules/apps/coredns/resources/db.zone.tpl @@ -0,0 +1,13 @@ +$ORIGIN ${fqdn}. +@ IN SOA dns.${fqdn}. ${domain_master}.${fqdn}. ( + ${timestamp} ; serial + ${refresh_rate} ; refresh + ${retry_rate} ; retry + ${expire_time} ; expire + ${ttl} ) ; minimum + + IN NS dns.${fqdn}. + +%{ for r in records ~} +${r.name} IN ${r.type} ${r.content} +%{ endfor ~} diff --git a/terraform/modules/apps/coredns/variables.tf b/terraform/modules/apps/coredns/variables.tf new file mode 100644 index 0000000..4241c9d --- /dev/null +++ b/terraform/modules/apps/coredns/variables.tf @@ -0,0 +1,29 @@ +variable "tag" { + description = "The version of app to install" + type = string + default = "latest" +} + +variable "forward_dns_servers" { + description = "Forward DNS servers" + type = list(string) + default = ["8.8.8.8", "8.8.4.4"] +} + +variable "zones" { + description = "Zones to be managed by coredns" + type = list(object({ + fqdn = string + domain_master = optional(string, "admin") + refresh_rate = optional(number, 7200) + retry_rate = optional(number, 3600) + expire_time = optional(number, 1209600) + ttl = optional(number, 3600) + records = list(object({ + name = string + type = string + content = string + })) + })) + default = [] +} diff --git a/terraform/modules/apps/nginx-reverse-proxy/config.tf b/terraform/modules/apps/nginx-reverse-proxy/config.tf index 9d06e37..fbe4ed0 100644 --- a/terraform/modules/apps/nginx-reverse-proxy/config.tf +++ b/terraform/modules/apps/nginx-reverse-proxy/config.tf @@ -27,6 +27,7 @@ locals { http_port = v.http_port https_port = v.https_port upstream = v.upstream + resolver = v.resolver tls = v.tls fqdn = join(" ", v.fqdn) auth = length([for l in v.locations : true if l.private]) > 0 @@ -35,6 +36,7 @@ locals { headers = merge(l.headers, coalesce(v.default_headers, local.default_headers)) private = l.private upstream = l.upstream + resolver = l.resolver rewrite = l.rewrite }] custom_snippet = v.custom_snippet diff --git a/terraform/modules/apps/nginx-reverse-proxy/main.tf b/terraform/modules/apps/nginx-reverse-proxy/main.tf index a590ca7..9934180 100644 --- a/terraform/modules/apps/nginx-reverse-proxy/main.tf +++ b/terraform/modules/apps/nginx-reverse-proxy/main.tf @@ -310,6 +310,27 @@ resource "kubernetes_deployment" "reverse_proxy" { allow_privilege_escalation = false } + readiness_probe { + http_get { + path = "/health" + port = 8888 + } + initial_delay_seconds = 5 + period_seconds = 10 + } + + liveness_probe { + http_get { + path = "/health" + port = 8888 + } + initial_delay_seconds = 5 + period_seconds = 10 + failure_threshold = 10 + timeout_seconds = 5 + success_threshold = 1 + } + dynamic "port" { for_each = { for k, v in var.services : k => v.http_port } content { diff --git a/terraform/modules/apps/nginx-reverse-proxy/resources/nginx/conf.d/default.conf.tpl b/terraform/modules/apps/nginx-reverse-proxy/resources/nginx/conf.d/default.conf.tpl index a8acd21..3071c9b 100644 --- a/terraform/modules/apps/nginx-reverse-proxy/resources/nginx/conf.d/default.conf.tpl +++ b/terraform/modules/apps/nginx-reverse-proxy/resources/nginx/conf.d/default.conf.tpl @@ -21,13 +21,21 @@ location = /auth { %{ endif } %{ for l in locations ~} location ${l.path} { +%{ if l.resolver == "" ~} +%{ if resolver != "" ~} + resolver ${resolver}; +%{ endif ~} +%{ else ~} + resolver ${l.resolver}; +%{ endif ~} %{ if l.private ~} - auth_request /auth; + auth_request /auth<; %{ endif ~} %{ if l.rewrite != "" ~} rewrite ${l.rewrite}; %{ endif ~} - proxy_pass %{ if l.upstream != "" ~}${l.upstream}%{ else ~}${upstream}%{ endif ~}; + set $upstream "%{ if l.upstream != "" ~}${l.upstream}%{ else ~}${upstream}%{ endif ~}"; + proxy_pass $upstream; proxy_http_version 1.1; %{ for k,v in l.headers ~} proxy_set_header ${k} ${v}; diff --git a/terraform/modules/apps/nginx-reverse-proxy/resources/nginx/nginx.conf b/terraform/modules/apps/nginx-reverse-proxy/resources/nginx/nginx.conf index 185ece8..757f89b 100644 --- a/terraform/modules/apps/nginx-reverse-proxy/resources/nginx/nginx.conf +++ b/terraform/modules/apps/nginx-reverse-proxy/resources/nginx/nginx.conf @@ -3,7 +3,6 @@ worker_processes auto; error_log /var/log/nginx/error.log notice; pid /tmp/nginx.pid; - events { worker_connections 1024; } @@ -32,5 +31,15 @@ http { keepalive_timeout 65; + server { + listen 8888; + server_name _; + location /health { + access_log off; + return 200 "OK"; + } + } + include /etc/nginx/conf.d/*.conf; -} \ No newline at end of file +} + diff --git a/terraform/modules/apps/nginx-reverse-proxy/variables.tf b/terraform/modules/apps/nginx-reverse-proxy/variables.tf index 5790645..b2a5a4c 100644 --- a/terraform/modules/apps/nginx-reverse-proxy/variables.tf +++ b/terraform/modules/apps/nginx-reverse-proxy/variables.tf @@ -31,6 +31,7 @@ variable "services" { type = map(object({ fqdn = list(string) upstream = string + resolver = optional(string, "") default_headers = optional(map(string), {}), http_port = optional(number, 80) https_port = optional(number, 443) @@ -40,6 +41,7 @@ variable "services" { headers = optional(map(string), {}), private = optional(bool, false), upstream = optional(string, ""), + resolver = optional(string, "") rewrite = optional(string, "") })), []) custom_snippet = optional(string, "")