From c162944a73c02f5f3405be30afd731c0ae3a7250 Mon Sep 17 00:00:00 2001 From: Alexandre Pires Date: Mon, 16 Jun 2025 23:19:25 +0200 Subject: [PATCH] Add support for nginx streams --- terraform/apps/dev-01/apps.tf | 1 + terraform/apps/dev-01/config.tf | 8 +++ terraform/modules/apps/coredns/main.tf | 17 +++-- terraform/modules/apps/coredns/variables.tf | 6 ++ .../apps/nginx-reverse-proxy/config.tf | 13 ++++ .../modules/apps/nginx-reverse-proxy/main.tf | 64 +++++++++++++++++++ .../resources/nginx/nginx.conf | 2 + .../resources/nginx/streams.d/stream.conf.tpl | 15 +++++ .../apps/nginx-reverse-proxy/variables.tf | 13 ++++ terraform/modules/apps/samba/main.tf | 4 ++ 10 files changed, 136 insertions(+), 7 deletions(-) create mode 100644 terraform/modules/apps/nginx-reverse-proxy/resources/nginx/streams.d/stream.conf.tpl diff --git a/terraform/apps/dev-01/apps.tf b/terraform/apps/dev-01/apps.tf index 46f2f9c..ee46fc1 100644 --- a/terraform/apps/dev-01/apps.tf +++ b/terraform/apps/dev-01/apps.tf @@ -23,6 +23,7 @@ module "reverse_proxy" { source = "../../modules/apps/nginx-reverse-proxy" services = local.reverse_proxy_services + streams = local.streams } diff --git a/terraform/apps/dev-01/config.tf b/terraform/apps/dev-01/config.tf index 4b7a5b0..65b4be0 100644 --- a/terraform/apps/dev-01/config.tf +++ b/terraform/apps/dev-01/config.tf @@ -95,6 +95,14 @@ locals { } } + streams = { + "dns" = { + port = 53 + upstream = "dns.dns.svc.cluster.local" + protocol = "udp" + } + } + scaleway_smtp_host = "smtp.tem.scw.cloud" nextcloud_fqdn = "cloud.alexpires.me" diff --git a/terraform/modules/apps/coredns/main.tf b/terraform/modules/apps/coredns/main.tf index 8ad2435..710eb98 100644 --- a/terraform/modules/apps/coredns/main.tf +++ b/terraform/modules/apps/coredns/main.tf @@ -45,7 +45,10 @@ resource "kubernetes_deployment" "dns" { } strategy { - type = "Recreate" + type = "RollingUpdate" + rolling_update { + max_unavailable = 1 + } } template { @@ -112,14 +115,12 @@ resource "kubernetes_deployment" "dns" { port { name = "dns-tcp" container_port = 53 - host_port = 53 protocol = "TCP" } port { name = "dns-udp" container_port = 53 - host_port = 53 protocol = "UDP" } @@ -157,11 +158,15 @@ resource "kubernetes_deployment" "dns" { } } } + + lifecycle { + ignore_changes = [spec[0].template[0].metadata[0].annotations["kubectl.kubernetes.io/restartedAt"]] + } } -resource "kubernetes_service" "dns" { +resource "kubernetes_service" "dns_internal" { metadata { - name = "dns-ports" + name = "dns" namespace = kubernetes_namespace.dns.metadata[0].name } @@ -183,8 +188,6 @@ resource "kubernetes_service" "dns" { port = 53 target_port = 53 } - - type = "LoadBalancer" } lifecycle { diff --git a/terraform/modules/apps/coredns/variables.tf b/terraform/modules/apps/coredns/variables.tf index b587f79..c249bab 100644 --- a/terraform/modules/apps/coredns/variables.tf +++ b/terraform/modules/apps/coredns/variables.tf @@ -36,3 +36,9 @@ variable "zones" { })) default = [] } + +variable "external_dns_enabled" { + description = "Enable listening on port 53 on the loadbalancer for external DNS resolution" + type = bool + default = false +} diff --git a/terraform/modules/apps/nginx-reverse-proxy/config.tf b/terraform/modules/apps/nginx-reverse-proxy/config.tf index afaff0c..3bfb34c 100644 --- a/terraform/modules/apps/nginx-reverse-proxy/config.tf +++ b/terraform/modules/apps/nginx-reverse-proxy/config.tf @@ -45,6 +45,19 @@ locals { } services_config_checksum = sha256(jsonencode(local.services_config)) + streams_config = { + for k, v in var.streams : "${k}.conf" => templatefile("${path.module}/resources/nginx/streams.d/stream.conf.tpl", { + name = k + port = v.port + upstream = v.upstream + custom_snippet = v.custom_snippet + upstream_port = coalesce(v.upstream_port, v.port) + resolver = v.resolver + protocol = upper(v.protocol) + }) + } + streams_config_checksum = sha256(jsonencode(local.streams_config)) + auth_script = file("${path.module}/resources/auth/auth_server.py") auth_script_checksum = sha256(local.auth_script) diff --git a/terraform/modules/apps/nginx-reverse-proxy/main.tf b/terraform/modules/apps/nginx-reverse-proxy/main.tf index 62f02bc..4352594 100644 --- a/terraform/modules/apps/nginx-reverse-proxy/main.tf +++ b/terraform/modules/apps/nginx-reverse-proxy/main.tf @@ -102,6 +102,15 @@ resource "kubernetes_config_map" "nginx_default_config" { data = local.services_config } +resource "kubernetes_config_map" "nginx_streams_config" { + metadata { + name = "nginx-streams-config" + namespace = kubernetes_namespace.reverse_proxy.metadata[0].name + } + + data = local.streams_config +} + resource "kubernetes_config_map" "nginx_custom_50x_page" { metadata { @@ -220,6 +229,7 @@ resource "kubernetes_deployment" "reverse_proxy" { "checksum/wireguard" = local.wireguard_config_checksum "checksum/auth_server" = local.auth_script_checksum "checksum/custom_50x" = local.errors_pages_checksum + "checksum/streams" = local.streams_config_checksum } } spec { @@ -367,6 +377,16 @@ resource "kubernetes_deployment" "reverse_proxy" { } } + dynamic "port" { + for_each = var.streams + content { + container_port = port.value.port + name = substr(format("stream-%s", replace(port.key, ".", "-")), 0, 15) + host_port = port.value.port + protocol = upper(port.value.protocol) + } + } + volume_mount { name = "nginx" mount_path = "/etc/nginx/nginx.conf" @@ -380,6 +400,12 @@ resource "kubernetes_deployment" "reverse_proxy" { read_only = true } + volume_mount { + name = "streams" + mount_path = "/etc/nginx/streams.d/" + read_only = true + } + volume_mount { name = "nginx-errors" mount_path = "/usr/share/nginx/html/errors" @@ -437,6 +463,13 @@ resource "kubernetes_deployment" "reverse_proxy" { } } + volume { + name = "streams" + config_map { + name = kubernetes_config_map.nginx_streams_config.metadata[0].name + } + } + volume { name = "nginx-errors" config_map { @@ -471,3 +504,34 @@ resource "kubernetes_deployment" "reverse_proxy" { } } + +# resource "kubernetes_service" "reverse_proxy_streams" { +# count = length(var.streams) > 0 ? 1 : 0 +# metadata { +# name = "stream-ports" +# namespace = kubernetes_namespace.reverse_proxy.metadata[0].name +# } + +# spec { +# selector = { +# app = "reverse-proxy" +# } + +# dynamic "port" { +# for_each = var.streams +# content { +# name = format("%s-%s-%d", port.key, port.value.protocol, port.value.port) +# port = port.value.port +# target_port = port.value.port +# protocol = upper(port.value.protocol) +# } +# } +# type = "LoadBalancer" +# } + +# lifecycle { +# ignore_changes = [ +# metadata[0].annotations +# ] +# } +# } diff --git a/terraform/modules/apps/nginx-reverse-proxy/resources/nginx/nginx.conf b/terraform/modules/apps/nginx-reverse-proxy/resources/nginx/nginx.conf index 757f89b..4f1f0c0 100644 --- a/terraform/modules/apps/nginx-reverse-proxy/resources/nginx/nginx.conf +++ b/terraform/modules/apps/nginx-reverse-proxy/resources/nginx/nginx.conf @@ -43,3 +43,5 @@ http { include /etc/nginx/conf.d/*.conf; } +include /etc/nginx/streams.d/*.conf; + diff --git a/terraform/modules/apps/nginx-reverse-proxy/resources/nginx/streams.d/stream.conf.tpl b/terraform/modules/apps/nginx-reverse-proxy/resources/nginx/streams.d/stream.conf.tpl new file mode 100644 index 0000000..d7aba8d --- /dev/null +++ b/terraform/modules/apps/nginx-reverse-proxy/resources/nginx/streams.d/stream.conf.tpl @@ -0,0 +1,15 @@ +stream { + upstream ${name} { +%{ if resolver != "" ~} + resolver ${resolver}; +%{ endif ~} + server ${upstream}:${upstream_port}; + } + server { + listen %{ if protocol == "UDP" ~}${port} udp;%{ else ~}${port};%{ endif } +%{ if custom_snippet != "" ~} +${custom_snippet} +%{ endif ~} + proxy_pass ${name}; + } +} diff --git a/terraform/modules/apps/nginx-reverse-proxy/variables.tf b/terraform/modules/apps/nginx-reverse-proxy/variables.tf index b105017..76fc960 100644 --- a/terraform/modules/apps/nginx-reverse-proxy/variables.tf +++ b/terraform/modules/apps/nginx-reverse-proxy/variables.tf @@ -57,3 +57,16 @@ variable "auth_token" { sensitive = true default = "" } + +variable "streams" { + description = "A list of streams to be deployed" + type = map(object({ + port = number + upstream = string + custom_snippet = optional(string, "") + resolver = optional(string, "") + upstream_port = optional(number, null) + protocol = optional(string, "TCP") + })) + default = {} +} diff --git a/terraform/modules/apps/samba/main.tf b/terraform/modules/apps/samba/main.tf index e328b2a..f38d046 100644 --- a/terraform/modules/apps/samba/main.tf +++ b/terraform/modules/apps/samba/main.tf @@ -117,6 +117,10 @@ resource "kubernetes_deployment" "samba" { } } } + + lifecycle { + ignore_changes = [spec[0].template[0].metadata[0].annotations["kubectl.kubernetes.io/restartedAt"]] + } } resource "kubernetes_service" "samba" {