Add custom error pages and Nginx configuration for proxy module

- Introduced custom HTML error pages for various HTTP status codes (400, 401, 403, 404, 408, 429, 500, 502, 503, 504) in the proxy module.
- Created a .gitignore file to exclude non-HTML files in the error resources directory.
- Updated Nginx configuration to serve custom error pages for upstream errors.
- Added new Nginx configuration templates for handling HTTP and stream protocols.
- Implemented Wireguard configuration templates and associated variables for secure connections.
- Established Kubernetes resources for Stunnel and Wireguard, including deployment, service accounts, and config maps.
- Defined variables for the Stunnel and Wireguard modules to facilitate configuration.
- Created a new Wol Proxy module with Kubernetes resources for deployment and service management.

ansible/host_vars/dev-01.lab.alexpires.me.yml

@@ -41,6 +41,7 @@ fw_allowed_ports:
   - { rule: "allow", port: "11434", proto: "tcp", from: "10.19.4.0/24" } # Ollama (local network only)
   - { rule: "allow", port: "11434", proto: "tcp", from: "10.5.5.5/32" } # Ollama (Laptop)
   - { rule: "allow", port: "11434", proto: "tcp", from: "10.5.5.2/32" } # Ollama (Contabo VPN)
+  - { rule: "allow", port: "9090", proto: "tcp", from: "10.5.5.2/32" } # Prometheus (Contabo VPN)
   - { rule: "allow", port: "443", proto: "tcp", from: "10.5.5.2/32" } # HTTPS (Contabo VPN)
   - { rule: "allow", port: "53", proto: "udp", from: "10.19.4.0/24" } # DNS (local network)
   - { rule: "allow", port: "53", proto: "udp", from: "10.5.5.2/32" } # DNS (Contabo VPN)
@@ -52,6 +53,7 @@ ufw_outgoing_traffic:
   - 80 # HTTP
   - 123 # NTP
   - 443 # HTTPS
+  - 465 # SMTPS
   - 853 # DNS over TLS
   - 10250 # Kubelet
   - 8880 # Kokoro-FastAPI
@@ -60,7 +62,8 @@ ufw_outgoing_traffic:
   - 8188 # comfyui
   - 4443
   - 9443
-  - 995 # POP3S
+  - 993 # IMAPS
+  - 9090 # Prometheus
 
 ufw_default_forward_policy: "ACCEPT"
 
@@ -84,7 +87,10 @@ fail2ban_jails:
     bantime: 3600
 
 # aide
-install_aide: false
+install_aide: true
+aide_exclude:
+  - "!/var/lib/snapd"
+  - "!/srv"
 
 misc_modules_blocklist:
   - bluetooth

ansible/host_vars/prod-01.alexpires.me.yml

@@ -60,7 +60,7 @@ ufw_outgoing_traffic:
   - 16443 # Kube API Server
   - 4443
   - 9443
-  - 995 # POP3S
+  - 993 # IMAPS
 
 ufw_default_forward_policy: "ACCEPT"
 
@@ -96,7 +96,15 @@ fail2ban_jails:
     filter: "postfix-k8s"
     logpath: "/var/log/postfix/current"
     maxretry: 3
-    findtime: 18000
+    findtime: 600
+    bantime: -1
+  dovecot:
+    enabled: true
+    port: "imap,imaps,pop3,pop3s"
+    filter: "dovecot-k8s"
+    logpath: "/var/log/dovecot/info.log"
+    maxretry: 3
+    findtime: 600
     bantime: -1
 
 fail2ban_filters:
@@ -104,6 +112,10 @@ fail2ban_filters:
     Definition:
       failregex: "^.*?postfix/smtps/smtpd\\[\\d+\\]:.*\\[<HOST>\\]: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed.*$"
       ignoreregex: ""
+  dovecot-k8s:
+    Definition:
+      failregex: "(?: pop3-login|imap-login): .*(?:Authentication failure|Aborted login \\(auth failed|Aborted login \\(tried to use disabled|Disconnected \\(auth failed|Aborted login \\(\\d+ authentication attempts).*rip=`<HOST>"
+      ignoreregex: ""
 
 # aide
 install_aide: true

ansible/playbook_mail.yml

@@ -8,8 +8,11 @@
     - name: Set required facts
       ansible.builtin.set_fact:
         mail_home: "{{ persistent_data | default('/var/lib') }}/mail"
+        log_home: "/var/log"
         mail_user_id: 1000
         mail_group_id: 1000
+        dovecot_user_id: 100
+        dovecot_group_id: 101
         postfix_user_id: 100
         postfix_group_id: 101
       tags: always
@@ -59,3 +62,17 @@
       tags:
         - tasks
         - tasks::folders
+
+    - name: Create required dovecot log
+      become: true
+      ansible.builtin.file:
+        state: directory
+        path: "{{ item }}"
+        mode: "0700"
+        owner: "{{ dovecot_user_id }}"
+        group: "{{ dovecot_group_id }}"
+      loop:
+        - "{{ log_home }}/dovecot"
+      tags:
+        - tasks
+        - tasks::folders

ansible/playbook_monitoring.yml

@@ -1,16 +1,13 @@
 ---
-- name: Monitoring Playbook (K8s)
+- name: Prometheus Playbook (K8s)
-  hosts: monitoring
+  hosts: prometheus
   gather_facts: true
 
 # Specific tasks for this playbook
   tasks:
     - name: Set required facts
       ansible.builtin.set_fact:
-        monitoring_home: "/var/lib/monitoring"
+        monitoring_home: "{{ persistent_data | default('/var/lib') }}/monitoring"
-        influxdb_home: "/var/lib/monitoring/influxdb.data"
-        kapacitor_home: "/var/lib/monitoring/kapacitor.data"
-        cronograf_home: "{{ persistent_data | default('/var/lib') }}/chronograf.data"
         monitoring_user_id: 1000
         monitoring_group_id: 1000
       tags: always
@@ -25,9 +22,6 @@
         group: "{{ monitoring_group_id }}"
       with_items:
         - "{{ monitoring_home }}"
-        - "{{ influxdb_home }}"
-        - "{{ kapacitor_home }}"
-        - "{{ cronograf_home }}"
       tags:
         - tasks
         - tasks::folders

inventory/hosts

@@ -101,3 +101,7 @@ game-01.lab.alexpires.me
 
 [windows_game]
 game-01.lab.alexpires.me
+
+[prometheus]
+prod-01.alexpires.me
+dev-01.lab.alexpires.me

terraform/apps/dev-01/apps.tf

@@ -21,10 +21,10 @@ module "dns" {
   zones               = local.zones
 }
 
-module "reverse_proxy" {
+module "proxy" {
-  source = "../../modules/apps/nginx-reverse-proxy"
+  source = "../../modules/utils/proxy"
 
-  services = local.reverse_proxy_services
+  services = local.proxy_services
   streams  = local.streams
 }
 
@@ -42,9 +42,23 @@ module "nextcloud" {
 }
 
 module "comfyui_proxy" {
-  source = "../../modules/apps/wol_proxy"
+  source = "../../modules/utils/wol_proxy"
 
   upstream   = local.comfyui_proxy
   proxy_port = local.comfyui_proxy.port
   namespace  = "comfyui"
 }
+
+module "prometheus" {
+  source = "../../modules/utils/prometheus"
+
+  persistent_folder = "${local.persistent_folder}/monitoring"
+  config            = file("${path.module}/resources/prometheus.yml")
+}
+
+module "grafana" {
+  source = "../../modules/apps/grafana"
+
+  persistent_folder = "${local.persistent_folder}/monitoring"
+  fqdn              = local.grafana_fqdn
+}

terraform/apps/dev-01/config.tf

@@ -72,16 +72,31 @@ locals {
           type    = "CNAME"
           content = "dev-01.${local.lab_domain}."
         },
+        {
+          name    = "grafana"
+          type    = "CNAME"
+          content = "dev-01.${local.lab_domain}."
+        },
+        {
+          name    = "prometheus"
+          type    = "CNAME"
+          content = "dev-01.${local.lab_domain}."
+        },
         {
           name    = "game-01"
           type    = "A"
           content = "10.19.4.245"
         },
+        {
+          name    = "prod-01"
+          type    = "A"
+          content = "10.5.5.2"
+        },
       ]
     }
   ]
 
-  reverse_proxy_services = {
+  proxy_services = {
     "comfyui" = {
       http_port  = 8080
       https_port = 8443
@@ -103,6 +118,7 @@ locals {
   streams = {
     "dns" = {
       port      = 53
+      host_port = 53
       upstream  = "dns.dns.svc.cluster.local"
       protocol  = "udp"
     }
@@ -150,6 +166,17 @@ locals {
   backup_app_api_secret_key = data.terraform_remote_state.scaleway.outputs.backup_app_api_secret_key
   backup_app_api_access_key = data.terraform_remote_state.scaleway.outputs.backup_app_api_access_key
   mail_app_api_secret_key   = data.terraform_remote_state.scaleway.outputs.mail_app_api_secret_key
+
+  grafana_fqdn = "grafana.${local.lab_domain}"
+
+  remote_targets = {
+    "prometheus" = {
+      listen_port = 9090
+      target_host = "10.5.5.5"
+      target_port = 9090
+      protocol    = "TCP"
+    }
+  }
 }
 
 data "terraform_remote_state" "scaleway" {

terraform/apps/dev-01/resources/prometheus.yml

@@ -0,0 +1,38 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+rule_files:
+  - /etc/prometheus/prometheus.rules
+scrape_configs:
+  - job_name: kubernetes-nodes-cadvisor
+    scrape_interval: 10s
+    scrape_timeout: 10s
+    scheme: https # remove if you want to scrape metrics on insecure port
+    tls_config:
+      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+      insecure_skip_verify: true
+    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+    kubernetes_sd_configs:
+      - role: node
+    relabel_configs:
+      - action: labelmap
+        regex: __meta_kubernetes_node_label_(.+)
+      # Only for Kubernetes ^1.7.3.
+      # See: https://github.com/prometheus/prometheus/issues/2916
+      - target_label: __address__
+        replacement: kubernetes.default.svc:443
+      - source_labels: [__meta_kubernetes_node_name]
+        regex: (.+)
+        target_label: __metrics_path__
+        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
+    metric_relabel_configs:
+      - action: replace
+        source_labels: [id]
+        regex: '^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$'
+        target_label: rkt_container_name
+        replacement: "${2}-${1}"
+      - action: replace
+        source_labels: [id]
+        regex: '^/system\.slice/(.+)\.service$'
+        target_label: systemd_service_name
+        replacement: "${1}"

terraform/apps/prod-01/apps.tf

@@ -54,6 +54,10 @@ module "mail" {
   mail_crypt_public_key  = var.mail_crypt_public_key
 
   ca_pem = var.root_ca_pem
+
+  # For now we need to use host networking because of the fail2ban integration
+  postfix_use_host_networking = true
+  dovecot_use_host_networking = true
 }
 
 module "gitea" {
@@ -116,11 +120,18 @@ module "ssh_locker_web" {
   socket_path       = "/var/run/ssh_locker/provision.sock"
 }
 
-module "reverse_proxy" {
+module "proxy" {
-  source = "../../modules/apps/nginx-reverse-proxy"
+  source = "../../modules/utils/proxy"
 
-  services         = local.reverse_proxy_services
-  wireguard_config = local.reverse_proxy_wireguard_config
   auth_token       = var.nginx_token_bearer
+  services         = local.proxy_services
   streams          = local.streams
+  wireguard_config = local.wireguard_config
+}
+
+module "prometheus" {
+  source = "../../modules/utils/prometheus"
+
+  persistent_folder = "${local.persistent_folder}/monitoring"
+  config            = file("${path.module}/resources/prometheus.yml")
 }

terraform/apps/prod-01/config.tf

@@ -79,39 +79,52 @@ locals {
   #reverse proxy settings
 
   streams = {
-    "imap" = {
+    # For now disabled, because this messes with fail2ban on the host
-      port     = 993
+    # since the ip is lost and replaced with the nginx ingress controller ip
-      upstream = "imap.mail.svc.cluster.local"
+    # to fix this we might need to setup logging on the stream and cross reference
-      protocol = "tcp"
+    # with postfix/dovecot logs and then ban the ip on the host
-    }
+    # "imap" = {
-    "smtps" = {
+    #   port     = 993
-      port     = 465
+    #   upstream = "imap.mail.svc.cluster.local"
-      upstream = "smtps.mail.svc.cluster.local"
+    #   protocol = "tcp"
-      protocol = "tcp"
+    # }
-    }
+    # "smtps" = {
+    #   port     = 465
+    #   upstream = "smtps.mail.svc.cluster.local"
+    #   protocol = "tcp"
+    # }
     "21115-tcp" = {
       port      = 21115
+      host_port = 21115
       upstream  = "tcp.rustdesk.svc.cluster.local"
       protocol  = "tcp"
     }
     "21116-tcp" = {
       port      = 21116
+      host_port = 21116
       upstream  = "tcp.rustdesk.svc.cluster.local"
       protocol  = "tcp"
     }
     "21116-udp" = {
       port      = 21116
+      host_port = 21116
       upstream  = "udp.rustdesk.svc.cluster.local"
       protocol  = "udp"
     }
     "21117-tcp" = {
       port      = 21117
+      host_port = 21117
       upstream  = "tcp.rustdesk.svc.cluster.local"
       protocol  = "tcp"
     }
+    "9090-tcp" = {
+      port     = 9090
+      upstream = "prometheus.prometheus.svc.cluster.local"
+      protocol = "tcp"
+    }
   }
 
-  reverse_proxy_wireguard_config = {
+  wireguard_config = {
     private_key    = var.wireguard_1_private_key
     public_key     = var.wireguard_1_public_key
     pre_shared_key = var.wireguard_1_pre_shared_key
@@ -125,7 +138,7 @@ locals {
     ]
   }
 
-  reverse_proxy_services = {
+  proxy_services = {
     "open-webui" = {
       http_port  = 8080
       https_port = 8443

terraform/apps/prod-01/resources/prometheus.yml

@@ -0,0 +1,40 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+rule_files:
+  - /etc/prometheus/prometheus.rules
+scrape_configs:
+  - job_name: kubernetes-nodes-cadvisor
+    scrape_interval: 10s
+    scrape_timeout: 10s
+    scheme: https # remove if you want to scrape metrics on insecure port
+    tls_config:
+      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+      insecure_skip_verify: true
+    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+    kubernetes_sd_configs:
+      - role: node
+    relabel_configs:
+      - action: labelmap
+        regex: __meta_kubernetes_node_label_(.+)
+      # Only for Kubernetes ^1.7.3.
+      # See: https://github.com/prometheus/prometheus/issues/2916
+      - target_label: __address__
+        replacement: kubernetes.default.svc:443
+      - source_labels: [__meta_kubernetes_node_name]
+        regex: (.+)
+        target_label: __metrics_path__
+        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
+    metric_relabel_configs:
+      - action: replace
+        source_labels: [id]
+        regex: '^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$'
+        target_label: rkt_container_name
+        replacement: "${2}-${1}"
+      - action: replace
+        source_labels: [id]
+        regex: '^/system\.slice/(.+)\.service$'
+        target_label: systemd_service_name
+        replacement: "${1}"
+remote_write:
+  - url: "http://prometheus.reverse-proxy.svc.cluster.local:9090/api/v1/write"

terraform/modules/apps/grafana/config.tf

@@ -0,0 +1,4 @@
+locals {
+  app_version       = var.tag
+  monitoring_folder = "${var.persistent_folder}/grafana"
+}

terraform/modules/apps/grafana/grafana.tf

@@ -0,0 +1,136 @@
+resource "kubernetes_deployment" "grafana" {
+
+  metadata {
+    name      = "grafana"
+    namespace = kubernetes_namespace.grafana.metadata[0].name
+    labels = {
+      app = "grafana"
+    }
+  }
+  spec {
+    selector {
+      match_labels = {
+        app = "grafana"
+      }
+    }
+    template {
+      metadata {
+        labels = {
+          app = "grafana"
+        }
+      }
+      spec {
+        container {
+          name              = "grafana"
+          image             = "grafana/grafana:${local.app_version}"
+          image_pull_policy = "IfNotPresent"
+          security_context {
+            run_as_user                = 1000
+            run_as_non_root            = true
+            privileged                 = false
+            allow_privilege_escalation = false
+          }
+          port {
+            name           = "http-grafana"
+            container_port = 3000
+            protocol       = "TCP"
+          }
+          readiness_probe {
+            http_get {
+              path   = "/robots.txt"
+              port   = 3000
+              scheme = "HTTP"
+            }
+            failure_threshold     = 3
+            initial_delay_seconds = 10
+            period_seconds        = 30
+            success_threshold     = 1
+            timeout_seconds       = 2
+          }
+          liveness_probe {
+            tcp_socket {
+              port = 3000
+            }
+            failure_threshold     = 3
+            initial_delay_seconds = 30
+            period_seconds        = 10
+            success_threshold     = 1
+            timeout_seconds       = 1
+          }
+          resources {
+            requests = {
+              cpu    = "250m"
+              memory = "750Mi"
+            }
+          }
+          volume_mount {
+            name       = "grafana-pv"
+            mount_path = "/var/lib/grafana"
+          }
+        }
+        volume {
+          name = "grafana-pv"
+          host_path {
+            path = local.monitoring_folder
+          }
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_service" "grafana" {
+
+  metadata {
+    name      = "grafana"
+    namespace = kubernetes_namespace.grafana.metadata[0].name
+  }
+  spec {
+    selector = {
+      app = "grafana"
+    }
+    port {
+      port        = 3000
+      protocol    = "TCP"
+      target_port = "http-grafana"
+    }
+    session_affinity = "None"
+    type             = "ClusterIP"
+  }
+}
+
+resource "kubernetes_ingress_v1" "grafana" {
+
+  metadata {
+    name      = "ingress"
+    namespace = kubernetes_namespace.grafana.metadata[0].name
+
+    annotations = {
+      "kubernetes.io/ingress.class"              = "public"
+      "cert-manager.io/cluster-issuer"           = "letsencrypt-prod"
+      "kubernetes.io/tls-acme"                   = "true"
+      "nginx.ingress.kubernetes.io/ssl-redirect" = "true"
+    }
+  }
+  spec {
+    tls {
+      hosts       = [var.fqdn]
+      secret_name = "grafana-tls"
+    }
+    rule {
+      host = var.fqdn
+      http {
+        path {
+          backend {
+            service {
+              name = kubernetes_service.grafana.metadata[0].name
+              port {
+                number = 3000
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

terraform/modules/apps/grafana/main.tf

@@ -0,0 +1,5 @@
+resource "kubernetes_namespace" "grafana" {
+  metadata {
+    name = "grafana"
+  }
+}

terraform/modules/apps/grafana/resources/default.yml

@@ -0,0 +1,38 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+rule_files:
+  - /etc/prometheus/prometheus.rules
+scrape_configs:
+  - job_name: kubernetes-nodes-cadvisor
+    scrape_interval: 10s
+    scrape_timeout: 10s
+    scheme: https # remove if you want to scrape metrics on insecure port
+    tls_config:
+      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+      insecure_skip_verify: true
+    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+    kubernetes_sd_configs:
+      - role: node
+    relabel_configs:
+      - action: labelmap
+        regex: __meta_kubernetes_node_label_(.+)
+      # Only for Kubernetes ^1.7.3.
+      # See: https://github.com/prometheus/prometheus/issues/2916
+      - target_label: __address__
+        replacement: kubernetes.default.svc:443
+      - source_labels: [__meta_kubernetes_node_name]
+        regex: (.+)
+        target_label: __metrics_path__
+        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
+    metric_relabel_configs:
+      - action: replace
+        source_labels: [id]
+        regex: '^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$'
+        target_label: rkt_container_name
+        replacement: "${2}-${1}"
+      - action: replace
+        source_labels: [id]
+        regex: '^/system\.slice/(.+)\.service$'
+        target_label: systemd_service_name
+        replacement: "${1}"

terraform/modules/apps/grafana/resources/prometheus.bak.yml

@@ -0,0 +1,126 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+rule_files:
+  - /etc/prometheus/prometheus.rules
+alerting:
+  alertmanagers:
+    - scheme: http
+      static_configs:
+        - targets:
+            - "alertmanager.monitoring.svc:9093"
+scrape_configs:
+  - job_name: "node-exporter"
+    kubernetes_sd_configs:
+      - role: endpoints
+    relabel_configs:
+      - source_labels: [__meta_kubernetes_endpoints_name]
+        regex: "node-exporter"
+        action: keep
+  - job_name: "kubernetes-apiservers"
+    kubernetes_sd_configs:
+      - role: endpoints
+    scheme: https
+    tls_config:
+      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+    relabel_configs:
+      - source_labels:
+          [
+            __meta_kubernetes_namespace,
+            __meta_kubernetes_service_name,
+            __meta_kubernetes_endpoint_port_name,
+          ]
+        action: keep
+        regex: default;kubernetes;https
+  - job_name: "kubernetes-nodes"
+    scheme: https
+    tls_config:
+      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+    kubernetes_sd_configs:
+      - role: node
+    relabel_configs:
+      - action: labelmap
+        regex: __meta_kubernetes_node_label_(.+)
+      - target_label: __address__
+        replacement: kubernetes.default.svc:443
+      - source_labels: [__meta_kubernetes_node_name]
+        regex: (.+)
+        target_label: __metrics_path__
+        replacement: /api/v1/nodes/$${1}/proxy/metrics
+  - job_name: "kubernetes-pods"
+    kubernetes_sd_configs:
+      - role: pod
+    relabel_configs:
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+        action: keep
+        regex: true
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+        action: replace
+        target_label: __metrics_path__
+        regex: (.+)
+      - source_labels:
+          [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+        action: replace
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+        target_label: __address__
+      - action: labelmap
+        regex: __meta_kubernetes_pod_label_(.+)
+      - source_labels: [__meta_kubernetes_namespace]
+        action: replace
+        target_label: kubernetes_namespace
+      - source_labels: [__meta_kubernetes_pod_name]
+        action: replace
+        target_label: kubernetes_pod_name
+  - job_name: "kube-state-metrics"
+    static_configs:
+      - targets: ["kube-state-metrics.kube-system.svc.cluster.local:8080"]
+  - job_name: "kubernetes-cadvisor"
+    scheme: https
+    tls_config:
+      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+    kubernetes_sd_configs:
+      - role: node
+    relabel_configs:
+      - action: labelmap
+        regex: __meta_kubernetes_node_label_(.+)
+      - target_label: __address__
+        replacement: kubernetes.default.svc:443
+      - source_labels: [__meta_kubernetes_node_name]
+        regex: (.+)
+        target_label: __metrics_path__
+        replacement: /api/v1/nodes/$${1}/proxy/metrics/cadvisor
+  - job_name: "kubernetes-service-endpoints"
+    kubernetes_sd_configs:
+      - role: endpoints
+    relabel_configs:
+      - source_labels:
+          [__meta_kubernetes_service_annotation_prometheus_io_scrape]
+        action: keep
+        regex: true
+      - source_labels:
+          [__meta_kubernetes_service_annotation_prometheus_io_scheme]
+        action: replace
+        target_label: __scheme__
+        regex: (https?)
+      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
+        action: replace
+        target_label: __metrics_path__
+        regex: (.+)
+      - source_labels:
+          [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
+        action: replace
+        target_label: __address__
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+      - action: labelmap
+        regex: __meta_kubernetes_service_label_(.+)
+      - source_labels: [__meta_kubernetes_namespace]
+        action: replace
+        target_label: kubernetes_namespace
+      - source_labels: [__meta_kubernetes_service_name]
+        action: replace
+        target_label: kubernetes_name

terraform/modules/apps/grafana/variables.tf

@@ -0,0 +1,22 @@
+variable "fqdn" {
+  description = "The fully qualified domain name for the app server"
+  type        = string
+  default     = ""
+}
+
+variable "issuer" {
+  description = "The issuer for the certificate"
+  type        = string
+  default     = "internal-ca"
+}
+
+variable "tag" {
+  description = "Grafana image tag"
+  type        = string
+  default     = "latest"
+}
+
+variable "persistent_folder" {
+  description = "The path to the persistent folder"
+  type        = string
+}

terraform/modules/apps/mail/dovecot.tf

@@ -59,7 +59,7 @@ resource "kubernetes_deployment" "dovecot" {
       }
     }
     strategy {
-      type = "RollingUpdate"
+      type = var.dovecot_use_host_networking ? "Recreate" : "RollingUpdate"
     }
     template {
       metadata {
@@ -81,6 +81,7 @@ resource "kubernetes_deployment" "dovecot" {
           image = "dovecot/dovecot:${var.dovecot_tag}"
           port {
             container_port = 31993
+            host_port      = var.dovecot_use_host_networking ? 993 : null
           }
           port {
             name           = "lmtp"
@@ -118,6 +119,19 @@ resource "kubernetes_deployment" "dovecot" {
             mount_path = "/etc/mail-crypt"
             read_only  = true
           }
+          volume_mount {
+            name       = "dovecot-logs"
+            mount_path = "/var/log/dovecot"
+            read_only  = false
+          }
+          volume_mount {
+            name       = "tmp"
+            mount_path = "/tmp"
+          }
+          volume_mount {
+            name       = "dovecot-run"
+            mount_path = "/var/run/dovecot"
+          }
         }
         container {
           name  = "stunnel"
@@ -194,6 +208,20 @@ resource "kubernetes_deployment" "dovecot" {
             }
           }
         }
+        volume {
+          name = "dovecot-logs"
+          host_path {
+            path = "/var/log/dovecot"
+          }
+        }
+        volume {
+          name = "tmp"
+          empty_dir {}
+        }
+        volume {
+          name = "dovecot-run"
+          empty_dir {}
+        }
       }
     }
   }
@@ -246,7 +274,6 @@ resource "kubernetes_service" "dovecot_lmtp" {
       target_port = 31024
     }
     type = "ClusterIP"
-    cluster_ip = "None"
   }
 }
 
@@ -266,6 +293,5 @@ resource "kubernetes_service" "dovecot_auth" {
       target_port = 31000
     }
     type = "ClusterIP"
-    cluster_ip = "None"
   }
 }

terraform/modules/apps/mail/postfix.tf

@@ -36,7 +36,7 @@ resource "kubernetes_deployment" "postfix" {
       }
     }
     strategy {
-      type = "RollingUpdate"
+      type = "Recreate"
     }
     template {
       metadata {
@@ -56,6 +56,7 @@ resource "kubernetes_deployment" "postfix" {
           image = "tozd/postfix:${var.postfix_tag}"
           port {
             container_port = 465
+            host_port      = var.postfix_use_host_networking ? 465 : null
           }
           dynamic "env" {
             for_each = local.postfix_env_vars
@@ -190,7 +191,6 @@ resource "kubernetes_service" "postfix" {
     }
 
     type = "ClusterIP"
-    cluster_ip = "None"
   }
 
   lifecycle {

terraform/modules/apps/mail/resources/dovecot/dovecot.conf

@@ -71,8 +71,6 @@ mail_attribute {
   }
 }
 
-log_path = /dev/stdout
-
 imap_hibernate_timeout = 5s
 
 mail_plugins {
@@ -242,9 +240,10 @@ crypt_global_private_key main {
 }
 crypt_global_public_key_file = /etc/mail-crypt/ecpubkey.pem
 
-log_path = /dev/stdout
+
-info_log_path = /dev/stdout
+log_path = /var/log/dovecot/default.log
-debug_log_path = /dev/stdout
+info_log_path = /var/log/dovecot/info.log
+debug_log_path = /var/log/dovecot/debug.log
 
 # Enable verbose logging
 # auth_verbose = yes

terraform/modules/apps/mail/resources/postfix/run.config

@@ -39,7 +39,7 @@ postconf -e "smtpd_sasl_type=dovecot"
 postconf -e "smtpd_sasl_path=inet:localhost:31000"
 postconf -e "smtpd_sasl_auth_enable=yes"
 postconf -e "smtpd_sasl_security_options=noanonymous"
-postconf -e "smtpd_recipient_restrictions=permit_sasl_authenticated, reject"
+postconf -e "smtpd_recipient_restrictions=permit_sasl_authenticated, reject_unauth_destination"
 postconf -e "smtpd_relay_restrictions=permit_sasl_authenticated, reject_unauth_destination"
 postconf -e "smtpd_sasl_authenticated_header=yes"
 postconf -e "smtpd_tls_wrappermode=yes"
@@ -68,7 +68,8 @@ smtps     inet  n       -       n       -       -       smtpd
   -o syslog_name=postfix/smtps
   -o smtpd_tls_wrappermode=yes
   -o smtpd_sasl_auth_enable=yes
-  -o smtpd_recipient_restrictions=permit_sasl_authenticated,reject
+  -o smtpd_recipient_restrictions=permit_sasl_authenticated,reject_unauth_destination
+  -o smtpd_relay_restrictions=permit_sasl_authenticated,reject_unauth_destination
 " | sudo tee -a /etc/postfix/master.cf
 
 /usr/sbin/postfix -c /etc/postfix start

terraform/modules/apps/mail/variables.tf

@@ -92,3 +92,15 @@ variable "ca_pem" {
   type        = string
   sensitive   = true
 }
+
+variable "postfix_use_host_networking" {
+  description = "Whether to use host networking for the Postfix container"
+  type        = bool
+  default     = false
+}
+
+variable "dovecot_use_host_networking" {
+  description = "Whether to use host networking for the Dovecot container"
+  type        = bool
+  default     = false
+}

terraform/modules/apps/nginx/main.tf

@@ -47,7 +47,6 @@ resource "kubernetes_service" "website" {
     }
 
     type = "ClusterIP"
-    cluster_ip = "None"
   }
 }
 

terraform/modules/utils/prometheus/config.tf

@@ -0,0 +1,17 @@
+locals {
+  app_version = var.tag
+  data_folder = "${var.persistent_folder}/prometheus"
+
+  prometheus_config          = coalesce(var.config, file("${path.module}/resources/default.yml"))
+  prometheus_config_checksum = md5(local.prometheus_config)
+
+  prometheus_rules = yamlencode({
+    groups = [
+      {
+        name  = "custom-rules"
+        rules = var.rules
+      }
+    ]
+  })
+  prometheus_rules_checksum = md5(local.prometheus_rules)
+}

terraform/modules/utils/prometheus/main.tf

@@ -0,0 +1,5 @@
+resource "kubernetes_namespace" "monitoring" {
+  metadata {
+    name = "prometheus"
+  }
+}

terraform/modules/utils/prometheus/prometheus.tf

@@ -0,0 +1,216 @@
+resource "kubernetes_service_account" "prometheus" {
+  metadata {
+    name      = "prometheus"
+    namespace = kubernetes_namespace.monitoring.metadata[0].name
+  }
+}
+
+resource "kubernetes_cluster_role" "prometheus" {
+  metadata {
+    name = "prometheus"
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["nodes", "nodes/proxy", "services", "endpoints", "pods"]
+    verbs      = ["get", "list", "watch"]
+  }
+
+  rule {
+    api_groups = ["extensions"]
+    resources  = ["ingresses"]
+    verbs      = ["get", "list", "watch"]
+  }
+
+  rule {
+    api_groups = ["networking.k8s.io"]
+    resources  = ["ingresses"]
+    verbs      = ["get", "list", "watch"]
+  }
+
+  rule {
+    non_resource_urls = ["/metrics"]
+    verbs             = ["get"]
+  }
+}
+
+resource "kubernetes_cluster_role_binding" "prometheus" {
+  metadata {
+    name = "prometheus"
+  }
+
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "ClusterRole"
+    name      = kubernetes_cluster_role.prometheus.metadata[0].name
+  }
+
+  subject {
+    kind      = "ServiceAccount"
+    name      = kubernetes_service_account.prometheus.metadata[0].name
+    namespace = kubernetes_namespace.monitoring.metadata[0].name
+  }
+}
+
+resource "kubernetes_config_map" "prometheus" {
+  metadata {
+    name      = "prometheus-server-conf"
+    namespace = kubernetes_namespace.monitoring.metadata[0].name
+    labels = {
+      name = "prometheus-server-conf"
+    }
+  }
+
+  data = {
+    "prometheus.rules" = local.prometheus_rules
+    "prometheus.yml"   = local.prometheus_config
+  }
+}
+
+resource "kubernetes_deployment" "prometheus" {
+  metadata {
+    name      = "prometheus"
+    namespace = kubernetes_namespace.monitoring.metadata[0].name
+    labels = {
+      app = "prometheus"
+    }
+  }
+
+  spec {
+    replicas = 1
+    strategy {
+      type = "Recreate"
+    }
+
+    selector {
+      match_labels = {
+        app = "prometheus"
+      }
+    }
+
+    template {
+
+      metadata {
+        labels = {
+          app = "prometheus"
+        }
+        annotations = {
+          "checksum/config" = local.prometheus_config_checksum
+          "checksum/rules"  = local.prometheus_rules_checksum
+        }
+      }
+
+      spec {
+        service_account_name = kubernetes_service_account.prometheus.metadata[0].name
+        container {
+          name  = "prometheus"
+          image = "prom/prometheus:${local.app_version}"
+          args = [
+            "--config.file=/etc/prometheus/prometheus.yml",
+            "--storage.tsdb.path=/prometheus/",
+            "--storage.tsdb.retention.time=${var.retention_time}",
+          ]
+
+          port {
+            name           = "http-prometheus"
+            container_port = 9090
+          }
+
+          security_context {
+            run_as_user                = 1000
+            run_as_group               = 1000
+            run_as_non_root            = true
+            allow_privilege_escalation = false
+          }
+
+          volume_mount {
+            name       = "prometheus-config-volume"
+            mount_path = "/etc/prometheus/"
+          }
+          volume_mount {
+            name       = "prometheus-storage-volume"
+            mount_path = "/prometheus"
+          }
+        }
+
+        volume {
+          name = "prometheus-config-volume"
+          config_map {
+            name = kubernetes_config_map.prometheus.metadata[0].name
+          }
+        }
+
+        volume {
+          name = "prometheus-storage-volume"
+          host_path {
+            path = local.data_folder
+            type = "DirectoryOrCreate"
+          }
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_service" "prometheus" {
+  metadata {
+    name      = "prometheus"
+    namespace = kubernetes_namespace.monitoring.metadata[0].name
+    annotations = {
+      "prometheus.io/scrape" = "true"
+      "prometheus.io/port"   = "9090"
+    }
+  }
+
+  spec {
+    selector = {
+      app = "prometheus"
+    }
+
+    type = "ClusterIP"
+
+    port {
+      port        = 9090
+      protocol    = "TCP"
+      target_port = "http-prometheus"
+    }
+  }
+}
+
+# resource "kubernetes_ingress_v1" "prometheus" {
+#   metadata {
+#     name      = "prometheus-ui"
+#     namespace = kubernetes_namespace.monitoring.metadata[0].name
+#     annotations = {
+#       "kubernetes.io/ingress.class" = var.ingress_class
+#     }
+#   }
+
+#   spec {
+#     rule {
+#       host = var.fqdn
+#       http {
+#         path {
+#           path      = "/"
+#           path_type = "Prefix"
+#           backend {
+#             service {
+#               name = kubernetes_service.prometheus.metadata[0].name
+#               port {
+#                 number = 8080
+#               }
+#             }
+#           }
+#         }
+#       }
+#     }
+
+#     dynamic "tls" {
+#       for_each = local.tls_crt_data != "" && local.tls_key_data != "" ? [1] : []
+#       content {
+#         hosts       = [var.fqdn]
+#         secret_name = local.tls_secret_name
+#       }
+#     }
+#   }
+# }

terraform/modules/utils/prometheus/provider.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = "~>1.8"
+  required_providers {
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "2.36.0"
+    }
+  }
+}

terraform/modules/utils/prometheus/resources/default.yml

@@ -0,0 +1,38 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+rule_files:
+  - /etc/prometheus/prometheus.rules
+scrape_configs:
+  - job_name: kubernetes-nodes-cadvisor
+    scrape_interval: 10s
+    scrape_timeout: 10s
+    scheme: https # remove if you want to scrape metrics on insecure port
+    tls_config:
+      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+      insecure_skip_verify: true
+    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+    kubernetes_sd_configs:
+      - role: node
+    relabel_configs:
+      - action: labelmap
+        regex: __meta_kubernetes_node_label_(.+)
+      # Only for Kubernetes ^1.7.3.
+      # See: https://github.com/prometheus/prometheus/issues/2916
+      - target_label: __address__
+        replacement: kubernetes.default.svc:443
+      - source_labels: [__meta_kubernetes_node_name]
+        regex: (.+)
+        target_label: __metrics_path__
+        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
+    metric_relabel_configs:
+      - action: replace
+        source_labels: [id]
+        regex: '^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$'
+        target_label: rkt_container_name
+        replacement: "${2}-${1}"
+      - action: replace
+        source_labels: [id]
+        regex: '^/system\.slice/(.+)\.service$'
+        target_label: systemd_service_name
+        replacement: "${1}"

terraform/modules/utils/prometheus/resources/prometheus.bak.yml

@@ -0,0 +1,126 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+rule_files:
+  - /etc/prometheus/prometheus.rules
+alerting:
+  alertmanagers:
+    - scheme: http
+      static_configs:
+        - targets:
+            - "alertmanager.monitoring.svc:9093"
+scrape_configs:
+  - job_name: "node-exporter"
+    kubernetes_sd_configs:
+      - role: endpoints
+    relabel_configs:
+      - source_labels: [__meta_kubernetes_endpoints_name]
+        regex: "node-exporter"
+        action: keep
+  - job_name: "kubernetes-apiservers"
+    kubernetes_sd_configs:
+      - role: endpoints
+    scheme: https
+    tls_config:
+      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+    relabel_configs:
+      - source_labels:
+          [
+            __meta_kubernetes_namespace,
+            __meta_kubernetes_service_name,
+            __meta_kubernetes_endpoint_port_name,
+          ]
+        action: keep
+        regex: default;kubernetes;https
+  - job_name: "kubernetes-nodes"
+    scheme: https
+    tls_config:
+      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+    kubernetes_sd_configs:
+      - role: node
+    relabel_configs:
+      - action: labelmap
+        regex: __meta_kubernetes_node_label_(.+)
+      - target_label: __address__
+        replacement: kubernetes.default.svc:443
+      - source_labels: [__meta_kubernetes_node_name]
+        regex: (.+)
+        target_label: __metrics_path__
+        replacement: /api/v1/nodes/$${1}/proxy/metrics
+  - job_name: "kubernetes-pods"
+    kubernetes_sd_configs:
+      - role: pod
+    relabel_configs:
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+        action: keep
+        regex: true
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+        action: replace
+        target_label: __metrics_path__
+        regex: (.+)
+      - source_labels:
+          [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+        action: replace
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+        target_label: __address__
+      - action: labelmap
+        regex: __meta_kubernetes_pod_label_(.+)
+      - source_labels: [__meta_kubernetes_namespace]
+        action: replace
+        target_label: kubernetes_namespace
+      - source_labels: [__meta_kubernetes_pod_name]
+        action: replace
+        target_label: kubernetes_pod_name
+  - job_name: "kube-state-metrics"
+    static_configs:
+      - targets: ["kube-state-metrics.kube-system.svc.cluster.local:8080"]
+  - job_name: "kubernetes-cadvisor"
+    scheme: https
+    tls_config:
+      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+    kubernetes_sd_configs:
+      - role: node
+    relabel_configs:
+      - action: labelmap
+        regex: __meta_kubernetes_node_label_(.+)
+      - target_label: __address__
+        replacement: kubernetes.default.svc:443
+      - source_labels: [__meta_kubernetes_node_name]
+        regex: (.+)
+        target_label: __metrics_path__
+        replacement: /api/v1/nodes/$${1}/proxy/metrics/cadvisor
+  - job_name: "kubernetes-service-endpoints"
+    kubernetes_sd_configs:
+      - role: endpoints
+    relabel_configs:
+      - source_labels:
+          [__meta_kubernetes_service_annotation_prometheus_io_scrape]
+        action: keep
+        regex: true
+      - source_labels:
+          [__meta_kubernetes_service_annotation_prometheus_io_scheme]
+        action: replace
+        target_label: __scheme__
+        regex: (https?)
+      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
+        action: replace
+        target_label: __metrics_path__
+        regex: (.+)
+      - source_labels:
+          [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
+        action: replace
+        target_label: __address__
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+      - action: labelmap
+        regex: __meta_kubernetes_service_label_(.+)
+      - source_labels: [__meta_kubernetes_namespace]
+        action: replace
+        target_label: kubernetes_namespace
+      - source_labels: [__meta_kubernetes_service_name]
+        action: replace
+        target_label: kubernetes_name

terraform/modules/utils/prometheus/variables.tf

@@ -0,0 +1,34 @@
+variable "tag" {
+  description = "Prometheus image tag"
+  type        = string
+  default     = "latest"
+}
+
+variable "retention_time" {
+  description = "Prometheus data retention time"
+  type        = string
+  default     = "15d"
+}
+
+variable "rules" {
+  description = "Prometheus alerting rules"
+  type = list(object({
+    alert       = string
+    expr        = string
+    for         = string
+    labels      = map(string)
+    annotations = map(string)
+  }))
+  default = []
+}
+
+variable "config" {
+  description = "Custom Prometheus configuration in YAML format"
+  type        = string
+  default     = ""
+}
+
+variable "persistent_folder" {
+  description = "The path to the persistent folder"
+  type        = string
+}

terraform/modules/utils/proxy/main.tf

@@ -1,22 +1,22 @@
-resource "kubernetes_namespace" "reverse_proxy" {
+resource "kubernetes_namespace" "proxy" {
   metadata {
     annotations = {
-      name = "reverse-proxy"
+      name = "proxy"
     }
 
     labels = {
-      name = "reverse-proxy"
+      name = "proxy"
     }
 
-    name = "reverse-proxy"
+    name = "proxy"
   }
 }
 
-resource "kubernetes_service_account" "reverse-proxy_service_account" {
+resource "kubernetes_service_account" "service_account" {
 
   metadata {
-    name      = "reverse-proxy-app-sa"
+    name      = "proxy-app-sa"
-    namespace = kubernetes_namespace.reverse_proxy.metadata[0].name
+    namespace = kubernetes_namespace.proxy.metadata[0].name
   }
 
   automount_service_account_token = false
@@ -26,7 +26,7 @@ resource "kubernetes_service" "services" {
   for_each = var.services
   metadata {
     name      = format("%s", replace(each.key, ".", "-"))
-    namespace = kubernetes_namespace.reverse_proxy.metadata[0].name
+    namespace = kubernetes_namespace.proxy.metadata[0].name
   }
   spec {
     port {
@@ -43,14 +43,34 @@ resource "kubernetes_service" "services" {
       }
     }
     selector = {
-      app = "reverse-proxy"
+      app = "proxy"
     }
 
     type = "ClusterIP"
-    cluster_ip = "None"
   }
 }
 
+resource "kubernetes_service" "streams" {
+  for_each = var.streams
+  metadata {
+    name      = substr(format("stream-%s", replace(each.key, ".", "-")), 0, 15)
+    namespace = kubernetes_namespace.proxy.metadata[0].name
+  }
+  spec {
+    port {
+      name        = substr(format("stream-%s", replace(each.key, ".", "-")), 0, 15)
+      port        = each.value.port
+      target_port = substr(format("stream-%s", replace(each.key, ".", "-")), 0, 15)
+    }
+    selector = {
+      app = "proxy"
+    }
+
+    type = "ClusterIP"
+  }
+}
+
+
 resource "kubernetes_manifest" "internal_cert" {
 
   manifest = {
@@ -58,7 +78,7 @@ resource "kubernetes_manifest" "internal_cert" {
     kind       = "Certificate"
     metadata = {
       name      = "internal-cert"
-      namespace = kubernetes_namespace.reverse_proxy.metadata[0].name
+      namespace = kubernetes_namespace.proxy.metadata[0].name
     }
     spec = {
       secretName = "internal-cert"
@@ -66,7 +86,7 @@ resource "kubernetes_manifest" "internal_cert" {
         name = var.issuer_name
         kind = "ClusterIssuer"
       }
-      commonName = "reverse-proxy.internal"
+      commonName = "proxy.internal"
       dnsNames   = flatten([for _, v in var.services : v.fqdn if v.tls == true])
     }
   }
@@ -76,7 +96,7 @@ resource "kubernetes_config_map" "wireguard_config" {
   count = local.wireguard_config != "null" ? 1 : 0
   metadata {
     name      = "wireguard-config"
-    namespace = kubernetes_namespace.reverse_proxy.metadata[0].name
+    namespace = kubernetes_namespace.proxy.metadata[0].name
   }
 
   data = {
@@ -87,7 +107,7 @@ resource "kubernetes_config_map" "wireguard_config" {
 resource "kubernetes_config_map" "nginx_config" {
   metadata {
     name      = "nginx-config"
-    namespace = kubernetes_namespace.reverse_proxy.metadata[0].name
+    namespace = kubernetes_namespace.proxy.metadata[0].name
   }
 
   data = {
@@ -98,7 +118,7 @@ resource "kubernetes_config_map" "nginx_config" {
 resource "kubernetes_config_map" "nginx_default_config" {
   metadata {
     name      = "nginx-default-config"
-    namespace = kubernetes_namespace.reverse_proxy.metadata[0].name
+    namespace = kubernetes_namespace.proxy.metadata[0].name
   }
 
   data = local.services_config
@@ -107,7 +127,7 @@ resource "kubernetes_config_map" "nginx_default_config" {
 resource "kubernetes_config_map" "nginx_streams_config" {
   metadata {
     name      = "nginx-streams-config"
-    namespace = kubernetes_namespace.reverse_proxy.metadata[0].name
+    namespace = kubernetes_namespace.proxy.metadata[0].name
   }
 
   data = local.streams_config
@@ -117,7 +137,7 @@ resource "kubernetes_config_map" "nginx_custom_50x_page" {
 
   metadata {
     name      = "nginx-errors"
-    namespace = kubernetes_namespace.reverse_proxy.metadata[0].name
+    namespace = kubernetes_namespace.proxy.metadata[0].name
   }
 
   data = {
@@ -129,7 +149,7 @@ resource "kubernetes_config_map" "nginx_custom_50x_page" {
 resource "kubernetes_config_map" "auth_script" {
   metadata {
     name      = "auth-script"
-    namespace = kubernetes_namespace.reverse_proxy.metadata[0].name
+    namespace = kubernetes_namespace.proxy.metadata[0].name
   }
 
   data = {
@@ -141,7 +161,7 @@ resource "kubernetes_ingress_v1" "ingress" {
   for_each = { for k, v in var.services : k => v if lookup(v, "ingress", true) && length(v.fqdn) > 0 }
   metadata {
     name      = format("%s", replace(each.key, ".", "-"))
-    namespace = kubernetes_namespace.reverse_proxy.metadata[0].name
+    namespace = kubernetes_namespace.proxy.metadata[0].name
 
     annotations = merge({
       "kubernetes.io/ingress.class"    = "public"
@@ -159,7 +179,7 @@ resource "kubernetes_ingress_v1" "ingress" {
   spec {
     tls {
       hosts       = each.value.fqdn
-      secret_name = format("reverse-proxy-%s-tls", replace(each.key, ".", "-"))
+      secret_name = format("proxy-%s-tls", replace(each.key, ".", "-"))
     }
     dynamic "rule" {
       for_each = each.value.fqdn
@@ -186,7 +206,7 @@ resource "kubernetes_secret" "auth_token" {
   count = var.auth_token != "" ? 1 : 0
   metadata {
     name      = "auth-token-secret"
-    namespace = kubernetes_namespace.reverse_proxy.metadata[0].name
+    namespace = kubernetes_namespace.proxy.metadata[0].name
   }
 
   data = {
@@ -196,11 +216,11 @@ resource "kubernetes_secret" "auth_token" {
   type = "Opaque"
 }
 
-resource "kubernetes_deployment" "reverse_proxy" {
+resource "kubernetes_deployment" "proxy" {
 
   metadata {
-    name      = "reverse-proxy"
+    name      = "proxy"
-    namespace = kubernetes_namespace.reverse_proxy.metadata[0].name
+    namespace = kubernetes_namespace.proxy.metadata[0].name
 
     annotations = {
       "security.alpha.kubernetes.io/unsafe-sysctls" = "net.ipv4.conf.all.src_valid_mark=1"
@@ -212,7 +232,7 @@ resource "kubernetes_deployment" "reverse_proxy" {
 
     selector {
       match_labels = {
-        app = "reverse-proxy"
+        app = "proxy"
       }
     }
 
@@ -223,7 +243,7 @@ resource "kubernetes_deployment" "reverse_proxy" {
     template {
       metadata {
         labels = {
-          app = "reverse-proxy"
+          app = "proxy"
         }
         annotations = {
           "checksum/nginx"       = local.nginx_config_checksum
@@ -342,7 +362,7 @@ resource "kubernetes_deployment" "reverse_proxy" {
         }
 
         container {
-          name              = "reverse-proxy"
+          name              = "proxy"
           image             = "nginx:${var.tag}"
           image_pull_policy = "Always"
 
@@ -399,7 +419,7 @@ resource "kubernetes_deployment" "reverse_proxy" {
             content {
               container_port = port.value.port
               name           = substr(format("stream-%s", replace(port.key, ".", "-")), 0, 15)
-              host_port      = port.value.port
+              host_port      = port.value.host_port
               protocol       = upper(port.value.protocol)
             }
           }

terraform/modules/utils/proxy/variables.tf

@@ -70,6 +70,7 @@ variable "streams" {
     resolver       = optional(string, "")
     upstream_port  = optional(number, null)
     protocol       = optional(string, "TCP")
+    host_port      = optional(number, null)
   }))
   default = {}
 }

terraform/modules/utils/stunnel/config.tf

@@ -0,0 +1,20 @@
+locals {
+  wireguard_version = var.wireguard_version
+  wireguard_config = try(templatefile("${path.module}/resources/wireguard.conf.tpl", {
+    wireguard_private_key    = var.wireguard_config.private_key
+    wireguard_public_key     = var.wireguard_config.public_key
+    wireguard_pre_shared_key = var.wireguard_config.pre_shared_key
+    wireguard_address        = var.wireguard_config.address
+    wireguard_dns            = var.wireguard_config.dns
+    wireguard_allowed_ips    = join(", ", var.wireguard_config.allowed_ips)
+    wireguard_endpoint       = "${var.wireguard_config.endpoint}:${var.wireguard_config.endpoint_port}"
+  }), "null")
+  wireguard_config_checksum = nonsensitive(sha256(local.wireguard_config))
+
+  stunnel_version = var.stunnel_version
+  stunnel_config = templatefile("${path.module}/resources/stunnel.conf.tpl", {
+    local_targets  = var.local_targets
+    remote_targets = var.remote_targets
+  })
+  stunnel_config_checksum = sha256(local.stunnel_config)
+}

terraform/modules/utils/stunnel/main.tf

@@ -0,0 +1,289 @@
+resource "kubernetes_namespace" "stunnel" {
+  metadata {
+    name = "stunnel"
+    annotations = {
+      name = "stunnel"
+    }
+    labels = {
+      name = "stunnel"
+    }
+  }
+}
+
+resource "kubernetes_service_account" "stunnel_service_account" {
+
+  metadata {
+    name      = "stunnel-app-sa"
+    namespace = kubernetes_namespace.stunnel.metadata[0].name
+  }
+
+  automount_service_account_token = false
+}
+
+
+resource "kubernetes_config_map" "wireguard_config" {
+  count = local.wireguard_config != "null" ? 1 : 0
+  metadata {
+    name      = "wireguard-config"
+    namespace = kubernetes_namespace.stunnel.metadata[0].name
+  }
+
+  data = {
+    "wireguard.conf" = local.wireguard_config
+  }
+}
+
+resource "kubernetes_config_map" "stunnel_config" {
+  metadata {
+    name      = "stunnel-config"
+    namespace = kubernetes_namespace.stunnel.metadata[0].name
+  }
+
+  data = {
+    "stunnel.conf" = local.stunnel_config
+  }
+}
+
+resource "kubernetes_deployment" "stunnel" {
+
+  metadata {
+    name      = "stunnel"
+    namespace = kubernetes_namespace.stunnel.metadata[0].name
+
+    annotations = {
+      "security.alpha.kubernetes.io/unsafe-sysctls" = "net.ipv4.conf.all.src_valid_mark=1"
+    }
+  }
+
+  spec {
+    replicas = 1
+
+    selector {
+      match_labels = {
+        app = "stunnel"
+      }
+    }
+
+    strategy {
+      type = "Recreate"
+    }
+
+    template {
+      metadata {
+        labels = {
+          app = "stunnel"
+        }
+        annotations = {
+          "checksum/stunnel"   = local.stunnel_config_checksum
+          "checksum/wireguard" = local.wireguard_config_checksum
+        }
+      }
+      spec {
+
+        dynamic "container" {
+          for_each = nonsensitive(local.wireguard_config) != "null" ? [1] : []
+          content {
+
+            name  = "wireguard"
+            image = "linuxserver/wireguard:${local.wireguard_version}"
+
+            resources {
+              requests = {
+                memory = "64Mi"
+              }
+            }
+
+            liveness_probe {
+              exec {
+                command = [
+                  "/bin/sh",
+                  "-c",
+                  "wg show | grep -q transfer"
+                ]
+              }
+              initial_delay_seconds = 90
+              period_seconds        = 90
+            }
+
+            security_context {
+              privileged = true
+              capabilities {
+                add = ["NET_ADMIN"]
+              }
+              read_only_root_filesystem = true
+            }
+
+            env {
+              name  = "PUID"
+              value = "1000"
+            }
+
+            env {
+              name  = "PGID"
+              value = "1000"
+            }
+
+            env {
+              name  = "TZ"
+              value = "Europe/Berlin"
+            }
+
+            volume_mount {
+              name       = "wireguard-config"
+              mount_path = "/config/wg_confs"
+              read_only  = true
+            }
+
+            volume_mount {
+              name       = "lib-modules"
+              mount_path = "/lib/modules"
+              read_only  = true
+            }
+
+            volume_mount {
+              name       = "wireguard-run"
+              mount_path = "/run"
+            }
+          }
+        }
+
+        container {
+          name  = "stunnel"
+          image = "chainguard/stunnel:${local.stunnel_version}"
+          args  = ["/etc/conf/stunnel.conf"]
+          volume_mount {
+            name       = "stunnel-config"
+            mount_path = "/etc/conf"
+            read_only  = true
+          }
+          volume_mount {
+            name       = "stunnel-certs"
+            mount_path = "/etc/ssl/certs/private"
+            read_only  = true
+          }
+          dynamic "port" {
+            for_each = var.remote_targets
+            content {
+              container_port = port.value.listen_port
+              name           = substr(replace(port.key, ".", "-"), 0, 15)
+              protocol       = coalesce(upper(port.value.protocol), "TCP")
+            }
+          }
+        }
+
+        dynamic "volume" {
+          for_each = nonsensitive(local.wireguard_config) != "null" ? [1] : []
+          content {
+            name = "wireguard-config"
+            config_map {
+              name = nonsensitive(kubernetes_config_map.wireguard_config[0].metadata[0].name)
+            }
+          }
+        }
+
+        dynamic "volume" {
+          for_each = nonsensitive(local.wireguard_config) != "null" ? [1] : []
+          content {
+            name = "lib-modules"
+            host_path {
+              path = "/lib/modules"
+            }
+          }
+        }
+
+        dynamic "volume" {
+          for_each = nonsensitive(local.wireguard_config) != "null" ? [1] : []
+          content {
+            name = "wireguard-run"
+            empty_dir {
+              medium = "Memory"
+            }
+          }
+        }
+
+        volume {
+          name = "stunnel-config"
+          config_map {
+            name = kubernetes_config_map.stunnel_config.metadata[0].name
+          }
+        }
+
+        volume {
+          name = "stunnel-certs"
+          secret {
+            secret_name = "remote-targets-cert"
+          }
+        }
+      }
+    }
+  }
+
+  lifecycle {
+    ignore_changes = [spec[0].template[0].metadata[0].annotations["kubectl.kubernetes.io/restartedAt"]]
+  }
+}
+
+resource "kubernetes_manifest" "remote_targets_cert" {
+
+  manifest = {
+    apiVersion = "cert-manager.io/v1"
+    kind       = "Certificate"
+    metadata = {
+      name      = "remote-targets-cert"
+      namespace = kubernetes_namespace.stunnel.metadata[0].name
+    }
+    spec = {
+      secretName = "remote-targets-cert"
+      issuerRef = {
+        name = var.issuer_name
+        kind = "ClusterIssuer"
+      }
+      commonName = "stunnel.remote_targets.internal"
+      dnsNames = flatten([
+        for k, v in var.remote_targets : "${k}.stunnel.svc.cluster.local"
+      ])
+    }
+  }
+}
+
+resource "kubernetes_manifest" "local_targets_cert" {
+
+  manifest = {
+    apiVersion = "cert-manager.io/v1"
+    kind       = "Certificate"
+    metadata = {
+      name      = "local-targets-cert"
+      namespace = kubernetes_namespace.stunnel.metadata[0].name
+    }
+    spec = {
+      secretName = "local-targets-cert"
+      issuerRef = {
+        name = var.issuer_name
+        kind = "ClusterIssuer"
+      }
+      commonName = "stunnel.local_targets.internal"
+      dnsNames = flatten([
+        for _, v in var.local_targets : v.target_host
+      ])
+    }
+  }
+}
+
+resource "kubernetes_service" "services" {
+  for_each = var.remote_targets
+  metadata {
+    name      = format("%s", replace(each.key, ".", "-"))
+    namespace = kubernetes_namespace.stunnel.metadata[0].name
+  }
+  spec {
+    port {
+      name        = substr(format("%s", replace(each.key, ".", "-")), 0, 15)
+      port        = each.value.listen_port
+      target_port = substr(format("%s", replace(each.key, ".", "-")), 0, 15)
+    }
+    selector = {
+      app = "stunnel"
+    }
+    type = "ClusterIP"
+  }
+}

terraform/modules/utils/stunnel/provider.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = "~>1.8"
+  required_providers {
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "2.36.0"
+    }
+  }
+}

terraform/modules/utils/stunnel/resources/stunnel.conf.tpl

@@ -0,0 +1,16 @@
+foreground = yes
+
+%{ for target in local_targets ~}
+[server-${target.listen_port}]
+accept = 0.0.0.0:${target.listen_port}
+connect = ${target.target_host}:${target.target_port}
+cert = /etc/ssl/certs/private/tls.crt
+key = /etc/ssl/certs/private/tls.key
+%{ endfor ~}
+
+%{ for target in remote_targets ~}
+[client-${target.listen_port}]
+client = yes
+accept = 0.0.0.0:${target.listen_port}
+connect = ${target.target_host}:${target.target_port}
+%{ endfor ~}

terraform/modules/utils/stunnel/resources/wireguard.conf.tpl

@@ -0,0 +1,10 @@
+[Interface]
+PrivateKey = ${wireguard_private_key}
+Address = ${wireguard_address}
+
+[Peer]
+PublicKey = ${wireguard_public_key}
+PresharedKey = ${wireguard_pre_shared_key}
+AllowedIPs = ${wireguard_allowed_ips}
+Endpoint = ${wireguard_endpoint}
+PersistentKeepalive = 25

terraform/modules/utils/stunnel/variables.tf

@@ -0,0 +1,55 @@
+variable "stunnel_version" {
+  description = "Stunnel image version"
+  type        = string
+  default     = "latest"
+}
+
+variable "wireguard_version" {
+  description = "Wireguard image version"
+  type        = string
+  default     = "1.0.20210914-r4-ls54"
+}
+
+variable "issuer_name" {
+  description = "The name of the issuer to use for TLS certificates, internal certificates"
+  type        = string
+  default     = "internal-ca"
+}
+
+variable "local_targets" {
+  description = "Map of local services to expose via stunnel"
+  type = map(object({
+    listen_port = number
+    target_host = string
+    target_port = number
+    protocol    = string
+  }))
+  default = {}
+}
+
+variable "wireguard_config" {
+  description = "The Wireguard configuration file"
+  type = object({
+    private_key    = string
+    public_key     = string
+    pre_shared_key = string
+    endpoint       = string
+    endpoint_port  = number
+    address        = string
+    dns            = string
+    allowed_ips    = list(string)
+  })
+  default   = null
+  sensitive = true
+}
+
+variable "remote_targets" {
+  description = "Map of remote services to connect to via stunnel"
+  type = map(object({
+    listen_port = number
+    target_host = string
+    target_port = number
+    protocol    = string
+  }))
+  default = {}
+}

terraform/modules/utils/wol_proxy/provider.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = "~>1.8"
+  required_providers {
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "2.36.0"
+    }
+  }
+}