Add cert checker module: implement Kubernetes resources, certificate validation logic, and deployment restart functionality

terraform/apps/prod-01/config.tf

@@ -52,7 +52,6 @@ locals {
   sftpgo_files_backup_cron_schedule = "50 2 * * *"
   mail_files_backup_cron_schedule   = "0 3 * * *"
 
-
   # email settings
   fetch_emails_schedule = "*/2 * * * *"
   scaleway_smtp_host    = "smtp.tem.scw.cloud"
@@ -76,7 +75,7 @@ locals {
     }
   ]
 
-  #reverse proxy settings
+  # proxy settings
 
   streams = {
     # For now disabled, because this messes with fail2ban on the host
@@ -193,6 +192,7 @@ locals {
     }
   }
 
+  # get secrets from remote state
   backup_bucket_name        = data.terraform_remote_state.scaleway.outputs.backup_bucket_name
   backup_app_api_secret_key = data.terraform_remote_state.scaleway.outputs.backup_app_api_secret_key
   backup_app_api_access_key = data.terraform_remote_state.scaleway.outputs.backup_app_api_access_key

terraform/modules/apps/mail/config.tf

@@ -88,6 +88,19 @@ locals {
 
   postfix_stunnel_config          = file("${path.module}/resources/postfix/stunnel.conf")
   postfix_stunnel_config_checksum = sha256(local.postfix_stunnel_config)
+
+  deployments = {
+    "dovecot" = {
+      service_name = "imap"
+      service_port = 993
+      secret_name  = "mail-tls"
+    },
+    "postfix" = {
+      service_name = "smtps"
+      service_port = 465
+      secret_name  = "mail-tls"
+    }
+  }
 }
 
 data "external" "password_hasher" {

terraform/modules/apps/mail/main.tf

@@ -47,7 +47,17 @@ resource "kubernetes_manifest" "mail_certificate" {
         kind = "ClusterIssuer"
       }
       commonName = "imap.${var.primary_domain}"
-      dnsNames   = flatten([for account in var.email_accounts : ["imap.${account.domain}", "smtp.${account.domain}"]])
+      dnsNames = flatten([for account in var.email_accounts : [
+        "imap.${account.domain}",
+        "smtp.${account.domain}"
+      ]])
     }
   }
 }
+
+module "cert_checker" {
+  source            = "../../utils/cert_checker"
+  namespace         = kubernetes_namespace.mail.metadata[0].name
+  persistent_folder = local.mail_path
+  deployments       = local.deployments
+}

terraform/modules/utils/cert_checker/config.tf

@@ -0,0 +1,10 @@
+locals {
+  app_version = var.tag
+  workdir     = "${var.persistent_folder}/cert-checker"
+  commands = [
+    "python -m venv /tmp/venv",
+    ". /tmp/venv/bin/activate",
+    "pip install -r /config/requirements.txt",
+    "python /config/check.py"
+  ]
+}

terraform/modules/utils/cert_checker/main.tf

@@ -0,0 +1,149 @@
+
+resource "kubernetes_service_account" "cert_checker" {
+  metadata {
+    name      = "cert-checker-sa"
+    namespace = var.namespace
+  }
+}
+
+resource "kubernetes_role" "cert_checker" {
+  metadata {
+    name = "cert-checker-role"
+  }
+  rule {
+    api_groups = ["apps"]
+    resources  = ["deployments"]
+    verbs      = ["get", "list", "patch", "update"]
+  }
+}
+
+resource "kubernetes_role_binding" "cert_checker_auth_reader" {
+  metadata {
+    name      = "cert-checker-biding"
+    namespace = var.namespace
+  }
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "Role"
+    name      = "cert-checker-role"
+  }
+  subject {
+    kind      = "ServiceAccount"
+    name      = "cert-checker-sa"
+    namespace = var.namespace
+  }
+}
+
+resource "kubernetes_config_map" "cert_checker" {
+  metadata {
+    name      = "cert-checker-config"
+    namespace = var.namespace
+  }
+  data = {
+    "check.py"         = file("${path.module}/resources/check.py")
+    "requirements.txt" = file("${path.module}/resources/requirements.txt")
+  }
+}
+
+resource "kubernetes_cron_job_v1" "cert_checker" {
+  for_each = var.deployments
+  metadata {
+    name      = "cert-checker-${each.key}"
+    namespace = var.namespace
+    labels = {
+      app = "cert-checker"
+    }
+  }
+  spec {
+    schedule                      = coalesce(each.value.schedule, "0 0 * * *") # Default to daily at midnight
+    successful_jobs_history_limit = 2
+    failed_jobs_history_limit     = 1
+
+    job_template {
+      metadata {
+        name = "cert-checker"
+      }
+      spec {
+        backoff_limit = 1
+        parallelism   = 1
+        completions   = 1
+
+        template {
+          metadata {
+            labels = {
+              app = "cert-checker"
+            }
+          }
+          spec {
+            service_account_name = kubernetes_service_account.cert_checker.metadata[0].name
+            restart_policy       = "Never"
+            container {
+              name    = "cert-checker"
+              image   = var.tag
+              command = ["sh", "-c", join(" && ", local.commands)]
+              volume_mount {
+                name       = "config-volume"
+                mount_path = "/config"
+                read_only  = true
+              }
+              volume_mount {
+                name       = "cert-volume"
+                mount_path = "/certs"
+                read_only  = true
+              }
+              volume_mount {
+                name       = "workdir"
+                mount_path = "/tmp"
+              }
+              security_context {
+                run_as_user                = 1000
+                run_as_group               = 1000
+                allow_privilege_escalation = false
+                read_only_root_filesystem  = true
+                run_as_non_root            = true
+                capabilities {
+                  drop = ["ALL"]
+                }
+              }
+              env {
+                name  = "DEPLOYMENT_NAME"
+                value = each.key
+              }
+              env {
+                name  = "NAMESPACE"
+                value = var.namespace
+              }
+              env {
+                name  = "SERVICE_HOST"
+                value = "${each.value.service_name}.${var.namespace}.svc.cluster.local"
+              }
+              env {
+                name  = "SERVICE_PORT"
+                value = tostring(each.value.service_port)
+              }
+            }
+            volume {
+              name = "config-volume"
+              config_map {
+                name = kubernetes_config_map.cert_checker.metadata[0].name
+              }
+            }
+            volume {
+              name = "cert-volume"
+              secret {
+                secret_name = each.value.secret_name
+              }
+            }
+            volume {
+              name = "workdir"
+              host_path {
+                path = local.workdir
+                type = "DirectoryOrCreate"
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

terraform/modules/utils/cert_checker/provider.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = "~>1.8"
+  required_providers {
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "2.36.0"
+    }
+  }
+}

terraform/modules/utils/cert_checker/resources/check.py

@@ -0,0 +1,129 @@
+import ssl
+import socket
+from datetime import datetime, timezone
+import os
+import logging
+from kubernetes import client, config
+import tempfile
+
+# Setup logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s"
+)
+
+DEPLOYMENT = os.getenv("DEPLOYMENT_NAME", "my-deployment")
+NAMESPACE = os.getenv("NAMESPACE", "default")
+SECRET_CERT_PATH = os.getenv("SECRET_CERT_PATH", "/certs/tls.crt")
+SERVICE_HOST = os.getenv("SERVICE_HOST", "my-app.default.svc")
+SERVICE_PORT = int(os.getenv("SERVICE_PORT", 443))
+
+def read_secret_cert():
+    logging.info(f"Reading certificate from secret path: {SECRET_CERT_PATH}")
+    return ssl._ssl._test_decode_cert(SECRET_CERT_PATH)
+
+def get_secret_cert_expiry():
+    cert = read_secret_cert()
+    expiry = datetime.strptime(cert['notAfter'], "%b %d %H:%M:%S %Y %Z")
+    logging.info(f"Secret certificate expires at: {expiry}")
+    return expiry
+
+def get_common_name(cert):
+    """
+    Extract Common Name (CN) from a cert dict as returned by ssl.getpeercert()
+    or ssl._ssl._test_decode_cert(). Returns None if not found.
+    """
+    if not cert:
+        return None
+    subject = cert.get('subject', [])
+    for rdn in subject:
+        # rdn is a tuple of tuples, e.g. (('commonName', 'example.com'),)
+        for attr in rdn:
+            if isinstance(attr, tuple) and len(attr) == 2:
+                key, val = attr
+                if key.lower() in ('commonname', 'cn'):
+                    return val
+    return None
+
+def get_live_cert_expiry():
+    logging.info(f"Connecting to service {SERVICE_HOST}:{SERVICE_PORT} to get live certificate")
+    # Try the secure default context first (will verify hostname)
+    ctx = ssl.create_default_context()
+    try:
+        with ctx.wrap_socket(socket.socket(), server_hostname=SERVICE_HOST) as s:
+            s.connect((SERVICE_HOST, SERVICE_PORT))
+            # request both the dict view and the binary DER form
+            cert = s.getpeercert()  # may be empty
+            der = s.getpeercert(binary_form=True)
+    except ssl.SSLCertVerificationError as e:
+        logging.warning("Certificate verification failed (%s). Falling back to non-verifying connection to read cert (insecure).", e)
+        # Insecure fallback: disable hostname and certificate verification just to read the cert
+        ctx = ssl.create_default_context()
+        ctx.check_hostname = False
+        ctx.verify_mode = ssl.CERT_NONE
+        with ctx.wrap_socket(socket.socket(), server_hostname=SERVICE_HOST) as s:
+            s.connect((SERVICE_HOST, SERVICE_PORT))
+            cert = s.getpeercert()
+            der = s.getpeercert(binary_form=True)
+
+    # If the dict view lacks 'notAfter', try decoding the DER to get a full dict
+    if not cert or 'notAfter' not in cert:
+        if der:
+            pem = ssl.DER_cert_to_PEM_cert(der)
+            # write PEM to a temp file and decode using ssl test decoder to get 'notAfter'
+            with tempfile.NamedTemporaryFile(mode="w", delete=True) as tf:
+                tf.write(pem)
+                tf.flush()
+                cert = ssl._ssl._test_decode_cert(tf.name)
+        else:
+            raise RuntimeError("Failed to obtain certificate from peer")
+
+    expiry = datetime.strptime(cert['notAfter'], "%b %d %H:%M:%S %Y %Z")
+    logging.info(f"Live certificate expires at: {expiry}")
+    return expiry, cert
+
+def restart_deployment():
+    logging.info(f"Restarting deployment {DEPLOYMENT} in namespace {NAMESPACE}")
+    config.load_incluster_config()
+    apps = client.AppsV1Api()
+    body = {
+        "spec": {
+            "template": {
+                "metadata": {
+                    "annotations": {
+                        "cert-checker/restartedAt": datetime.now(timezone.utc).isoformat()
+                    }
+                }
+            }
+        }
+    }
+    apps.patch_namespaced_deployment(
+        name=DEPLOYMENT,
+        namespace=NAMESPACE,
+        body=body
+    )
+    logging.info(f"Deployment {DEPLOYMENT} restarted")
+
+if __name__ == "__main__":
+    logging.info("Starting certificate check")
+    secret_cert = read_secret_cert()
+    secret_expiry = datetime.strptime(secret_cert['notAfter'], "%b %d %H:%M:%S %Y %Z")
+    live_expiry, live_cert = get_live_cert_expiry()
+
+    secret_cn = get_common_name(secret_cert)
+    live_cn = get_common_name(live_cert)
+
+    logging.info(f"Secret expiry: {secret_expiry}")
+    logging.info(f"Live cert expiry: {live_expiry}")
+    logging.info(f"Secret CN: {secret_cn}")
+    logging.info(f"Live CN: {live_cn}")
+
+    if secret_cn != live_cn:
+        logging.warning("Certificate mismatch detected (CN), check your setup.")
+        exit(0)
+
+    if secret_expiry != live_expiry:
+        logging.warning("Certificate mismatch detected (expiry), restarting deployment")
+        restart_deployment()
+    else:
+        logging.info("Certificates match, nothing to do.")

terraform/modules/utils/cert_checker/resources/requirements.txt

@@ -0,0 +1 @@
+kubernetes==34.1.0

terraform/modules/utils/cert_checker/variables.tf

@@ -0,0 +1,26 @@
+variable "tag" {
+  description = "Python image tag"
+  type        = string
+  default     = "python:3.12-slim"
+}
+
+variable "namespace" {
+  description = "Kubernetes namespace to deploy the metrics server"
+  type        = string
+}
+
+variable "persistent_folder" {
+  description = "The path to the persistent folder"
+  type        = string
+}
+
+variable "deployments" {
+  description = "A map of configuration items for the cert checker. Each item should include deployment_name, service_name, and service_port."
+  type = map(object({
+    service_name = string
+    service_port = number
+    secret_name  = string
+    schedule     = optional(string) # Cron schedule expression
+  }))
+  default = {}
+}

terraform/modules/utils/metrics_server/config.tf

@@ -0,0 +1,3 @@
+locals {
+  app_version = var.tag
+}

terraform/modules/utils/metrics_server/main.tf

@@ -0,0 +1,259 @@
+
+resource "kubernetes_service_account" "metrics_server" {
+  metadata {
+    name      = "metrics-server"
+    namespace = var.namespace
+    labels = {
+      k8s-app = "metrics-server"
+    }
+  }
+}
+
+resource "kubernetes_cluster_role" "aggregated_metrics_reader" {
+  metadata {
+    name = "system:aggregated-metrics-reader"
+    labels = {
+      k8s-app                                        = "metrics-server"
+      "rbac.authorization.k8s.io/aggregate-to-admin" = "true"
+      "rbac.authorization.k8s.io/aggregate-to-edit"  = "true"
+      "rbac.authorization.k8s.io/aggregate-to-view"  = "true"
+    }
+  }
+  rule {
+    api_groups = ["metrics.k8s.io"]
+    resources  = ["pods", "nodes"]
+    verbs      = ["get", "list", "watch"]
+  }
+}
+
+resource "kubernetes_cluster_role" "metrics_server" {
+  metadata {
+    name = "system:metrics-server"
+    labels = {
+      k8s-app = "metrics-server"
+    }
+  }
+  rule {
+    api_groups = [""]
+    resources  = ["nodes/metrics"]
+    verbs      = ["get"]
+  }
+  rule {
+    api_groups = [""]
+    resources  = ["pods", "nodes"]
+    verbs      = ["get", "list", "watch"]
+  }
+}
+
+resource "kubernetes_role_binding" "metrics_server_auth_reader" {
+  metadata {
+    name      = "metrics-server-auth-reader"
+    namespace = var.namespace
+    labels = {
+      k8s-app = "metrics-server"
+    }
+  }
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "Role"
+    name      = "extension-apiserver-authentication-reader"
+  }
+  subject {
+    kind      = "ServiceAccount"
+    name      = "metrics-server"
+    namespace = var.namespace
+  }
+}
+
+resource "kubernetes_cluster_role_binding" "metrics_server_auth_delegator" {
+  metadata {
+    name = "metrics-server:system:auth-delegator"
+    labels = {
+      k8s-app = "metrics-server"
+    }
+  }
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "ClusterRole"
+    name      = "system:auth-delegator"
+  }
+  subject {
+    kind      = "ServiceAccount"
+    name      = "metrics-server"
+    namespace = var.namespace
+  }
+}
+
+resource "kubernetes_cluster_role_binding" "metrics_server" {
+  metadata {
+    name = "system:metrics-server"
+    labels = {
+      k8s-app = "metrics-server"
+    }
+  }
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "ClusterRole"
+    name      = "system:metrics-server"
+  }
+  subject {
+    kind      = "ServiceAccount"
+    name      = "metrics-server"
+    namespace = var.namespace
+  }
+}
+
+resource "kubernetes_service" "metrics_server" {
+  metadata {
+    name      = "metrics-server"
+    namespace = var.namespace
+    labels = {
+      k8s-app = "metrics-server"
+    }
+  }
+  spec {
+    selector = {
+      k8s-app = "metrics-server"
+    }
+    port {
+      name        = "https"
+      port        = 443
+      protocol    = "TCP"
+      target_port = "https"
+    }
+  }
+}
+
+resource "kubernetes_deployment" "metrics_server" {
+  metadata {
+    name      = "metrics-server"
+    namespace = var.namespace
+    labels = {
+      k8s-app = "metrics-server"
+    }
+  }
+  spec {
+    selector {
+      match_labels = {
+        k8s-app = "metrics-server"
+      }
+    }
+    strategy {
+      type = "RollingUpdate"
+      rolling_update {
+        max_unavailable = 0
+      }
+    }
+    template {
+      metadata {
+        labels = {
+          k8s-app = "metrics-server"
+        }
+      }
+      spec {
+        service_account_name = "metrics-server"
+        priority_class_name  = "system-cluster-critical"
+        node_selector = {
+          "kubernetes.io/os"   = "linux"
+          "kubernetes.io/arch" = "$ARCH"
+        }
+        container {
+          name              = "metrics-server"
+          image             = "registry.k8s.io/metrics-server/metrics-server:${local.app_version}"
+          image_pull_policy = "IfNotPresent"
+          args = [
+            "--cert-dir=/tmp",
+            "--secure-port=4443",
+            "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
+            "--kubelet-use-node-status-port",
+            "--metric-resolution=15s",
+            "--kubelet-insecure-tls"
+          ]
+          port {
+            container_port = 4443
+            name           = "https"
+            protocol       = "TCP"
+          }
+          liveness_probe {
+            http_get {
+              path   = "/livez"
+              port   = "https"
+              scheme = "HTTPS"
+            }
+            failure_threshold = 3
+            period_seconds    = 10
+          }
+          readiness_probe {
+            http_get {
+              path   = "/readyz"
+              port   = "https"
+              scheme = "HTTPS"
+            }
+            failure_threshold     = 3
+            initial_delay_seconds = 20
+            period_seconds        = 10
+          }
+          resources {
+            requests = {
+              cpu    = "100m"
+              memory = "200Mi"
+            }
+          }
+          security_context {
+            read_only_root_filesystem = true
+            run_as_non_root           = true
+            run_as_user               = 1000
+          }
+          volume_mount {
+            name       = "tmp-dir"
+            mount_path = "/tmp"
+          }
+        }
+        volume {
+          name = "tmp-dir"
+          empty_dir {}
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_manifest" "metrics_server_apiservice" {
+  manifest = {
+    apiVersion = "apiregistration.k8s.io/v1"
+    kind       = "APIService"
+    metadata = {
+      name = "v1beta1.metrics.k8s.io"
+      labels = {
+        k8s-app = "metrics-server"
+      }
+    }
+    spec = {
+      group                 = "metrics.k8s.io"
+      groupPriorityMinimum  = 100
+      insecureSkipTLSVerify = true
+      service = {
+        name      = "metrics-server"
+        namespace = var.namespace
+      }
+      version         = "v1beta1"
+      versionPriority = 100
+    }
+  }
+}
+
+# resource "kubernetes_cluster_role_binding" "microk8s_admin" {
+#   metadata {
+#     name = "microk8s-admin"
+#   }
+#   role_ref {
+#     api_group = "rbac.authorization.k8s.io"
+#     kind      = "ClusterRole"
+#     name      = "admin"
+#   }
+#   subject {
+#     kind      = "User"
+#     name      = "front-proxy-client"
+#     api_group = "rbac.authorization.k8s.io"
+#   }
+# }

terraform/modules/utils/metrics_server/provider.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = "~>1.8"
+  required_providers {
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "2.36.0"
+    }
+  }
+}

terraform/modules/utils/metrics_server/variables.tf

@@ -0,0 +1,11 @@
+variable "tag" {
+  description = "Prometheus image tag"
+  type        = string
+  default     = "v0.8.0"
+}
+
+variable "namespace" {
+  description = "Kubernetes namespace to deploy the metrics server"
+  type        = string
+  default     = "kube-system"
+}

terraform/modules/utils/prometheus/main.tf

@@ -1,5 +1,10 @@
-resource "kubernetes_namespace" "monitoring" {
+resource "kubernetes_namespace" "prometheus" {
   metadata {
     name = "prometheus"
   }
 }
+
+moved {
+  from = kubernetes_namespace.monitoring
+  to   = kubernetes_namespace.prometheus
+}

terraform/modules/utils/prometheus/prometheus.tf

@@ -1,7 +1,7 @@
 resource "kubernetes_service_account" "prometheus" {
   metadata {
     name      = "prometheus"
-    namespace = kubernetes_namespace.monitoring.metadata[0].name
+    namespace = kubernetes_namespace.prometheus.metadata[0].name
   }
 }
 
@@ -48,14 +48,14 @@ resource "kubernetes_cluster_role_binding" "prometheus" {
   subject {
     kind      = "ServiceAccount"
     name      = kubernetes_service_account.prometheus.metadata[0].name
-    namespace = kubernetes_namespace.monitoring.metadata[0].name
+    namespace = kubernetes_namespace.prometheus.metadata[0].name
   }
 }
 
 resource "kubernetes_config_map" "prometheus" {
   metadata {
     name      = "prometheus-server-conf"
-    namespace = kubernetes_namespace.monitoring.metadata[0].name
+    namespace = kubernetes_namespace.prometheus.metadata[0].name
     labels = {
       name = "prometheus-server-conf"
     }
@@ -70,7 +70,7 @@ resource "kubernetes_config_map" "prometheus" {
 resource "kubernetes_deployment" "prometheus" {
   metadata {
     name      = "prometheus"
-    namespace = kubernetes_namespace.monitoring.metadata[0].name
+    namespace = kubernetes_namespace.prometheus.metadata[0].name
     labels = {
       app = "prometheus"
     }
@@ -155,7 +155,7 @@ resource "kubernetes_deployment" "prometheus" {
 resource "kubernetes_service" "prometheus" {
   metadata {
     name      = "prometheus"
-    namespace = kubernetes_namespace.monitoring.metadata[0].name
+    namespace = kubernetes_namespace.prometheus.metadata[0].name
     annotations = {
       "prometheus.io/scrape" = "true"
       "prometheus.io/port"   = "9090"
@@ -180,7 +180,7 @@ resource "kubernetes_service" "prometheus" {
 # resource "kubernetes_ingress_v1" "prometheus" {
 #   metadata {
 #     name      = "prometheus-ui"
-#     namespace = kubernetes_namespace.monitoring.metadata[0].name
+#     namespace = kubernetes_namespace.prometheus.metadata[0].name
 #     annotations = {
 #       "kubernetes.io/ingress.class" = var.ingress_class
 #     }