diff --git a/terraform/apps/prod-01/config.tf b/terraform/apps/prod-01/config.tf index b1237e3..f33cbd8 100644 --- a/terraform/apps/prod-01/config.tf +++ b/terraform/apps/prod-01/config.tf @@ -52,7 +52,6 @@ locals { sftpgo_files_backup_cron_schedule = "50 2 * * *" mail_files_backup_cron_schedule = "0 3 * * *" - # email settings fetch_emails_schedule = "*/2 * * * *" scaleway_smtp_host = "smtp.tem.scw.cloud" @@ -76,7 +75,7 @@ locals { } ] - #reverse proxy settings + # proxy settings streams = { # For now disabled, because this messes with fail2ban on the host @@ -193,6 +192,7 @@ locals { } } + # get secrets from remote state backup_bucket_name = data.terraform_remote_state.scaleway.outputs.backup_bucket_name backup_app_api_secret_key = data.terraform_remote_state.scaleway.outputs.backup_app_api_secret_key backup_app_api_access_key = data.terraform_remote_state.scaleway.outputs.backup_app_api_access_key diff --git a/terraform/modules/apps/mail/config.tf b/terraform/modules/apps/mail/config.tf index 3697172..0f3c343 100644 --- a/terraform/modules/apps/mail/config.tf +++ b/terraform/modules/apps/mail/config.tf @@ -88,6 +88,19 @@ locals { postfix_stunnel_config = file("${path.module}/resources/postfix/stunnel.conf") postfix_stunnel_config_checksum = sha256(local.postfix_stunnel_config) + + deployments = { + "dovecot" = { + service_name = "imap" + service_port = 993 + secret_name = "mail-tls" + }, + "postfix" = { + service_name = "smtps" + service_port = 465 + secret_name = "mail-tls" + } + } } data "external" "password_hasher" { diff --git a/terraform/modules/apps/mail/main.tf b/terraform/modules/apps/mail/main.tf index 0484a77..06291be 100644 --- a/terraform/modules/apps/mail/main.tf +++ b/terraform/modules/apps/mail/main.tf @@ -47,7 +47,17 @@ resource "kubernetes_manifest" "mail_certificate" { kind = "ClusterIssuer" } commonName = "imap.${var.primary_domain}" - dnsNames = flatten([for account in var.email_accounts : ["imap.${account.domain}", "smtp.${account.domain}"]]) + dnsNames = flatten([for account in var.email_accounts : [ + "imap.${account.domain}", + "smtp.${account.domain}" + ]]) } } } + +module "cert_checker" { + source = "../../utils/cert_checker" + namespace = kubernetes_namespace.mail.metadata[0].name + persistent_folder = local.mail_path + deployments = local.deployments +} diff --git a/terraform/modules/utils/cert_checker/config.tf b/terraform/modules/utils/cert_checker/config.tf new file mode 100644 index 0000000..3e6a36c --- /dev/null +++ b/terraform/modules/utils/cert_checker/config.tf @@ -0,0 +1,10 @@ +locals { + app_version = var.tag + workdir = "${var.persistent_folder}/cert-checker" + commands = [ + "python -m venv /tmp/venv", + ". /tmp/venv/bin/activate", + "pip install -r /config/requirements.txt", + "python /config/check.py" + ] +} diff --git a/terraform/modules/utils/cert_checker/main.tf b/terraform/modules/utils/cert_checker/main.tf new file mode 100644 index 0000000..eb9b135 --- /dev/null +++ b/terraform/modules/utils/cert_checker/main.tf @@ -0,0 +1,149 @@ + +resource "kubernetes_service_account" "cert_checker" { + metadata { + name = "cert-checker-sa" + namespace = var.namespace + } +} + +resource "kubernetes_role" "cert_checker" { + metadata { + name = "cert-checker-role" + } + rule { + api_groups = ["apps"] + resources = ["deployments"] + verbs = ["get", "list", "patch", "update"] + } +} + +resource "kubernetes_role_binding" "cert_checker_auth_reader" { + metadata { + name = "cert-checker-biding" + namespace = var.namespace + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "Role" + name = "cert-checker-role" + } + subject { + kind = "ServiceAccount" + name = "cert-checker-sa" + namespace = var.namespace + } +} + +resource "kubernetes_config_map" "cert_checker" { + metadata { + name = "cert-checker-config" + namespace = var.namespace + } + data = { + "check.py" = file("${path.module}/resources/check.py") + "requirements.txt" = file("${path.module}/resources/requirements.txt") + } +} + +resource "kubernetes_cron_job_v1" "cert_checker" { + for_each = var.deployments + metadata { + name = "cert-checker-${each.key}" + namespace = var.namespace + labels = { + app = "cert-checker" + } + } + spec { + schedule = coalesce(each.value.schedule, "0 0 * * *") # Default to daily at midnight + successful_jobs_history_limit = 2 + failed_jobs_history_limit = 1 + + job_template { + metadata { + name = "cert-checker" + } + spec { + backoff_limit = 1 + parallelism = 1 + completions = 1 + + template { + metadata { + labels = { + app = "cert-checker" + } + } + spec { + service_account_name = kubernetes_service_account.cert_checker.metadata[0].name + restart_policy = "Never" + container { + name = "cert-checker" + image = var.tag + command = ["sh", "-c", join(" && ", local.commands)] + volume_mount { + name = "config-volume" + mount_path = "/config" + read_only = true + } + volume_mount { + name = "cert-volume" + mount_path = "/certs" + read_only = true + } + volume_mount { + name = "workdir" + mount_path = "/tmp" + } + security_context { + run_as_user = 1000 + run_as_group = 1000 + allow_privilege_escalation = false + read_only_root_filesystem = true + run_as_non_root = true + capabilities { + drop = ["ALL"] + } + } + env { + name = "DEPLOYMENT_NAME" + value = each.key + } + env { + name = "NAMESPACE" + value = var.namespace + } + env { + name = "SERVICE_HOST" + value = "${each.value.service_name}.${var.namespace}.svc.cluster.local" + } + env { + name = "SERVICE_PORT" + value = tostring(each.value.service_port) + } + } + volume { + name = "config-volume" + config_map { + name = kubernetes_config_map.cert_checker.metadata[0].name + } + } + volume { + name = "cert-volume" + secret { + secret_name = each.value.secret_name + } + } + volume { + name = "workdir" + host_path { + path = local.workdir + type = "DirectoryOrCreate" + } + } + } + } + } + } + } +} diff --git a/terraform/modules/utils/cert_checker/provider.tf b/terraform/modules/utils/cert_checker/provider.tf new file mode 100644 index 0000000..2eaa508 --- /dev/null +++ b/terraform/modules/utils/cert_checker/provider.tf @@ -0,0 +1,9 @@ +terraform { + required_version = "~>1.8" + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.36.0" + } + } +} diff --git a/terraform/modules/utils/cert_checker/resources/check.py b/terraform/modules/utils/cert_checker/resources/check.py new file mode 100644 index 0000000..0093e2f --- /dev/null +++ b/terraform/modules/utils/cert_checker/resources/check.py @@ -0,0 +1,129 @@ +import ssl +import socket +from datetime import datetime, timezone +import os +import logging +from kubernetes import client, config +import tempfile + +# Setup logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s" +) + +DEPLOYMENT = os.getenv("DEPLOYMENT_NAME", "my-deployment") +NAMESPACE = os.getenv("NAMESPACE", "default") +SECRET_CERT_PATH = os.getenv("SECRET_CERT_PATH", "/certs/tls.crt") +SERVICE_HOST = os.getenv("SERVICE_HOST", "my-app.default.svc") +SERVICE_PORT = int(os.getenv("SERVICE_PORT", 443)) + +def read_secret_cert(): + logging.info(f"Reading certificate from secret path: {SECRET_CERT_PATH}") + return ssl._ssl._test_decode_cert(SECRET_CERT_PATH) + +def get_secret_cert_expiry(): + cert = read_secret_cert() + expiry = datetime.strptime(cert['notAfter'], "%b %d %H:%M:%S %Y %Z") + logging.info(f"Secret certificate expires at: {expiry}") + return expiry + +def get_common_name(cert): + """ + Extract Common Name (CN) from a cert dict as returned by ssl.getpeercert() + or ssl._ssl._test_decode_cert(). Returns None if not found. + """ + if not cert: + return None + subject = cert.get('subject', []) + for rdn in subject: + # rdn is a tuple of tuples, e.g. (('commonName', 'example.com'),) + for attr in rdn: + if isinstance(attr, tuple) and len(attr) == 2: + key, val = attr + if key.lower() in ('commonname', 'cn'): + return val + return None + +def get_live_cert_expiry(): + logging.info(f"Connecting to service {SERVICE_HOST}:{SERVICE_PORT} to get live certificate") + # Try the secure default context first (will verify hostname) + ctx = ssl.create_default_context() + try: + with ctx.wrap_socket(socket.socket(), server_hostname=SERVICE_HOST) as s: + s.connect((SERVICE_HOST, SERVICE_PORT)) + # request both the dict view and the binary DER form + cert = s.getpeercert() # may be empty + der = s.getpeercert(binary_form=True) + except ssl.SSLCertVerificationError as e: + logging.warning("Certificate verification failed (%s). Falling back to non-verifying connection to read cert (insecure).", e) + # Insecure fallback: disable hostname and certificate verification just to read the cert + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + with ctx.wrap_socket(socket.socket(), server_hostname=SERVICE_HOST) as s: + s.connect((SERVICE_HOST, SERVICE_PORT)) + cert = s.getpeercert() + der = s.getpeercert(binary_form=True) + + # If the dict view lacks 'notAfter', try decoding the DER to get a full dict + if not cert or 'notAfter' not in cert: + if der: + pem = ssl.DER_cert_to_PEM_cert(der) + # write PEM to a temp file and decode using ssl test decoder to get 'notAfter' + with tempfile.NamedTemporaryFile(mode="w", delete=True) as tf: + tf.write(pem) + tf.flush() + cert = ssl._ssl._test_decode_cert(tf.name) + else: + raise RuntimeError("Failed to obtain certificate from peer") + + expiry = datetime.strptime(cert['notAfter'], "%b %d %H:%M:%S %Y %Z") + logging.info(f"Live certificate expires at: {expiry}") + return expiry, cert + +def restart_deployment(): + logging.info(f"Restarting deployment {DEPLOYMENT} in namespace {NAMESPACE}") + config.load_incluster_config() + apps = client.AppsV1Api() + body = { + "spec": { + "template": { + "metadata": { + "annotations": { + "cert-checker/restartedAt": datetime.now(timezone.utc).isoformat() + } + } + } + } + } + apps.patch_namespaced_deployment( + name=DEPLOYMENT, + namespace=NAMESPACE, + body=body + ) + logging.info(f"Deployment {DEPLOYMENT} restarted") + +if __name__ == "__main__": + logging.info("Starting certificate check") + secret_cert = read_secret_cert() + secret_expiry = datetime.strptime(secret_cert['notAfter'], "%b %d %H:%M:%S %Y %Z") + live_expiry, live_cert = get_live_cert_expiry() + + secret_cn = get_common_name(secret_cert) + live_cn = get_common_name(live_cert) + + logging.info(f"Secret expiry: {secret_expiry}") + logging.info(f"Live cert expiry: {live_expiry}") + logging.info(f"Secret CN: {secret_cn}") + logging.info(f"Live CN: {live_cn}") + + if secret_cn != live_cn: + logging.warning("Certificate mismatch detected (CN), check your setup.") + exit(0) + + if secret_expiry != live_expiry: + logging.warning("Certificate mismatch detected (expiry), restarting deployment") + restart_deployment() + else: + logging.info("Certificates match, nothing to do.") diff --git a/terraform/modules/utils/cert_checker/resources/requirements.txt b/terraform/modules/utils/cert_checker/resources/requirements.txt new file mode 100644 index 0000000..6148330 --- /dev/null +++ b/terraform/modules/utils/cert_checker/resources/requirements.txt @@ -0,0 +1 @@ +kubernetes==34.1.0 \ No newline at end of file diff --git a/terraform/modules/utils/cert_checker/variables.tf b/terraform/modules/utils/cert_checker/variables.tf new file mode 100644 index 0000000..8f2948f --- /dev/null +++ b/terraform/modules/utils/cert_checker/variables.tf @@ -0,0 +1,26 @@ +variable "tag" { + description = "Python image tag" + type = string + default = "python:3.12-slim" +} + +variable "namespace" { + description = "Kubernetes namespace to deploy the metrics server" + type = string +} + +variable "persistent_folder" { + description = "The path to the persistent folder" + type = string +} + +variable "deployments" { + description = "A map of configuration items for the cert checker. Each item should include deployment_name, service_name, and service_port." + type = map(object({ + service_name = string + service_port = number + secret_name = string + schedule = optional(string) # Cron schedule expression + })) + default = {} +} diff --git a/terraform/modules/utils/metrics_server/config.tf b/terraform/modules/utils/metrics_server/config.tf new file mode 100644 index 0000000..cc246c7 --- /dev/null +++ b/terraform/modules/utils/metrics_server/config.tf @@ -0,0 +1,3 @@ +locals { + app_version = var.tag +} diff --git a/terraform/modules/utils/metrics_server/main.tf b/terraform/modules/utils/metrics_server/main.tf new file mode 100644 index 0000000..046527f --- /dev/null +++ b/terraform/modules/utils/metrics_server/main.tf @@ -0,0 +1,259 @@ + +resource "kubernetes_service_account" "metrics_server" { + metadata { + name = "metrics-server" + namespace = var.namespace + labels = { + k8s-app = "metrics-server" + } + } +} + +resource "kubernetes_cluster_role" "aggregated_metrics_reader" { + metadata { + name = "system:aggregated-metrics-reader" + labels = { + k8s-app = "metrics-server" + "rbac.authorization.k8s.io/aggregate-to-admin" = "true" + "rbac.authorization.k8s.io/aggregate-to-edit" = "true" + "rbac.authorization.k8s.io/aggregate-to-view" = "true" + } + } + rule { + api_groups = ["metrics.k8s.io"] + resources = ["pods", "nodes"] + verbs = ["get", "list", "watch"] + } +} + +resource "kubernetes_cluster_role" "metrics_server" { + metadata { + name = "system:metrics-server" + labels = { + k8s-app = "metrics-server" + } + } + rule { + api_groups = [""] + resources = ["nodes/metrics"] + verbs = ["get"] + } + rule { + api_groups = [""] + resources = ["pods", "nodes"] + verbs = ["get", "list", "watch"] + } +} + +resource "kubernetes_role_binding" "metrics_server_auth_reader" { + metadata { + name = "metrics-server-auth-reader" + namespace = var.namespace + labels = { + k8s-app = "metrics-server" + } + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "Role" + name = "extension-apiserver-authentication-reader" + } + subject { + kind = "ServiceAccount" + name = "metrics-server" + namespace = var.namespace + } +} + +resource "kubernetes_cluster_role_binding" "metrics_server_auth_delegator" { + metadata { + name = "metrics-server:system:auth-delegator" + labels = { + k8s-app = "metrics-server" + } + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "system:auth-delegator" + } + subject { + kind = "ServiceAccount" + name = "metrics-server" + namespace = var.namespace + } +} + +resource "kubernetes_cluster_role_binding" "metrics_server" { + metadata { + name = "system:metrics-server" + labels = { + k8s-app = "metrics-server" + } + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "system:metrics-server" + } + subject { + kind = "ServiceAccount" + name = "metrics-server" + namespace = var.namespace + } +} + +resource "kubernetes_service" "metrics_server" { + metadata { + name = "metrics-server" + namespace = var.namespace + labels = { + k8s-app = "metrics-server" + } + } + spec { + selector = { + k8s-app = "metrics-server" + } + port { + name = "https" + port = 443 + protocol = "TCP" + target_port = "https" + } + } +} + +resource "kubernetes_deployment" "metrics_server" { + metadata { + name = "metrics-server" + namespace = var.namespace + labels = { + k8s-app = "metrics-server" + } + } + spec { + selector { + match_labels = { + k8s-app = "metrics-server" + } + } + strategy { + type = "RollingUpdate" + rolling_update { + max_unavailable = 0 + } + } + template { + metadata { + labels = { + k8s-app = "metrics-server" + } + } + spec { + service_account_name = "metrics-server" + priority_class_name = "system-cluster-critical" + node_selector = { + "kubernetes.io/os" = "linux" + "kubernetes.io/arch" = "$ARCH" + } + container { + name = "metrics-server" + image = "registry.k8s.io/metrics-server/metrics-server:${local.app_version}" + image_pull_policy = "IfNotPresent" + args = [ + "--cert-dir=/tmp", + "--secure-port=4443", + "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", + "--kubelet-use-node-status-port", + "--metric-resolution=15s", + "--kubelet-insecure-tls" + ] + port { + container_port = 4443 + name = "https" + protocol = "TCP" + } + liveness_probe { + http_get { + path = "/livez" + port = "https" + scheme = "HTTPS" + } + failure_threshold = 3 + period_seconds = 10 + } + readiness_probe { + http_get { + path = "/readyz" + port = "https" + scheme = "HTTPS" + } + failure_threshold = 3 + initial_delay_seconds = 20 + period_seconds = 10 + } + resources { + requests = { + cpu = "100m" + memory = "200Mi" + } + } + security_context { + read_only_root_filesystem = true + run_as_non_root = true + run_as_user = 1000 + } + volume_mount { + name = "tmp-dir" + mount_path = "/tmp" + } + } + volume { + name = "tmp-dir" + empty_dir {} + } + } + } + } +} + +resource "kubernetes_manifest" "metrics_server_apiservice" { + manifest = { + apiVersion = "apiregistration.k8s.io/v1" + kind = "APIService" + metadata = { + name = "v1beta1.metrics.k8s.io" + labels = { + k8s-app = "metrics-server" + } + } + spec = { + group = "metrics.k8s.io" + groupPriorityMinimum = 100 + insecureSkipTLSVerify = true + service = { + name = "metrics-server" + namespace = var.namespace + } + version = "v1beta1" + versionPriority = 100 + } + } +} + +# resource "kubernetes_cluster_role_binding" "microk8s_admin" { +# metadata { +# name = "microk8s-admin" +# } +# role_ref { +# api_group = "rbac.authorization.k8s.io" +# kind = "ClusterRole" +# name = "admin" +# } +# subject { +# kind = "User" +# name = "front-proxy-client" +# api_group = "rbac.authorization.k8s.io" +# } +# } diff --git a/terraform/modules/utils/metrics_server/provider.tf b/terraform/modules/utils/metrics_server/provider.tf new file mode 100644 index 0000000..2eaa508 --- /dev/null +++ b/terraform/modules/utils/metrics_server/provider.tf @@ -0,0 +1,9 @@ +terraform { + required_version = "~>1.8" + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.36.0" + } + } +} diff --git a/terraform/modules/utils/metrics_server/variables.tf b/terraform/modules/utils/metrics_server/variables.tf new file mode 100644 index 0000000..84d99e4 --- /dev/null +++ b/terraform/modules/utils/metrics_server/variables.tf @@ -0,0 +1,11 @@ +variable "tag" { + description = "Prometheus image tag" + type = string + default = "v0.8.0" +} + +variable "namespace" { + description = "Kubernetes namespace to deploy the metrics server" + type = string + default = "kube-system" +} diff --git a/terraform/modules/utils/prometheus/main.tf b/terraform/modules/utils/prometheus/main.tf index 3f79f52..c215702 100644 --- a/terraform/modules/utils/prometheus/main.tf +++ b/terraform/modules/utils/prometheus/main.tf @@ -1,5 +1,10 @@ -resource "kubernetes_namespace" "monitoring" { +resource "kubernetes_namespace" "prometheus" { metadata { name = "prometheus" } } + +moved { + from = kubernetes_namespace.monitoring + to = kubernetes_namespace.prometheus +} diff --git a/terraform/modules/utils/prometheus/prometheus.tf b/terraform/modules/utils/prometheus/prometheus.tf index 45eee5c..8d63e1f 100644 --- a/terraform/modules/utils/prometheus/prometheus.tf +++ b/terraform/modules/utils/prometheus/prometheus.tf @@ -1,7 +1,7 @@ resource "kubernetes_service_account" "prometheus" { metadata { name = "prometheus" - namespace = kubernetes_namespace.monitoring.metadata[0].name + namespace = kubernetes_namespace.prometheus.metadata[0].name } } @@ -48,14 +48,14 @@ resource "kubernetes_cluster_role_binding" "prometheus" { subject { kind = "ServiceAccount" name = kubernetes_service_account.prometheus.metadata[0].name - namespace = kubernetes_namespace.monitoring.metadata[0].name + namespace = kubernetes_namespace.prometheus.metadata[0].name } } resource "kubernetes_config_map" "prometheus" { metadata { name = "prometheus-server-conf" - namespace = kubernetes_namespace.monitoring.metadata[0].name + namespace = kubernetes_namespace.prometheus.metadata[0].name labels = { name = "prometheus-server-conf" } @@ -70,7 +70,7 @@ resource "kubernetes_config_map" "prometheus" { resource "kubernetes_deployment" "prometheus" { metadata { name = "prometheus" - namespace = kubernetes_namespace.monitoring.metadata[0].name + namespace = kubernetes_namespace.prometheus.metadata[0].name labels = { app = "prometheus" } @@ -155,7 +155,7 @@ resource "kubernetes_deployment" "prometheus" { resource "kubernetes_service" "prometheus" { metadata { name = "prometheus" - namespace = kubernetes_namespace.monitoring.metadata[0].name + namespace = kubernetes_namespace.prometheus.metadata[0].name annotations = { "prometheus.io/scrape" = "true" "prometheus.io/port" = "9090" @@ -180,7 +180,7 @@ resource "kubernetes_service" "prometheus" { # resource "kubernetes_ingress_v1" "prometheus" { # metadata { # name = "prometheus-ui" -# namespace = kubernetes_namespace.monitoring.metadata[0].name +# namespace = kubernetes_namespace.prometheus.metadata[0].name # annotations = { # "kubernetes.io/ingress.class" = var.ingress_class # }