diff --git a/ansible/host_vars/dev-01.lab.alexpires.me.yml b/ansible/host_vars/dev-01.lab.alexpires.me.yml index ce2bb59..4eec4b9 100644 --- a/ansible/host_vars/dev-01.lab.alexpires.me.yml +++ b/ansible/host_vars/dev-01.lab.alexpires.me.yml @@ -41,6 +41,7 @@ fw_allowed_ports: - { rule: "allow", port: "11434", proto: "tcp", from: "10.19.4.0/24" } # Ollama (local network only) - { rule: "allow", port: "11434", proto: "tcp", from: "10.5.5.5/32" } # Ollama (Laptop) - { rule: "allow", port: "11434", proto: "tcp", from: "10.5.5.2/32" } # Ollama (Contabo VPN) + - { rule: "allow", port: "9090", proto: "tcp", from: "10.5.5.2/32" } # Prometheus (Contabo VPN) - { rule: "allow", port: "443", proto: "tcp", from: "10.5.5.2/32" } # HTTPS (Contabo VPN) - { rule: "allow", port: "53", proto: "udp", from: "10.19.4.0/24" } # DNS (local network) - { rule: "allow", port: "53", proto: "udp", from: "10.5.5.2/32" } # DNS (Contabo VPN) @@ -52,6 +53,7 @@ ufw_outgoing_traffic: - 80 # HTTP - 123 # NTP - 443 # HTTPS + - 465 # SMTPS - 853 # DNS over TLS - 10250 # Kubelet - 8880 # Kokoro-FastAPI @@ -60,7 +62,8 @@ ufw_outgoing_traffic: - 8188 # comfyui - 4443 - 9443 - - 995 # POP3S + - 993 # IMAPS + - 9090 # Prometheus ufw_default_forward_policy: "ACCEPT" @@ -84,7 +87,10 @@ fail2ban_jails: bantime: 3600 # aide -install_aide: false +install_aide: true +aide_exclude: + - "!/var/lib/snapd" + - "!/srv" misc_modules_blocklist: - bluetooth diff --git a/ansible/host_vars/prod-01.alexpires.me.yml b/ansible/host_vars/prod-01.alexpires.me.yml index b282b39..caa0b1f 100644 --- a/ansible/host_vars/prod-01.alexpires.me.yml +++ b/ansible/host_vars/prod-01.alexpires.me.yml @@ -60,7 +60,7 @@ ufw_outgoing_traffic: - 16443 # Kube API Server - 4443 - 9443 - - 995 # POP3S + - 993 # IMAPS ufw_default_forward_policy: "ACCEPT" @@ -96,7 +96,15 @@ fail2ban_jails: filter: "postfix-k8s" logpath: "/var/log/postfix/current" maxretry: 3 - findtime: 18000 + findtime: 600 + bantime: -1 + dovecot: + enabled: true + port: "imap,imaps,pop3,pop3s" + filter: "dovecot-k8s" + logpath: "/var/log/dovecot/info.log" + maxretry: 3 + findtime: 600 bantime: -1 fail2ban_filters: @@ -104,6 +112,10 @@ fail2ban_filters: Definition: failregex: "^.*?postfix/smtps/smtpd\\[\\d+\\]:.*\\[\\]: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed.*$" ignoreregex: "" + dovecot-k8s: + Definition: + failregex: "(?: pop3-login|imap-login): .*(?:Authentication failure|Aborted login \\(auth failed|Aborted login \\(tried to use disabled|Disconnected \\(auth failed|Aborted login \\(\\d+ authentication attempts).*rip=`" + ignoreregex: "" # aide install_aide: true diff --git a/ansible/playbook_mail.yml b/ansible/playbook_mail.yml index 6e1f1be..7d3d153 100644 --- a/ansible/playbook_mail.yml +++ b/ansible/playbook_mail.yml @@ -8,8 +8,11 @@ - name: Set required facts ansible.builtin.set_fact: mail_home: "{{ persistent_data | default('/var/lib') }}/mail" + log_home: "/var/log" mail_user_id: 1000 mail_group_id: 1000 + dovecot_user_id: 100 + dovecot_group_id: 101 postfix_user_id: 100 postfix_group_id: 101 tags: always @@ -59,3 +62,17 @@ tags: - tasks - tasks::folders + + - name: Create required dovecot log + become: true + ansible.builtin.file: + state: directory + path: "{{ item }}" + mode: "0700" + owner: "{{ dovecot_user_id }}" + group: "{{ dovecot_group_id }}" + loop: + - "{{ log_home }}/dovecot" + tags: + - tasks + - tasks::folders diff --git a/ansible/playbook_monitoring.yml b/ansible/playbook_monitoring.yml index 6b6cae9..76891ef 100644 --- a/ansible/playbook_monitoring.yml +++ b/ansible/playbook_monitoring.yml @@ -1,16 +1,13 @@ --- -- name: Monitoring Playbook (K8s) - hosts: monitoring +- name: Prometheus Playbook (K8s) + hosts: prometheus gather_facts: true # Specific tasks for this playbook tasks: - name: Set required facts ansible.builtin.set_fact: - monitoring_home: "/var/lib/monitoring" - influxdb_home: "/var/lib/monitoring/influxdb.data" - kapacitor_home: "/var/lib/monitoring/kapacitor.data" - cronograf_home: "{{ persistent_data | default('/var/lib') }}/chronograf.data" + monitoring_home: "{{ persistent_data | default('/var/lib') }}/monitoring" monitoring_user_id: 1000 monitoring_group_id: 1000 tags: always @@ -25,9 +22,6 @@ group: "{{ monitoring_group_id }}" with_items: - "{{ monitoring_home }}" - - "{{ influxdb_home }}" - - "{{ kapacitor_home }}" - - "{{ cronograf_home }}" tags: - tasks - tasks::folders diff --git a/inventory/hosts b/inventory/hosts index 00568a9..59e6599 100644 --- a/inventory/hosts +++ b/inventory/hosts @@ -101,3 +101,7 @@ game-01.lab.alexpires.me [windows_game] game-01.lab.alexpires.me + +[prometheus] +prod-01.alexpires.me +dev-01.lab.alexpires.me diff --git a/terraform/apps/dev-01/apps.tf b/terraform/apps/dev-01/apps.tf index c2a09a4..a644eee 100644 --- a/terraform/apps/dev-01/apps.tf +++ b/terraform/apps/dev-01/apps.tf @@ -21,10 +21,10 @@ module "dns" { zones = local.zones } -module "reverse_proxy" { - source = "../../modules/apps/nginx-reverse-proxy" +module "proxy" { + source = "../../modules/utils/proxy" - services = local.reverse_proxy_services + services = local.proxy_services streams = local.streams } @@ -42,9 +42,23 @@ module "nextcloud" { } module "comfyui_proxy" { - source = "../../modules/apps/wol_proxy" + source = "../../modules/utils/wol_proxy" upstream = local.comfyui_proxy proxy_port = local.comfyui_proxy.port namespace = "comfyui" } + +module "prometheus" { + source = "../../modules/utils/prometheus" + + persistent_folder = "${local.persistent_folder}/monitoring" + config = file("${path.module}/resources/prometheus.yml") +} + +module "grafana" { + source = "../../modules/apps/grafana" + + persistent_folder = "${local.persistent_folder}/monitoring" + fqdn = local.grafana_fqdn +} diff --git a/terraform/apps/dev-01/config.tf b/terraform/apps/dev-01/config.tf index 4172bfc..315442a 100644 --- a/terraform/apps/dev-01/config.tf +++ b/terraform/apps/dev-01/config.tf @@ -72,16 +72,31 @@ locals { type = "CNAME" content = "dev-01.${local.lab_domain}." }, + { + name = "grafana" + type = "CNAME" + content = "dev-01.${local.lab_domain}." + }, + { + name = "prometheus" + type = "CNAME" + content = "dev-01.${local.lab_domain}." + }, { name = "game-01" type = "A" content = "10.19.4.245" }, + { + name = "prod-01" + type = "A" + content = "10.5.5.2" + }, ] } ] - reverse_proxy_services = { + proxy_services = { "comfyui" = { http_port = 8080 https_port = 8443 @@ -102,14 +117,15 @@ locals { streams = { "dns" = { - port = 53 - upstream = "dns.dns.svc.cluster.local" - protocol = "udp" + port = 53 + host_port = 53 + upstream = "dns.dns.svc.cluster.local" + protocol = "udp" } } ollama_proxy = { - enabled = true + enabled = true mac = "04:7c:16:d6:63:66" ip = "10.19.4.106" port = "11434" @@ -117,10 +133,10 @@ locals { } comfyui_proxy = { - mac = "04:7c:16:d6:63:66" - ip = "10.19.4.106" - port = "8188" - scheme = "http" + mac = "04:7c:16:d6:63:66" + ip = "10.19.4.106" + port = "8188" + scheme = "http" } scaleway_smtp_host = "smtp.tem.scw.cloud" @@ -150,6 +166,17 @@ locals { backup_app_api_secret_key = data.terraform_remote_state.scaleway.outputs.backup_app_api_secret_key backup_app_api_access_key = data.terraform_remote_state.scaleway.outputs.backup_app_api_access_key mail_app_api_secret_key = data.terraform_remote_state.scaleway.outputs.mail_app_api_secret_key + + grafana_fqdn = "grafana.${local.lab_domain}" + + remote_targets = { + "prometheus" = { + listen_port = 9090 + target_host = "10.5.5.5" + target_port = 9090 + protocol = "TCP" + } + } } data "terraform_remote_state" "scaleway" { diff --git a/terraform/apps/dev-01/resources/prometheus.yml b/terraform/apps/dev-01/resources/prometheus.yml new file mode 100644 index 0000000..6119791 --- /dev/null +++ b/terraform/apps/dev-01/resources/prometheus.yml @@ -0,0 +1,38 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s +rule_files: + - /etc/prometheus/prometheus.rules +scrape_configs: + - job_name: kubernetes-nodes-cadvisor + scrape_interval: 10s + scrape_timeout: 10s + scheme: https # remove if you want to scrape metrics on insecure port + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + # Only for Kubernetes ^1.7.3. + # See: https://github.com/prometheus/prometheus/issues/2916 + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + metric_relabel_configs: + - action: replace + source_labels: [id] + regex: '^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$' + target_label: rkt_container_name + replacement: "${2}-${1}" + - action: replace + source_labels: [id] + regex: '^/system\.slice/(.+)\.service$' + target_label: systemd_service_name + replacement: "${1}" diff --git a/terraform/apps/prod-01/apps.tf b/terraform/apps/prod-01/apps.tf index 58a3ce2..d2d6336 100644 --- a/terraform/apps/prod-01/apps.tf +++ b/terraform/apps/prod-01/apps.tf @@ -54,6 +54,10 @@ module "mail" { mail_crypt_public_key = var.mail_crypt_public_key ca_pem = var.root_ca_pem + + # For now we need to use host networking because of the fail2ban integration + postfix_use_host_networking = true + dovecot_use_host_networking = true } module "gitea" { @@ -116,11 +120,18 @@ module "ssh_locker_web" { socket_path = "/var/run/ssh_locker/provision.sock" } -module "reverse_proxy" { - source = "../../modules/apps/nginx-reverse-proxy" +module "proxy" { + source = "../../modules/utils/proxy" - services = local.reverse_proxy_services - wireguard_config = local.reverse_proxy_wireguard_config auth_token = var.nginx_token_bearer + services = local.proxy_services streams = local.streams + wireguard_config = local.wireguard_config +} + +module "prometheus" { + source = "../../modules/utils/prometheus" + + persistent_folder = "${local.persistent_folder}/monitoring" + config = file("${path.module}/resources/prometheus.yml") } diff --git a/terraform/apps/prod-01/config.tf b/terraform/apps/prod-01/config.tf index 0be2a72..b1237e3 100644 --- a/terraform/apps/prod-01/config.tf +++ b/terraform/apps/prod-01/config.tf @@ -79,39 +79,52 @@ locals { #reverse proxy settings streams = { - "imap" = { - port = 993 - upstream = "imap.mail.svc.cluster.local" - protocol = "tcp" - } - "smtps" = { - port = 465 - upstream = "smtps.mail.svc.cluster.local" - protocol = "tcp" - } + # For now disabled, because this messes with fail2ban on the host + # since the ip is lost and replaced with the nginx ingress controller ip + # to fix this we might need to setup logging on the stream and cross reference + # with postfix/dovecot logs and then ban the ip on the host + # "imap" = { + # port = 993 + # upstream = "imap.mail.svc.cluster.local" + # protocol = "tcp" + # } + # "smtps" = { + # port = 465 + # upstream = "smtps.mail.svc.cluster.local" + # protocol = "tcp" + # } "21115-tcp" = { - port = 21115 - upstream = "tcp.rustdesk.svc.cluster.local" - protocol = "tcp" + port = 21115 + host_port = 21115 + upstream = "tcp.rustdesk.svc.cluster.local" + protocol = "tcp" } "21116-tcp" = { - port = 21116 - upstream = "tcp.rustdesk.svc.cluster.local" - protocol = "tcp" + port = 21116 + host_port = 21116 + upstream = "tcp.rustdesk.svc.cluster.local" + protocol = "tcp" } "21116-udp" = { - port = 21116 - upstream = "udp.rustdesk.svc.cluster.local" - protocol = "udp" + port = 21116 + host_port = 21116 + upstream = "udp.rustdesk.svc.cluster.local" + protocol = "udp" } "21117-tcp" = { - port = 21117 - upstream = "tcp.rustdesk.svc.cluster.local" + port = 21117 + host_port = 21117 + upstream = "tcp.rustdesk.svc.cluster.local" + protocol = "tcp" + } + "9090-tcp" = { + port = 9090 + upstream = "prometheus.prometheus.svc.cluster.local" protocol = "tcp" } } - reverse_proxy_wireguard_config = { + wireguard_config = { private_key = var.wireguard_1_private_key public_key = var.wireguard_1_public_key pre_shared_key = var.wireguard_1_pre_shared_key @@ -125,7 +138,7 @@ locals { ] } - reverse_proxy_services = { + proxy_services = { "open-webui" = { http_port = 8080 https_port = 8443 diff --git a/terraform/apps/prod-01/resources/prometheus.yml b/terraform/apps/prod-01/resources/prometheus.yml new file mode 100644 index 0000000..440cd4d --- /dev/null +++ b/terraform/apps/prod-01/resources/prometheus.yml @@ -0,0 +1,40 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s +rule_files: + - /etc/prometheus/prometheus.rules +scrape_configs: + - job_name: kubernetes-nodes-cadvisor + scrape_interval: 10s + scrape_timeout: 10s + scheme: https # remove if you want to scrape metrics on insecure port + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + # Only for Kubernetes ^1.7.3. + # See: https://github.com/prometheus/prometheus/issues/2916 + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + metric_relabel_configs: + - action: replace + source_labels: [id] + regex: '^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$' + target_label: rkt_container_name + replacement: "${2}-${1}" + - action: replace + source_labels: [id] + regex: '^/system\.slice/(.+)\.service$' + target_label: systemd_service_name + replacement: "${1}" +remote_write: + - url: "http://prometheus.reverse-proxy.svc.cluster.local:9090/api/v1/write" diff --git a/terraform/modules/apps/grafana/config.tf b/terraform/modules/apps/grafana/config.tf new file mode 100644 index 0000000..8991f75 --- /dev/null +++ b/terraform/modules/apps/grafana/config.tf @@ -0,0 +1,4 @@ +locals { + app_version = var.tag + monitoring_folder = "${var.persistent_folder}/grafana" +} diff --git a/terraform/modules/apps/grafana/grafana.tf b/terraform/modules/apps/grafana/grafana.tf new file mode 100644 index 0000000..6363b06 --- /dev/null +++ b/terraform/modules/apps/grafana/grafana.tf @@ -0,0 +1,136 @@ +resource "kubernetes_deployment" "grafana" { + + metadata { + name = "grafana" + namespace = kubernetes_namespace.grafana.metadata[0].name + labels = { + app = "grafana" + } + } + spec { + selector { + match_labels = { + app = "grafana" + } + } + template { + metadata { + labels = { + app = "grafana" + } + } + spec { + container { + name = "grafana" + image = "grafana/grafana:${local.app_version}" + image_pull_policy = "IfNotPresent" + security_context { + run_as_user = 1000 + run_as_non_root = true + privileged = false + allow_privilege_escalation = false + } + port { + name = "http-grafana" + container_port = 3000 + protocol = "TCP" + } + readiness_probe { + http_get { + path = "/robots.txt" + port = 3000 + scheme = "HTTP" + } + failure_threshold = 3 + initial_delay_seconds = 10 + period_seconds = 30 + success_threshold = 1 + timeout_seconds = 2 + } + liveness_probe { + tcp_socket { + port = 3000 + } + failure_threshold = 3 + initial_delay_seconds = 30 + period_seconds = 10 + success_threshold = 1 + timeout_seconds = 1 + } + resources { + requests = { + cpu = "250m" + memory = "750Mi" + } + } + volume_mount { + name = "grafana-pv" + mount_path = "/var/lib/grafana" + } + } + volume { + name = "grafana-pv" + host_path { + path = local.monitoring_folder + } + } + } + } + } +} + +resource "kubernetes_service" "grafana" { + + metadata { + name = "grafana" + namespace = kubernetes_namespace.grafana.metadata[0].name + } + spec { + selector = { + app = "grafana" + } + port { + port = 3000 + protocol = "TCP" + target_port = "http-grafana" + } + session_affinity = "None" + type = "ClusterIP" + } +} + +resource "kubernetes_ingress_v1" "grafana" { + + metadata { + name = "ingress" + namespace = kubernetes_namespace.grafana.metadata[0].name + + annotations = { + "kubernetes.io/ingress.class" = "public" + "cert-manager.io/cluster-issuer" = "letsencrypt-prod" + "kubernetes.io/tls-acme" = "true" + "nginx.ingress.kubernetes.io/ssl-redirect" = "true" + } + } + spec { + tls { + hosts = [var.fqdn] + secret_name = "grafana-tls" + } + rule { + host = var.fqdn + http { + path { + backend { + service { + name = kubernetes_service.grafana.metadata[0].name + port { + number = 3000 + } + } + } + } + } + } + } +} diff --git a/terraform/modules/apps/grafana/main.tf b/terraform/modules/apps/grafana/main.tf new file mode 100644 index 0000000..1ff0db0 --- /dev/null +++ b/terraform/modules/apps/grafana/main.tf @@ -0,0 +1,5 @@ +resource "kubernetes_namespace" "grafana" { + metadata { + name = "grafana" + } +} diff --git a/terraform/modules/apps/wol_proxy/provider.tf b/terraform/modules/apps/grafana/provider.tf similarity index 100% rename from terraform/modules/apps/wol_proxy/provider.tf rename to terraform/modules/apps/grafana/provider.tf diff --git a/terraform/modules/apps/grafana/resources/default.yml b/terraform/modules/apps/grafana/resources/default.yml new file mode 100644 index 0000000..6119791 --- /dev/null +++ b/terraform/modules/apps/grafana/resources/default.yml @@ -0,0 +1,38 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s +rule_files: + - /etc/prometheus/prometheus.rules +scrape_configs: + - job_name: kubernetes-nodes-cadvisor + scrape_interval: 10s + scrape_timeout: 10s + scheme: https # remove if you want to scrape metrics on insecure port + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + # Only for Kubernetes ^1.7.3. + # See: https://github.com/prometheus/prometheus/issues/2916 + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + metric_relabel_configs: + - action: replace + source_labels: [id] + regex: '^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$' + target_label: rkt_container_name + replacement: "${2}-${1}" + - action: replace + source_labels: [id] + regex: '^/system\.slice/(.+)\.service$' + target_label: systemd_service_name + replacement: "${1}" diff --git a/terraform/modules/apps/grafana/resources/prometheus.bak.yml b/terraform/modules/apps/grafana/resources/prometheus.bak.yml new file mode 100644 index 0000000..ac7400d --- /dev/null +++ b/terraform/modules/apps/grafana/resources/prometheus.bak.yml @@ -0,0 +1,126 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s +rule_files: + - /etc/prometheus/prometheus.rules +alerting: + alertmanagers: + - scheme: http + static_configs: + - targets: + - "alertmanager.monitoring.svc:9093" +scrape_configs: + - job_name: "node-exporter" + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: [__meta_kubernetes_endpoints_name] + regex: "node-exporter" + action: keep + - job_name: "kubernetes-apiservers" + kubernetes_sd_configs: + - role: endpoints + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: + [ + __meta_kubernetes_namespace, + __meta_kubernetes_service_name, + __meta_kubernetes_endpoint_port_name, + ] + action: keep + regex: default;kubernetes;https + - job_name: "kubernetes-nodes" + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/$${1}/proxy/metrics + - job_name: "kubernetes-pods" + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: + [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name + - job_name: "kube-state-metrics" + static_configs: + - targets: ["kube-state-metrics.kube-system.svc.cluster.local:8080"] + - job_name: "kubernetes-cadvisor" + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/$${1}/proxy/metrics/cadvisor + - job_name: "kubernetes-service-endpoints" + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: + [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: + [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: + [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: kubernetes_name diff --git a/terraform/modules/apps/grafana/variables.tf b/terraform/modules/apps/grafana/variables.tf new file mode 100644 index 0000000..c1a5fd3 --- /dev/null +++ b/terraform/modules/apps/grafana/variables.tf @@ -0,0 +1,22 @@ +variable "fqdn" { + description = "The fully qualified domain name for the app server" + type = string + default = "" +} + +variable "issuer" { + description = "The issuer for the certificate" + type = string + default = "internal-ca" +} + +variable "tag" { + description = "Grafana image tag" + type = string + default = "latest" +} + +variable "persistent_folder" { + description = "The path to the persistent folder" + type = string +} diff --git a/terraform/modules/apps/mail/dovecot.tf b/terraform/modules/apps/mail/dovecot.tf index fd1c275..56555a3 100644 --- a/terraform/modules/apps/mail/dovecot.tf +++ b/terraform/modules/apps/mail/dovecot.tf @@ -59,7 +59,7 @@ resource "kubernetes_deployment" "dovecot" { } } strategy { - type = "RollingUpdate" + type = var.dovecot_use_host_networking ? "Recreate" : "RollingUpdate" } template { metadata { @@ -81,6 +81,7 @@ resource "kubernetes_deployment" "dovecot" { image = "dovecot/dovecot:${var.dovecot_tag}" port { container_port = 31993 + host_port = var.dovecot_use_host_networking ? 993 : null } port { name = "lmtp" @@ -118,6 +119,19 @@ resource "kubernetes_deployment" "dovecot" { mount_path = "/etc/mail-crypt" read_only = true } + volume_mount { + name = "dovecot-logs" + mount_path = "/var/log/dovecot" + read_only = false + } + volume_mount { + name = "tmp" + mount_path = "/tmp" + } + volume_mount { + name = "dovecot-run" + mount_path = "/var/run/dovecot" + } } container { name = "stunnel" @@ -194,6 +208,20 @@ resource "kubernetes_deployment" "dovecot" { } } } + volume { + name = "dovecot-logs" + host_path { + path = "/var/log/dovecot" + } + } + volume { + name = "tmp" + empty_dir {} + } + volume { + name = "dovecot-run" + empty_dir {} + } } } } @@ -245,8 +273,7 @@ resource "kubernetes_service" "dovecot_lmtp" { port = 31024 target_port = 31024 } - type = "ClusterIP" - cluster_ip = "None" + type = "ClusterIP" } } @@ -265,7 +292,6 @@ resource "kubernetes_service" "dovecot_auth" { port = 31000 target_port = 31000 } - type = "ClusterIP" - cluster_ip = "None" + type = "ClusterIP" } } diff --git a/terraform/modules/apps/mail/postfix.tf b/terraform/modules/apps/mail/postfix.tf index 6e72858..1547ace 100644 --- a/terraform/modules/apps/mail/postfix.tf +++ b/terraform/modules/apps/mail/postfix.tf @@ -36,7 +36,7 @@ resource "kubernetes_deployment" "postfix" { } } strategy { - type = "RollingUpdate" + type = "Recreate" } template { metadata { @@ -56,6 +56,7 @@ resource "kubernetes_deployment" "postfix" { image = "tozd/postfix:${var.postfix_tag}" port { container_port = 465 + host_port = var.postfix_use_host_networking ? 465 : null } dynamic "env" { for_each = local.postfix_env_vars @@ -189,8 +190,7 @@ resource "kubernetes_service" "postfix" { target_port = 465 } - type = "ClusterIP" - cluster_ip = "None" + type = "ClusterIP" } lifecycle { diff --git a/terraform/modules/apps/mail/resources/dovecot/dovecot.conf b/terraform/modules/apps/mail/resources/dovecot/dovecot.conf index 945de02..095708e 100644 --- a/terraform/modules/apps/mail/resources/dovecot/dovecot.conf +++ b/terraform/modules/apps/mail/resources/dovecot/dovecot.conf @@ -71,8 +71,6 @@ mail_attribute { } } -log_path = /dev/stdout - imap_hibernate_timeout = 5s mail_plugins { @@ -242,9 +240,10 @@ crypt_global_private_key main { } crypt_global_public_key_file = /etc/mail-crypt/ecpubkey.pem -log_path = /dev/stdout -info_log_path = /dev/stdout -debug_log_path = /dev/stdout + +log_path = /var/log/dovecot/default.log +info_log_path = /var/log/dovecot/info.log +debug_log_path = /var/log/dovecot/debug.log # Enable verbose logging # auth_verbose = yes diff --git a/terraform/modules/apps/mail/resources/postfix/run.config b/terraform/modules/apps/mail/resources/postfix/run.config index 9b1dce3..89312bb 100644 --- a/terraform/modules/apps/mail/resources/postfix/run.config +++ b/terraform/modules/apps/mail/resources/postfix/run.config @@ -39,7 +39,7 @@ postconf -e "smtpd_sasl_type=dovecot" postconf -e "smtpd_sasl_path=inet:localhost:31000" postconf -e "smtpd_sasl_auth_enable=yes" postconf -e "smtpd_sasl_security_options=noanonymous" -postconf -e "smtpd_recipient_restrictions=permit_sasl_authenticated, reject" +postconf -e "smtpd_recipient_restrictions=permit_sasl_authenticated, reject_unauth_destination" postconf -e "smtpd_relay_restrictions=permit_sasl_authenticated, reject_unauth_destination" postconf -e "smtpd_sasl_authenticated_header=yes" postconf -e "smtpd_tls_wrappermode=yes" @@ -68,7 +68,8 @@ smtps inet n - n - - smtpd -o syslog_name=postfix/smtps -o smtpd_tls_wrappermode=yes -o smtpd_sasl_auth_enable=yes - -o smtpd_recipient_restrictions=permit_sasl_authenticated,reject + -o smtpd_recipient_restrictions=permit_sasl_authenticated,reject_unauth_destination + -o smtpd_relay_restrictions=permit_sasl_authenticated,reject_unauth_destination " | sudo tee -a /etc/postfix/master.cf /usr/sbin/postfix -c /etc/postfix start diff --git a/terraform/modules/apps/mail/variables.tf b/terraform/modules/apps/mail/variables.tf index 3eb6244..c9bf195 100644 --- a/terraform/modules/apps/mail/variables.tf +++ b/terraform/modules/apps/mail/variables.tf @@ -92,3 +92,15 @@ variable "ca_pem" { type = string sensitive = true } + +variable "postfix_use_host_networking" { + description = "Whether to use host networking for the Postfix container" + type = bool + default = false +} + +variable "dovecot_use_host_networking" { + description = "Whether to use host networking for the Dovecot container" + type = bool + default = false +} diff --git a/terraform/modules/apps/nginx/main.tf b/terraform/modules/apps/nginx/main.tf index 91245bb..021d117 100644 --- a/terraform/modules/apps/nginx/main.tf +++ b/terraform/modules/apps/nginx/main.tf @@ -46,8 +46,7 @@ resource "kubernetes_service" "website" { app = format("website-%s", replace(each.key, ".", "-")) } - type = "ClusterIP" - cluster_ip = "None" + type = "ClusterIP" } } diff --git a/terraform/modules/utils/prometheus/config.tf b/terraform/modules/utils/prometheus/config.tf new file mode 100644 index 0000000..5d1dd79 --- /dev/null +++ b/terraform/modules/utils/prometheus/config.tf @@ -0,0 +1,17 @@ +locals { + app_version = var.tag + data_folder = "${var.persistent_folder}/prometheus" + + prometheus_config = coalesce(var.config, file("${path.module}/resources/default.yml")) + prometheus_config_checksum = md5(local.prometheus_config) + + prometheus_rules = yamlencode({ + groups = [ + { + name = "custom-rules" + rules = var.rules + } + ] + }) + prometheus_rules_checksum = md5(local.prometheus_rules) +} diff --git a/terraform/modules/utils/prometheus/main.tf b/terraform/modules/utils/prometheus/main.tf new file mode 100644 index 0000000..3f79f52 --- /dev/null +++ b/terraform/modules/utils/prometheus/main.tf @@ -0,0 +1,5 @@ +resource "kubernetes_namespace" "monitoring" { + metadata { + name = "prometheus" + } +} diff --git a/terraform/modules/utils/prometheus/prometheus.tf b/terraform/modules/utils/prometheus/prometheus.tf new file mode 100644 index 0000000..45eee5c --- /dev/null +++ b/terraform/modules/utils/prometheus/prometheus.tf @@ -0,0 +1,216 @@ +resource "kubernetes_service_account" "prometheus" { + metadata { + name = "prometheus" + namespace = kubernetes_namespace.monitoring.metadata[0].name + } +} + +resource "kubernetes_cluster_role" "prometheus" { + metadata { + name = "prometheus" + } + + rule { + api_groups = [""] + resources = ["nodes", "nodes/proxy", "services", "endpoints", "pods"] + verbs = ["get", "list", "watch"] + } + + rule { + api_groups = ["extensions"] + resources = ["ingresses"] + verbs = ["get", "list", "watch"] + } + + rule { + api_groups = ["networking.k8s.io"] + resources = ["ingresses"] + verbs = ["get", "list", "watch"] + } + + rule { + non_resource_urls = ["/metrics"] + verbs = ["get"] + } +} + +resource "kubernetes_cluster_role_binding" "prometheus" { + metadata { + name = "prometheus" + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = kubernetes_cluster_role.prometheus.metadata[0].name + } + + subject { + kind = "ServiceAccount" + name = kubernetes_service_account.prometheus.metadata[0].name + namespace = kubernetes_namespace.monitoring.metadata[0].name + } +} + +resource "kubernetes_config_map" "prometheus" { + metadata { + name = "prometheus-server-conf" + namespace = kubernetes_namespace.monitoring.metadata[0].name + labels = { + name = "prometheus-server-conf" + } + } + + data = { + "prometheus.rules" = local.prometheus_rules + "prometheus.yml" = local.prometheus_config + } +} + +resource "kubernetes_deployment" "prometheus" { + metadata { + name = "prometheus" + namespace = kubernetes_namespace.monitoring.metadata[0].name + labels = { + app = "prometheus" + } + } + + spec { + replicas = 1 + strategy { + type = "Recreate" + } + + selector { + match_labels = { + app = "prometheus" + } + } + + template { + + metadata { + labels = { + app = "prometheus" + } + annotations = { + "checksum/config" = local.prometheus_config_checksum + "checksum/rules" = local.prometheus_rules_checksum + } + } + + spec { + service_account_name = kubernetes_service_account.prometheus.metadata[0].name + container { + name = "prometheus" + image = "prom/prometheus:${local.app_version}" + args = [ + "--config.file=/etc/prometheus/prometheus.yml", + "--storage.tsdb.path=/prometheus/", + "--storage.tsdb.retention.time=${var.retention_time}", + ] + + port { + name = "http-prometheus" + container_port = 9090 + } + + security_context { + run_as_user = 1000 + run_as_group = 1000 + run_as_non_root = true + allow_privilege_escalation = false + } + + volume_mount { + name = "prometheus-config-volume" + mount_path = "/etc/prometheus/" + } + volume_mount { + name = "prometheus-storage-volume" + mount_path = "/prometheus" + } + } + + volume { + name = "prometheus-config-volume" + config_map { + name = kubernetes_config_map.prometheus.metadata[0].name + } + } + + volume { + name = "prometheus-storage-volume" + host_path { + path = local.data_folder + type = "DirectoryOrCreate" + } + } + } + } + } +} + +resource "kubernetes_service" "prometheus" { + metadata { + name = "prometheus" + namespace = kubernetes_namespace.monitoring.metadata[0].name + annotations = { + "prometheus.io/scrape" = "true" + "prometheus.io/port" = "9090" + } + } + + spec { + selector = { + app = "prometheus" + } + + type = "ClusterIP" + + port { + port = 9090 + protocol = "TCP" + target_port = "http-prometheus" + } + } +} + +# resource "kubernetes_ingress_v1" "prometheus" { +# metadata { +# name = "prometheus-ui" +# namespace = kubernetes_namespace.monitoring.metadata[0].name +# annotations = { +# "kubernetes.io/ingress.class" = var.ingress_class +# } +# } + +# spec { +# rule { +# host = var.fqdn +# http { +# path { +# path = "/" +# path_type = "Prefix" +# backend { +# service { +# name = kubernetes_service.prometheus.metadata[0].name +# port { +# number = 8080 +# } +# } +# } +# } +# } +# } + +# dynamic "tls" { +# for_each = local.tls_crt_data != "" && local.tls_key_data != "" ? [1] : [] +# content { +# hosts = [var.fqdn] +# secret_name = local.tls_secret_name +# } +# } +# } +# } diff --git a/terraform/modules/utils/prometheus/provider.tf b/terraform/modules/utils/prometheus/provider.tf new file mode 100644 index 0000000..2eaa508 --- /dev/null +++ b/terraform/modules/utils/prometheus/provider.tf @@ -0,0 +1,9 @@ +terraform { + required_version = "~>1.8" + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.36.0" + } + } +} diff --git a/terraform/modules/utils/prometheus/resources/default.yml b/terraform/modules/utils/prometheus/resources/default.yml new file mode 100644 index 0000000..6119791 --- /dev/null +++ b/terraform/modules/utils/prometheus/resources/default.yml @@ -0,0 +1,38 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s +rule_files: + - /etc/prometheus/prometheus.rules +scrape_configs: + - job_name: kubernetes-nodes-cadvisor + scrape_interval: 10s + scrape_timeout: 10s + scheme: https # remove if you want to scrape metrics on insecure port + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + # Only for Kubernetes ^1.7.3. + # See: https://github.com/prometheus/prometheus/issues/2916 + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + metric_relabel_configs: + - action: replace + source_labels: [id] + regex: '^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$' + target_label: rkt_container_name + replacement: "${2}-${1}" + - action: replace + source_labels: [id] + regex: '^/system\.slice/(.+)\.service$' + target_label: systemd_service_name + replacement: "${1}" diff --git a/terraform/modules/utils/prometheus/resources/prometheus.bak.yml b/terraform/modules/utils/prometheus/resources/prometheus.bak.yml new file mode 100644 index 0000000..ac7400d --- /dev/null +++ b/terraform/modules/utils/prometheus/resources/prometheus.bak.yml @@ -0,0 +1,126 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s +rule_files: + - /etc/prometheus/prometheus.rules +alerting: + alertmanagers: + - scheme: http + static_configs: + - targets: + - "alertmanager.monitoring.svc:9093" +scrape_configs: + - job_name: "node-exporter" + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: [__meta_kubernetes_endpoints_name] + regex: "node-exporter" + action: keep + - job_name: "kubernetes-apiservers" + kubernetes_sd_configs: + - role: endpoints + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: + [ + __meta_kubernetes_namespace, + __meta_kubernetes_service_name, + __meta_kubernetes_endpoint_port_name, + ] + action: keep + regex: default;kubernetes;https + - job_name: "kubernetes-nodes" + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/$${1}/proxy/metrics + - job_name: "kubernetes-pods" + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: + [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name + - job_name: "kube-state-metrics" + static_configs: + - targets: ["kube-state-metrics.kube-system.svc.cluster.local:8080"] + - job_name: "kubernetes-cadvisor" + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/$${1}/proxy/metrics/cadvisor + - job_name: "kubernetes-service-endpoints" + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: + [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: + [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: + [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: kubernetes_name diff --git a/terraform/modules/utils/prometheus/variables.tf b/terraform/modules/utils/prometheus/variables.tf new file mode 100644 index 0000000..f0c037d --- /dev/null +++ b/terraform/modules/utils/prometheus/variables.tf @@ -0,0 +1,34 @@ +variable "tag" { + description = "Prometheus image tag" + type = string + default = "latest" +} + +variable "retention_time" { + description = "Prometheus data retention time" + type = string + default = "15d" +} + +variable "rules" { + description = "Prometheus alerting rules" + type = list(object({ + alert = string + expr = string + for = string + labels = map(string) + annotations = map(string) + })) + default = [] +} + +variable "config" { + description = "Custom Prometheus configuration in YAML format" + type = string + default = "" +} + +variable "persistent_folder" { + description = "The path to the persistent folder" + type = string +} diff --git a/terraform/modules/apps/nginx-reverse-proxy/config.tf b/terraform/modules/utils/proxy/config.tf similarity index 100% rename from terraform/modules/apps/nginx-reverse-proxy/config.tf rename to terraform/modules/utils/proxy/config.tf diff --git a/terraform/modules/apps/nginx-reverse-proxy/main.tf b/terraform/modules/utils/proxy/main.tf similarity index 87% rename from terraform/modules/apps/nginx-reverse-proxy/main.tf rename to terraform/modules/utils/proxy/main.tf index 20e12bc..c348274 100644 --- a/terraform/modules/apps/nginx-reverse-proxy/main.tf +++ b/terraform/modules/utils/proxy/main.tf @@ -1,22 +1,22 @@ -resource "kubernetes_namespace" "reverse_proxy" { +resource "kubernetes_namespace" "proxy" { metadata { annotations = { - name = "reverse-proxy" + name = "proxy" } labels = { - name = "reverse-proxy" + name = "proxy" } - name = "reverse-proxy" + name = "proxy" } } -resource "kubernetes_service_account" "reverse-proxy_service_account" { +resource "kubernetes_service_account" "service_account" { metadata { - name = "reverse-proxy-app-sa" - namespace = kubernetes_namespace.reverse_proxy.metadata[0].name + name = "proxy-app-sa" + namespace = kubernetes_namespace.proxy.metadata[0].name } automount_service_account_token = false @@ -26,7 +26,7 @@ resource "kubernetes_service" "services" { for_each = var.services metadata { name = format("%s", replace(each.key, ".", "-")) - namespace = kubernetes_namespace.reverse_proxy.metadata[0].name + namespace = kubernetes_namespace.proxy.metadata[0].name } spec { port { @@ -43,14 +43,34 @@ resource "kubernetes_service" "services" { } } selector = { - app = "reverse-proxy" + app = "proxy" } - type = "ClusterIP" - cluster_ip = "None" + type = "ClusterIP" } } +resource "kubernetes_service" "streams" { + for_each = var.streams + metadata { + name = substr(format("stream-%s", replace(each.key, ".", "-")), 0, 15) + namespace = kubernetes_namespace.proxy.metadata[0].name + } + spec { + port { + name = substr(format("stream-%s", replace(each.key, ".", "-")), 0, 15) + port = each.value.port + target_port = substr(format("stream-%s", replace(each.key, ".", "-")), 0, 15) + } + selector = { + app = "proxy" + } + + type = "ClusterIP" + } +} + + resource "kubernetes_manifest" "internal_cert" { manifest = { @@ -58,7 +78,7 @@ resource "kubernetes_manifest" "internal_cert" { kind = "Certificate" metadata = { name = "internal-cert" - namespace = kubernetes_namespace.reverse_proxy.metadata[0].name + namespace = kubernetes_namespace.proxy.metadata[0].name } spec = { secretName = "internal-cert" @@ -66,7 +86,7 @@ resource "kubernetes_manifest" "internal_cert" { name = var.issuer_name kind = "ClusterIssuer" } - commonName = "reverse-proxy.internal" + commonName = "proxy.internal" dnsNames = flatten([for _, v in var.services : v.fqdn if v.tls == true]) } } @@ -76,7 +96,7 @@ resource "kubernetes_config_map" "wireguard_config" { count = local.wireguard_config != "null" ? 1 : 0 metadata { name = "wireguard-config" - namespace = kubernetes_namespace.reverse_proxy.metadata[0].name + namespace = kubernetes_namespace.proxy.metadata[0].name } data = { @@ -87,7 +107,7 @@ resource "kubernetes_config_map" "wireguard_config" { resource "kubernetes_config_map" "nginx_config" { metadata { name = "nginx-config" - namespace = kubernetes_namespace.reverse_proxy.metadata[0].name + namespace = kubernetes_namespace.proxy.metadata[0].name } data = { @@ -98,7 +118,7 @@ resource "kubernetes_config_map" "nginx_config" { resource "kubernetes_config_map" "nginx_default_config" { metadata { name = "nginx-default-config" - namespace = kubernetes_namespace.reverse_proxy.metadata[0].name + namespace = kubernetes_namespace.proxy.metadata[0].name } data = local.services_config @@ -107,7 +127,7 @@ resource "kubernetes_config_map" "nginx_default_config" { resource "kubernetes_config_map" "nginx_streams_config" { metadata { name = "nginx-streams-config" - namespace = kubernetes_namespace.reverse_proxy.metadata[0].name + namespace = kubernetes_namespace.proxy.metadata[0].name } data = local.streams_config @@ -117,7 +137,7 @@ resource "kubernetes_config_map" "nginx_custom_50x_page" { metadata { name = "nginx-errors" - namespace = kubernetes_namespace.reverse_proxy.metadata[0].name + namespace = kubernetes_namespace.proxy.metadata[0].name } data = { @@ -129,7 +149,7 @@ resource "kubernetes_config_map" "nginx_custom_50x_page" { resource "kubernetes_config_map" "auth_script" { metadata { name = "auth-script" - namespace = kubernetes_namespace.reverse_proxy.metadata[0].name + namespace = kubernetes_namespace.proxy.metadata[0].name } data = { @@ -141,7 +161,7 @@ resource "kubernetes_ingress_v1" "ingress" { for_each = { for k, v in var.services : k => v if lookup(v, "ingress", true) && length(v.fqdn) > 0 } metadata { name = format("%s", replace(each.key, ".", "-")) - namespace = kubernetes_namespace.reverse_proxy.metadata[0].name + namespace = kubernetes_namespace.proxy.metadata[0].name annotations = merge({ "kubernetes.io/ingress.class" = "public" @@ -159,7 +179,7 @@ resource "kubernetes_ingress_v1" "ingress" { spec { tls { hosts = each.value.fqdn - secret_name = format("reverse-proxy-%s-tls", replace(each.key, ".", "-")) + secret_name = format("proxy-%s-tls", replace(each.key, ".", "-")) } dynamic "rule" { for_each = each.value.fqdn @@ -186,7 +206,7 @@ resource "kubernetes_secret" "auth_token" { count = var.auth_token != "" ? 1 : 0 metadata { name = "auth-token-secret" - namespace = kubernetes_namespace.reverse_proxy.metadata[0].name + namespace = kubernetes_namespace.proxy.metadata[0].name } data = { @@ -196,11 +216,11 @@ resource "kubernetes_secret" "auth_token" { type = "Opaque" } -resource "kubernetes_deployment" "reverse_proxy" { +resource "kubernetes_deployment" "proxy" { metadata { - name = "reverse-proxy" - namespace = kubernetes_namespace.reverse_proxy.metadata[0].name + name = "proxy" + namespace = kubernetes_namespace.proxy.metadata[0].name annotations = { "security.alpha.kubernetes.io/unsafe-sysctls" = "net.ipv4.conf.all.src_valid_mark=1" @@ -212,7 +232,7 @@ resource "kubernetes_deployment" "reverse_proxy" { selector { match_labels = { - app = "reverse-proxy" + app = "proxy" } } @@ -223,7 +243,7 @@ resource "kubernetes_deployment" "reverse_proxy" { template { metadata { labels = { - app = "reverse-proxy" + app = "proxy" } annotations = { "checksum/nginx" = local.nginx_config_checksum @@ -342,7 +362,7 @@ resource "kubernetes_deployment" "reverse_proxy" { } container { - name = "reverse-proxy" + name = "proxy" image = "nginx:${var.tag}" image_pull_policy = "Always" @@ -399,7 +419,7 @@ resource "kubernetes_deployment" "reverse_proxy" { content { container_port = port.value.port name = substr(format("stream-%s", replace(port.key, ".", "-")), 0, 15) - host_port = port.value.port + host_port = port.value.host_port protocol = upper(port.value.protocol) } } diff --git a/terraform/modules/apps/nginx-reverse-proxy/provider.tf b/terraform/modules/utils/proxy/provider.tf similarity index 100% rename from terraform/modules/apps/nginx-reverse-proxy/provider.tf rename to terraform/modules/utils/proxy/provider.tf diff --git a/terraform/modules/apps/nginx-reverse-proxy/resources/auth/auth_server.py b/terraform/modules/utils/proxy/resources/auth/auth_server.py similarity index 100% rename from terraform/modules/apps/nginx-reverse-proxy/resources/auth/auth_server.py rename to terraform/modules/utils/proxy/resources/auth/auth_server.py diff --git a/terraform/modules/apps/nginx-reverse-proxy/resources/errors/.gitignore b/terraform/modules/utils/proxy/resources/errors/.gitignore similarity index 100% rename from terraform/modules/apps/nginx-reverse-proxy/resources/errors/.gitignore rename to terraform/modules/utils/proxy/resources/errors/.gitignore diff --git a/terraform/modules/apps/nginx-reverse-proxy/resources/errors/400.html b/terraform/modules/utils/proxy/resources/errors/400.html similarity index 100% rename from terraform/modules/apps/nginx-reverse-proxy/resources/errors/400.html rename to terraform/modules/utils/proxy/resources/errors/400.html diff --git a/terraform/modules/apps/nginx-reverse-proxy/resources/errors/401.html b/terraform/modules/utils/proxy/resources/errors/401.html similarity index 100% rename from terraform/modules/apps/nginx-reverse-proxy/resources/errors/401.html rename to terraform/modules/utils/proxy/resources/errors/401.html diff --git a/terraform/modules/apps/nginx-reverse-proxy/resources/errors/403.html b/terraform/modules/utils/proxy/resources/errors/403.html similarity index 100% rename from terraform/modules/apps/nginx-reverse-proxy/resources/errors/403.html rename to terraform/modules/utils/proxy/resources/errors/403.html diff --git a/terraform/modules/apps/nginx-reverse-proxy/resources/errors/404.html b/terraform/modules/utils/proxy/resources/errors/404.html similarity index 100% rename from terraform/modules/apps/nginx-reverse-proxy/resources/errors/404.html rename to terraform/modules/utils/proxy/resources/errors/404.html diff --git a/terraform/modules/apps/nginx-reverse-proxy/resources/errors/408.html b/terraform/modules/utils/proxy/resources/errors/408.html similarity index 100% rename from terraform/modules/apps/nginx-reverse-proxy/resources/errors/408.html rename to terraform/modules/utils/proxy/resources/errors/408.html diff --git a/terraform/modules/apps/nginx-reverse-proxy/resources/errors/429.html b/terraform/modules/utils/proxy/resources/errors/429.html similarity index 100% rename from terraform/modules/apps/nginx-reverse-proxy/resources/errors/429.html rename to terraform/modules/utils/proxy/resources/errors/429.html diff --git a/terraform/modules/apps/nginx-reverse-proxy/resources/errors/500.html b/terraform/modules/utils/proxy/resources/errors/500.html similarity index 100% rename from terraform/modules/apps/nginx-reverse-proxy/resources/errors/500.html rename to terraform/modules/utils/proxy/resources/errors/500.html diff --git a/terraform/modules/apps/nginx-reverse-proxy/resources/errors/502.html b/terraform/modules/utils/proxy/resources/errors/502.html similarity index 100% rename from terraform/modules/apps/nginx-reverse-proxy/resources/errors/502.html rename to terraform/modules/utils/proxy/resources/errors/502.html diff --git a/terraform/modules/apps/nginx-reverse-proxy/resources/errors/503.html b/terraform/modules/utils/proxy/resources/errors/503.html similarity index 100% rename from terraform/modules/apps/nginx-reverse-proxy/resources/errors/503.html rename to terraform/modules/utils/proxy/resources/errors/503.html diff --git a/terraform/modules/apps/nginx-reverse-proxy/resources/errors/504.html b/terraform/modules/utils/proxy/resources/errors/504.html similarity index 100% rename from terraform/modules/apps/nginx-reverse-proxy/resources/errors/504.html rename to terraform/modules/utils/proxy/resources/errors/504.html diff --git a/terraform/modules/apps/nginx-reverse-proxy/resources/nginx/conf.d/default.conf.tpl b/terraform/modules/utils/proxy/resources/nginx/conf.d/default.conf.tpl similarity index 100% rename from terraform/modules/apps/nginx-reverse-proxy/resources/nginx/conf.d/default.conf.tpl rename to terraform/modules/utils/proxy/resources/nginx/conf.d/default.conf.tpl diff --git a/terraform/modules/apps/nginx-reverse-proxy/resources/nginx/nginx.conf b/terraform/modules/utils/proxy/resources/nginx/nginx.conf similarity index 100% rename from terraform/modules/apps/nginx-reverse-proxy/resources/nginx/nginx.conf rename to terraform/modules/utils/proxy/resources/nginx/nginx.conf diff --git a/terraform/modules/apps/nginx-reverse-proxy/resources/nginx/streams.d/stream.conf.tpl b/terraform/modules/utils/proxy/resources/nginx/streams.d/stream.conf.tpl similarity index 100% rename from terraform/modules/apps/nginx-reverse-proxy/resources/nginx/streams.d/stream.conf.tpl rename to terraform/modules/utils/proxy/resources/nginx/streams.d/stream.conf.tpl diff --git a/terraform/modules/apps/nginx-reverse-proxy/resources/wireguard/wireguard.conf.tpl b/terraform/modules/utils/proxy/resources/wireguard/wireguard.conf.tpl similarity index 100% rename from terraform/modules/apps/nginx-reverse-proxy/resources/wireguard/wireguard.conf.tpl rename to terraform/modules/utils/proxy/resources/wireguard/wireguard.conf.tpl diff --git a/terraform/modules/apps/nginx-reverse-proxy/variables.tf b/terraform/modules/utils/proxy/variables.tf similarity index 98% rename from terraform/modules/apps/nginx-reverse-proxy/variables.tf rename to terraform/modules/utils/proxy/variables.tf index 586add7..c4d34f3 100644 --- a/terraform/modules/apps/nginx-reverse-proxy/variables.tf +++ b/terraform/modules/utils/proxy/variables.tf @@ -70,6 +70,7 @@ variable "streams" { resolver = optional(string, "") upstream_port = optional(number, null) protocol = optional(string, "TCP") + host_port = optional(number, null) })) default = {} } diff --git a/terraform/modules/utils/stunnel/config.tf b/terraform/modules/utils/stunnel/config.tf new file mode 100644 index 0000000..14467e2 --- /dev/null +++ b/terraform/modules/utils/stunnel/config.tf @@ -0,0 +1,20 @@ +locals { + wireguard_version = var.wireguard_version + wireguard_config = try(templatefile("${path.module}/resources/wireguard.conf.tpl", { + wireguard_private_key = var.wireguard_config.private_key + wireguard_public_key = var.wireguard_config.public_key + wireguard_pre_shared_key = var.wireguard_config.pre_shared_key + wireguard_address = var.wireguard_config.address + wireguard_dns = var.wireguard_config.dns + wireguard_allowed_ips = join(", ", var.wireguard_config.allowed_ips) + wireguard_endpoint = "${var.wireguard_config.endpoint}:${var.wireguard_config.endpoint_port}" + }), "null") + wireguard_config_checksum = nonsensitive(sha256(local.wireguard_config)) + + stunnel_version = var.stunnel_version + stunnel_config = templatefile("${path.module}/resources/stunnel.conf.tpl", { + local_targets = var.local_targets + remote_targets = var.remote_targets + }) + stunnel_config_checksum = sha256(local.stunnel_config) +} diff --git a/terraform/modules/utils/stunnel/main.tf b/terraform/modules/utils/stunnel/main.tf new file mode 100644 index 0000000..e86651d --- /dev/null +++ b/terraform/modules/utils/stunnel/main.tf @@ -0,0 +1,289 @@ +resource "kubernetes_namespace" "stunnel" { + metadata { + name = "stunnel" + annotations = { + name = "stunnel" + } + labels = { + name = "stunnel" + } + } +} + +resource "kubernetes_service_account" "stunnel_service_account" { + + metadata { + name = "stunnel-app-sa" + namespace = kubernetes_namespace.stunnel.metadata[0].name + } + + automount_service_account_token = false +} + + +resource "kubernetes_config_map" "wireguard_config" { + count = local.wireguard_config != "null" ? 1 : 0 + metadata { + name = "wireguard-config" + namespace = kubernetes_namespace.stunnel.metadata[0].name + } + + data = { + "wireguard.conf" = local.wireguard_config + } +} + +resource "kubernetes_config_map" "stunnel_config" { + metadata { + name = "stunnel-config" + namespace = kubernetes_namespace.stunnel.metadata[0].name + } + + data = { + "stunnel.conf" = local.stunnel_config + } +} + +resource "kubernetes_deployment" "stunnel" { + + metadata { + name = "stunnel" + namespace = kubernetes_namespace.stunnel.metadata[0].name + + annotations = { + "security.alpha.kubernetes.io/unsafe-sysctls" = "net.ipv4.conf.all.src_valid_mark=1" + } + } + + spec { + replicas = 1 + + selector { + match_labels = { + app = "stunnel" + } + } + + strategy { + type = "Recreate" + } + + template { + metadata { + labels = { + app = "stunnel" + } + annotations = { + "checksum/stunnel" = local.stunnel_config_checksum + "checksum/wireguard" = local.wireguard_config_checksum + } + } + spec { + + dynamic "container" { + for_each = nonsensitive(local.wireguard_config) != "null" ? [1] : [] + content { + + name = "wireguard" + image = "linuxserver/wireguard:${local.wireguard_version}" + + resources { + requests = { + memory = "64Mi" + } + } + + liveness_probe { + exec { + command = [ + "/bin/sh", + "-c", + "wg show | grep -q transfer" + ] + } + initial_delay_seconds = 90 + period_seconds = 90 + } + + security_context { + privileged = true + capabilities { + add = ["NET_ADMIN"] + } + read_only_root_filesystem = true + } + + env { + name = "PUID" + value = "1000" + } + + env { + name = "PGID" + value = "1000" + } + + env { + name = "TZ" + value = "Europe/Berlin" + } + + volume_mount { + name = "wireguard-config" + mount_path = "/config/wg_confs" + read_only = true + } + + volume_mount { + name = "lib-modules" + mount_path = "/lib/modules" + read_only = true + } + + volume_mount { + name = "wireguard-run" + mount_path = "/run" + } + } + } + + container { + name = "stunnel" + image = "chainguard/stunnel:${local.stunnel_version}" + args = ["/etc/conf/stunnel.conf"] + volume_mount { + name = "stunnel-config" + mount_path = "/etc/conf" + read_only = true + } + volume_mount { + name = "stunnel-certs" + mount_path = "/etc/ssl/certs/private" + read_only = true + } + dynamic "port" { + for_each = var.remote_targets + content { + container_port = port.value.listen_port + name = substr(replace(port.key, ".", "-"), 0, 15) + protocol = coalesce(upper(port.value.protocol), "TCP") + } + } + } + + dynamic "volume" { + for_each = nonsensitive(local.wireguard_config) != "null" ? [1] : [] + content { + name = "wireguard-config" + config_map { + name = nonsensitive(kubernetes_config_map.wireguard_config[0].metadata[0].name) + } + } + } + + dynamic "volume" { + for_each = nonsensitive(local.wireguard_config) != "null" ? [1] : [] + content { + name = "lib-modules" + host_path { + path = "/lib/modules" + } + } + } + + dynamic "volume" { + for_each = nonsensitive(local.wireguard_config) != "null" ? [1] : [] + content { + name = "wireguard-run" + empty_dir { + medium = "Memory" + } + } + } + + volume { + name = "stunnel-config" + config_map { + name = kubernetes_config_map.stunnel_config.metadata[0].name + } + } + + volume { + name = "stunnel-certs" + secret { + secret_name = "remote-targets-cert" + } + } + } + } + } + + lifecycle { + ignore_changes = [spec[0].template[0].metadata[0].annotations["kubectl.kubernetes.io/restartedAt"]] + } +} + +resource "kubernetes_manifest" "remote_targets_cert" { + + manifest = { + apiVersion = "cert-manager.io/v1" + kind = "Certificate" + metadata = { + name = "remote-targets-cert" + namespace = kubernetes_namespace.stunnel.metadata[0].name + } + spec = { + secretName = "remote-targets-cert" + issuerRef = { + name = var.issuer_name + kind = "ClusterIssuer" + } + commonName = "stunnel.remote_targets.internal" + dnsNames = flatten([ + for k, v in var.remote_targets : "${k}.stunnel.svc.cluster.local" + ]) + } + } +} + +resource "kubernetes_manifest" "local_targets_cert" { + + manifest = { + apiVersion = "cert-manager.io/v1" + kind = "Certificate" + metadata = { + name = "local-targets-cert" + namespace = kubernetes_namespace.stunnel.metadata[0].name + } + spec = { + secretName = "local-targets-cert" + issuerRef = { + name = var.issuer_name + kind = "ClusterIssuer" + } + commonName = "stunnel.local_targets.internal" + dnsNames = flatten([ + for _, v in var.local_targets : v.target_host + ]) + } + } +} + +resource "kubernetes_service" "services" { + for_each = var.remote_targets + metadata { + name = format("%s", replace(each.key, ".", "-")) + namespace = kubernetes_namespace.stunnel.metadata[0].name + } + spec { + port { + name = substr(format("%s", replace(each.key, ".", "-")), 0, 15) + port = each.value.listen_port + target_port = substr(format("%s", replace(each.key, ".", "-")), 0, 15) + } + selector = { + app = "stunnel" + } + type = "ClusterIP" + } +} diff --git a/terraform/modules/utils/stunnel/provider.tf b/terraform/modules/utils/stunnel/provider.tf new file mode 100644 index 0000000..2eaa508 --- /dev/null +++ b/terraform/modules/utils/stunnel/provider.tf @@ -0,0 +1,9 @@ +terraform { + required_version = "~>1.8" + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.36.0" + } + } +} diff --git a/terraform/modules/utils/stunnel/resources/stunnel.conf.tpl b/terraform/modules/utils/stunnel/resources/stunnel.conf.tpl new file mode 100644 index 0000000..67db2fc --- /dev/null +++ b/terraform/modules/utils/stunnel/resources/stunnel.conf.tpl @@ -0,0 +1,16 @@ +foreground = yes + +%{ for target in local_targets ~} +[server-${target.listen_port}] +accept = 0.0.0.0:${target.listen_port} +connect = ${target.target_host}:${target.target_port} +cert = /etc/ssl/certs/private/tls.crt +key = /etc/ssl/certs/private/tls.key +%{ endfor ~} + +%{ for target in remote_targets ~} +[client-${target.listen_port}] +client = yes +accept = 0.0.0.0:${target.listen_port} +connect = ${target.target_host}:${target.target_port} +%{ endfor ~} \ No newline at end of file diff --git a/terraform/modules/utils/stunnel/resources/wireguard.conf.tpl b/terraform/modules/utils/stunnel/resources/wireguard.conf.tpl new file mode 100644 index 0000000..f3f495c --- /dev/null +++ b/terraform/modules/utils/stunnel/resources/wireguard.conf.tpl @@ -0,0 +1,10 @@ +[Interface] +PrivateKey = ${wireguard_private_key} +Address = ${wireguard_address} + +[Peer] +PublicKey = ${wireguard_public_key} +PresharedKey = ${wireguard_pre_shared_key} +AllowedIPs = ${wireguard_allowed_ips} +Endpoint = ${wireguard_endpoint} +PersistentKeepalive = 25 diff --git a/terraform/modules/utils/stunnel/variables.tf b/terraform/modules/utils/stunnel/variables.tf new file mode 100644 index 0000000..1865757 --- /dev/null +++ b/terraform/modules/utils/stunnel/variables.tf @@ -0,0 +1,55 @@ +variable "stunnel_version" { + description = "Stunnel image version" + type = string + default = "latest" +} + +variable "wireguard_version" { + description = "Wireguard image version" + type = string + default = "1.0.20210914-r4-ls54" +} + +variable "issuer_name" { + description = "The name of the issuer to use for TLS certificates, internal certificates" + type = string + default = "internal-ca" +} + +variable "local_targets" { + description = "Map of local services to expose via stunnel" + type = map(object({ + listen_port = number + target_host = string + target_port = number + protocol = string + })) + default = {} +} + +variable "wireguard_config" { + description = "The Wireguard configuration file" + type = object({ + private_key = string + public_key = string + pre_shared_key = string + endpoint = string + endpoint_port = number + address = string + dns = string + allowed_ips = list(string) + }) + default = null + sensitive = true +} + +variable "remote_targets" { + description = "Map of remote services to connect to via stunnel" + type = map(object({ + listen_port = number + target_host = string + target_port = number + protocol = string + })) + default = {} +} diff --git a/terraform/modules/apps/wol_proxy/ingress.tf b/terraform/modules/utils/wol_proxy/ingress.tf similarity index 100% rename from terraform/modules/apps/wol_proxy/ingress.tf rename to terraform/modules/utils/wol_proxy/ingress.tf diff --git a/terraform/modules/apps/wol_proxy/main.tf b/terraform/modules/utils/wol_proxy/main.tf similarity index 100% rename from terraform/modules/apps/wol_proxy/main.tf rename to terraform/modules/utils/wol_proxy/main.tf diff --git a/terraform/modules/utils/wol_proxy/provider.tf b/terraform/modules/utils/wol_proxy/provider.tf new file mode 100644 index 0000000..2eaa508 --- /dev/null +++ b/terraform/modules/utils/wol_proxy/provider.tf @@ -0,0 +1,9 @@ +terraform { + required_version = "~>1.8" + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.36.0" + } + } +} diff --git a/terraform/modules/apps/wol_proxy/proxy.tf b/terraform/modules/utils/wol_proxy/proxy.tf similarity index 100% rename from terraform/modules/apps/wol_proxy/proxy.tf rename to terraform/modules/utils/wol_proxy/proxy.tf diff --git a/terraform/modules/apps/wol_proxy/variables.tf b/terraform/modules/utils/wol_proxy/variables.tf similarity index 100% rename from terraform/modules/apps/wol_proxy/variables.tf rename to terraform/modules/utils/wol_proxy/variables.tf