diff --git a/terraform/apps/dev-01/apps.tf b/terraform/apps/dev-01/apps.tf index e764ca6..2e138ec 100644 --- a/terraform/apps/dev-01/apps.tf +++ b/terraform/apps/dev-01/apps.tf @@ -52,8 +52,18 @@ module "comfyui_proxy" { module "prometheus" { source = "../../modules/utils/prometheus" - persistent_folder = "${local.persistent_folder}/monitoring" - config = file("${path.module}/resources/prometheus.yml") + cluster_name = "dev-01" + persistent_folder = "${local.persistent_folder}/monitoring" + config = local.prometheus_config + rules = local.prometheus_rules + alertmanager_config = local.alertmanager_config + + email_config = local.alertmanager_email_config + telegram_config = { + bot_token = var.telegram_bot_token + chat_id = local.telegram_chat_id + message = local.telegram_message + } } module "grafana" { diff --git a/terraform/apps/dev-01/config.tf b/terraform/apps/dev-01/config.tf index 315442a..7233011 100644 --- a/terraform/apps/dev-01/config.tf +++ b/terraform/apps/dev-01/config.tf @@ -167,16 +167,22 @@ locals { backup_app_api_access_key = data.terraform_remote_state.scaleway.outputs.backup_app_api_access_key mail_app_api_secret_key = data.terraform_remote_state.scaleway.outputs.mail_app_api_secret_key - grafana_fqdn = "grafana.${local.lab_domain}" - - remote_targets = { - "prometheus" = { - listen_port = 9090 - target_host = "10.5.5.5" - target_port = 9090 - protocol = "TCP" - } + # prometheus and grafana + prometheus_config = file("${path.module}/resources/prometheus.yml") + prometheus_rules = file("${path.module}/resources/prometheus-rules.yml") + alertmanager_config = file("${path.module}/resources/alertmanager.yml") + alertmanager_email_config = { + to = "c.alexandre.pires@alexpires.me" + from = "alertmanager@${local.lab_domain}" + smarthost = "smtp.tem.scw.cloud:587" + auth_username = var.scaleway_project_id + auth_password = local.mail_app_api_secret_key } + + telegram_chat_id = "5432313610" # a13labs channel + telegram_message = file("${path.module}/resources/telegram.tpl") + + grafana_fqdn = "grafana.${local.lab_domain}" } data "terraform_remote_state" "scaleway" { diff --git a/terraform/apps/dev-01/resources/alertmanager.yml b/terraform/apps/dev-01/resources/alertmanager.yml new file mode 100644 index 0000000..320a381 --- /dev/null +++ b/terraform/apps/dev-01/resources/alertmanager.yml @@ -0,0 +1,11 @@ +route: + group_by: ["alertname"] + receiver: "default" + group_wait: 10s + group_interval: 10m + repeat_interval: 1h + routes: + - receiver: "telegram" + matchers: + - 'alertname="UserLoggedIn"' + continue: false diff --git a/terraform/apps/dev-01/resources/prometheus-rules.yml b/terraform/apps/dev-01/resources/prometheus-rules.yml new file mode 100644 index 0000000..96f975a --- /dev/null +++ b/terraform/apps/dev-01/resources/prometheus-rules.yml @@ -0,0 +1,10 @@ +groups: + - name: auth.rules + rules: + - alert: UserLoggedIn + expr: increase(auth_sessions_active_by_user[1m]) > 0 + labels: + severity: info + annotations: + summary: "Login: {{ $labels.username }} on {{ $labels.instance }}" + description: "User {{ $labels.username }} logged in via {{ $labels.source }} on {{ $labels.instance }}." diff --git a/terraform/apps/dev-01/resources/prometheus.yml b/terraform/apps/dev-01/resources/prometheus.yml index 76387f7..4c34948 100644 --- a/terraform/apps/dev-01/resources/prometheus.yml +++ b/terraform/apps/dev-01/resources/prometheus.yml @@ -1,41 +1,4 @@ -global: - scrape_interval: 5s - evaluation_interval: 5s -rule_files: - - /etc/prometheus/prometheus.rules scrape_configs: - - job_name: kubernetes-nodes-cadvisor - scrape_interval: 10s - scrape_timeout: 10s - scheme: https # remove if you want to scrape metrics on insecure port - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: true - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - kubernetes_sd_configs: - - role: node - relabel_configs: - - action: labelmap - regex: __meta_kubernetes_node_label_(.+) - # Only for Kubernetes ^1.7.3. - # See: https://github.com/prometheus/prometheus/issues/2916 - - target_label: __address__ - replacement: kubernetes.default.svc:443 - - source_labels: [__meta_kubernetes_node_name] - regex: (.+) - target_label: __metrics_path__ - replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor - metric_relabel_configs: - - action: replace - source_labels: [id] - regex: '^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$' - target_label: rkt_container_name - replacement: "${2}-${1}" - - action: replace - source_labels: [id] - regex: '^/system\.slice/(.+)\.service$' - target_label: systemd_service_name - replacement: "${1}" - job_name: node-exporter scrape_interval: 10s static_configs: diff --git a/terraform/apps/dev-01/resources/telegram.tpl b/terraform/apps/dev-01/resources/telegram.tpl new file mode 100644 index 0000000..88a6606 --- /dev/null +++ b/terraform/apps/dev-01/resources/telegram.tpl @@ -0,0 +1,13 @@ +{{/* Always produce a non-empty message. Handles single or multiple firing alerts and missing username. */}} +{{ if gt (len .Alerts.Firing) 1 -}} +{{ len .Alerts.Firing }} logins on {{ with .CommonLabels.cluster }}{{ . }}{{ else }}unknown host{{ end }}: +{{- range .Alerts.Firing }} +• {{ with .Labels.username }}{{ . }}{{ else }}unknown{{ end }} — Source: {{ with .Labels.source }}{{ . }}{{ else }}?{{ end }}{{ with .Labels.method }}, Method: {{ . }}{{ else }}, Method: ?{{ end }} +{{- end }} +{{ else if eq (len .Alerts.Firing) 1 -}} +{{ $a := index .Alerts.Firing 0 -}} +Login: {{ with $a.Labels.username }}{{ . }}{{ else }}unknown{{ end }} on {{ $a.Labels.cluster }} +Source: {{ with $a.Labels.source }}{{ . }}{{ else }}?{{ end }}{{ with $a.Labels.method }}, Method: {{ . }}{{ else }}, Method: ?{{ end }} +{{ else -}} +Login detected — details unavailable. +{{ end -}} \ No newline at end of file diff --git a/terraform/apps/dev-01/secrets.tf b/terraform/apps/dev-01/secrets.tf index 70bcf54..491ff1e 100644 --- a/terraform/apps/dev-01/secrets.tf +++ b/terraform/apps/dev-01/secrets.tf @@ -15,3 +15,9 @@ variable "nextcloud_admin_password" { type = string sensitive = true } + +variable "telegram_bot_token" { + description = "Telegram bot token for Alertmanager notifications" + type = string + sensitive = true +} diff --git a/terraform/apps/dev-01/sectool.env b/terraform/apps/dev-01/sectool.env index 794536d..eb9320c 100644 --- a/terraform/apps/dev-01/sectool.env +++ b/terraform/apps/dev-01/sectool.env @@ -4,3 +4,4 @@ TF_VAR_nextcloud_db_passwd=$MYSQL_NEXTCLOUD_PASSWORD TF_VAR_nextcloud_db_root_password=$MYSQL_ROOT_PASSWORD TF_VAR_nextcloud_admin_password=$NEXTCLOUD_ADMIN_PASSWORD TF_VAR_scaleway_project_id=$SCALEWAY_PROJECT_ID +TF_VAR_telegram_bot_token=$TELEGRAM_BOT_TOKEN diff --git a/terraform/apps/prod-01/apps.tf b/terraform/apps/prod-01/apps.tf index da2671f..acd59cd 100644 --- a/terraform/apps/prod-01/apps.tf +++ b/terraform/apps/prod-01/apps.tf @@ -132,8 +132,18 @@ module "proxy" { module "prometheus" { source = "../../modules/utils/prometheus" - persistent_folder = "${local.persistent_folder}/monitoring" - config = file("${path.module}/resources/prometheus.yml") + cluster_name = "prod-01" + persistent_folder = "${local.persistent_folder}/monitoring" + config = file("${path.module}/resources/prometheus.yml") + rules = local.prometheus_rules + alertmanager_config = local.alertmanager_config + + email_config = local.alertmanager_email_config + telegram_config = { + bot_token = var.telegram_bot_token + chat_id = local.telegram_chat_id + message = local.telegram_message + } } module "fail2ban_exporter" { diff --git a/terraform/apps/prod-01/config.tf b/terraform/apps/prod-01/config.tf index f33cbd8..bc81228 100644 --- a/terraform/apps/prod-01/config.tf +++ b/terraform/apps/prod-01/config.tf @@ -192,6 +192,21 @@ locals { } } + # prometheus + prometheus_config = file("${path.module}/resources/prometheus.yml") + prometheus_rules = file("${path.module}/resources/prometheus-rules.yml") + alertmanager_config = file("${path.module}/resources/alertmanager.yml") + alertmanager_email_config = { + to = "c.alexandre.pires@alexpires.me" + from = "alertmanager@${local.primary_domain}" + smarthost = "smtp.tem.scw.cloud:587" + auth_username = var.scaleway_project_id + auth_password = local.mail_app_api_secret_key + } + + telegram_chat_id = "5432313610" # a13labs channel + telegram_message = file("${path.module}/resources/telegram.tpl") + # get secrets from remote state backup_bucket_name = data.terraform_remote_state.scaleway.outputs.backup_bucket_name backup_app_api_secret_key = data.terraform_remote_state.scaleway.outputs.backup_app_api_secret_key diff --git a/terraform/apps/prod-01/resources/alertmanager.yml b/terraform/apps/prod-01/resources/alertmanager.yml new file mode 100644 index 0000000..320a381 --- /dev/null +++ b/terraform/apps/prod-01/resources/alertmanager.yml @@ -0,0 +1,11 @@ +route: + group_by: ["alertname"] + receiver: "default" + group_wait: 10s + group_interval: 10m + repeat_interval: 1h + routes: + - receiver: "telegram" + matchers: + - 'alertname="UserLoggedIn"' + continue: false diff --git a/terraform/apps/prod-01/resources/prometheus-rules.yml b/terraform/apps/prod-01/resources/prometheus-rules.yml new file mode 100644 index 0000000..96f975a --- /dev/null +++ b/terraform/apps/prod-01/resources/prometheus-rules.yml @@ -0,0 +1,10 @@ +groups: + - name: auth.rules + rules: + - alert: UserLoggedIn + expr: increase(auth_sessions_active_by_user[1m]) > 0 + labels: + severity: info + annotations: + summary: "Login: {{ $labels.username }} on {{ $labels.instance }}" + description: "User {{ $labels.username }} logged in via {{ $labels.source }} on {{ $labels.instance }}." diff --git a/terraform/apps/prod-01/resources/prometheus.yml b/terraform/apps/prod-01/resources/prometheus.yml index 0833657..942fa94 100644 --- a/terraform/apps/prod-01/resources/prometheus.yml +++ b/terraform/apps/prod-01/resources/prometheus.yml @@ -1,41 +1,4 @@ -global: - scrape_interval: 5s - evaluation_interval: 5s -rule_files: - - /etc/prometheus/prometheus.rules scrape_configs: - - job_name: kubernetes-nodes-cadvisor - scrape_interval: 10s - scrape_timeout: 10s - scheme: https # remove if you want to scrape metrics on insecure port - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: true - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - kubernetes_sd_configs: - - role: node - relabel_configs: - - action: labelmap - regex: __meta_kubernetes_node_label_(.+) - # Only for Kubernetes ^1.7.3. - # See: https://github.com/prometheus/prometheus/issues/2916 - - target_label: __address__ - replacement: kubernetes.default.svc:443 - - source_labels: [__meta_kubernetes_node_name] - regex: (.+) - target_label: __metrics_path__ - replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor - metric_relabel_configs: - - action: replace - source_labels: [id] - regex: '^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$' - target_label: rkt_container_name - replacement: "${2}-${1}" - - action: replace - source_labels: [id] - regex: '^/system\.slice/(.+)\.service$' - target_label: systemd_service_name - replacement: "${1}" - job_name: node-exporter scrape_interval: 10s static_configs: diff --git a/terraform/apps/prod-01/resources/telegram.tpl b/terraform/apps/prod-01/resources/telegram.tpl new file mode 100644 index 0000000..88a6606 --- /dev/null +++ b/terraform/apps/prod-01/resources/telegram.tpl @@ -0,0 +1,13 @@ +{{/* Always produce a non-empty message. Handles single or multiple firing alerts and missing username. */}} +{{ if gt (len .Alerts.Firing) 1 -}} +{{ len .Alerts.Firing }} logins on {{ with .CommonLabels.cluster }}{{ . }}{{ else }}unknown host{{ end }}: +{{- range .Alerts.Firing }} +• {{ with .Labels.username }}{{ . }}{{ else }}unknown{{ end }} — Source: {{ with .Labels.source }}{{ . }}{{ else }}?{{ end }}{{ with .Labels.method }}, Method: {{ . }}{{ else }}, Method: ?{{ end }} +{{- end }} +{{ else if eq (len .Alerts.Firing) 1 -}} +{{ $a := index .Alerts.Firing 0 -}} +Login: {{ with $a.Labels.username }}{{ . }}{{ else }}unknown{{ end }} on {{ $a.Labels.cluster }} +Source: {{ with $a.Labels.source }}{{ . }}{{ else }}?{{ end }}{{ with $a.Labels.method }}, Method: {{ . }}{{ else }}, Method: ?{{ end }} +{{ else -}} +Login detected — details unavailable. +{{ end -}} \ No newline at end of file diff --git a/terraform/apps/prod-01/secrets.tf b/terraform/apps/prod-01/secrets.tf index c5f5400..4b4cc0f 100644 --- a/terraform/apps/prod-01/secrets.tf +++ b/terraform/apps/prod-01/secrets.tf @@ -128,3 +128,9 @@ variable "nginx_token_bearer" { type = string sensitive = true } + +variable "telegram_bot_token" { + description = "Telegram bot token for Alertmanager notifications" + type = string + sensitive = true +} diff --git a/terraform/apps/prod-01/sectool.env b/terraform/apps/prod-01/sectool.env index e2ab170..4cf4889 100644 --- a/terraform/apps/prod-01/sectool.env +++ b/terraform/apps/prod-01/sectool.env @@ -32,4 +32,4 @@ TF_VAR_wireguard_1_private_key=$WIREGUARD_1_PRIVATE_KEY TF_VAR_wireguard_1_public_key=$WIREGUARD_1_PUBLIC_KEY TF_VAR_wireguard_1_pre_shared_key=$WIREGUARD_1_PRE_SHARED_KEY TF_VAR_nginx_token_bearer=$NGINX_TOKEN_BEARER - +TF_VAR_telegram_bot_token=$TELEGRAM_BOT_TOKEN diff --git a/terraform/modules/utils/prometheus/alertmanager.tf b/terraform/modules/utils/prometheus/alertmanager.tf new file mode 100644 index 0000000..c48fa4e --- /dev/null +++ b/terraform/modules/utils/prometheus/alertmanager.tf @@ -0,0 +1,118 @@ +resource "kubernetes_config_map" "alertmanager" { + metadata { + name = "alertmanager-conf" + namespace = kubernetes_namespace.prometheus.metadata[0].name + labels = { + name = "alertmanager-conf" + } + } + + data = { + "alertmanager.yml" = local.alertmanager_config + } +} + +resource "kubernetes_deployment" "alertmanager" { + metadata { + name = "alertmanager" + namespace = kubernetes_namespace.prometheus.metadata[0].name + labels = { + app = "alertmanager" + } + } + + spec { + replicas = 1 + + selector { + match_labels = { + app = "alertmanager" + } + } + + template { + metadata { + labels = { + app = "alertmanager" + } + annotations = { + "checksum/config" = local.alertmanager_config_checksum + } + } + + spec { + container { + name = "alertmanager" + image = "prom/alertmanager:${local.app_version}" + + args = [ + "--config.file=/etc/alertmanager/alertmanager.yml", + "--storage.path=/alertmanager" + ] + + port { + name = "http" + container_port = 9093 + } + + volume_mount { + name = "alertmanager-config" + mount_path = "/etc/alertmanager" + } + + volume_mount { + name = "alertmanager-storage" + mount_path = "/alertmanager" + } + + security_context { + run_as_user = 1000 + run_as_group = 1000 + run_as_non_root = true + allow_privilege_escalation = false + } + } + + volume { + name = "alertmanager-config" + config_map { + name = kubernetes_config_map.alertmanager.metadata[0].name + } + } + + volume { + name = "alertmanager-storage" + host_path { + path = local.alertmanager_folder + type = "DirectoryOrCreate" + } + } + } + } + } +} + +resource "kubernetes_service" "alertmanager" { + metadata { + name = "alertmanager" + namespace = kubernetes_namespace.prometheus.metadata[0].name + labels = { + app = "alertmanager" + } + } + + spec { + selector = { + app = "alertmanager" + } + + type = "ClusterIP" + + port { + name = "web" + port = 9093 + protocol = "TCP" + target_port = "http" + } + } +} diff --git a/terraform/modules/utils/prometheus/config.tf b/terraform/modules/utils/prometheus/config.tf index 5d1dd79..51bdd4a 100644 --- a/terraform/modules/utils/prometheus/config.tf +++ b/terraform/modules/utils/prometheus/config.tf @@ -1,17 +1,112 @@ locals { - app_version = var.tag - data_folder = "${var.persistent_folder}/prometheus" + app_version = var.tag + prometheus_folder = "${var.persistent_folder}/prometheus" + alertmanager_folder = "${var.persistent_folder}/alertmanager" - prometheus_config = coalesce(var.config, file("${path.module}/resources/default.yml")) + prometheus_default_config = yamldecode(file("${path.module}/resources/prometheus.yml")) + alertmanager_default_config = yamldecode(file("${path.module}/resources/alertmanager.yml")) + + prometheus_user_config = try( + yamldecode(var.config), + {} + ) + prometheus_config = yamlencode({ + global = merge( + try(local.prometheus_default_config.global, {}), + try(local.prometheus_user_config.global, {}), + { + scrape_interval = "15s" + evaluation_interval = "15s" + external_labels = { + cluster = var.cluster_name + } + } + ), + rule_files = concat( + try(local.prometheus_default_config.rule_files, []), + try(local.prometheus_user_config.rule_files, []) + ), + scrape_configs = concat( + try(local.prometheus_default_config.scrape_configs, []), + try(local.prometheus_user_config.scrape_configs, []) + ), + alerting = merge( + try(local.prometheus_default_config.alerting, {}), + try(local.prometheus_user_config.alerting, {}), + { + alertmanagers = [ + { + static_configs = [ + { + targets = ["alertmanager.prometheus.svc.cluster.local:9093"] + } + ] + } + ] + } + ) + }) prometheus_config_checksum = md5(local.prometheus_config) - prometheus_rules = yamlencode({ - groups = [ - { - name = "custom-rules" - rules = var.rules - } - ] - }) + prometheus_user_rules = try( + yamldecode(var.rules), + {} + ) + prometheus_merged_rules = merge({ groups = [] }, local.prometheus_user_rules) + prometheus_rules = yamlencode(local.prometheus_merged_rules) prometheus_rules_checksum = md5(local.prometheus_rules) + + alertmanager_user_config = try( + yamldecode(var.alertmanager_config), + {} + ) + alertmanager_config = yamlencode({ + global = merge( + try(local.alertmanager_default_config.global, {}), + try(local.alertmanager_user_config.global, {}) + ), + route = merge( + try(local.alertmanager_default_config.route, {}), + try(local.alertmanager_user_config.route, { + receiver = "default" + }) + ), + receivers = concat( + try(local.alertmanager_default_config.receivers, []), + try(local.alertmanager_user_config.receivers, []), + [ + { + name = "default" + email_configs = [ + { + to = var.email_config.to + from = var.email_config.from + smarthost = var.email_config.smarthost + auth_username = var.email_config.auth_username + auth_password = var.email_config.auth_password + } + ] + } + ], + try([ + { + name = "telegram" + telegram_configs = [ + { + send_resolved = true + parse_mode = var.telegram_config.parse_mode + bot_token = var.telegram_config.bot_token + chat_id = tonumber(var.telegram_config.chat_id) + message = var.telegram_config.message != "" ? var.telegram_config.message : "{{ template \"telegram.default.message\" . }}" + } + ] + } + ], []) + ) + templates = concat( + try(local.prometheus_default_config.templates, []), + try(local.prometheus_user_config.templates, []) + ) + }) + alertmanager_config_checksum = md5(local.alertmanager_config) } diff --git a/terraform/modules/utils/prometheus/prometheus.tf b/terraform/modules/utils/prometheus/prometheus.tf index 8d63e1f..2cf3484 100644 --- a/terraform/modules/utils/prometheus/prometheus.tf +++ b/terraform/modules/utils/prometheus/prometheus.tf @@ -143,7 +143,7 @@ resource "kubernetes_deployment" "prometheus" { volume { name = "prometheus-storage-volume" host_path { - path = local.data_folder + path = local.prometheus_folder type = "DirectoryOrCreate" } } diff --git a/terraform/modules/utils/prometheus/resources/alertmanager.yml b/terraform/modules/utils/prometheus/resources/alertmanager.yml new file mode 100644 index 0000000..69f0d75 --- /dev/null +++ b/terraform/modules/utils/prometheus/resources/alertmanager.yml @@ -0,0 +1,7 @@ +global: + resolve_timeout: 5m +route: + receiver: "default" + group_wait: 30s + group_interval: 5m + repeat_interval: 12h diff --git a/terraform/modules/utils/prometheus/resources/prometheus.bak.yml b/terraform/modules/utils/prometheus/resources/prometheus.bak.yml deleted file mode 100644 index ac7400d..0000000 --- a/terraform/modules/utils/prometheus/resources/prometheus.bak.yml +++ /dev/null @@ -1,126 +0,0 @@ -global: - scrape_interval: 5s - evaluation_interval: 5s -rule_files: - - /etc/prometheus/prometheus.rules -alerting: - alertmanagers: - - scheme: http - static_configs: - - targets: - - "alertmanager.monitoring.svc:9093" -scrape_configs: - - job_name: "node-exporter" - kubernetes_sd_configs: - - role: endpoints - relabel_configs: - - source_labels: [__meta_kubernetes_endpoints_name] - regex: "node-exporter" - action: keep - - job_name: "kubernetes-apiservers" - kubernetes_sd_configs: - - role: endpoints - scheme: https - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - relabel_configs: - - source_labels: - [ - __meta_kubernetes_namespace, - __meta_kubernetes_service_name, - __meta_kubernetes_endpoint_port_name, - ] - action: keep - regex: default;kubernetes;https - - job_name: "kubernetes-nodes" - scheme: https - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - kubernetes_sd_configs: - - role: node - relabel_configs: - - action: labelmap - regex: __meta_kubernetes_node_label_(.+) - - target_label: __address__ - replacement: kubernetes.default.svc:443 - - source_labels: [__meta_kubernetes_node_name] - regex: (.+) - target_label: __metrics_path__ - replacement: /api/v1/nodes/$${1}/proxy/metrics - - job_name: "kubernetes-pods" - kubernetes_sd_configs: - - role: pod - relabel_configs: - - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] - action: keep - regex: true - - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] - action: replace - target_label: __metrics_path__ - regex: (.+) - - source_labels: - [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] - action: replace - regex: ([^:]+)(?::\d+)?;(\d+) - replacement: $1:$2 - target_label: __address__ - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - source_labels: [__meta_kubernetes_namespace] - action: replace - target_label: kubernetes_namespace - - source_labels: [__meta_kubernetes_pod_name] - action: replace - target_label: kubernetes_pod_name - - job_name: "kube-state-metrics" - static_configs: - - targets: ["kube-state-metrics.kube-system.svc.cluster.local:8080"] - - job_name: "kubernetes-cadvisor" - scheme: https - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - kubernetes_sd_configs: - - role: node - relabel_configs: - - action: labelmap - regex: __meta_kubernetes_node_label_(.+) - - target_label: __address__ - replacement: kubernetes.default.svc:443 - - source_labels: [__meta_kubernetes_node_name] - regex: (.+) - target_label: __metrics_path__ - replacement: /api/v1/nodes/$${1}/proxy/metrics/cadvisor - - job_name: "kubernetes-service-endpoints" - kubernetes_sd_configs: - - role: endpoints - relabel_configs: - - source_labels: - [__meta_kubernetes_service_annotation_prometheus_io_scrape] - action: keep - regex: true - - source_labels: - [__meta_kubernetes_service_annotation_prometheus_io_scheme] - action: replace - target_label: __scheme__ - regex: (https?) - - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] - action: replace - target_label: __metrics_path__ - regex: (.+) - - source_labels: - [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] - action: replace - target_label: __address__ - regex: ([^:]+)(?::\d+)?;(\d+) - replacement: $1:$2 - - action: labelmap - regex: __meta_kubernetes_service_label_(.+) - - source_labels: [__meta_kubernetes_namespace] - action: replace - target_label: kubernetes_namespace - - source_labels: [__meta_kubernetes_service_name] - action: replace - target_label: kubernetes_name diff --git a/terraform/modules/utils/prometheus/resources/default.yml b/terraform/modules/utils/prometheus/resources/prometheus.yml similarity index 92% rename from terraform/modules/utils/prometheus/resources/default.yml rename to terraform/modules/utils/prometheus/resources/prometheus.yml index 6119791..9c152c9 100644 --- a/terraform/modules/utils/prometheus/resources/default.yml +++ b/terraform/modules/utils/prometheus/resources/prometheus.yml @@ -36,3 +36,7 @@ scrape_configs: regex: '^/system\.slice/(.+)\.service$' target_label: systemd_service_name replacement: "${1}" +alerting: + alertmanagers: + - static_configs: + - targets: ["alertmanager.prometheus.svc.cluster.local:9093"] diff --git a/terraform/modules/utils/prometheus/variables.tf b/terraform/modules/utils/prometheus/variables.tf index f0c037d..9dcfa31 100644 --- a/terraform/modules/utils/prometheus/variables.tf +++ b/terraform/modules/utils/prometheus/variables.tf @@ -11,15 +11,9 @@ variable "retention_time" { } variable "rules" { - description = "Prometheus alerting rules" - type = list(object({ - alert = string - expr = string - for = string - labels = map(string) - annotations = map(string) - })) - default = [] + description = "Prometheus alerting rules in YAML format" + type = string + default = "" } variable "config" { @@ -32,3 +26,44 @@ variable "persistent_folder" { description = "The path to the persistent folder" type = string } + +variable "alertmanager_config" { + description = "Custom Alertmanager configuration in YAML format" + type = string + default = "" +} + +variable "cluster_name" { + description = "The name of the Kubernetes cluster" + type = string + default = "kubernetes" +} + +variable "email_config" { + description = "Email configuration for Alertmanager in YAML format" + type = object({ + to = optional(string, "root@localhost") + from = optional(string, "root@localhost") + smarthost = optional(string, "localhost:25") + auth_username = optional(string, "") + auth_password = optional(string, "") + }) + default = { + to = "root@localhost" + from = "root@localhost" + smarthost = "localhost:25" + auth_username = "" + auth_password = "" + } +} + +variable "telegram_config" { + description = "Telegram configuration for Alertmanager in YAML format" + type = object({ + bot_token = string + chat_id = string + parse_mode = optional(string, "HTML") + message = optional(string, "") + }) + default = null +}