Add Prometheus and Alertmanager configurations: implement custom rules, alerting, and Telegram notifications

terraform/apps/dev-01/apps.tf

@@ -52,8 +52,18 @@ module "comfyui_proxy" {
 module "prometheus" {
   source = "../../modules/utils/prometheus"
 
-  persistent_folder = "${local.persistent_folder}/monitoring"
+  cluster_name        = "dev-01"
-  config            = file("${path.module}/resources/prometheus.yml")
+  persistent_folder   = "${local.persistent_folder}/monitoring"
+  config              = local.prometheus_config
+  rules               = local.prometheus_rules
+  alertmanager_config = local.alertmanager_config
+
+  email_config = local.alertmanager_email_config
+  telegram_config = {
+    bot_token = var.telegram_bot_token
+    chat_id   = local.telegram_chat_id
+    message   = local.telegram_message
+  }
 }
 
 module "grafana" {

terraform/apps/dev-01/config.tf

@@ -167,16 +167,22 @@ locals {
   backup_app_api_access_key = data.terraform_remote_state.scaleway.outputs.backup_app_api_access_key
   mail_app_api_secret_key   = data.terraform_remote_state.scaleway.outputs.mail_app_api_secret_key
 
-  grafana_fqdn = "grafana.${local.lab_domain}"
+  # prometheus and grafana
-
+  prometheus_config   = file("${path.module}/resources/prometheus.yml")
-  remote_targets = {
+  prometheus_rules    = file("${path.module}/resources/prometheus-rules.yml")
-    "prometheus" = {
+  alertmanager_config = file("${path.module}/resources/alertmanager.yml")
-      listen_port = 9090
+  alertmanager_email_config = {
-      target_host = "10.5.5.5"
+    to            = "c.alexandre.pires@alexpires.me"
-      target_port = 9090
+    from          = "alertmanager@${local.lab_domain}"
-      protocol    = "TCP"
+    smarthost     = "smtp.tem.scw.cloud:587"
-    }
+    auth_username = var.scaleway_project_id
+    auth_password = local.mail_app_api_secret_key
   }
+
+  telegram_chat_id = "5432313610" # a13labs channel
+  telegram_message = file("${path.module}/resources/telegram.tpl")
+
+  grafana_fqdn = "grafana.${local.lab_domain}"
 }
 
 data "terraform_remote_state" "scaleway" {

terraform/apps/dev-01/resources/alertmanager.yml

@@ -0,0 +1,11 @@
+route:
+  group_by: ["alertname"]
+  receiver: "default"
+  group_wait: 10s
+  group_interval: 10m
+  repeat_interval: 1h
+  routes:
+    - receiver: "telegram"
+      matchers:
+        - 'alertname="UserLoggedIn"'
+      continue: false

terraform/apps/dev-01/resources/prometheus-rules.yml

@@ -0,0 +1,10 @@
+groups:
+  - name: auth.rules
+    rules:
+      - alert: UserLoggedIn
+        expr: increase(auth_sessions_active_by_user[1m]) > 0
+        labels:
+          severity: info
+        annotations:
+          summary: "Login: {{ $labels.username }} on {{ $labels.instance }}"
+          description: "User {{ $labels.username }} logged in via {{ $labels.source }} on {{ $labels.instance }}."

terraform/apps/dev-01/resources/prometheus.yml

@@ -1,41 +1,4 @@
-global:
-  scrape_interval: 5s
-  evaluation_interval: 5s
-rule_files:
-  - /etc/prometheus/prometheus.rules
 scrape_configs:
-  - job_name: kubernetes-nodes-cadvisor
-    scrape_interval: 10s
-    scrape_timeout: 10s
-    scheme: https # remove if you want to scrape metrics on insecure port
-    tls_config:
-      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-      insecure_skip_verify: true
-    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-    kubernetes_sd_configs:
-      - role: node
-    relabel_configs:
-      - action: labelmap
-        regex: __meta_kubernetes_node_label_(.+)
-      # Only for Kubernetes ^1.7.3.
-      # See: https://github.com/prometheus/prometheus/issues/2916
-      - target_label: __address__
-        replacement: kubernetes.default.svc:443
-      - source_labels: [__meta_kubernetes_node_name]
-        regex: (.+)
-        target_label: __metrics_path__
-        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
-    metric_relabel_configs:
-      - action: replace
-        source_labels: [id]
-        regex: '^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$'
-        target_label: rkt_container_name
-        replacement: "${2}-${1}"
-      - action: replace
-        source_labels: [id]
-        regex: '^/system\.slice/(.+)\.service$'
-        target_label: systemd_service_name
-        replacement: "${1}"
   - job_name: node-exporter
     scrape_interval: 10s
     static_configs:

terraform/apps/dev-01/resources/telegram.tpl

@@ -0,0 +1,13 @@
+{{/* Always produce a non-empty message. Handles single or multiple firing alerts and missing username. */}}
+{{ if gt (len .Alerts.Firing) 1 -}}
+<b>{{ len .Alerts.Firing }} logins</b> on {{ with .CommonLabels.cluster }}<b>{{ . }}</b>{{ else }}<b>unknown host</b>{{ end }}:
+{{- range .Alerts.Firing }}
+• {{ with .Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} — Source: <code>{{ with .Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with .Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
+{{- end }}
+{{ else if eq (len .Alerts.Firing) 1 -}}
+{{ $a := index .Alerts.Firing 0 -}}
+<b>Login:</b> {{ with $a.Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} on <b>{{ $a.Labels.cluster }}</b>
+Source: <code>{{ with $a.Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with $a.Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
+{{ else -}}
+<b>Login detected</b> — details unavailable.
+{{ end -}}

terraform/apps/dev-01/secrets.tf

@@ -15,3 +15,9 @@ variable "nextcloud_admin_password" {
   type        = string
   sensitive   = true
 }
+
+variable "telegram_bot_token" {
+  description = "Telegram bot token for Alertmanager notifications"
+  type        = string
+  sensitive   = true
+}

terraform/apps/dev-01/sectool.env

@@ -4,3 +4,4 @@ TF_VAR_nextcloud_db_passwd=$MYSQL_NEXTCLOUD_PASSWORD
 TF_VAR_nextcloud_db_root_password=$MYSQL_ROOT_PASSWORD
 TF_VAR_nextcloud_admin_password=$NEXTCLOUD_ADMIN_PASSWORD
 TF_VAR_scaleway_project_id=$SCALEWAY_PROJECT_ID
+TF_VAR_telegram_bot_token=$TELEGRAM_BOT_TOKEN

terraform/apps/prod-01/apps.tf

@@ -132,8 +132,18 @@ module "proxy" {
 module "prometheus" {
   source = "../../modules/utils/prometheus"
 
-  persistent_folder = "${local.persistent_folder}/monitoring"
+  cluster_name        = "prod-01"
-  config            = file("${path.module}/resources/prometheus.yml")
+  persistent_folder   = "${local.persistent_folder}/monitoring"
+  config              = file("${path.module}/resources/prometheus.yml")
+  rules               = local.prometheus_rules
+  alertmanager_config = local.alertmanager_config
+
+  email_config = local.alertmanager_email_config
+  telegram_config = {
+    bot_token = var.telegram_bot_token
+    chat_id   = local.telegram_chat_id
+    message   = local.telegram_message
+  }
 }
 
 module "fail2ban_exporter" {

terraform/apps/prod-01/config.tf

@@ -192,6 +192,21 @@ locals {
     }
   }
 
+  # prometheus
+  prometheus_config   = file("${path.module}/resources/prometheus.yml")
+  prometheus_rules    = file("${path.module}/resources/prometheus-rules.yml")
+  alertmanager_config = file("${path.module}/resources/alertmanager.yml")
+  alertmanager_email_config = {
+    to            = "c.alexandre.pires@alexpires.me"
+    from          = "alertmanager@${local.primary_domain}"
+    smarthost     = "smtp.tem.scw.cloud:587"
+    auth_username = var.scaleway_project_id
+    auth_password = local.mail_app_api_secret_key
+  }
+
+  telegram_chat_id = "5432313610" # a13labs channel
+  telegram_message = file("${path.module}/resources/telegram.tpl")
+
   # get secrets from remote state
   backup_bucket_name        = data.terraform_remote_state.scaleway.outputs.backup_bucket_name
   backup_app_api_secret_key = data.terraform_remote_state.scaleway.outputs.backup_app_api_secret_key

terraform/apps/prod-01/resources/alertmanager.yml

@@ -0,0 +1,11 @@
+route:
+  group_by: ["alertname"]
+  receiver: "default"
+  group_wait: 10s
+  group_interval: 10m
+  repeat_interval: 1h
+  routes:
+    - receiver: "telegram"
+      matchers:
+        - 'alertname="UserLoggedIn"'
+      continue: false

terraform/apps/prod-01/resources/prometheus-rules.yml

@@ -0,0 +1,10 @@
+groups:
+  - name: auth.rules
+    rules:
+      - alert: UserLoggedIn
+        expr: increase(auth_sessions_active_by_user[1m]) > 0
+        labels:
+          severity: info
+        annotations:
+          summary: "Login: {{ $labels.username }} on {{ $labels.instance }}"
+          description: "User {{ $labels.username }} logged in via {{ $labels.source }} on {{ $labels.instance }}."

terraform/apps/prod-01/resources/prometheus.yml

@@ -1,41 +1,4 @@
-global:
-  scrape_interval: 5s
-  evaluation_interval: 5s
-rule_files:
-  - /etc/prometheus/prometheus.rules
 scrape_configs:
-  - job_name: kubernetes-nodes-cadvisor
-    scrape_interval: 10s
-    scrape_timeout: 10s
-    scheme: https # remove if you want to scrape metrics on insecure port
-    tls_config:
-      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-      insecure_skip_verify: true
-    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-    kubernetes_sd_configs:
-      - role: node
-    relabel_configs:
-      - action: labelmap
-        regex: __meta_kubernetes_node_label_(.+)
-      # Only for Kubernetes ^1.7.3.
-      # See: https://github.com/prometheus/prometheus/issues/2916
-      - target_label: __address__
-        replacement: kubernetes.default.svc:443
-      - source_labels: [__meta_kubernetes_node_name]
-        regex: (.+)
-        target_label: __metrics_path__
-        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
-    metric_relabel_configs:
-      - action: replace
-        source_labels: [id]
-        regex: '^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$'
-        target_label: rkt_container_name
-        replacement: "${2}-${1}"
-      - action: replace
-        source_labels: [id]
-        regex: '^/system\.slice/(.+)\.service$'
-        target_label: systemd_service_name
-        replacement: "${1}"
   - job_name: node-exporter
     scrape_interval: 10s
     static_configs:

terraform/apps/prod-01/resources/telegram.tpl

@@ -0,0 +1,13 @@
+{{/* Always produce a non-empty message. Handles single or multiple firing alerts and missing username. */}}
+{{ if gt (len .Alerts.Firing) 1 -}}
+<b>{{ len .Alerts.Firing }} logins</b> on {{ with .CommonLabels.cluster }}<b>{{ . }}</b>{{ else }}<b>unknown host</b>{{ end }}:
+{{- range .Alerts.Firing }}
+• {{ with .Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} — Source: <code>{{ with .Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with .Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
+{{- end }}
+{{ else if eq (len .Alerts.Firing) 1 -}}
+{{ $a := index .Alerts.Firing 0 -}}
+<b>Login:</b> {{ with $a.Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} on <b>{{ $a.Labels.cluster }}</b>
+Source: <code>{{ with $a.Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with $a.Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
+{{ else -}}
+<b>Login detected</b> — details unavailable.
+{{ end -}}

terraform/apps/prod-01/secrets.tf

@@ -128,3 +128,9 @@ variable "nginx_token_bearer" {
   type        = string
   sensitive   = true
 }
+
+variable "telegram_bot_token" {
+  description = "Telegram bot token for Alertmanager notifications"
+  type        = string
+  sensitive   = true
+}

terraform/apps/prod-01/sectool.env

@@ -32,4 +32,4 @@ TF_VAR_wireguard_1_private_key=$WIREGUARD_1_PRIVATE_KEY
 TF_VAR_wireguard_1_public_key=$WIREGUARD_1_PUBLIC_KEY
 TF_VAR_wireguard_1_pre_shared_key=$WIREGUARD_1_PRE_SHARED_KEY
 TF_VAR_nginx_token_bearer=$NGINX_TOKEN_BEARER
-
+TF_VAR_telegram_bot_token=$TELEGRAM_BOT_TOKEN

terraform/modules/utils/prometheus/alertmanager.tf

@@ -0,0 +1,118 @@
+resource "kubernetes_config_map" "alertmanager" {
+  metadata {
+    name      = "alertmanager-conf"
+    namespace = kubernetes_namespace.prometheus.metadata[0].name
+    labels = {
+      name = "alertmanager-conf"
+    }
+  }
+
+  data = {
+    "alertmanager.yml" = local.alertmanager_config
+  }
+}
+
+resource "kubernetes_deployment" "alertmanager" {
+  metadata {
+    name      = "alertmanager"
+    namespace = kubernetes_namespace.prometheus.metadata[0].name
+    labels = {
+      app = "alertmanager"
+    }
+  }
+
+  spec {
+    replicas = 1
+
+    selector {
+      match_labels = {
+        app = "alertmanager"
+      }
+    }
+
+    template {
+      metadata {
+        labels = {
+          app = "alertmanager"
+        }
+        annotations = {
+          "checksum/config" = local.alertmanager_config_checksum
+        }
+      }
+
+      spec {
+        container {
+          name  = "alertmanager"
+          image = "prom/alertmanager:${local.app_version}"
+
+          args = [
+            "--config.file=/etc/alertmanager/alertmanager.yml",
+            "--storage.path=/alertmanager"
+          ]
+
+          port {
+            name           = "http"
+            container_port = 9093
+          }
+
+          volume_mount {
+            name       = "alertmanager-config"
+            mount_path = "/etc/alertmanager"
+          }
+
+          volume_mount {
+            name       = "alertmanager-storage"
+            mount_path = "/alertmanager"
+          }
+
+          security_context {
+            run_as_user                = 1000
+            run_as_group               = 1000
+            run_as_non_root            = true
+            allow_privilege_escalation = false
+          }
+        }
+
+        volume {
+          name = "alertmanager-config"
+          config_map {
+            name = kubernetes_config_map.alertmanager.metadata[0].name
+          }
+        }
+
+        volume {
+          name = "alertmanager-storage"
+          host_path {
+            path = local.alertmanager_folder
+            type = "DirectoryOrCreate"
+          }
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_service" "alertmanager" {
+  metadata {
+    name      = "alertmanager"
+    namespace = kubernetes_namespace.prometheus.metadata[0].name
+    labels = {
+      app = "alertmanager"
+    }
+  }
+
+  spec {
+    selector = {
+      app = "alertmanager"
+    }
+
+    type = "ClusterIP"
+
+    port {
+      name        = "web"
+      port        = 9093
+      protocol    = "TCP"
+      target_port = "http"
+    }
+  }
+}

terraform/modules/utils/prometheus/config.tf

@@ -1,17 +1,112 @@
 locals {
-  app_version = var.tag
+  app_version         = var.tag
-  data_folder = "${var.persistent_folder}/prometheus"
+  prometheus_folder   = "${var.persistent_folder}/prometheus"
+  alertmanager_folder = "${var.persistent_folder}/alertmanager"
 
-  prometheus_config          = coalesce(var.config, file("${path.module}/resources/default.yml"))
+  prometheus_default_config   = yamldecode(file("${path.module}/resources/prometheus.yml"))
+  alertmanager_default_config = yamldecode(file("${path.module}/resources/alertmanager.yml"))
+
+  prometheus_user_config = try(
+    yamldecode(var.config),
+    {}
+  )
+  prometheus_config = yamlencode({
+    global = merge(
+      try(local.prometheus_default_config.global, {}),
+      try(local.prometheus_user_config.global, {}),
+      {
+        scrape_interval     = "15s"
+        evaluation_interval = "15s"
+        external_labels = {
+          cluster = var.cluster_name
+        }
+      }
+    ),
+    rule_files = concat(
+      try(local.prometheus_default_config.rule_files, []),
+      try(local.prometheus_user_config.rule_files, [])
+    ),
+    scrape_configs = concat(
+      try(local.prometheus_default_config.scrape_configs, []),
+      try(local.prometheus_user_config.scrape_configs, [])
+    ),
+    alerting = merge(
+      try(local.prometheus_default_config.alerting, {}),
+      try(local.prometheus_user_config.alerting, {}),
+      {
+        alertmanagers = [
+          {
+            static_configs = [
+              {
+                targets = ["alertmanager.prometheus.svc.cluster.local:9093"]
+              }
+            ]
+          }
+        ]
+      }
+    )
+  })
   prometheus_config_checksum = md5(local.prometheus_config)
 
-  prometheus_rules = yamlencode({
+  prometheus_user_rules = try(
-    groups = [
+    yamldecode(var.rules),
-      {
+    {}
-        name  = "custom-rules"
+  )
-        rules = var.rules
+  prometheus_merged_rules   = merge({ groups = [] }, local.prometheus_user_rules)
-      }
+  prometheus_rules          = yamlencode(local.prometheus_merged_rules)
-    ]
-  })
   prometheus_rules_checksum = md5(local.prometheus_rules)
+
+  alertmanager_user_config = try(
+    yamldecode(var.alertmanager_config),
+    {}
+  )
+  alertmanager_config = yamlencode({
+    global = merge(
+      try(local.alertmanager_default_config.global, {}),
+      try(local.alertmanager_user_config.global, {})
+    ),
+    route = merge(
+      try(local.alertmanager_default_config.route, {}),
+      try(local.alertmanager_user_config.route, {
+        receiver = "default"
+      })
+    ),
+    receivers = concat(
+      try(local.alertmanager_default_config.receivers, []),
+      try(local.alertmanager_user_config.receivers, []),
+      [
+        {
+          name = "default"
+          email_configs = [
+            {
+              to            = var.email_config.to
+              from          = var.email_config.from
+              smarthost     = var.email_config.smarthost
+              auth_username = var.email_config.auth_username
+              auth_password = var.email_config.auth_password
+            }
+          ]
+        }
+      ],
+      try([
+        {
+          name = "telegram"
+          telegram_configs = [
+            {
+              send_resolved = true
+              parse_mode    = var.telegram_config.parse_mode
+              bot_token     = var.telegram_config.bot_token
+              chat_id       = tonumber(var.telegram_config.chat_id)
+              message       = var.telegram_config.message != "" ? var.telegram_config.message : "{{ template \"telegram.default.message\" . }}"
+            }
+          ]
+        }
+      ], [])
+    )
+    templates = concat(
+      try(local.prometheus_default_config.templates, []),
+      try(local.prometheus_user_config.templates, [])
+    )
+  })
+  alertmanager_config_checksum = md5(local.alertmanager_config)
 }

terraform/modules/utils/prometheus/prometheus.tf

@@ -143,7 +143,7 @@ resource "kubernetes_deployment" "prometheus" {
         volume {
           name = "prometheus-storage-volume"
           host_path {
-            path = local.data_folder
+            path = local.prometheus_folder
             type = "DirectoryOrCreate"
           }
         }

terraform/modules/utils/prometheus/resources/alertmanager.yml

@@ -0,0 +1,7 @@
+global:
+  resolve_timeout: 5m
+route:
+  receiver: "default"
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 12h

terraform/modules/utils/prometheus/resources/prometheus.bak.yml

@@ -1,126 +0,0 @@
-global:
-  scrape_interval: 5s
-  evaluation_interval: 5s
-rule_files:
-  - /etc/prometheus/prometheus.rules
-alerting:
-  alertmanagers:
-    - scheme: http
-      static_configs:
-        - targets:
-            - "alertmanager.monitoring.svc:9093"
-scrape_configs:
-  - job_name: "node-exporter"
-    kubernetes_sd_configs:
-      - role: endpoints
-    relabel_configs:
-      - source_labels: [__meta_kubernetes_endpoints_name]
-        regex: "node-exporter"
-        action: keep
-  - job_name: "kubernetes-apiservers"
-    kubernetes_sd_configs:
-      - role: endpoints
-    scheme: https
-    tls_config:
-      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-    relabel_configs:
-      - source_labels:
-          [
-            __meta_kubernetes_namespace,
-            __meta_kubernetes_service_name,
-            __meta_kubernetes_endpoint_port_name,
-          ]
-        action: keep
-        regex: default;kubernetes;https
-  - job_name: "kubernetes-nodes"
-    scheme: https
-    tls_config:
-      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-    kubernetes_sd_configs:
-      - role: node
-    relabel_configs:
-      - action: labelmap
-        regex: __meta_kubernetes_node_label_(.+)
-      - target_label: __address__
-        replacement: kubernetes.default.svc:443
-      - source_labels: [__meta_kubernetes_node_name]
-        regex: (.+)
-        target_label: __metrics_path__
-        replacement: /api/v1/nodes/$${1}/proxy/metrics
-  - job_name: "kubernetes-pods"
-    kubernetes_sd_configs:
-      - role: pod
-    relabel_configs:
-      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
-        action: keep
-        regex: true
-      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
-        action: replace
-        target_label: __metrics_path__
-        regex: (.+)
-      - source_labels:
-          [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
-        action: replace
-        regex: ([^:]+)(?::\d+)?;(\d+)
-        replacement: $1:$2
-        target_label: __address__
-      - action: labelmap
-        regex: __meta_kubernetes_pod_label_(.+)
-      - source_labels: [__meta_kubernetes_namespace]
-        action: replace
-        target_label: kubernetes_namespace
-      - source_labels: [__meta_kubernetes_pod_name]
-        action: replace
-        target_label: kubernetes_pod_name
-  - job_name: "kube-state-metrics"
-    static_configs:
-      - targets: ["kube-state-metrics.kube-system.svc.cluster.local:8080"]
-  - job_name: "kubernetes-cadvisor"
-    scheme: https
-    tls_config:
-      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-    kubernetes_sd_configs:
-      - role: node
-    relabel_configs:
-      - action: labelmap
-        regex: __meta_kubernetes_node_label_(.+)
-      - target_label: __address__
-        replacement: kubernetes.default.svc:443
-      - source_labels: [__meta_kubernetes_node_name]
-        regex: (.+)
-        target_label: __metrics_path__
-        replacement: /api/v1/nodes/$${1}/proxy/metrics/cadvisor
-  - job_name: "kubernetes-service-endpoints"
-    kubernetes_sd_configs:
-      - role: endpoints
-    relabel_configs:
-      - source_labels:
-          [__meta_kubernetes_service_annotation_prometheus_io_scrape]
-        action: keep
-        regex: true
-      - source_labels:
-          [__meta_kubernetes_service_annotation_prometheus_io_scheme]
-        action: replace
-        target_label: __scheme__
-        regex: (https?)
-      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
-        action: replace
-        target_label: __metrics_path__
-        regex: (.+)
-      - source_labels:
-          [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
-        action: replace
-        target_label: __address__
-        regex: ([^:]+)(?::\d+)?;(\d+)
-        replacement: $1:$2
-      - action: labelmap
-        regex: __meta_kubernetes_service_label_(.+)
-      - source_labels: [__meta_kubernetes_namespace]
-        action: replace
-        target_label: kubernetes_namespace
-      - source_labels: [__meta_kubernetes_service_name]
-        action: replace
-        target_label: kubernetes_name

terraform/modules/utils/prometheus/resources/prometheus.yml

@@ -36,3 +36,7 @@ scrape_configs:
         regex: '^/system\.slice/(.+)\.service$'
         target_label: systemd_service_name
         replacement: "${1}"
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: ["alertmanager.prometheus.svc.cluster.local:9093"]

terraform/modules/utils/prometheus/variables.tf

@@ -11,15 +11,9 @@ variable "retention_time" {
 }
 
 variable "rules" {
-  description = "Prometheus alerting rules"
+  description = "Prometheus alerting rules in YAML format"
-  type = list(object({
+  type        = string
-    alert       = string
+  default     = ""
-    expr        = string
-    for         = string
-    labels      = map(string)
-    annotations = map(string)
-  }))
-  default = []
 }
 
 variable "config" {
@@ -32,3 +26,44 @@ variable "persistent_folder" {
   description = "The path to the persistent folder"
   type        = string
 }
+
+variable "alertmanager_config" {
+  description = "Custom Alertmanager configuration in YAML format"
+  type        = string
+  default     = ""
+}
+
+variable "cluster_name" {
+  description = "The name of the Kubernetes cluster"
+  type        = string
+  default     = "kubernetes"
+}
+
+variable "email_config" {
+  description = "Email configuration for Alertmanager in YAML format"
+  type = object({
+    to            = optional(string, "root@localhost")
+    from          = optional(string, "root@localhost")
+    smarthost     = optional(string, "localhost:25")
+    auth_username = optional(string, "")
+    auth_password = optional(string, "")
+  })
+  default = {
+    to            = "root@localhost"
+    from          = "root@localhost"
+    smarthost     = "localhost:25"
+    auth_username = ""
+    auth_password = ""
+  }
+}
+
+variable "telegram_config" {
+  description = "Telegram configuration for Alertmanager in YAML format"
+  type = object({
+    bot_token  = string
+    chat_id    = string
+    parse_mode = optional(string, "HTML")
+    message    = optional(string, "")
+  })
+  default = null
+}