alexpires.me/a13labs.infra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add Prometheus and Alertmanager configurations: implement custom rules, alerting, and Telegram notifications

Browse Source
This commit is contained in:
Alexandre Pires
2025-10-04 18:05:10 +02:00
parent 7cad175406
commit 0d3fb56f78
23 changed files with 416 additions and 235 deletions
Download Patch File Download Diff File Expand all files Collapse all files
terraform/modules/utils/prometheus/alertmanager.tf
+118
View File
@@ -0,0 +1,118 @@
resource "kubernetes_config_map" "alertmanager" {
metadata {
name = "alertmanager-conf"
namespace = kubernetes_namespace.prometheus.metadata[0].name
labels = {
name = "alertmanager-conf"
}
}
data = {
"alertmanager.yml" = local.alertmanager_config
}
}
resource "kubernetes_deployment" "alertmanager" {
metadata {
name = "alertmanager"
namespace = kubernetes_namespace.prometheus.metadata[0].name
labels = {
app = "alertmanager"
}
}
spec {
replicas = 1
selector {
match_labels = {
app = "alertmanager"
}
}
template {
metadata {
labels = {
app = "alertmanager"
}
annotations = {
"checksum/config" = local.alertmanager_config_checksum
}
}
spec {
container {
name = "alertmanager"
image = "prom/alertmanager:${local.app_version}"
args = [
"--config.file=/etc/alertmanager/alertmanager.yml",
"--storage.path=/alertmanager"
]
port {
name = "http"
container_port = 9093
}
volume_mount {
name = "alertmanager-config"
mount_path = "/etc/alertmanager"
}
volume_mount {
name = "alertmanager-storage"
mount_path = "/alertmanager"
}
security_context {
run_as_user = 1000
run_as_group = 1000
run_as_non_root = true
allow_privilege_escalation = false
}
}
volume {
name = "alertmanager-config"
config_map {
name = kubernetes_config_map.alertmanager.metadata[0].name
}
}
volume {
name = "alertmanager-storage"
host_path {
path = local.alertmanager_folder
type = "DirectoryOrCreate"
}
}
}
}
}
}
resource "kubernetes_service" "alertmanager" {
metadata {
name = "alertmanager"
namespace = kubernetes_namespace.prometheus.metadata[0].name
labels = {
app = "alertmanager"
}
}
spec {
selector = {
app = "alertmanager"
}
type = "ClusterIP"
port {
name = "web"
port = 9093
protocol = "TCP"
target_port = "http"
}
}
}
terraform/modules/utils/prometheus/config.tf
+106 -11
View File
@@ -1,17 +1,112 @@
locals {
app_version = var.tag
data_folder = "${var.persistent_folder}/prometheus"
app_version = var.tag
prometheus_folder = "${var.persistent_folder}/prometheus"
alertmanager_folder = "${var.persistent_folder}/alertmanager"
prometheus_config = coalesce(var.config, file("${path.module}/resources/default.yml"))
prometheus_default_config = yamldecode(file("${path.module}/resources/prometheus.yml"))
alertmanager_default_config = yamldecode(file("${path.module}/resources/alertmanager.yml"))
prometheus_user_config = try(
yamldecode(var.config),
{}
)
prometheus_config = yamlencode({
global = merge(
try(local.prometheus_default_config.global, {}),
try(local.prometheus_user_config.global, {}),
{
scrape_interval = "15s"
evaluation_interval = "15s"
external_labels = {
cluster = var.cluster_name
}
}
),
rule_files = concat(
try(local.prometheus_default_config.rule_files, []),
try(local.prometheus_user_config.rule_files, [])
),
scrape_configs = concat(
try(local.prometheus_default_config.scrape_configs, []),
try(local.prometheus_user_config.scrape_configs, [])
),
alerting = merge(
try(local.prometheus_default_config.alerting, {}),
try(local.prometheus_user_config.alerting, {}),
{
alertmanagers = [
{
static_configs = [
{
targets = ["alertmanager.prometheus.svc.cluster.local:9093"]
}
]
}
]
}
)
})
prometheus_config_checksum = md5(local.prometheus_config)
prometheus_rules = yamlencode({
groups = [
{
name = "custom-rules"
rules = var.rules
}
]
})
prometheus_user_rules = try(
yamldecode(var.rules),
{}
)
prometheus_merged_rules = merge({ groups = [] }, local.prometheus_user_rules)
prometheus_rules = yamlencode(local.prometheus_merged_rules)
prometheus_rules_checksum = md5(local.prometheus_rules)
alertmanager_user_config = try(
yamldecode(var.alertmanager_config),
{}
)
alertmanager_config = yamlencode({
global = merge(
try(local.alertmanager_default_config.global, {}),
try(local.alertmanager_user_config.global, {})
),
route = merge(
try(local.alertmanager_default_config.route, {}),
try(local.alertmanager_user_config.route, {
receiver = "default"
})
),
receivers = concat(
try(local.alertmanager_default_config.receivers, []),
try(local.alertmanager_user_config.receivers, []),
[
{
name = "default"
email_configs = [
{
to = var.email_config.to
from = var.email_config.from
smarthost = var.email_config.smarthost
auth_username = var.email_config.auth_username
auth_password = var.email_config.auth_password
}
]
}
],
try([
{
name = "telegram"
telegram_configs = [
{
send_resolved = true
parse_mode = var.telegram_config.parse_mode
bot_token = var.telegram_config.bot_token
chat_id = tonumber(var.telegram_config.chat_id)
message = var.telegram_config.message != "" ? var.telegram_config.message : "{{ template \"telegram.default.message\" . }}"
}
]
}
], [])
)
templates = concat(
try(local.prometheus_default_config.templates, []),
try(local.prometheus_user_config.templates, [])
)
})
alertmanager_config_checksum = md5(local.alertmanager_config)
}
terraform/modules/utils/prometheus/prometheus.tf
+1 -1
View File
@@ -143,7 +143,7 @@ resource "kubernetes_deployment" "prometheus" {
volume {
name = "prometheus-storage-volume"
host_path {
path = local.data_folder
path = local.prometheus_folder
type = "DirectoryOrCreate"
}
}
terraform/modules/utils/prometheus/resources/alertmanager.yml
+7
View File
@@ -0,0 +1,7 @@
global:
resolve_timeout: 5m
route:
receiver: "default"
group_wait: 30s
group_interval: 5m
repeat_interval: 12h
terraform/modules/utils/prometheus/resources/prometheus.bak.yml
-126
View File
@@ -1,126 +0,0 @@
global:
scrape_interval: 5s
evaluation_interval: 5s
rule_files:
- /etc/prometheus/prometheus.rules
alerting:
alertmanagers:
- scheme: http
static_configs:
- targets:
- "alertmanager.monitoring.svc:9093"
scrape_configs:
- job_name: "node-exporter"
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- source_labels: [__meta_kubernetes_endpoints_name]
regex: "node-exporter"
action: keep
- job_name: "kubernetes-apiservers"
kubernetes_sd_configs:
- role: endpoints
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- source_labels:
[
__meta_kubernetes_namespace,
__meta_kubernetes_service_name,
__meta_kubernetes_endpoint_port_name,
]
action: keep
regex: default;kubernetes;https
- job_name: "kubernetes-nodes"
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
- role: node
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/$${1}/proxy/metrics
- job_name: "kubernetes-pods"
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels:
[__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
action: replace
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
target_label: __address__
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_pod_name]
action: replace
target_label: kubernetes_pod_name
- job_name: "kube-state-metrics"
static_configs:
- targets: ["kube-state-metrics.kube-system.svc.cluster.local:8080"]
- job_name: "kubernetes-cadvisor"
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
- role: node
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/$${1}/proxy/metrics/cadvisor
- job_name: "kubernetes-service-endpoints"
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- source_labels:
[__meta_kubernetes_service_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels:
[__meta_kubernetes_service_annotation_prometheus_io_scheme]
action: replace
target_label: __scheme__
regex: (https?)
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels:
[__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
action: replace
target_label: __address__
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_service_name]
action: replace
target_label: kubernetes_name
terraform/modules/utils/prometheus/resources/default.yml → terraform/modules/utils/prometheus/resources/prometheus.yml
+4
View File
@@ -36,3 +36,7 @@ scrape_configs:
regex: '^/system\.slice/(.+)\.service$'
target_label: systemd_service_name
replacement: "${1}"
alerting:
alertmanagers:
- static_configs:
- targets: ["alertmanager.prometheus.svc.cluster.local:9093"]
terraform/modules/utils/prometheus/variables.tf
+44 -9
View File
@@ -11,15 +11,9 @@ variable "retention_time" {
}
variable "rules" {
description = "Prometheus alerting rules"
type = list(object({
alert = string
expr = string
for = string
labels = map(string)
annotations = map(string)
}))
default = []
description = "Prometheus alerting rules in YAML format"
type = string
default = ""
}
variable "config" {
@@ -32,3 +26,44 @@ variable "persistent_folder" {
description = "The path to the persistent folder"
type = string
}
variable "alertmanager_config" {
description = "Custom Alertmanager configuration in YAML format"
type = string
default = ""
}
variable "cluster_name" {
description = "The name of the Kubernetes cluster"
type = string
default = "kubernetes"
}
variable "email_config" {
description = "Email configuration for Alertmanager in YAML format"
type = object({
to = optional(string, "root@localhost")
from = optional(string, "root@localhost")
smarthost = optional(string, "localhost:25")
auth_username = optional(string, "")
auth_password = optional(string, "")
})
default = {
to = "root@localhost"
from = "root@localhost"
smarthost = "localhost:25"
auth_username = ""
auth_password = ""
}
}
variable "telegram_config" {
description = "Telegram configuration for Alertmanager in YAML format"
type = object({
bot_token = string
chat_id = string
parse_mode = optional(string, "HTML")
message = optional(string, "")
})
default = null
}