alexpires.me/a13labs.infra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(trivy): Enhance Trivy report generation and metrics handling

Browse Source 
- Updated `generate_report.sh` to include additional scanners and output paths.
- Enhanced `send_report.py` with new data classes for misconfigurations and vulnerabilities.
- Improved parsing logic for Trivy output, supporting both Kubernetes and image scans.
- Added Prometheus metrics generation for various scan results, including misconfigurations and vulnerabilities.
- Introduced unit tests for parsing Trivy output and generating Prometheus metrics.
- Modified Terraform variables for SMTP port type and added Pushgateway URL variable.
This commit is contained in:
Alexandre Pires
2026-05-22 23:41:26 +02:00
parent 829ab48919
commit 255ba6aec9
25 changed files with 410938 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ansible/playbook_podman_ollama_rocm.yml
+1 -1
View File
@@ -12,7 +12,7 @@
build:
dockerfile: "{{ lookup('file', 'dockerfiles/Dockerfile.ollama.rocm') }}"
env:
OLLAMA_CONTEXT_LENGTH: 4096
OLLAMA_CONTEXT_LENGTH: 32000
OLLAMA_NUM_THREADS: 10
OLLAMA_NUM_PARALLEL: 2
OLLAMA_FLASH_ATTENTION: 1
requirements/dev.txt
+1
View File
@@ -4,3 +4,4 @@ molecule
molecule-podman
passlib
pytest-molecule
requests
terraform/apps/prod-01/apps.tf
+2 -1
View File
@@ -98,9 +98,10 @@ module "trivy" {
persistent_folder = local.persistent_folder
cron_schedule = local.trivy_cronjob_schedule
tag = "0.70.0"
smtp_host = local.scaleway_smtp_host
smtp_port = local.scaleway_smtp_port
smtp_port = tostring(local.scaleway_smtp_port)
smtp_username = var.scaleway_project_id
smtp_password = local.mail_app_api_secret_key
from_email = "trivy@${local.primary_domain}"
terraform/modules/apps/grafana/config.tf
+4
View File
@@ -1,4 +1,8 @@
locals {
app_version = var.tag
monitoring_folder = "${var.persistent_folder}/grafana"
grafana_datasource_config = file("${path.module}/resources/default.yml")
grafana_dashboard_provider = file("${path.module}/resources/dashboard-provider.yml")
grafana_trivy_dashboard = file("${path.module}/resources/trivy-vulnerabilities-dashboard.json")
}
terraform/modules/apps/grafana/grafana.tf
+66
View File
@@ -1,3 +1,36 @@
resource "kubernetes_config_map" "grafana_datasources" {
metadata {
name = "grafana-datasources"
namespace = kubernetes_namespace.grafana.metadata[0].name
}
data = {
"default.yml" = replace(local.grafana_datasource_config, "__PROMETHEUS_URL__", var.prometheus_url)
}
}
resource "kubernetes_config_map" "grafana_dashboard_provider" {
metadata {
name = "grafana-dashboard-provider"
namespace = kubernetes_namespace.grafana.metadata[0].name
}
data = {
"dashboard-provider.yml" = local.grafana_dashboard_provider
}
}
resource "kubernetes_config_map" "grafana_dashboards" {
metadata {
name = "grafana-dashboards"
namespace = kubernetes_namespace.grafana.metadata[0].name
}
data = {
"trivy-vulnerabilities-dashboard.json" = local.grafana_trivy_dashboard
}
}
resource "kubernetes_deployment" "grafana" {
metadata {
@@ -67,6 +100,21 @@ resource "kubernetes_deployment" "grafana" {
name = "grafana-pv"
mount_path = "/var/lib/grafana"
}
volume_mount {
name = "grafana-datasources"
mount_path = "/etc/grafana/provisioning/datasources"
read_only = true
}
volume_mount {
name = "grafana-dashboard-provider"
mount_path = "/etc/grafana/provisioning/dashboards"
read_only = true
}
volume_mount {
name = "grafana-dashboards"
mount_path = "/var/lib/grafana/dashboards"
read_only = true
}
}
volume {
name = "grafana-pv"
@@ -74,6 +122,24 @@ resource "kubernetes_deployment" "grafana" {
path = local.monitoring_folder
}
}
volume {
name = "grafana-datasources"
config_map {
name = kubernetes_config_map.grafana_datasources.metadata[0].name
}
}
volume {
name = "grafana-dashboard-provider"
config_map {
name = kubernetes_config_map.grafana_dashboard_provider.metadata[0].name
}
}
volume {
name = "grafana-dashboards"
config_map {
name = kubernetes_config_map.grafana_dashboards.metadata[0].name
}
}
}
}
}
terraform/modules/apps/grafana/resources/auth-exportyer-dashboard.json
+580
View File
@@ -0,0 +1,580 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 5,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 2,
"panels": [],
"title": "Sessions",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 1
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "auth_sessions_active{}",
"legendFormat": "{{source}}",
"range": true,
"refId": "A"
}
],
"title": "Active sessions (by source)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 1
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum by (source, method) (rate(auth_sessions_total[5m]))",
"legendFormat": "{{source}} / {{method}}",
"range": true,
"refId": "A"
}
],
"title": "Auth sessions started (rate)",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 8
},
"id": 6,
"panels": [],
"title": "Users",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 9
},
"id": 5,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "auth_sessions_by_user_total{}",
"legendFormat": "{{username}}:{{source}}",
"range": true,
"refId": "A"
}
],
"title": "Top users by session starts (last 5m)",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 17
},
"id": 3,
"panels": [],
"title": "Exporter metrics",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 18
},
"id": 7,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "rate(auth_exporter_parse_errors_total[5m])",
"legendFormat": "{{instance}}",
"range": true,
"refId": "A"
}
],
"title": "Parse errors (rate)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezn5wieufi80d"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 18
},
"id": 8,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"editorMode": "code",
"expr": "rate(auth_exporter_user_labels_dropped_total[5m])",
"legendFormat": "{{instance}}",
"range": true,
"refId": "A"
}
],
"title": "User label drops (rate)",
"type": "timeseries"
}
],
"preload": false,
"schemaVersion": 42,
"tags": [
"security"
],
"templating": {
"list": [
{
"current": {
"text": "prometheus-dev-01",
"value": "bezn5wieufi80d"
},
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"type": "datasource"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Auth Exporter",
"uid": "7381c747-ecf6-4f2f-ac4b-551e56f24cf0",
"version": 5
}
terraform/modules/apps/grafana/resources/dashboard-provider.yml
+12
View File
@@ -0,0 +1,12 @@
apiVersion: 1
providers:
- name: trivy-vulnerabilities
orgId: 1
folder: Security
type: file
disableDeletion: false
updateIntervalSeconds: 30
allowUiUpdates: true
options:
path: /var/lib/grafana/dashboards
terraform/modules/apps/grafana/resources/default.yml
+9 -38
View File
@@ -1,38 +1,9 @@
global:
scrape_interval: 5s
evaluation_interval: 5s
rule_files:
- /etc/prometheus/prometheus.rules
scrape_configs:
- job_name: kubernetes-nodes-cadvisor
scrape_interval: 10s
scrape_timeout: 10s
scheme: https # remove if you want to scrape metrics on insecure port
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
- role: node
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
# Only for Kubernetes ^1.7.3.
# See: https://github.com/prometheus/prometheus/issues/2916
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
metric_relabel_configs:
- action: replace
source_labels: [id]
regex: '^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$'
target_label: rkt_container_name
replacement: "${2}-${1}"
- action: replace
source_labels: [id]
regex: '^/system\.slice/(.+)\.service$'
target_label: systemd_service_name
replacement: "${1}"
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
access: proxy
url: __PROMETHEUS_URL__
isDefault: true
editable: true
terraform/modules/apps/grafana/resources/devolo-powerline-dashboard.json
+551
View File
@@ -0,0 +1,551 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 8,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"decimals": 0,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "text",
"value": 0
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 5,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "devolo_uptime_seconds",
"instant": true,
"range": false,
"refId": "A"
}
],
"title": "Device Uptime",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezn5wieufi80d"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"max": 200,
"thresholds": {
"mode": "percentage",
"steps": [
{
"color": "red",
"value": 0
},
{
"color": "orange",
"value": 70
},
{
"color": "green",
"value": 85
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 7,
"x": 5,
"y": 0
},
"id": 5,
"options": {
"displayMode": "lcd",
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"maxVizHeight": 300,
"minVizHeight": 16,
"minVizWidth": 8,
"namePlacement": "auto",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": true,
"sizing": "auto",
"valueMode": "color"
},
"pluginVersion": "12.3.1",
"targets": [
{
"editorMode": "code",
"expr": "devolo_rx_mbps{device_id=~\"$device_id\", mac=~\"$mac\"}",
"legendFormat": "RX",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezn5wieufi80d"
},
"editorMode": "code",
"expr": "devolo_tx_mbps{device_id=~\"$device_id\", mac=~\"$mac\"}",
"hide": false,
"instant": false,
"legendFormat": "TX",
"range": true,
"refId": "B"
}
],
"title": "RX Rate",
"type": "bargauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"footer": {
"reducers": []
},
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 12,
"x": 12,
"y": 0
},
"id": 3,
"options": {
"cellHeight": "sm",
"showHeader": true
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "devolo_device_info_info",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Devolo Device Info",
"transformations": [
{
"id": "labelsToFields",
"options": {
"keepLabels": [
"firmware",
"mac",
"name",
"serial"
],
"mode": "columns"
}
},
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true,
"Value": true,
"devolo_device_info_info": true
},
"includeByName": {},
"indexByName": {},
"renameByName": {
"firmware": "Firmware",
"mac": "MAC",
"name": "Name",
"serial": "Serial"
}
}
}
],
"type": "table"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "continuous-RdYlGr"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 20,
"gradientMode": "scheme",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 3,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"max": 200,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": 0
},
{
"color": "green",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 6
},
"id": 6,
"options": {
"legend": {
"calcs": [],
"displayMode": "hidden",
"placement": "right",
"showLegend": false
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "devolo_rx_mbps{device_id=~\"$device_id\", mac=~\"$mac\"}",
"instant": false,
"legendFormat": "TX",
"range": true,
"refId": "A"
}
],
"title": "RX Throughput",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "continuous-RdYlGr",
"seriesBy": "max"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 20,
"gradientMode": "scheme",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 3,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"max": 200,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": 0
},
{
"color": "green",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 6
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "hidden",
"placement": "right",
"showLegend": false
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "devolo_tx_mbps{device_id=~\"$device_id\", mac=~\"$mac\"}",
"instant": false,
"legendFormat": "TX",
"range": true,
"refId": "A"
}
],
"title": "TX Throughput",
"type": "timeseries"
}
],
"preload": false,
"refresh": "10s",
"schemaVersion": 42,
"tags": [
"devolo",
"powerline",
"network"
],
"templating": {
"list": [
{
"current": {
"text": "Prometheus",
"value": "PBFA97CFB590B2093"
},
"label": "Prometheus",
"name": "DS_PROMETHEUS",
"options": [],
"query": "prometheus",
"refresh": 1,
"type": "datasource"
},
{
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"definition": "label_values(devolo_tx_mbps, device_id)",
"includeAll": true,
"label": "Device ID",
"multi": true,
"name": "device_id",
"options": [],
"query": {
"query": "label_values(devolo_tx_mbps, device_id)",
"refId": "StandardVariableQuery"
},
"refresh": 2,
"type": "query"
},
{
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"definition": "label_values(devolo_tx_mbps{device_id=~\"$device_id\"}, mac)",
"includeAll": true,
"label": "MAC",
"multi": true,
"name": "mac",
"options": [],
"query": {
"query": "label_values(devolo_tx_mbps{device_id=~\"$device_id\"}, mac)",
"refId": "StandardVariableQuery"
},
"refresh": 2,
"type": "query"
}
]
},
"time": {
"from": "now-24h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Devolo Powerline",
"uid": "devolo-powerline",
"version": 10
}
terraform/modules/apps/grafana/resources/disk-health-dashboard.json
+652
View File
@@ -0,0 +1,652 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 7,
"links": [],
"panels": [
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 3,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.1",
"targets": [
{
"editorMode": "code",
"expr": "count(disk_info{})-count(disk_healthy{})",
"instant": true,
"legendFormat": "__auto",
"refId": "A"
}
],
"title": "Unhealthy Disks",
"type": "stat"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "orange",
"value": 60
},
{
"color": "red",
"value": 70
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 9,
"x": 3,
"y": 0
},
"id": 3,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"expr": "disk_smart_attr_normalized_value{name=\"Temperature_Celsius\",device=~\"$device\"}",
"legendFormat": "{{device}}",
"refId": "A"
}
],
"title": "Temperature (Normalized)",
"type": "timeseries"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "continuous-GrYlRd"
},
"custom": {
"axisPlacement": "auto",
"fillOpacity": 70,
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineWidth": 0,
"spanNulls": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 11,
"x": 12,
"y": 0
},
"id": 2,
"options": {
"alignValue": "left",
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"mergeValues": true,
"rowHeight": 0.9,
"showValue": "auto",
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"expr": "disk_healthy{device=~\"$device\"}",
"legendFormat": "{{device}}",
"refId": "A"
}
],
"title": "Disk Health State",
"type": "state-timeline"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 23,
"x": 0,
"y": 7
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"expr": "disk_reallocated_sector_count{device=~\"$device\"}",
"legendFormat": "{{device}}",
"refId": "A"
}
],
"title": "Reallocated Sectors",
"type": "timeseries"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 23,
"x": 0,
"y": 14
},
"id": 5,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"expr": "disk_offline_uncorrectable{device=~\"$device\"}",
"legendFormat": "{{device}}",
"refId": "A"
}
],
"title": "Offline Uncorrectable",
"type": "timeseries"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 23,
"x": 0,
"y": 21
},
"id": 6,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"expr": "disk_smart_attr_raw_value{name=\"Power_On_Hours\",device=~\"$device\"}",
"legendFormat": "{{device}}",
"refId": "A"
}
],
"title": "Power On Hours",
"type": "timeseries"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 23,
"x": 0,
"y": 27
},
"id": 7,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"expr": "disk_smart_attr_raw_value{name=~\"Media_Wearout_Indicator|Available_Reservd_Space\",device=~\"$device\"}",
"legendFormat": "{{device}} {{name}}",
"refId": "A"
}
],
"title": "Media / Wear Indicators",
"type": "timeseries"
}
],
"preload": false,
"refresh": "",
"schemaVersion": 42,
"tags": [
"disks",
"smart"
],
"templating": {
"list": [
{
"current": {
"text": "All",
"value": "$__all"
},
"datasource": "$datasource",
"includeAll": true,
"multi": true,
"name": "device",
"options": [],
"query": "label_values(disk_healthy, device)",
"refresh": 1,
"type": "query"
},
{
"current": {
"text": "prometheus-dev-01",
"value": "bezn5wieufi80d"
},
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"type": "datasource"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Disk Health (smartctl)",
"uid": "disk-health-smartctl",
"version": 2
}
terraform/modules/apps/grafana/resources/fail2ban-dashboard.json
+648
View File
@@ -0,0 +1,648 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 2,
"id": 3,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "aezs0pa1i5xq8f"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"footer": {
"reducers": []
},
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": ".*Time"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
}
]
},
"gridPos": {
"h": 6,
"w": 24,
"x": 0,
"y": 0
},
"id": 206,
"options": {
"cellHeight": "sm",
"showHeader": true
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "aezs0pa1i5xq8f"
},
"editorMode": "code",
"exemplar": false,
"expr": "f2b_config_jail_max_retries{instance=~\"$instance\"}",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "{{jail}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "aezs0pa1i5xq8f"
},
"editorMode": "code",
"exemplar": false,
"expr": "f2b_config_jail_ban_time{instance=~\"$instance\"}",
"format": "table",
"hide": false,
"instant": true,
"interval": "",
"legendFormat": "{{jail}}",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "aezs0pa1i5xq8f"
},
"editorMode": "code",
"exemplar": false,
"expr": "f2b_config_jail_find_time{instance=~\"$instance\"}",
"format": "table",
"hide": false,
"instant": true,
"interval": "",
"legendFormat": "{{jail}}",
"refId": "C"
}
],
"title": "F2B Config",
"transformations": [
{
"id": "merge",
"options": {}
},
{
"id": "groupBy",
"options": {
"fields": {
"Value #A": {
"aggregations": [
"lastNotNull"
],
"operation": "aggregate"
},
"Value #B": {
"aggregations": [
"lastNotNull"
],
"operation": "aggregate"
},
"Value #C": {
"aggregations": [
"lastNotNull"
],
"operation": "aggregate"
},
"instance": {
"aggregations": [],
"operation": "groupby"
},
"jail": {
"aggregations": [],
"operation": "groupby"
}
}
}
},
{
"id": "organize",
"options": {
"excludeByName": {},
"indexByName": {},
"renameByName": {
"Value #A (lastNotNull)": "Max Retries",
"Value #B (lastNotNull)": "Ban Time",
"Value #C (lastNotNull)": "Find Time",
"jail": "Jail"
}
}
}
],
"transparent": true,
"type": "table"
},
{
"datasource": {
"type": "prometheus",
"uid": "aezs0pa1i5xq8f"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 6
},
"id": 190,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "aezs0pa1i5xq8f"
},
"editorMode": "code",
"exemplar": true,
"expr": "f2b_jail_failed_total{instance=~\"$instance\"}",
"hide": false,
"interval": "",
"legendFormat": "{{jail}}",
"range": true,
"refId": "A"
}
],
"title": "Fail2Ban Failures (Total)",
"transparent": true,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "aezs0pa1i5xq8f"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 6
},
"id": 191,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "aezs0pa1i5xq8f"
},
"editorMode": "code",
"exemplar": true,
"expr": "f2b_jail_banned_total{instance=~\"$instance\"}",
"interval": "",
"legendFormat": "{{jail}}",
"range": true,
"refId": "A"
}
],
"title": "Fail2Ban Bans (Total)",
"transparent": true,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "aezs0pa1i5xq8f"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 14
},
"id": 208,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "aezs0pa1i5xq8f"
},
"editorMode": "code",
"exemplar": true,
"expr": "f2b_jail_failed_current{instance=~\"$instance\"}",
"interval": "",
"legendFormat": "{{jail}}",
"range": true,
"refId": "A"
}
],
"title": "Fail2Ban Failures (Current)",
"transparent": true,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "aezs0pa1i5xq8f"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 14
},
"id": 209,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "aezs0pa1i5xq8f"
},
"editorMode": "code",
"exemplar": true,
"expr": "f2b_jail_banned_current{instance=~\"$instance\"}",
"interval": "",
"legendFormat": "{{jail}}",
"range": true,
"refId": "A"
}
],
"title": "Fail2Ban Bans (Current)",
"transparent": true,
"type": "timeseries"
}
],
"preload": false,
"refresh": "30s",
"schemaVersion": 42,
"tags": [
"security"
],
"templating": {
"list": [
{
"current": {
"text": [
"fail2ban-exporter.prometheus.svc.cluster.local:9100"
],
"value": [
"fail2ban-exporter.prometheus.svc.cluster.local:9100"
]
},
"datasource": {
"type": "prometheus",
"uid": "aezs0pa1i5xq8f"
},
"definition": "f2b_up",
"description": "Select which instance(s) to show",
"includeAll": false,
"label": "Instance",
"multi": true,
"name": "instance",
"options": [],
"query": {
"query": "f2b_up",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "/.*instance=\"([^\"]+)\"/",
"type": "query"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Fail2Ban Exporter",
"uid": "cTkH9AT7z",
"version": 4
}
terraform/modules/apps/grafana/resources/kubernetes-dashboard.json
+1691
View File
File diff suppressed because it is too large Load Diff
terraform/modules/apps/grafana/resources/node-exporter-dashboard.json
+14836
View File
File diff suppressed because it is too large Load Diff
terraform/modules/apps/grafana/resources/trivy-vulnerabilities-dashboard.json
+1072
View File
File diff suppressed because it is too large Load Diff
terraform/modules/apps/grafana/resources/windows-exporter-dashboard.json
+2448
View File
File diff suppressed because it is too large Load Diff
terraform/modules/apps/grafana/variables.tf
+6
View File
@@ -20,3 +20,9 @@ variable "persistent_folder" {
description = "The path to the persistent folder"
type = string
}
variable "prometheus_url" {
description = "Prometheus service URL used by Grafana datasource provisioning"
type = string
default = "http://prometheus.prometheus.svc.cluster.local:9090"
}
terraform/modules/utils/prometheus/config.tf
+1
View File
@@ -1,6 +1,7 @@
locals {
app_version = var.tag
prometheus_folder = "${var.persistent_folder}/prometheus"
pushgateway_folder = "${var.persistent_folder}/pushgateway"
alertmanager_folder = "${var.persistent_folder}/alertmanager"
prometheus_default_config = yamldecode(file("${path.module}/resources/prometheus.yml"))
terraform/modules/utils/prometheus/pushgateway.tf
+104
View File
@@ -0,0 +1,104 @@
resource "kubernetes_deployment" "pushgateway" {
metadata {
name = "pushgateway"
namespace = kubernetes_namespace.prometheus.metadata[0].name
labels = {
app = "pushgateway"
}
}
spec {
replicas = 1
strategy {
type = "Recreate"
}
selector {
match_labels = {
app = "pushgateway"
}
}
template {
metadata {
labels = {
app = "pushgateway"
}
annotations = {
"prometheus.io/scrape" = "true"
"prometheus.io/port" = "9091"
}
}
spec {
automount_service_account_token = false
security_context {
run_as_non_root = true
run_as_user = 65534
run_as_group = 65534
seccomp_profile {
type = "RuntimeDefault"
}
}
container {
name = "pushgateway"
image = "prom/pushgateway:${local.app_version}"
args = ["--persistence.file=/pushgateway/metrics"]
port {
name = "http"
container_port = 9091
}
security_context {
run_as_non_root = true
run_as_user = 65534
run_as_group = 65534
allow_privilege_escalation = false
privileged = false
read_only_root_filesystem = true
capabilities {
drop = ["ALL"]
}
}
volume_mount {
name = "pushgateway-storage-volume"
mount_path = "/pushgateway"
}
}
volume {
name = "pushgateway-storage-volume"
host_path {
path = local.pushgateway_folder
type = "DirectoryOrCreate"
}
}
}
}
}
}
resource "kubernetes_service" "pushgateway" {
metadata {
name = "pushgateway"
namespace = kubernetes_namespace.prometheus.metadata[0].name
}
spec {
selector = {
app = "pushgateway"
}
type = "ClusterIP"
port {
port = 9091
protocol = "TCP"
target_port = "http"
}
}
}
terraform/modules/utils/prometheus/resources/prometheus.yml
+4
View File
@@ -36,6 +36,10 @@ scrape_configs:
regex: '^/system\.slice/(.+)\.service$'
target_label: systemd_service_name
replacement: "${1}"
- job_name: pushgateway
honor_labels: true
static_configs:
- targets: ["pushgateway.prometheus.svc.cluster.local:9091"]
alerting:
alertmanagers:
- static_configs:
terraform/modules/utils/trivy/main.tf
+16 -4
View File
@@ -85,7 +85,7 @@ resource "kubernetes_cron_job_v1" "trivy" {
backoff_limit = 1
parallelism = 1
completions = 1
active_deadline_seconds = 250
active_deadline_seconds = 600
template {
metadata {
@@ -126,9 +126,9 @@ resource "kubernetes_cron_job_v1" "trivy" {
}
}
container {
name = "send-report"
image = "python:${local.python_version}"
args = ["/scripts/send_report.py"]
name = "send-report"
image = "python:${local.python_version}"
command = ["python3", "/scripts/send_report.py"]
env {
name = "SMTP_HOST"
value = var.smtp_host
@@ -163,6 +163,18 @@ resource "kubernetes_cron_job_v1" "trivy" {
name = "EMAIL_TO"
value = var.to_email
}
env {
name = "PUSHGATEWAY_URL"
value = var.pushgateway_url
}
env {
name = "REPORT_PATH"
value = "/tmp/report.html"
}
env {
name = "TRIVY_OUTPUT_FILE"
value = "/tmp/trivy.json"
}
security_context {
run_as_non_root = true
run_as_user = 1000
terraform/modules/utils/trivy/resources/fixtures/trivy.json
+387194
View File
File diff suppressed because it is too large Load Diff
terraform/modules/utils/trivy/resources/generate_report.sh
+3 -2
View File
@@ -8,8 +8,9 @@ trivy plugin install scan2html
trivy k8s --report=${REPORT_TYPE:-all} \
--exit-code=0 \
--severity=${LEVELS:-CRITICAL,HIGH,MEDIUM} \
-o tmp/trivy.json \
--scanners ${SCANNERS:-vuln,misconfig} \
-o /tmp/trivy.json \
--format json \
--disable-node-collector \
--parallel ${PARALLEL:-5}
trivy scan2html generate --scan2html-flags --output tmp/report.html --from tmp/trivy.json
trivy scan2html generate --scan2html-flags --output /tmp/report.html --from /tmp/trivy.json
terraform/modules/utils/trivy/resources/send_report.py
+687 -23
View File
@@ -3,6 +3,455 @@ import os
import time
import smtplib
from email.message import EmailMessage
import json
import re
import urllib.request
import urllib.error
from datetime import datetime
from typing import Dict, List, Optional, Any
from dataclasses import dataclass, field
# Imports and Configuration
@dataclass
class Misconfiguration:
"""Represents a Trivy misconfiguration finding."""
status: str
message: str
severity: str
check_id: str = ""
title: str = ""
check_namespace: str = ""
primary_url: str = ""
@dataclass
class Vulnerability:
"""Represents a Trivy vulnerability finding."""
vulnerability_id: str
package_name: str
severity: str
target: str = ""
vuln_type: str = ""
class_name: str = ""
installed_version: str = ""
fixed_version: str = ""
title: str = ""
@dataclass
class Resource:
"""Represents a Trivy scan resource."""
type: str
name: str
kind: Optional[str] = None
namespace: Optional[str] = None
image_digests: List[str] = field(default_factory=list)
metadata: List[dict] = field(default_factory=list)
misconf_summary: dict = field(default_factory=dict)
misconfigurations: List[Misconfiguration] = field(default_factory=list)
vulnerabilities: List[Vulnerability] = field(default_factory=list)
vulnerability_summary: dict = field(default_factory=dict)
successes_count: int = 0
failures_count: int = 0
@dataclass
class ParseResult:
"""Parsed Trivy scan result."""
cluster_name: Optional[str] = None
resources: List[Resource] = field(default_factory=list)
start_time: Optional[datetime] = None
end_time: Optional[datetime] = None
duration: Optional[float] = None
image: Optional[str] = None
# Configuration
PUSHGATEWAY_URL = os.environ.get("PUSHGATEWAY_URL", "http://pushgateway:9091/metrics")
PUSHGATEWAY_NAMESPACE = os.environ.get("PUSHGATEWAY_NAMESPACE", "trivy")
PUSHGATEWAY_JOB = os.environ.get("PUSHGATEWAY_JOB", "trivy-metrics")
METRICS_PREFIX = "trivy"
REQUEST_TIMEOUT = 30
REPORT_PATH = os.environ.get("REPORT_PATH", "/tmp/report.html")
TRIVY_OUTPUT_FILE = os.environ.get("TRIVY_OUTPUT_FILE", "/tmp/trivy.json")
WAIT_TIMEOUT = int(os.environ.get("WAIT_TIMEOUT", "200"))
SLEEP_INTERVAL = int(os.environ.get("SLEEP_INTERVAL", "5") )
# Metric definitions
METRICS = {
"vulnerabilities_found": {
"type": "gauge",
"description": "Total number of vulnerabilities found",
"labels": ["severity", "type", "package", "vulnerability_id"]
},
"vulnerabilities_by_severity": {
"type": "gauge",
"description": "Count of vulnerabilities grouped by severity",
"labels": ["severity"]
},
"vulnerabilities_by_type": {
"type": "gauge",
"description": "Count of vulnerabilities grouped by type",
"labels": ["type"]
},
"scans_completed": {
"type": "counter",
"description": "Total number of scans completed",
"labels": ["status"]
},
"scan_duration_seconds": {
"type": "gauge",
"description": "Duration of the last scan in seconds",
"labels": ["status"]
},
"image_scanned": {
"type": "gauge",
"description": "Image that was scanned",
"labels": ["image"]
},
"scan_timestamp": {
"type": "gauge",
"description": "Timestamp of the scan",
"labels": []
},
"severities_found": {
"type": "gauge",
"description": "Severities found in the scan",
"labels": ["severity"]
}
}
def parse_trivy_output(output: str) -> ParseResult:
"""Parse Trivy scan output and extract metrics."""
# Try to parse JSON output if available
if output.strip().startswith("{"):
try:
data = json.loads(output)
# Check if it's Kubernetes/container scan JSON
if "Resources" in data or "ClusterName" in data:
result = ParseResult(cluster_name=data.get("ClusterName"))
for resource_data in data.get("Resources", []):
misconfigurations = []
vulnerabilities = []
misconf_summary: dict = {}
vulnerability_summary: dict = {}
resource_type = "kubernetes"
for res in resource_data.get("Results", []):
resource_type = res.get("Type") or resource_type
for k, v in res.get("MisconfSummary", {}).items():
misconf_summary[k] = misconf_summary.get(k, 0) + v
for mc in res.get("Misconfigurations", []):
misconfigurations.append(Misconfiguration(
status=mc.get("Status", ""),
message=mc.get("Message", ""),
severity=mc.get("Severity", ""),
check_id=mc.get("ID", ""),
title=mc.get("Title", ""),
check_namespace=mc.get("Namespace", ""),
primary_url=mc.get("PrimaryURL", ""),
))
for v in res.get("Vulnerabilities", []):
severity = v.get("Severity", "UNKNOWN")
vulnerability_summary[severity] = vulnerability_summary.get(severity, 0) + 1
vulnerabilities.append(Vulnerability(
vulnerability_id=v.get("VulnerabilityID", ""),
package_name=v.get("PkgName", ""),
severity=severity,
target=res.get("Target", ""),
vuln_type=res.get("Type", ""),
class_name=res.get("Class", ""),
installed_version=v.get("InstalledVersion", ""),
fixed_version=v.get("FixedVersion", ""),
title=v.get("Title", ""),
))
resource = Resource(
type=resource_type,
name=resource_data.get("Name", ""),
kind=resource_data.get("Kind"),
namespace=resource_data.get("Namespace"),
metadata=resource_data.get("Metadata", []),
misconf_summary=misconf_summary,
misconfigurations=misconfigurations,
vulnerabilities=vulnerabilities,
vulnerability_summary=vulnerability_summary,
successes_count=misconf_summary.get("Successes", 0),
failures_count=misconf_summary.get("Failures", 0),
)
result.resources.append(resource)
return result
except json.JSONDecodeError:
pass
# Parse text output for image scanning
metrics_data = {
"vulnerabilities": [],
"image": None,
"start_time": None,
"end_time": None,
"summary": {}
}
# Extract image name
image_match = re.search(r'Scanning image: (.+)', output)
if image_match:
metrics_data["image"] = image_match.group(1).strip()
# Extract scan duration
duration_match = re.search(r'Spent.*? (\d+(?:\.\d+)?)', output)
if duration_match:
metrics_data["scan_duration"] = float(duration_match.group(1))
# Extract severity counts
severity_patterns = {
"CRITICAL": r'Critical:.*?(\d+)',
"HIGH": r'High:.*?(\d+)',
"MEDIUM": r'Medium:.*?(\d+)',
"LOW": r'Low:.*?(\d+)',
"UNKNOWN": r'Unknown:.*?(\d+)',
"STYLE": r'Style:.*?(\d+)',
"NOT_SET": r'Not Set:.*?(\d+)'
}
severity_counts = {}
for severity, pattern in severity_patterns.items():
match = re.search(pattern, output)
if match:
severity_counts[severity] = int(match.group(1))
metrics_data["summary"] = severity_counts
for line in output.split('\n'):
# Match vulnerability lines
vuln_match = re.search(r'\[.*?\].*?CVE-?\d+\.\d+-\d+', line)
if vuln_match:
# Parse vulnerability details
vuln_data = {
"cve": None,
"package": None,
"type": None,
"severity": None,
"fix_version": None
}
# Extract CVE
cve_match = re.search(r'CVE-?\d+\.\d+-\d+', line)
if cve_match:
vuln_data["cve"] = cve_match.group()
# Extract package name
pkg_match = re.search(r'package: (\S+)', line)
if pkg_match:
vuln_data["package"] = pkg_match.group(1)
# Extract vulnerability ID
vuln_id_match = re.search(r'ID: ([\w\-]+)', line)
if vuln_id_match:
vuln_data["vulnerability_id"] = vuln_id_match.group(1)
# Extract severity
severity_match = re.search(r'severity: (\S+)', line)
if severity_match:
vuln_data["severity"] = severity_match.group(1)
if any(vuln_data.values()):
metrics_data["vulnerabilities"].append(vuln_data)
# Convert to ParseResult for backward compatibility
result = ParseResult()
result.image = metrics_data.get("image")
result.start_time = metrics_data.get("start_time")
result.end_time = metrics_data.get("end_time")
result.duration = metrics_data.get("scan_duration")
return result
def generate_prometheus_metrics(scan_data: Dict[str, Any], scan_duration: Optional[float] = None) -> str:
"""Generate Prometheus metrics format from scan data."""
lines = []
# Scan metadata
timestamp = datetime.utcnow().isoformat() + "Z"
lines.append(f'# HELP {METRICS_PREFIX}_scan_timestamp Timestamp of the scan')
lines.append(f'# TYPE {METRICS_PREFIX}_scan_timestamp gauge')
lines.append(f'{METRICS_PREFIX}_scan_timestamp 1')
# Image scanned
if scan_data.get("image"):
image = scan_data["image"].replace(":", "_").replace("/", "_")
lines.append(f'# HELP {METRICS_PREFIX}_image_scanned Image that was scanned')
lines.append(f'# TYPE {METRICS_PREFIX}_image_scanned gauge')
lines.append(f'{METRICS_PREFIX}_image_scanned{{image="{image}"}} 1')
# Duration
if scan_duration is not None:
lines.append(f'# HELP {METRICS_PREFIX}_scan_duration_seconds Duration of the last scan in seconds')
lines.append(f'# TYPE {METRICS_PREFIX}_scan_duration_seconds gauge')
lines.append(f'{METRICS_PREFIX}_scan_duration_seconds{{status="success"}} {scan_duration}')
# Vulnerabilities by severity
summary = scan_data.get("summary", {})
for severity, count in summary.items():
if count > 0:
lines.append(f'# HELP {METRICS_PREFIX}_vulnerabilities_by_severity Count of vulnerabilities grouped by severity')
lines.append(f'# TYPE {METRICS_PREFIX}_vulnerabilities_by_severity gauge')
lines.append(f'{METRICS_PREFIX}_vulnerabilities_by_severity{{severity="{severity}"}} {count}')
# Individual vulnerabilities
vulnerabilities = scan_data.get("vulnerabilities", [])
for vuln in vulnerabilities:
cve = vuln.get("cve", "unknown")
package = vuln.get("package", "unknown")
vuln_id = vuln.get("vulnerability_id", cve)
severity = vuln.get("severity", "unknown")
# Truncate CVE to fit in label
cve_short = cve[-12:] if len(cve) > 12 else cve
lines.append(f'# HELP {METRICS_PREFIX}_vulnerabilities_found Total number of vulnerabilities found')
lines.append(f'# TYPE {METRICS_PREFIX}_vulnerabilities_found gauge')
labels = {
"severity": severity,
"type": vuln.get("type", "unknown"),
"package": package,
"vulnerability_id": cve_short
}
# Sanitize labels for Prometheus
safe_labels = {k: str(v).replace('"', "").replace("'", "")[:20] for k, v in labels.items()}
label_string = ",".join(f'{k}="{v}"' for k, v in safe_labels.items())
lines.append(f'{METRICS_PREFIX}_vulnerabilities_found{{{label_string}}} 1')
# Clean up duplicate HELP lines
cleaned_lines = []
last_help = None
for line in lines:
if line.startswith('# HELP'):
if last_help and line.split()[2] == last_help.split()[2]:
continue # Skip duplicate HELP
last_help = line.split()[2]
cleaned_lines.append(line)
return '\n'.join(cleaned_lines)
def push_metrics_to_pushgateway(metrics: str):
"""Push metrics to PushGateway."""
try:
validated_metrics = sanitize_prometheus_metrics(metrics)
if not validated_metrics:
print("✗ No valid Prometheus metrics remained after sanitization")
return False
req = urllib.request.Request(
PUSHGATEWAY_URL,
data=(validated_metrics + "\n").encode("utf-8"),
headers={"Content-Type": "text/plain; charset=utf-8"},
method="POST",
)
with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT) as response:
status_code = response.getcode()
response_body = response.read().decode("utf-8", errors="replace")
if status_code == 200:
print(f"✓ Successfully pushed {len(validated_metrics.split(chr(10)))} lines of metrics to PushGateway")
return True
else:
print(f"✗ Failed to push metrics to PushGateway. Status code: {status_code}")
print(f"Response: {response_body}")
return False
except urllib.error.HTTPError as e:
body = e.read().decode("utf-8", errors="replace")
print(f"✗ Error pushing metrics to PushGateway: HTTP {e.code} {e.reason}")
print(f"Response: {body}")
return False
except urllib.error.URLError as e:
print(f"✗ Error pushing metrics to PushGateway: {e}")
return False
def _has_balanced_label_quotes(label_block: str) -> bool:
"""Return True when all double quotes in a label block are balanced."""
in_quotes = False
escaped = False
for ch in label_block:
if escaped:
escaped = False
continue
if ch == "\\":
escaped = True
continue
if ch == '"':
in_quotes = not in_quotes
return not in_quotes
def sanitize_prometheus_metrics(metrics: str) -> str:
"""Drop malformed metric lines so one bad sample does not break the whole push."""
sanitized_lines: List[str] = []
dropped_lines = 0
merged_duplicates = 0
series_totals: Dict[str, float] = {}
series_order: List[str] = []
def _format_value(value: float) -> str:
return str(int(value)) if value.is_integer() else format(value, "g")
for idx, raw_line in enumerate(metrics.splitlines(), start=1):
line = raw_line.rstrip("\r")
if not line:
continue
if line.startswith("#"):
sanitized_lines.append(line)
continue
if "{" in line:
open_brace = line.find("{")
close_brace = line.find("}", open_brace + 1)
if close_brace == -1:
print(f"⚠ Dropping malformed metric line {idx}: missing closing brace")
dropped_lines += 1
continue
label_block = line[open_brace + 1:close_brace]
if not _has_balanced_label_quotes(label_block):
print(f"⚠ Dropping malformed metric line {idx}: unbalanced label quotes")
dropped_lines += 1
continue
parts = line.rsplit(" ", 1)
if len(parts) != 2:
print(f"⚠ Dropping malformed metric line {idx}: missing value")
dropped_lines += 1
continue
series_id, value_raw = parts[0], parts[1].strip()
try:
value = float(value_raw)
except ValueError:
print(f"⚠ Dropping malformed metric line {idx}: invalid numeric value '{value_raw}'")
dropped_lines += 1
continue
if series_id in series_totals:
merged_duplicates += 1
series_totals[series_id] += value
else:
series_order.append(series_id)
series_totals[series_id] = value
if dropped_lines:
print(f"⚠ Dropped {dropped_lines} malformed metric line(s) before push")
for series_id in series_order:
sanitized_lines.append(f"{series_id} {_format_value(series_totals[series_id])}")
if merged_duplicates:
print(f"⚠ Merged {merged_duplicates} duplicate metric line(s) before push")
return "\n".join(sanitized_lines)
def dispatch_email(report_path, subject):
msg = EmailMessage()
@@ -26,38 +475,253 @@ def dispatch_email(report_path, subject):
smtp_user = os.environ.get("SMTP_USER", "user")
smtp_pass = os.environ.get("SMTP_PASS", "pass")
with smtplib.SMTP(smtp_host, smtp_port) as server:
if smtp_port == 465:
server = smtplib.SMTP_SSL(smtp_host, smtp_port)
else:
server = smtplib.SMTP(smtp_host, smtp_port)
if smtp_port != 465:
server.starttls()
try:
server.login(smtp_user, smtp_pass)
except smtplib.SMTPAuthenticationError:
print("SMTP Authentication Error: Check your SMTP credentials.")
return
server.send_message(msg)
print("Report sent.")
if smtp_port == 465:
server = smtplib.SMTP_SSL(smtp_host, smtp_port)
else:
server = smtplib.SMTP(smtp_host, smtp_port)
server.starttls()
try:
server.login(smtp_user, smtp_pass)
except smtplib.SMTPAuthenticationError:
print("SMTP Authentication Error: Check your SMTP credentials.")
server.quit()
return
server.send_message(msg)
server.quit()
print("Report sent.")
if __name__ == "__main__":
report_path = os.environ.get("REPORT_PATH", "/tmp/report.html")
wait_timeout = int(os.environ.get("WAIT_TIMEOUT", "200"))
sleep_interval = int(os.environ.get("SLEEP_INTERVAL", "5") )
print("Trivy report path:", report_path)
def generate_prometheus_metrics_from_parse_result(scan_result: ParseResult) -> str:
"""Generate Prometheus metrics from ParseResult object."""
lines = []
lines.append(f'# HELP {METRICS_PREFIX}_cluster_scan_timestamp Timestamp of the cluster scan')
lines.append(f'# TYPE {METRICS_PREFIX}_cluster_scan_timestamp gauge')
lines.append(f'{METRICS_PREFIX}_cluster_scan_timestamp 1')
lines.append(f'# HELP {METRICS_PREFIX}_cluster_name Name of the cluster')
lines.append(f'# TYPE {METRICS_PREFIX}_cluster_name gauge')
def _safe_label(value: str) -> str:
# Prometheus label values cannot contain unescaped control chars/newlines.
cleaned = re.sub(r"[\x00-\x1F\x7F]+", " ", value)
return cleaned.replace('\\', '\\\\').replace('"', '\\"')
def _safe_truncated_label(value: str, max_len: int) -> str:
# Truncate before escaping so we never cut through an escape sequence.
raw_truncated = re.sub(r"[\x00-\x1F\x7F]+", " ", value)[:max_len]
return _safe_label(raw_truncated)
safe_cluster_name = _safe_label(scan_result.cluster_name or "")
lines.append(f'{METRICS_PREFIX}_cluster_name{{cluster_name="{safe_cluster_name}"}} 1')
resource_type_header_written = False
image_digest_header_written = False
successes_header_written = False
failures_header_written = False
misconfig_header_written = False
vulnerability_total_header_written = False
vulnerability_severity_header_written = False
vulnerability_header_written = False
for resource in scan_result.resources:
rtype = _safe_label(resource.type or "")
rname = _safe_label(resource.name or "")
rkind = _safe_label(resource.kind or "")
rns = _safe_label(resource.namespace or "")
resource_labels = f'type="{rtype}",kind="{rkind}",name="{rname}",namespace="{rns}"'
if not resource_type_header_written:
lines.append(f'# HELP {METRICS_PREFIX}_resource_type Type of resource')
lines.append(f'# TYPE {METRICS_PREFIX}_resource_type gauge')
resource_type_header_written = True
lines.append(f'{METRICS_PREFIX}_resource_type{{{resource_labels}}} 1')
if resource.image_digests:
if not image_digest_header_written:
lines.append(f'# HELP {METRICS_PREFIX}_image_digest Image digest')
lines.append(f'# TYPE {METRICS_PREFIX}_image_digest gauge')
image_digest_header_written = True
for digest in resource.image_digests:
safe_digest = _safe_truncated_label(digest, 64)
lines.append(f'{METRICS_PREFIX}_image_digest{{{resource_labels},digest="{safe_digest}"}} 1')
if resource.misconf_summary:
summary = resource.misconf_summary
successes = summary.get('Successes', 0)
failures = summary.get('Failures', 0)
if not successes_header_written:
lines.append(f'# HELP {METRICS_PREFIX}_misconfiguration_successes Successful checks')
lines.append(f'# TYPE {METRICS_PREFIX}_misconfiguration_successes gauge')
successes_header_written = True
lines.append(f'{METRICS_PREFIX}_misconfiguration_successes{{{resource_labels}}} {successes}')
if not failures_header_written:
lines.append(f'# HELP {METRICS_PREFIX}_misconfiguration_failures Failed checks')
lines.append(f'# TYPE {METRICS_PREFIX}_misconfiguration_failures gauge')
failures_header_written = True
lines.append(f'{METRICS_PREFIX}_misconfiguration_failures{{{resource_labels}}} {failures}')
misconfig_counts = {}
for m in resource.misconfigurations:
key = (
_safe_label(m.severity or ""),
_safe_label(m.status or ""),
_safe_truncated_label(m.check_id or "", 32),
_safe_truncated_label(m.check_namespace or "", 64),
_safe_truncated_label(m.title or "", 100),
_safe_truncated_label(m.primary_url or "", 200),
)
misconfig_counts[key] = misconfig_counts.get(key, 0) + 1
for (safe_severity, safe_status, safe_check_id, safe_check_namespace, safe_title, safe_primary_url), count in misconfig_counts.items():
if not misconfig_header_written:
lines.append(f'# HELP {METRICS_PREFIX}_misconfiguration Misconfiguration finding')
lines.append(f'# TYPE {METRICS_PREFIX}_misconfiguration gauge')
misconfig_header_written = True
lines.append(
f'{METRICS_PREFIX}_misconfiguration{{{resource_labels},'
f'severity="{safe_severity}",status="{safe_status}",'
f'check_id="{safe_check_id}",check_namespace="{safe_check_namespace}",title="{safe_title}",primary_url="{safe_primary_url}"}} {count}'
)
if not vulnerability_total_header_written:
lines.append(f'# HELP {METRICS_PREFIX}_vulnerability_total Total vulnerabilities for resource')
lines.append(f'# TYPE {METRICS_PREFIX}_vulnerability_total gauge')
vulnerability_total_header_written = True
lines.append(f'{METRICS_PREFIX}_vulnerability_total{{{resource_labels}}} {len(resource.vulnerabilities)}')
for severity, count in resource.vulnerability_summary.items():
safe_severity = _safe_label(severity)
if not vulnerability_severity_header_written:
lines.append(f'# HELP {METRICS_PREFIX}_vulnerability_severity_count Vulnerabilities grouped by severity for resource')
lines.append(f'# TYPE {METRICS_PREFIX}_vulnerability_severity_count gauge')
vulnerability_severity_header_written = True
lines.append(
f'{METRICS_PREFIX}_vulnerability_severity_count{{{resource_labels},severity="{safe_severity}"}} {count}'
)
vulnerability_counts = {}
for v in resource.vulnerabilities:
key = (
_safe_label(v.severity or ""),
_safe_truncated_label(v.vulnerability_id or "", 32),
_safe_truncated_label(v.package_name or "", 64),
_safe_truncated_label(v.target or "", 96),
_safe_truncated_label(v.vuln_type or "", 24),
_safe_truncated_label(v.class_name or "", 24),
)
vulnerability_counts[key] = vulnerability_counts.get(key, 0) + 1
for (safe_severity, safe_vuln_id, safe_package, safe_target, safe_vuln_type, safe_class_name), count in vulnerability_counts.items():
if not vulnerability_header_written:
lines.append(f'# HELP {METRICS_PREFIX}_vulnerability Vulnerability finding')
lines.append(f'# TYPE {METRICS_PREFIX}_vulnerability gauge')
vulnerability_header_written = True
lines.append(
f'{METRICS_PREFIX}_vulnerability{{{resource_labels},'
f'severity="{safe_severity}",vulnerability_id="{safe_vuln_id}",'
f'package="{safe_package}",target="{safe_target}",'
f'vuln_type="{safe_vuln_type}",class="{safe_class_name}"}} {count}'
)
if scan_result.duration:
lines.append(f'# HELP {METRICS_PREFIX}_scan_duration Duration of the last scan in seconds')
lines.append(f'# TYPE {METRICS_PREFIX}_scan_duration gauge')
lines.append(f'{METRICS_PREFIX}_scan_duration {scan_result.duration}')
return '\n'.join(lines)
def main():
"""Main function to process Trivy scan output and push metrics."""
print("🔍 Trivy Metrics Exporter")
print(f"📍 PushGateway URL: {PUSHGATEWAY_URL}")
print(f"🏷️ Namespace: {PUSHGATEWAY_NAMESPACE}")
print(f"💼 Job: {PUSHGATEWAY_JOB}")
print(f"⏳ Request timeout: {REQUEST_TIMEOUT} seconds")
print()
print("Trivy report path:", REPORT_PATH)
print("Starting to wait for the report to be generated...")
start = time.time()
while not os.path.exists(report_path):
if time.time() - start > wait_timeout:
while not os.path.exists(REPORT_PATH):
if time.time() - start > WAIT_TIMEOUT:
print("Timeout waiting for report to be generated.")
dispatch_email(None, "Trivy Scan Report - Timeout")
exit(1)
print("Still waiting for report to be generated...")
time.sleep(sleep_interval)
time.sleep(SLEEP_INTERVAL)
print("Report generated, proceeding to send email...")
time.sleep(5) # Wait for a bit to ensure the file is fully written
dispatch_email(report_path, "Trivy Scan Report - " + time.strftime("%Y-%m-%d %H:%M:%S"))
dispatch_email(REPORT_PATH, "Trivy Scan Report - " + time.strftime("%Y-%m-%d %H:%M:%S"))
if not os.path.exists(TRIVY_OUTPUT_FILE):
print(f"✗ Output file not found: {TRIVY_OUTPUT_FILE}")
print(" Make sure Trivy has written its output before running this script.")
exit(1)
print(f"📄 Reading scan output from: {TRIVY_OUTPUT_FILE}")
with open(TRIVY_OUTPUT_FILE, 'r') as f:
output = f.read()
print(f"📊 Parsing scan output...")
# Parse the output
scan_data = parse_trivy_output(output)
# Handle both ParseResult (for Kubernetes scans) and old dict format (for image scans)
if hasattr(scan_data, 'resources'):
# ParseResult object (Kubernetes/container scan output)
resource_count = len(scan_data.resources) if scan_data.resources else 0
if resource_count == 0:
print("⚠️ No vulnerabilities found in scan output")
print(" Pushing basic scan metadata...")
scan_data = None
else:
print(f"Found {resource_count} resource(s) to scan")
# Generate Prometheus metrics
print("🔧 Generating Prometheus metrics...")
metrics = generate_prometheus_metrics_from_parse_result(scan_data)
# Push to PushGateway
print("🚀 Pushing metrics to PushGateway...")
success = push_metrics_to_pushgateway(metrics)
if success:
print("✓ Metrics successfully pushed to PushGateway")
# Also print a summary to stdout for dashboards
print("\n📈 Scan Summary:")
print(f" Resources scanned: {resource_count}")
total_vulns = sum(resource.misconf_summary.get('Failures', 0) for resource in scan_data.resources)
print(f" Misconfigurations found: {total_vulns}")
for resource in scan_data.resources:
if resource.misconf_summary:
print(f" Resource: {resource.name} ({resource.type})")
for severity, count in resource.misconf_summary.items():
if count > 0:
print(f" - {severity}: {count}")
else:
print("✗ Failed to push metrics")
exit(1)
else:
# Old dict format (image scan output) - keep existing behavior for backward compatibility
print("⚠️ No vulnerabilities found in scan output")
print(" Pushing basic scan metadata...")
# For backward compatibility with image scans
metrics = generate_prometheus_metrics(scan_data)
print("🚀 Pushing metrics to PushGateway...")
success = push_metrics_to_pushgateway(metrics)
if success:
print("✓ Metrics successfully pushed to PushGateway")
else:
print("✗ Failed to push metrics")
exit(1)
if __name__ == "__main__":
main()
terraform/modules/utils/trivy/resources/test_parse_trivy_output.py
+343
View File
@@ -0,0 +1,343 @@
"""Unit tests for parse_trivy_output function."""
import pytest
from pathlib import Path
from typing import Any
import json
from send_report import (
parse_trivy_output,
generate_prometheus_metrics_from_parse_result,
ParseResult,
Resource,
Misconfiguration,
)
class TestParseTrivyOutput:
"""Tests for parsing Trivy JSON output."""
@pytest.fixture
def fixture_path(self) -> Path:
"""Path to the Trivy fixture file."""
return Path(__file__).parent / "fixtures" / "trivy.json"
@pytest.fixture
def fixture_data(self, fixture_path: Path) -> dict[str, Any]:
"""Load fixture data."""
with open(fixture_path, "r") as f:
return json.load(f)
def test_parse_trivy_output_cluster_name(self, fixture_data: dict[str, Any]) -> None:
"""Test parsing cluster name from Trivy output."""
result = parse_trivy_output(json.dumps(fixture_data))
assert result.cluster_name == "microk8s-prod-01"
def test_parse_trivy_output_results_empty(self, fixture_data: dict[str, Any]) -> None:
"""Test when no resources are found."""
# Modify fixture to remove resources
modified_data = {**fixture_data, "Resources": []}
result = parse_trivy_output(json.dumps(modified_data))
assert result.cluster_name == "microk8s-prod-01"
assert result.resources == []
assert len(result.resources) == 0
def test_parse_trivy_output_single_resource(self, fixture_data: dict[str, Any]) -> None:
"""Test parsing a single resource."""
result = parse_trivy_output(json.dumps(fixture_data))
assert isinstance(result.resources, list)
assert len(result.resources) > 0
def test_parse_trivy_output_result_types(self, fixture_data: dict[str, Any]) -> None:
"""Test result types for different resources."""
result = parse_trivy_output(json.dumps(fixture_data))
for resource in result.resources:
assert hasattr(resource, "type")
assert isinstance(resource.type, str)
assert len(resource.type) > 0
def test_parse_trivy_output_metadata_list(self, fixture_data: dict[str, Any]) -> None:
"""Test that metadata is correctly parsed as a list."""
resource = result = parse_trivy_output(json.dumps(fixture_data)).resources[0]
assert hasattr(resource, "metadata")
assert isinstance(resource.metadata, list)
def test_parse_trivy_output_misconfiguration_summary(self, fixture_data: dict[str, Any]) -> None:
"""Test that misconfiguration summary is correctly parsed."""
resource = result = parse_trivy_output(json.dumps(fixture_data)).resources[0]
assert hasattr(resource, "misconf_summary")
assert isinstance(resource.misconf_summary, dict)
assert "Successes" in resource.misconf_summary
assert "Failures" in resource.misconf_summary
def test_parse_trivy_output_misconfigurations_list(self, fixture_data: dict[str, Any]) -> None:
"""Test that misconfigurations are correctly parsed as a list."""
resource = result = parse_trivy_output(json.dumps(fixture_data)).resources[0]
assert hasattr(resource, "misconfigurations")
assert isinstance(resource.misconfigurations, list)
def test_parse_trivy_output_successes_count(self, fixture_data: dict[str, Any]) -> None:
"""Test that successes count is accurately parsed."""
resource = result = parse_trivy_output(json.dumps(fixture_data)).resources[0]
for misconfig in resource.misconfigurations:
if misconfig.status == "PASS":
resource.successes_count += 1
assert resource.successes_count > 0
def test_parse_trivy_output_failures_count(self, fixture_data: dict[str, Any]) -> None:
"""Test that failures count is accurately parsed."""
resource = result = parse_trivy_output(json.dumps(fixture_data)).resources[0]
for misconfig in resource.misconfigurations:
if misconfig.status == "FAIL":
resource.failures_count += 1
assert resource.failures_count > 0
def test_parse_trivy_output_kubernetes_type(self, fixture_data: dict[str, Any]) -> None:
"""Test parsing of Kubernetes resource type."""
resource = result = parse_trivy_output(json.dumps(fixture_data)).resources[0]
assert resource.type == "kubernetes"
def test_parse_trivy_output_resource_name(self, fixture_data: dict[str, Any]) -> None:
"""Test parsing resource name (e.g., ClusterRole, Namespace)."""
resource = result = parse_trivy_output(json.dumps(fixture_data)).resources[0]
assert resource.name == "admin" or resource.name == "namespace"
def test_parse_trivy_output_namespace_field(self, fixture_data: dict[str, Any]) -> None:
"""Test that namespace field is correctly extracted."""
resource = result = parse_trivy_output(json.dumps(fixture_data)).resources[0]
# Namespace field may be None for cluster-scoped resources
assert hasattr(resource, "namespace")
assert resource.namespace is None or isinstance(resource.namespace, str)
def test_parse_trivy_output_empty_namespaces(self, fixture_data: dict[str, Any]) -> None:
"""Test parsing when namespaces array is empty."""
resource = result = parse_trivy_output(json.dumps(fixture_data)).resources[0]
if hasattr(resource, "namespaces"):
assert isinstance(resource.namespaces, list)
def test_parse_trivy_output_empty_image_list(self, fixture_data: dict[str, Any]) -> None:
"""Test parsing when images array is empty."""
resource = result = parse_trivy_output(json.dumps(fixture_data)).resources[0]
if hasattr(resource, "images"):
assert isinstance(resource.images, list)
def test_parse_trivy_output_empty_image_digest_list(self, fixture_data: dict[str, Any]) -> None:
"""Test parsing when image digests array is empty."""
resource = result = parse_trivy_output(json.dumps(fixture_data)).resources[0]
if hasattr(resource, "image_digests"):
assert isinstance(resource.image_digests, list)
def test_parse_trivy_output_parses_vulnerabilities(self) -> None:
"""Test parsing vulnerabilities from Kubernetes JSON results."""
sample = {
"ClusterName": "microk8s-prod-01",
"Resources": [
{
"Kind": "Deployment",
"Name": "sample-app",
"Namespace": "default",
"Results": [
{
"Class": "os-pkgs",
"Type": "alpine",
"Target": "sample-app:1.0",
"Vulnerabilities": [
{
"VulnerabilityID": "CVE-2024-0001",
"PkgName": "openssl",
"InstalledVersion": "3.0.0",
"FixedVersion": "3.0.1",
"Severity": "HIGH",
"Title": "Example vulnerability"
}
]
}
]
}
]
}
result = parse_trivy_output(json.dumps(sample))
assert len(result.resources) == 1
resource = result.resources[0]
assert len(resource.vulnerabilities) == 1
assert resource.vulnerabilities[0].vulnerability_id == "CVE-2024-0001"
assert resource.vulnerability_summary["HIGH"] == 1
class TestGeneratePrometheusMetricsFromParseResult:
"""Tests for generate_prometheus_metrics_from_parse_result."""
@pytest.fixture
def fixture_data(self) -> dict[str, Any]:
fixture_path = Path(__file__).parent / "fixtures" / "trivy.json"
with open(fixture_path) as f:
return json.load(f)
@pytest.fixture
def parsed_result(self, fixture_data: dict[str, Any]) -> ParseResult:
return parse_trivy_output(json.dumps(fixture_data))
@pytest.fixture
def simple_result(self) -> ParseResult:
return ParseResult(
cluster_name="test-cluster",
resources=[
Resource(
type="kubernetes",
name="my-role",
namespace="default",
misconf_summary={"Successes": 10, "Failures": 3},
misconfigurations=[
Misconfiguration(
status="FAIL",
message="Bad config",
severity="HIGH",
check_id="KSV-0041",
title="Manage secrets",
check_namespace="builtin.kubernetes.KSV041",
primary_url="https://avd.aquasec.com/misconfig/ksv-0041",
),
Misconfiguration(
status="FAIL",
message="Another issue",
severity="MEDIUM",
check_id="KSV-0114",
title="Limit wildcard access",
check_namespace="builtin.kubernetes.KSV114",
primary_url="https://avd.aquasec.com/misconfig/ksv-0114",
),
],
)
],
)
def test_output_is_string(self, parsed_result: ParseResult) -> None:
output = generate_prometheus_metrics_from_parse_result(parsed_result)
assert isinstance(output, str)
def test_contains_cluster_timestamp(self, parsed_result: ParseResult) -> None:
output = generate_prometheus_metrics_from_parse_result(parsed_result)
assert "trivy_cluster_scan_timestamp 1" in output
def test_contains_cluster_name(self, parsed_result: ParseResult) -> None:
output = generate_prometheus_metrics_from_parse_result(parsed_result)
assert 'cluster_name="microk8s-prod-01"' in output
def test_contains_resource_type_metric(self, simple_result: ParseResult) -> None:
output = generate_prometheus_metrics_from_parse_result(simple_result)
assert "trivy_resource_type" in output
assert 'name="my-role"' in output
assert 'namespace="default"' in output
def test_contains_misconfiguration_successes(self, simple_result: ParseResult) -> None:
output = generate_prometheus_metrics_from_parse_result(simple_result)
assert "trivy_misconfiguration_successes" in output
assert "} 10" in output
def test_contains_misconfiguration_failures(self, simple_result: ParseResult) -> None:
output = generate_prometheus_metrics_from_parse_result(simple_result)
assert "trivy_misconfiguration_failures" in output
assert "} 3" in output
def test_contains_individual_misconfigurations(self, simple_result: ParseResult) -> None:
output = generate_prometheus_metrics_from_parse_result(simple_result)
assert 'severity="HIGH"' in output
assert 'severity="MEDIUM"' in output
assert 'status="FAIL"' in output
def test_contains_misconfiguration_identity_labels(self, simple_result: ParseResult) -> None:
output = generate_prometheus_metrics_from_parse_result(simple_result)
assert 'check_id="KSV-0041"' in output
assert 'check_namespace="builtin.kubernetes.KSV041"' in output
assert 'title="Manage secrets"' in output
assert 'primary_url="https://avd.aquasec.com/misconfig/ksv-0041"' in output
def test_parse_trivy_output_primary_url(self, fixture_data: dict[str, Any]) -> None:
result = parse_trivy_output(json.dumps(fixture_data))
first_with_url = None
for resource in result.resources:
for misconfig in resource.misconfigurations:
if misconfig.primary_url:
first_with_url = misconfig
break
if first_with_url:
break
assert first_with_url is not None
assert first_with_url.primary_url.startswith("https://avd.aquasec.com/misconfig/")
def test_empty_resources_still_has_header(self) -> None:
result = ParseResult(cluster_name="empty-cluster", resources=[])
output = generate_prometheus_metrics_from_parse_result(result)
assert "trivy_cluster_scan_timestamp" in output
assert 'cluster_name="empty-cluster"' in output
assert "trivy_resource_type" not in output
def test_no_misconf_summary_skips_summary_metrics(self) -> None:
result = ParseResult(
cluster_name="c",
resources=[Resource(type="kubernetes", name="r", misconf_summary={})],
)
output = generate_prometheus_metrics_from_parse_result(result)
assert "trivy_misconfiguration_successes" not in output
assert "trivy_misconfiguration_failures" not in output
def test_duration_metric_present_when_set(self) -> None:
result = ParseResult(cluster_name="c", resources=[], duration=42.5)
output = generate_prometheus_metrics_from_parse_result(result)
assert "trivy_scan_duration 42.5" in output
def test_duration_metric_absent_when_not_set(self) -> None:
result = ParseResult(cluster_name="c", resources=[])
output = generate_prometheus_metrics_from_parse_result(result)
assert "trivy_scan_duration" not in output
def test_image_digest_metric_emitted(self) -> None:
result = ParseResult(
cluster_name="c",
resources=[
Resource(
type="kubernetes",
name="r",
image_digests=["sha256:abc123"],
)
],
)
output = generate_prometheus_metrics_from_parse_result(result)
assert "trivy_image_digest" in output
assert "sha256:abc123" in output
def test_fixture_produces_nonempty_output(self, parsed_result: ParseResult) -> None:
output = generate_prometheus_metrics_from_parse_result(parsed_result)
lines = [l for l in output.splitlines() if l and not l.startswith("#")]
assert len(lines) > 0
def test_prometheus_help_type_pairs(self, simple_result: ParseResult) -> None:
output = generate_prometheus_metrics_from_parse_result(simple_result)
lines = output.splitlines()
for i, line in enumerate(lines):
if line.startswith("# TYPE"):
metric_name = line.split()[2]
assert any(
l.startswith(metric_name) for l in lines[i + 1 :]
), f"No sample line found after TYPE declaration for {metric_name}"
def test_contains_vulnerability_metrics(self) -> None:
result = ParseResult(
cluster_name="test-cluster",
resources=[
Resource(
type="kubernetes",
kind="Deployment",
name="sample-app",
namespace="default",
vulnerability_summary={"CRITICAL": 1, "HIGH": 2},
vulnerabilities=[],
)
],
)
output = generate_prometheus_metrics_from_parse_result(result)
assert "trivy_vulnerability_total" in output
assert "trivy_vulnerability_severity_count" in output
assert 'severity="CRITICAL"' in output
terraform/modules/utils/trivy/variables.tf
+7 -2
View File
@@ -10,7 +10,7 @@ variable "smtp_host" {
variable "smtp_port" {
description = "The SMTP relay port"
type = number
type = string
}
variable "smtp_username" {
@@ -43,7 +43,6 @@ variable "cron_schedule" {
variable "tag" {
description = "The version of Trivy to install"
type = string
default = "latest"
}
variable "report_type" {
@@ -57,3 +56,9 @@ variable "levels" {
type = string
default = "HIGH,CRITICAL"
}
variable "pushgateway_url" {
description = "URL of the Prometheus Pushgateway including /metrics/job/<job> path"
type = string
default = "http://pushgateway.prometheus.svc.cluster.local:9091/metrics/job/trivy-metrics"
}