From 581f35cad359151c3632ef6cea78fac24ea660ce Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 17 Mar 2025 16:06:35 +0000 Subject: [PATCH] add alerting --- config/alertmanager.yml | 22 ++++++++ config/prometheus-alerts.yml | 18 +++++-- .../provisioning/alerting/contact_points.yml | 21 ++++++++ .../alerting/notification_policies.yml | 12 +++++ config/teams-template.tmpl | 51 +++++++++++++++++++ docker-compose.yml | 31 +++++++++-- 6 files changed, 146 insertions(+), 9 deletions(-) create mode 100644 config/alertmanager.yml create mode 100644 config/provisioning/alerting/contact_points.yml create mode 100644 config/provisioning/alerting/notification_policies.yml create mode 100644 config/teams-template.tmpl diff --git a/config/alertmanager.yml b/config/alertmanager.yml new file mode 100644 index 0000000..36d03d3 --- /dev/null +++ b/config/alertmanager.yml @@ -0,0 +1,22 @@ +global: + resolve_timeout: 5m + +route: + group_by: ['alertname', 'job'] + group_wait: 30s + group_interval: 5m + repeat_interval: 3h + receiver: 'teams' + +receivers: +- name: 'teams' + webhook_configs: + - url: 'http://webhook-adapter:8080/alert' + send_resolved: true + +inhibit_rules: + - source_matchers: + - severity = 'critical' + target_matchers: + - severity = 'warning' + equal: ['alertname'] diff --git a/config/prometheus-alerts.yml b/config/prometheus-alerts.yml index 8397224..6e6e397 100644 --- a/config/prometheus-alerts.yml +++ b/config/prometheus-alerts.yml @@ -40,14 +40,23 @@ groups: description: "Container {{ $labels.name }} CPU usage is > 80% for 5 minutes" - alert: ContainerHighMemory - expr: (container_memory_usage_bytes{name!=""} / container_spec_memory_limit_bytes{name!=""} * 100) > 80 - for: 5m + # First filter out containers with memory limits, then calculate percentage + expr: | + ( + container_memory_usage_bytes{name!=""} + / + container_spec_memory_limit_bytes{name!=""} + * 100 > 90 + ) + and + container_spec_memory_limit_bytes{name!=""} > 0 + for: 10m labels: severity: warning annotations: summary: "High memory usage in container {{ $labels.name }}" - description: "Container {{ $labels.name }} memory usage is > 80% for 5 minutes" - + description: "Container {{ $labels.name }} ({{ $labels.image }}) memory usage is {{ printf \"%.1f\" $value }}% of its limit for 10 minutes" + - alert: ContainerDown expr: absent(container_last_seen{name!=""}) for: 1m @@ -65,3 +74,4 @@ groups: annotations: summary: "Container {{ $labels.name }} restarting frequently" description: "Container {{ $labels.name }} has restarted more than 2 times in the last 15 minutes" + diff --git a/config/provisioning/alerting/contact_points.yml b/config/provisioning/alerting/contact_points.yml new file mode 100644 index 0000000..fc138a6 --- /dev/null +++ b/config/provisioning/alerting/contact_points.yml @@ -0,0 +1,21 @@ +apiVersion: 1 + +contactPoints: + - name: ms-teams + receivers: + - uid: teams-notifications + type: teams + settings: + url: https://prod-152.westeurope.logic.azure.com:443/workflows/6040474cfb314615aabd47694804d7e7/triggers/manual/paths/invoke?api-version=2016-06-01&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0&sig=ehfajgttd-2NC7rdEL5z9rylE4IiyVpgPjiOfDqF_z0 + title: "{{ if eq .Status \"firing\" }}🔥{{ else }}✅{{ end }} [{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}" + message: |- + {{ if ne .Status "firing" }}Resolved{{ else }}{{ end }}: {{ .CommonAnnotations.summary }} + + {{ .CommonAnnotations.description }} + + {{- if gt (len .Alerts.Firing) 0 }} + Alerts: + {{- range .Alerts.Firing }} + - {{ .Labels.instance }} {{ .Labels.job }}: {{ .Annotations.summary }} + {{- end }} + {{- end }} diff --git a/config/provisioning/alerting/notification_policies.yml b/config/provisioning/alerting/notification_policies.yml new file mode 100644 index 0000000..e5a6e10 --- /dev/null +++ b/config/provisioning/alerting/notification_policies.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +policies: + - receiver: ms-teams + group_by: ['alertname', 'job'] + repeat_interval: 3h + group_interval: 5m + routes: + - receiver: ms-teams + matchers: + - severity =~ "critical|warning|info" + continue: true diff --git a/config/teams-template.tmpl b/config/teams-template.tmpl new file mode 100644 index 0000000..c6758a6 --- /dev/null +++ b/config/teams-template.tmpl @@ -0,0 +1,51 @@ +{ + "type": "message", + "attachments": [ + { + "contentType": "application/vnd.microsoft.card.adaptive", + "content": { + "$schema": "http://adaptivecards.io/schemas/adaptive-card.json", + "type": "AdaptiveCard", + "version": "1.3", + "body": [ + { + "type": "TextBlock", + "text": "{{ if eq .Status "firing" }}🔥{{ else }}✅{{ end }} {{ .Status | title }}: {{ .CommonLabels.alertname }}", + "weight": "Bolder", + "size": "Large" + }, + { + "type": "TextBlock", + "text": "{{ .CommonAnnotations.summary }}", + "wrap": true + }, + { + "type": "FactSet", + "facts": [ + { + "title": "Status", + "value": "{{ .Status | title }}" + }, + { + "title": "Severity", + "value": "{{ .CommonLabels.severity | title }}" + } + ] + }, + { + "type": "TextBlock", + "text": "{{ .CommonAnnotations.description }}", + "wrap": true + } + ], + "actions": [ + { + "type": "Action.OpenUrl", + "title": "View in Alertmanager", + "url": "{{ .ExternalURL }}" + } + ] + } + } + ] +} diff --git a/docker-compose.yml b/docker-compose.yml index 974dccb..f96fd4d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,10 @@ services: # Grafana for visualization - grafanao: + grafana: image: grafana/grafana:latest container_name: grafana healthcheck: - test: ["CMD", "wget", "-q", "--spider", "http://localhost:3003"] + test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000"] interval: 30s timeout: 10s retries: 3 @@ -14,12 +14,15 @@ services: - GF_SECURITY_ADMIN_USER=admin - GF_SECURITY_ADMIN_PASSWORD=password # Change this in production - GF_USERS_ALLOW_SIGN_UP=false + - GF_UNIFIED_ALERTING_ENABLED=true + - GF_ALERTING_ENABLED=false + - GF_UNIFIED_ALERTING_PROMETHEUS_ALERTS_ENABLED=true volumes: - grafana-data:/var/lib/grafana - ./config/provisioning/dashboards:/etc/grafana/provisioning/dashboards - ./config/provisioning/datasources:/etc/grafana/provisioning/datasources - # Add this new mount to store the actual dashboard JSON - - ./config/provisioning/dashboards:/var/lib/grafana/dashboards + - ./config/provisioning/alerting:/etc/grafana/provisioning/alerting + - ./config/dashboards:/var/lib/grafana/dashboards depends_on: - loki - prometheus @@ -32,7 +35,9 @@ services: container_name: loki ports: - "9020:3100" - command: -config.file=/etc/loki/local-config.yaml + volumes: + - ./config/loki.yml:/etc/loki/local-config.yaml + - loki-data:/var/loki restart: unless-stopped networks: - monitoring @@ -54,6 +59,18 @@ services: networks: - monitoring + # AlertManager for handling alerts + alertmanager: + image: quay.io/prometheus/alertmanager:latest + container_name: alertmanager + ports: + - "9093:9093" + volumes: + - ./config/alertmanager.yml:/etc/alertmanager/config.yml + restart: unless-stopped + networks: + - monitoring + # Prometheus for metrics collection prometheus: image: prom/prometheus:latest @@ -129,6 +146,10 @@ volumes: prometheus-data: alloy-data: driver: local + loki-data: + driver: local + alertmanager-data: + driver: local networks: monitoring: