fix container surveillance

This commit is contained in:
Bartal Læarsson
2025-03-18 09:33:02 +00:00
6 changed files with 136 additions and 7 deletions
+22
View File
@@ -0,0 +1,22 @@
global:
resolve_timeout: 5m
route:
group_by: ['alertname', 'job']
group_wait: 30s
group_interval: 5m
repeat_interval: 3h
receiver: 'teams'
receivers:
- name: 'teams'
webhook_configs:
- url: 'http://webhook-adapter:8080/alert'
send_resolved: true
inhibit_rules:
- source_matchers:
- severity = 'critical'
target_matchers:
- severity = 'warning'
equal: ['alertname']
+4 -2
View File
@@ -40,10 +40,11 @@ groups:
description: "Container {{ $labels.name }} CPU usage is > 80% for 5 minutes" description: "Container {{ $labels.name }} CPU usage is > 80% for 5 minutes"
- alert: ContainerHighMemory - alert: ContainerHighMemory
# First filter out containers with memory limits, then calculate percentage
expr: | expr: |
( (
container_memory_usage_bytes{name!=""} container_memory_usage_bytes{name!=""}
/ /
container_spec_memory_limit_bytes{name!=""} container_spec_memory_limit_bytes{name!=""}
* 100 > 90 * 100 > 90
) )
@@ -54,7 +55,7 @@ groups:
severity: warning severity: warning
annotations: annotations:
summary: "High memory usage in container {{ $labels.name }}" summary: "High memory usage in container {{ $labels.name }}"
description: "Container {{ $labels.name }} ({{ $labels.image }}) memory usage is {{ printf \"%.1f\" $value }}% of its limit for 10 minutes" description: "Container {{ $labels.name }} ({{ $labels.image }}) memory usage is {{ printf \"%.1f\" $value }}% of its limit for 10 minutes"
- alert: ContainerDown - alert: ContainerDown
expr: absent(container_last_seen{name!=""}) expr: absent(container_last_seen{name!=""})
@@ -73,3 +74,4 @@ groups:
annotations: annotations:
summary: "Container {{ $labels.name }} restarting frequently" summary: "Container {{ $labels.name }} restarting frequently"
description: "Container {{ $labels.name }} has restarted more than 2 times in the last 15 minutes" description: "Container {{ $labels.name }} has restarted more than 2 times in the last 15 minutes"
@@ -0,0 +1,21 @@
apiVersion: 1
contactPoints:
- name: ms-teams
receivers:
- uid: teams-notifications
type: teams
settings:
url: https://prod-152.westeurope.logic.azure.com:443/workflows/6040474cfb314615aabd47694804d7e7/triggers/manual/paths/invoke?api-version=2016-06-01&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0&sig=ehfajgttd-2NC7rdEL5z9rylE4IiyVpgPjiOfDqF_z0
title: "{{ if eq .Status \"firing\" }}🔥{{ else }}✅{{ end }} [{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}"
message: |-
{{ if ne .Status "firing" }}Resolved{{ else }}{{ end }}: {{ .CommonAnnotations.summary }}
{{ .CommonAnnotations.description }}
{{- if gt (len .Alerts.Firing) 0 }}
Alerts:
{{- range .Alerts.Firing }}
- {{ .Labels.instance }} {{ .Labels.job }}: {{ .Annotations.summary }}
{{- end }}
{{- end }}
@@ -0,0 +1,12 @@
apiVersion: 1
policies:
- receiver: ms-teams
group_by: ['alertname', 'job']
repeat_interval: 3h
group_interval: 5m
routes:
- receiver: ms-teams
matchers:
- severity =~ "critical|warning|info"
continue: true
+51
View File
@@ -0,0 +1,51 @@
{
"type": "message",
"attachments": [
{
"contentType": "application/vnd.microsoft.card.adaptive",
"content": {
"$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
"type": "AdaptiveCard",
"version": "1.3",
"body": [
{
"type": "TextBlock",
"text": "{{ if eq .Status "firing" }}🔥{{ else }}✅{{ end }} {{ .Status | title }}: {{ .CommonLabels.alertname }}",
"weight": "Bolder",
"size": "Large"
},
{
"type": "TextBlock",
"text": "{{ .CommonAnnotations.summary }}",
"wrap": true
},
{
"type": "FactSet",
"facts": [
{
"title": "Status",
"value": "{{ .Status | title }}"
},
{
"title": "Severity",
"value": "{{ .CommonLabels.severity | title }}"
}
]
},
{
"type": "TextBlock",
"text": "{{ .CommonAnnotations.description }}",
"wrap": true
}
],
"actions": [
{
"type": "Action.OpenUrl",
"title": "View in Alertmanager",
"url": "{{ .ExternalURL }}"
}
]
}
}
]
}
+26 -5
View File
@@ -1,10 +1,10 @@
services: services:
# Grafana for visualization # Grafana for visualization
grafanao: grafana:
image: grafana/grafana:latest image: grafana/grafana:latest
container_name: grafana container_name: grafana
healthcheck: healthcheck:
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3003"] test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000"]
interval: 30s interval: 30s
timeout: 10s timeout: 10s
retries: 3 retries: 3
@@ -14,12 +14,15 @@ services:
- GF_SECURITY_ADMIN_USER=admin - GF_SECURITY_ADMIN_USER=admin
- GF_SECURITY_ADMIN_PASSWORD=password # Change this in production - GF_SECURITY_ADMIN_PASSWORD=password # Change this in production
- GF_USERS_ALLOW_SIGN_UP=false - GF_USERS_ALLOW_SIGN_UP=false
- GF_UNIFIED_ALERTING_ENABLED=true
- GF_ALERTING_ENABLED=false
- GF_UNIFIED_ALERTING_PROMETHEUS_ALERTS_ENABLED=true
volumes: volumes:
- grafana-data:/var/lib/grafana - grafana-data:/var/lib/grafana
- ./config/provisioning/dashboards:/etc/grafana/provisioning/dashboards - ./config/provisioning/dashboards:/etc/grafana/provisioning/dashboards
- ./config/provisioning/datasources:/etc/grafana/provisioning/datasources - ./config/provisioning/datasources:/etc/grafana/provisioning/datasources
# Add this new mount to store the actual dashboard JSON - ./config/provisioning/alerting:/etc/grafana/provisioning/alerting
- ./config/provisioning/dashboards:/var/lib/grafana/dashboards - ./config/dashboards:/var/lib/grafana/dashboards
depends_on: depends_on:
- loki - loki
- prometheus - prometheus
@@ -32,7 +35,9 @@ services:
container_name: loki container_name: loki
ports: ports:
- "9020:3100" - "9020:3100"
command: -config.file=/etc/loki/local-config.yaml volumes:
- ./config/loki.yml:/etc/loki/local-config.yaml
- loki-data:/var/loki
restart: unless-stopped restart: unless-stopped
networks: networks:
- monitoring - monitoring
@@ -54,6 +59,18 @@ services:
networks: networks:
- monitoring - monitoring
# AlertManager for handling alerts
alertmanager:
image: quay.io/prometheus/alertmanager:latest
container_name: alertmanager
ports:
- "9093:9093"
volumes:
- ./config/alertmanager.yml:/etc/alertmanager/config.yml
restart: unless-stopped
networks:
- monitoring
# Prometheus for metrics collection # Prometheus for metrics collection
prometheus: prometheus:
image: prom/prometheus:latest image: prom/prometheus:latest
@@ -129,6 +146,10 @@ volumes:
prometheus-data: prometheus-data:
alloy-data: alloy-data:
driver: local driver: local
loki-data:
driver: local
alertmanager-data:
driver: local
networks: networks:
monitoring: monitoring: