fix container surveillance
This commit is contained in:
@@ -0,0 +1,22 @@
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
|
||||
route:
|
||||
group_by: ['alertname', 'job']
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 3h
|
||||
receiver: 'teams'
|
||||
|
||||
receivers:
|
||||
- name: 'teams'
|
||||
webhook_configs:
|
||||
- url: 'http://webhook-adapter:8080/alert'
|
||||
send_resolved: true
|
||||
|
||||
inhibit_rules:
|
||||
- source_matchers:
|
||||
- severity = 'critical'
|
||||
target_matchers:
|
||||
- severity = 'warning'
|
||||
equal: ['alertname']
|
||||
@@ -40,6 +40,7 @@ groups:
|
||||
description: "Container {{ $labels.name }} CPU usage is > 80% for 5 minutes"
|
||||
|
||||
- alert: ContainerHighMemory
|
||||
# First filter out containers with memory limits, then calculate percentage
|
||||
expr: |
|
||||
(
|
||||
container_memory_usage_bytes{name!=""}
|
||||
@@ -73,3 +74,4 @@ groups:
|
||||
annotations:
|
||||
summary: "Container {{ $labels.name }} restarting frequently"
|
||||
description: "Container {{ $labels.name }} has restarted more than 2 times in the last 15 minutes"
|
||||
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
apiVersion: 1
|
||||
|
||||
contactPoints:
|
||||
- name: ms-teams
|
||||
receivers:
|
||||
- uid: teams-notifications
|
||||
type: teams
|
||||
settings:
|
||||
url: https://prod-152.westeurope.logic.azure.com:443/workflows/6040474cfb314615aabd47694804d7e7/triggers/manual/paths/invoke?api-version=2016-06-01&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0&sig=ehfajgttd-2NC7rdEL5z9rylE4IiyVpgPjiOfDqF_z0
|
||||
title: "{{ if eq .Status \"firing\" }}🔥{{ else }}✅{{ end }} [{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}"
|
||||
message: |-
|
||||
{{ if ne .Status "firing" }}Resolved{{ else }}{{ end }}: {{ .CommonAnnotations.summary }}
|
||||
|
||||
{{ .CommonAnnotations.description }}
|
||||
|
||||
{{- if gt (len .Alerts.Firing) 0 }}
|
||||
Alerts:
|
||||
{{- range .Alerts.Firing }}
|
||||
- {{ .Labels.instance }} {{ .Labels.job }}: {{ .Annotations.summary }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: 1
|
||||
|
||||
policies:
|
||||
- receiver: ms-teams
|
||||
group_by: ['alertname', 'job']
|
||||
repeat_interval: 3h
|
||||
group_interval: 5m
|
||||
routes:
|
||||
- receiver: ms-teams
|
||||
matchers:
|
||||
- severity =~ "critical|warning|info"
|
||||
continue: true
|
||||
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"type": "message",
|
||||
"attachments": [
|
||||
{
|
||||
"contentType": "application/vnd.microsoft.card.adaptive",
|
||||
"content": {
|
||||
"$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
|
||||
"type": "AdaptiveCard",
|
||||
"version": "1.3",
|
||||
"body": [
|
||||
{
|
||||
"type": "TextBlock",
|
||||
"text": "{{ if eq .Status "firing" }}🔥{{ else }}✅{{ end }} {{ .Status | title }}: {{ .CommonLabels.alertname }}",
|
||||
"weight": "Bolder",
|
||||
"size": "Large"
|
||||
},
|
||||
{
|
||||
"type": "TextBlock",
|
||||
"text": "{{ .CommonAnnotations.summary }}",
|
||||
"wrap": true
|
||||
},
|
||||
{
|
||||
"type": "FactSet",
|
||||
"facts": [
|
||||
{
|
||||
"title": "Status",
|
||||
"value": "{{ .Status | title }}"
|
||||
},
|
||||
{
|
||||
"title": "Severity",
|
||||
"value": "{{ .CommonLabels.severity | title }}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "TextBlock",
|
||||
"text": "{{ .CommonAnnotations.description }}",
|
||||
"wrap": true
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
{
|
||||
"type": "Action.OpenUrl",
|
||||
"title": "View in Alertmanager",
|
||||
"url": "{{ .ExternalURL }}"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+26
-5
@@ -1,10 +1,10 @@
|
||||
services:
|
||||
# Grafana for visualization
|
||||
grafanao:
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
container_name: grafana
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3003"]
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
@@ -14,12 +14,15 @@ services:
|
||||
- GF_SECURITY_ADMIN_USER=admin
|
||||
- GF_SECURITY_ADMIN_PASSWORD=password # Change this in production
|
||||
- GF_USERS_ALLOW_SIGN_UP=false
|
||||
- GF_UNIFIED_ALERTING_ENABLED=true
|
||||
- GF_ALERTING_ENABLED=false
|
||||
- GF_UNIFIED_ALERTING_PROMETHEUS_ALERTS_ENABLED=true
|
||||
volumes:
|
||||
- grafana-data:/var/lib/grafana
|
||||
- ./config/provisioning/dashboards:/etc/grafana/provisioning/dashboards
|
||||
- ./config/provisioning/datasources:/etc/grafana/provisioning/datasources
|
||||
# Add this new mount to store the actual dashboard JSON
|
||||
- ./config/provisioning/dashboards:/var/lib/grafana/dashboards
|
||||
- ./config/provisioning/alerting:/etc/grafana/provisioning/alerting
|
||||
- ./config/dashboards:/var/lib/grafana/dashboards
|
||||
depends_on:
|
||||
- loki
|
||||
- prometheus
|
||||
@@ -32,7 +35,9 @@ services:
|
||||
container_name: loki
|
||||
ports:
|
||||
- "9020:3100"
|
||||
command: -config.file=/etc/loki/local-config.yaml
|
||||
volumes:
|
||||
- ./config/loki.yml:/etc/loki/local-config.yaml
|
||||
- loki-data:/var/loki
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- monitoring
|
||||
@@ -54,6 +59,18 @@ services:
|
||||
networks:
|
||||
- monitoring
|
||||
|
||||
# AlertManager for handling alerts
|
||||
alertmanager:
|
||||
image: quay.io/prometheus/alertmanager:latest
|
||||
container_name: alertmanager
|
||||
ports:
|
||||
- "9093:9093"
|
||||
volumes:
|
||||
- ./config/alertmanager.yml:/etc/alertmanager/config.yml
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- monitoring
|
||||
|
||||
# Prometheus for metrics collection
|
||||
prometheus:
|
||||
image: prom/prometheus:latest
|
||||
@@ -129,6 +146,10 @@ volumes:
|
||||
prometheus-data:
|
||||
alloy-data:
|
||||
driver: local
|
||||
loki-data:
|
||||
driver: local
|
||||
alertmanager-data:
|
||||
driver: local
|
||||
|
||||
networks:
|
||||
monitoring:
|
||||
|
||||
Reference in New Issue
Block a user