fix container surveillance
This commit is contained in:
@@ -0,0 +1,22 @@
|
|||||||
|
global:
|
||||||
|
resolve_timeout: 5m
|
||||||
|
|
||||||
|
route:
|
||||||
|
group_by: ['alertname', 'job']
|
||||||
|
group_wait: 30s
|
||||||
|
group_interval: 5m
|
||||||
|
repeat_interval: 3h
|
||||||
|
receiver: 'teams'
|
||||||
|
|
||||||
|
receivers:
|
||||||
|
- name: 'teams'
|
||||||
|
webhook_configs:
|
||||||
|
- url: 'http://webhook-adapter:8080/alert'
|
||||||
|
send_resolved: true
|
||||||
|
|
||||||
|
inhibit_rules:
|
||||||
|
- source_matchers:
|
||||||
|
- severity = 'critical'
|
||||||
|
target_matchers:
|
||||||
|
- severity = 'warning'
|
||||||
|
equal: ['alertname']
|
||||||
@@ -40,6 +40,7 @@ groups:
|
|||||||
description: "Container {{ $labels.name }} CPU usage is > 80% for 5 minutes"
|
description: "Container {{ $labels.name }} CPU usage is > 80% for 5 minutes"
|
||||||
|
|
||||||
- alert: ContainerHighMemory
|
- alert: ContainerHighMemory
|
||||||
|
# First filter out containers with memory limits, then calculate percentage
|
||||||
expr: |
|
expr: |
|
||||||
(
|
(
|
||||||
container_memory_usage_bytes{name!=""}
|
container_memory_usage_bytes{name!=""}
|
||||||
@@ -73,3 +74,4 @@ groups:
|
|||||||
annotations:
|
annotations:
|
||||||
summary: "Container {{ $labels.name }} restarting frequently"
|
summary: "Container {{ $labels.name }} restarting frequently"
|
||||||
description: "Container {{ $labels.name }} has restarted more than 2 times in the last 15 minutes"
|
description: "Container {{ $labels.name }} has restarted more than 2 times in the last 15 minutes"
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
contactPoints:
|
||||||
|
- name: ms-teams
|
||||||
|
receivers:
|
||||||
|
- uid: teams-notifications
|
||||||
|
type: teams
|
||||||
|
settings:
|
||||||
|
url: https://prod-152.westeurope.logic.azure.com:443/workflows/6040474cfb314615aabd47694804d7e7/triggers/manual/paths/invoke?api-version=2016-06-01&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0&sig=ehfajgttd-2NC7rdEL5z9rylE4IiyVpgPjiOfDqF_z0
|
||||||
|
title: "{{ if eq .Status \"firing\" }}🔥{{ else }}✅{{ end }} [{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}"
|
||||||
|
message: |-
|
||||||
|
{{ if ne .Status "firing" }}Resolved{{ else }}{{ end }}: {{ .CommonAnnotations.summary }}
|
||||||
|
|
||||||
|
{{ .CommonAnnotations.description }}
|
||||||
|
|
||||||
|
{{- if gt (len .Alerts.Firing) 0 }}
|
||||||
|
Alerts:
|
||||||
|
{{- range .Alerts.Firing }}
|
||||||
|
- {{ .Labels.instance }} {{ .Labels.job }}: {{ .Annotations.summary }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
policies:
|
||||||
|
- receiver: ms-teams
|
||||||
|
group_by: ['alertname', 'job']
|
||||||
|
repeat_interval: 3h
|
||||||
|
group_interval: 5m
|
||||||
|
routes:
|
||||||
|
- receiver: ms-teams
|
||||||
|
matchers:
|
||||||
|
- severity =~ "critical|warning|info"
|
||||||
|
continue: true
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
{
|
||||||
|
"type": "message",
|
||||||
|
"attachments": [
|
||||||
|
{
|
||||||
|
"contentType": "application/vnd.microsoft.card.adaptive",
|
||||||
|
"content": {
|
||||||
|
"$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
|
||||||
|
"type": "AdaptiveCard",
|
||||||
|
"version": "1.3",
|
||||||
|
"body": [
|
||||||
|
{
|
||||||
|
"type": "TextBlock",
|
||||||
|
"text": "{{ if eq .Status "firing" }}🔥{{ else }}✅{{ end }} {{ .Status | title }}: {{ .CommonLabels.alertname }}",
|
||||||
|
"weight": "Bolder",
|
||||||
|
"size": "Large"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "TextBlock",
|
||||||
|
"text": "{{ .CommonAnnotations.summary }}",
|
||||||
|
"wrap": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "FactSet",
|
||||||
|
"facts": [
|
||||||
|
{
|
||||||
|
"title": "Status",
|
||||||
|
"value": "{{ .Status | title }}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "Severity",
|
||||||
|
"value": "{{ .CommonLabels.severity | title }}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "TextBlock",
|
||||||
|
"text": "{{ .CommonAnnotations.description }}",
|
||||||
|
"wrap": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"actions": [
|
||||||
|
{
|
||||||
|
"type": "Action.OpenUrl",
|
||||||
|
"title": "View in Alertmanager",
|
||||||
|
"url": "{{ .ExternalURL }}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
+26
-5
@@ -1,10 +1,10 @@
|
|||||||
services:
|
services:
|
||||||
# Grafana for visualization
|
# Grafana for visualization
|
||||||
grafanao:
|
grafana:
|
||||||
image: grafana/grafana:latest
|
image: grafana/grafana:latest
|
||||||
container_name: grafana
|
container_name: grafana
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3003"]
|
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000"]
|
||||||
interval: 30s
|
interval: 30s
|
||||||
timeout: 10s
|
timeout: 10s
|
||||||
retries: 3
|
retries: 3
|
||||||
@@ -14,12 +14,15 @@ services:
|
|||||||
- GF_SECURITY_ADMIN_USER=admin
|
- GF_SECURITY_ADMIN_USER=admin
|
||||||
- GF_SECURITY_ADMIN_PASSWORD=password # Change this in production
|
- GF_SECURITY_ADMIN_PASSWORD=password # Change this in production
|
||||||
- GF_USERS_ALLOW_SIGN_UP=false
|
- GF_USERS_ALLOW_SIGN_UP=false
|
||||||
|
- GF_UNIFIED_ALERTING_ENABLED=true
|
||||||
|
- GF_ALERTING_ENABLED=false
|
||||||
|
- GF_UNIFIED_ALERTING_PROMETHEUS_ALERTS_ENABLED=true
|
||||||
volumes:
|
volumes:
|
||||||
- grafana-data:/var/lib/grafana
|
- grafana-data:/var/lib/grafana
|
||||||
- ./config/provisioning/dashboards:/etc/grafana/provisioning/dashboards
|
- ./config/provisioning/dashboards:/etc/grafana/provisioning/dashboards
|
||||||
- ./config/provisioning/datasources:/etc/grafana/provisioning/datasources
|
- ./config/provisioning/datasources:/etc/grafana/provisioning/datasources
|
||||||
# Add this new mount to store the actual dashboard JSON
|
- ./config/provisioning/alerting:/etc/grafana/provisioning/alerting
|
||||||
- ./config/provisioning/dashboards:/var/lib/grafana/dashboards
|
- ./config/dashboards:/var/lib/grafana/dashboards
|
||||||
depends_on:
|
depends_on:
|
||||||
- loki
|
- loki
|
||||||
- prometheus
|
- prometheus
|
||||||
@@ -32,7 +35,9 @@ services:
|
|||||||
container_name: loki
|
container_name: loki
|
||||||
ports:
|
ports:
|
||||||
- "9020:3100"
|
- "9020:3100"
|
||||||
command: -config.file=/etc/loki/local-config.yaml
|
volumes:
|
||||||
|
- ./config/loki.yml:/etc/loki/local-config.yaml
|
||||||
|
- loki-data:/var/loki
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
networks:
|
networks:
|
||||||
- monitoring
|
- monitoring
|
||||||
@@ -54,6 +59,18 @@ services:
|
|||||||
networks:
|
networks:
|
||||||
- monitoring
|
- monitoring
|
||||||
|
|
||||||
|
# AlertManager for handling alerts
|
||||||
|
alertmanager:
|
||||||
|
image: quay.io/prometheus/alertmanager:latest
|
||||||
|
container_name: alertmanager
|
||||||
|
ports:
|
||||||
|
- "9093:9093"
|
||||||
|
volumes:
|
||||||
|
- ./config/alertmanager.yml:/etc/alertmanager/config.yml
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- monitoring
|
||||||
|
|
||||||
# Prometheus for metrics collection
|
# Prometheus for metrics collection
|
||||||
prometheus:
|
prometheus:
|
||||||
image: prom/prometheus:latest
|
image: prom/prometheus:latest
|
||||||
@@ -129,6 +146,10 @@ volumes:
|
|||||||
prometheus-data:
|
prometheus-data:
|
||||||
alloy-data:
|
alloy-data:
|
||||||
driver: local
|
driver: local
|
||||||
|
loki-data:
|
||||||
|
driver: local
|
||||||
|
alertmanager-data:
|
||||||
|
driver: local
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
monitoring:
|
monitoring:
|
||||||
|
|||||||
Reference in New Issue
Block a user