add alerting

This commit is contained in:
Your Name
2025-03-17 16:06:35 +00:00
parent d0ca70c546
commit 581f35cad3
6 changed files with 146 additions and 9 deletions
+22
View File
@@ -0,0 +1,22 @@
global:
resolve_timeout: 5m
route:
group_by: ['alertname', 'job']
group_wait: 30s
group_interval: 5m
repeat_interval: 3h
receiver: 'teams'
receivers:
- name: 'teams'
webhook_configs:
- url: 'http://webhook-adapter:8080/alert'
send_resolved: true
inhibit_rules:
- source_matchers:
- severity = 'critical'
target_matchers:
- severity = 'warning'
equal: ['alertname']
+14 -4
View File
@@ -40,14 +40,23 @@ groups:
description: "Container {{ $labels.name }} CPU usage is > 80% for 5 minutes"
- alert: ContainerHighMemory
expr: (container_memory_usage_bytes{name!=""} / container_spec_memory_limit_bytes{name!=""} * 100) > 80
for: 5m
# First filter out containers with memory limits, then calculate percentage
expr: |
(
container_memory_usage_bytes{name!=""}
/
container_spec_memory_limit_bytes{name!=""}
* 100 > 90
)
and
container_spec_memory_limit_bytes{name!=""} > 0
for: 10m
labels:
severity: warning
annotations:
summary: "High memory usage in container {{ $labels.name }}"
description: "Container {{ $labels.name }} memory usage is > 80% for 5 minutes"
description: "Container {{ $labels.name }} ({{ $labels.image }}) memory usage is {{ printf \"%.1f\" $value }}% of its limit for 10 minutes"
- alert: ContainerDown
expr: absent(container_last_seen{name!=""})
for: 1m
@@ -65,3 +74,4 @@ groups:
annotations:
summary: "Container {{ $labels.name }} restarting frequently"
description: "Container {{ $labels.name }} has restarted more than 2 times in the last 15 minutes"
@@ -0,0 +1,21 @@
apiVersion: 1
contactPoints:
- name: ms-teams
receivers:
- uid: teams-notifications
type: teams
settings:
url: https://prod-152.westeurope.logic.azure.com:443/workflows/6040474cfb314615aabd47694804d7e7/triggers/manual/paths/invoke?api-version=2016-06-01&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0&sig=ehfajgttd-2NC7rdEL5z9rylE4IiyVpgPjiOfDqF_z0
title: "{{ if eq .Status \"firing\" }}🔥{{ else }}✅{{ end }} [{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}"
message: |-
{{ if ne .Status "firing" }}Resolved{{ else }}{{ end }}: {{ .CommonAnnotations.summary }}
{{ .CommonAnnotations.description }}
{{- if gt (len .Alerts.Firing) 0 }}
Alerts:
{{- range .Alerts.Firing }}
- {{ .Labels.instance }} {{ .Labels.job }}: {{ .Annotations.summary }}
{{- end }}
{{- end }}
@@ -0,0 +1,12 @@
apiVersion: 1
policies:
- receiver: ms-teams
group_by: ['alertname', 'job']
repeat_interval: 3h
group_interval: 5m
routes:
- receiver: ms-teams
matchers:
- severity =~ "critical|warning|info"
continue: true
+51
View File
@@ -0,0 +1,51 @@
{
"type": "message",
"attachments": [
{
"contentType": "application/vnd.microsoft.card.adaptive",
"content": {
"$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
"type": "AdaptiveCard",
"version": "1.3",
"body": [
{
"type": "TextBlock",
"text": "{{ if eq .Status "firing" }}🔥{{ else }}✅{{ end }} {{ .Status | title }}: {{ .CommonLabels.alertname }}",
"weight": "Bolder",
"size": "Large"
},
{
"type": "TextBlock",
"text": "{{ .CommonAnnotations.summary }}",
"wrap": true
},
{
"type": "FactSet",
"facts": [
{
"title": "Status",
"value": "{{ .Status | title }}"
},
{
"title": "Severity",
"value": "{{ .CommonLabels.severity | title }}"
}
]
},
{
"type": "TextBlock",
"text": "{{ .CommonAnnotations.description }}",
"wrap": true
}
],
"actions": [
{
"type": "Action.OpenUrl",
"title": "View in Alertmanager",
"url": "{{ .ExternalURL }}"
}
]
}
}
]
}