add alerting
This commit is contained in:
@@ -0,0 +1,22 @@
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
|
||||
route:
|
||||
group_by: ['alertname', 'job']
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 3h
|
||||
receiver: 'teams'
|
||||
|
||||
receivers:
|
||||
- name: 'teams'
|
||||
webhook_configs:
|
||||
- url: 'http://webhook-adapter:8080/alert'
|
||||
send_resolved: true
|
||||
|
||||
inhibit_rules:
|
||||
- source_matchers:
|
||||
- severity = 'critical'
|
||||
target_matchers:
|
||||
- severity = 'warning'
|
||||
equal: ['alertname']
|
||||
@@ -40,14 +40,23 @@ groups:
|
||||
description: "Container {{ $labels.name }} CPU usage is > 80% for 5 minutes"
|
||||
|
||||
- alert: ContainerHighMemory
|
||||
expr: (container_memory_usage_bytes{name!=""} / container_spec_memory_limit_bytes{name!=""} * 100) > 80
|
||||
for: 5m
|
||||
# First filter out containers with memory limits, then calculate percentage
|
||||
expr: |
|
||||
(
|
||||
container_memory_usage_bytes{name!=""}
|
||||
/
|
||||
container_spec_memory_limit_bytes{name!=""}
|
||||
* 100 > 90
|
||||
)
|
||||
and
|
||||
container_spec_memory_limit_bytes{name!=""} > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High memory usage in container {{ $labels.name }}"
|
||||
description: "Container {{ $labels.name }} memory usage is > 80% for 5 minutes"
|
||||
|
||||
description: "Container {{ $labels.name }} ({{ $labels.image }}) memory usage is {{ printf \"%.1f\" $value }}% of its limit for 10 minutes"
|
||||
|
||||
- alert: ContainerDown
|
||||
expr: absent(container_last_seen{name!=""})
|
||||
for: 1m
|
||||
@@ -65,3 +74,4 @@ groups:
|
||||
annotations:
|
||||
summary: "Container {{ $labels.name }} restarting frequently"
|
||||
description: "Container {{ $labels.name }} has restarted more than 2 times in the last 15 minutes"
|
||||
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
apiVersion: 1
|
||||
|
||||
contactPoints:
|
||||
- name: ms-teams
|
||||
receivers:
|
||||
- uid: teams-notifications
|
||||
type: teams
|
||||
settings:
|
||||
url: https://prod-152.westeurope.logic.azure.com:443/workflows/6040474cfb314615aabd47694804d7e7/triggers/manual/paths/invoke?api-version=2016-06-01&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0&sig=ehfajgttd-2NC7rdEL5z9rylE4IiyVpgPjiOfDqF_z0
|
||||
title: "{{ if eq .Status \"firing\" }}🔥{{ else }}✅{{ end }} [{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}"
|
||||
message: |-
|
||||
{{ if ne .Status "firing" }}Resolved{{ else }}{{ end }}: {{ .CommonAnnotations.summary }}
|
||||
|
||||
{{ .CommonAnnotations.description }}
|
||||
|
||||
{{- if gt (len .Alerts.Firing) 0 }}
|
||||
Alerts:
|
||||
{{- range .Alerts.Firing }}
|
||||
- {{ .Labels.instance }} {{ .Labels.job }}: {{ .Annotations.summary }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: 1
|
||||
|
||||
policies:
|
||||
- receiver: ms-teams
|
||||
group_by: ['alertname', 'job']
|
||||
repeat_interval: 3h
|
||||
group_interval: 5m
|
||||
routes:
|
||||
- receiver: ms-teams
|
||||
matchers:
|
||||
- severity =~ "critical|warning|info"
|
||||
continue: true
|
||||
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"type": "message",
|
||||
"attachments": [
|
||||
{
|
||||
"contentType": "application/vnd.microsoft.card.adaptive",
|
||||
"content": {
|
||||
"$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
|
||||
"type": "AdaptiveCard",
|
||||
"version": "1.3",
|
||||
"body": [
|
||||
{
|
||||
"type": "TextBlock",
|
||||
"text": "{{ if eq .Status "firing" }}🔥{{ else }}✅{{ end }} {{ .Status | title }}: {{ .CommonLabels.alertname }}",
|
||||
"weight": "Bolder",
|
||||
"size": "Large"
|
||||
},
|
||||
{
|
||||
"type": "TextBlock",
|
||||
"text": "{{ .CommonAnnotations.summary }}",
|
||||
"wrap": true
|
||||
},
|
||||
{
|
||||
"type": "FactSet",
|
||||
"facts": [
|
||||
{
|
||||
"title": "Status",
|
||||
"value": "{{ .Status | title }}"
|
||||
},
|
||||
{
|
||||
"title": "Severity",
|
||||
"value": "{{ .CommonLabels.severity | title }}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "TextBlock",
|
||||
"text": "{{ .CommonAnnotations.description }}",
|
||||
"wrap": true
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
{
|
||||
"type": "Action.OpenUrl",
|
||||
"title": "View in Alertmanager",
|
||||
"url": "{{ .ExternalURL }}"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user