78 lines
2.7 KiB
YAML
78 lines
2.7 KiB
YAML
groups:
|
|
- name: host_alerts
|
|
rules:
|
|
- alert: HostHighCPULoad
|
|
expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High CPU load on {{ $labels.instance }}"
|
|
description: "CPU load is > 80% for 5 minutes"
|
|
|
|
- alert: HostOutOfMemory
|
|
expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Host out of memory on {{ $labels.instance }}"
|
|
description: "Node memory is filling up (< 10% left)"
|
|
|
|
- alert: HostOutOfDiskSpace
|
|
expr: (node_filesystem_avail_bytes / node_filesystem_size_bytes * 100) < 10
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Host out of disk space on {{ $labels.instance }}"
|
|
description: "Disk is almost full (< 10% left)"
|
|
|
|
- name: container_alerts
|
|
rules:
|
|
- alert: ContainerHighCPU
|
|
expr: (sum(rate(container_cpu_usage_seconds_total{name!=""}[3m])) by (name) * 100) > 80
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High CPU usage in container {{ $labels.name }}"
|
|
description: "Container {{ $labels.name }} CPU usage is > 80% for 5 minutes"
|
|
|
|
- alert: ContainerHighMemory
|
|
# First filter out containers with memory limits, then calculate percentage
|
|
expr: |
|
|
(
|
|
container_memory_usage_bytes{name!=""}
|
|
/
|
|
container_spec_memory_limit_bytes{name!=""}
|
|
* 100 > 90
|
|
)
|
|
and
|
|
container_spec_memory_limit_bytes{name!=""} > 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High memory usage in container {{ $labels.name }}"
|
|
description: "Container {{ $labels.name }} ({{ $labels.image }}) memory usage is {{ printf \"%.1f\" $value }}% of its limit for 10 minutes"
|
|
|
|
- alert: ContainerDown
|
|
expr: absent(container_last_seen{name!=""})
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Container {{ $labels.name }} down"
|
|
description: "Container {{ $labels.name }} has been down for more than 1 minute"
|
|
|
|
- alert: ContainerRestarting
|
|
expr: changes(container_start_time_seconds{name!=""}[15m]) > 2
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Container {{ $labels.name }} restarting frequently"
|
|
description: "Container {{ $labels.name }} has restarted more than 2 times in the last 15 minutes"
|
|
|