initial commit
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
logging {
|
||||
level = "info"
|
||||
format = "logfmt"
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
|
||||
common:
|
||||
path_prefix: /var/loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /var/loki/chunks
|
||||
rules_directory: /var/loki/rules
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2020-10-24
|
||||
store: boltdb-shipper
|
||||
object_store: filesystem
|
||||
schema: v11
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
storage_config:
|
||||
boltdb_shipper:
|
||||
active_index_directory: /var/loki/index
|
||||
cache_location: /var/loki/cache
|
||||
cache_ttl: 24h
|
||||
filesystem:
|
||||
directory: /var/loki/chunks
|
||||
|
||||
ingester:
|
||||
wal:
|
||||
dir: /var/loki/wal
|
||||
lifecycler:
|
||||
ring:
|
||||
replication_factor: 1
|
||||
|
||||
limits_config:
|
||||
reject_old_samples: true
|
||||
reject_old_samples_max_age: 168h
|
||||
max_entries_limit_per_query: 5000
|
||||
allow_structured_metadata: false
|
||||
|
||||
compactor:
|
||||
working_directory: /var/loki/compactor
|
||||
@@ -0,0 +1,67 @@
|
||||
groups:
|
||||
- name: host_alerts
|
||||
rules:
|
||||
- alert: HostHighCPULoad
|
||||
expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High CPU load on {{ $labels.instance }}"
|
||||
description: "CPU load is > 80% for 5 minutes"
|
||||
|
||||
- alert: HostOutOfMemory
|
||||
expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Host out of memory on {{ $labels.instance }}"
|
||||
description: "Node memory is filling up (< 10% left)"
|
||||
|
||||
- alert: HostOutOfDiskSpace
|
||||
expr: (node_filesystem_avail_bytes / node_filesystem_size_bytes * 100) < 10
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Host out of disk space on {{ $labels.instance }}"
|
||||
description: "Disk is almost full (< 10% left)"
|
||||
|
||||
- name: container_alerts
|
||||
rules:
|
||||
- alert: ContainerHighCPU
|
||||
expr: (sum(rate(container_cpu_usage_seconds_total{name!=""}[3m])) by (name) * 100) > 80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High CPU usage in container {{ $labels.name }}"
|
||||
description: "Container {{ $labels.name }} CPU usage is > 80% for 5 minutes"
|
||||
|
||||
- alert: ContainerHighMemory
|
||||
expr: (container_memory_usage_bytes{name!=""} / container_spec_memory_limit_bytes{name!=""} * 100) > 80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High memory usage in container {{ $labels.name }}"
|
||||
description: "Container {{ $labels.name }} memory usage is > 80% for 5 minutes"
|
||||
|
||||
- alert: ContainerDown
|
||||
expr: absent(container_last_seen{name!=""})
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Container {{ $labels.name }} down"
|
||||
description: "Container {{ $labels.name }} has been down for more than 1 minute"
|
||||
|
||||
- alert: ContainerRestarting
|
||||
expr: changes(container_start_time_seconds{name!=""}[15m]) > 2
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Container {{ $labels.name }} restarting frequently"
|
||||
description: "Container {{ $labels.name }} has restarted more than 2 times in the last 15 minutes"
|
||||
@@ -0,0 +1,47 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
# Rule files
|
||||
rule_files:
|
||||
- 'alerts.yml'
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets:
|
||||
# We're not setting up Alertmanager in this config
|
||||
# but you can add it later if needed
|
||||
|
||||
scrape_configs:
|
||||
# Prometheus itself
|
||||
- job_name: 'prometheus'
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
|
||||
# Container metrics via cAdvisor
|
||||
- job_name: 'cadvisor'
|
||||
static_configs:
|
||||
- targets: ['cadvisor:8080']
|
||||
metrics_path: /metrics
|
||||
scheme: http
|
||||
|
||||
# Host metrics via node-exporter
|
||||
- job_name: 'node-exporter'
|
||||
static_configs:
|
||||
- targets: ['node-exporter:9100']
|
||||
|
||||
# Loki metrics
|
||||
- job_name: 'loki'
|
||||
static_configs:
|
||||
- targets: ['loki:3100']
|
||||
|
||||
# Grafana metrics
|
||||
- job_name: 'grafana'
|
||||
static_configs:
|
||||
- targets: ['grafana:3000']
|
||||
|
||||
# Alloy metrics
|
||||
- job_name: 'alloy'
|
||||
static_configs:
|
||||
- targets: ['alloy:12345'] # Default metrics port for Alloy
|
||||
Reference in New Issue
Block a user