initial commit
This commit is contained in:
@@ -0,0 +1,4 @@
|
|||||||
|
logging {
|
||||||
|
level = "info"
|
||||||
|
format = "logfmt"
|
||||||
|
}
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
auth_enabled: false
|
||||||
|
|
||||||
|
server:
|
||||||
|
http_listen_port: 3100
|
||||||
|
|
||||||
|
common:
|
||||||
|
path_prefix: /var/loki
|
||||||
|
storage:
|
||||||
|
filesystem:
|
||||||
|
chunks_directory: /var/loki/chunks
|
||||||
|
rules_directory: /var/loki/rules
|
||||||
|
ring:
|
||||||
|
kvstore:
|
||||||
|
store: inmemory
|
||||||
|
|
||||||
|
schema_config:
|
||||||
|
configs:
|
||||||
|
- from: 2020-10-24
|
||||||
|
store: boltdb-shipper
|
||||||
|
object_store: filesystem
|
||||||
|
schema: v11
|
||||||
|
index:
|
||||||
|
prefix: index_
|
||||||
|
period: 24h
|
||||||
|
|
||||||
|
storage_config:
|
||||||
|
boltdb_shipper:
|
||||||
|
active_index_directory: /var/loki/index
|
||||||
|
cache_location: /var/loki/cache
|
||||||
|
cache_ttl: 24h
|
||||||
|
filesystem:
|
||||||
|
directory: /var/loki/chunks
|
||||||
|
|
||||||
|
ingester:
|
||||||
|
wal:
|
||||||
|
dir: /var/loki/wal
|
||||||
|
lifecycler:
|
||||||
|
ring:
|
||||||
|
replication_factor: 1
|
||||||
|
|
||||||
|
limits_config:
|
||||||
|
reject_old_samples: true
|
||||||
|
reject_old_samples_max_age: 168h
|
||||||
|
max_entries_limit_per_query: 5000
|
||||||
|
allow_structured_metadata: false
|
||||||
|
|
||||||
|
compactor:
|
||||||
|
working_directory: /var/loki/compactor
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
groups:
|
||||||
|
- name: host_alerts
|
||||||
|
rules:
|
||||||
|
- alert: HostHighCPULoad
|
||||||
|
expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High CPU load on {{ $labels.instance }}"
|
||||||
|
description: "CPU load is > 80% for 5 minutes"
|
||||||
|
|
||||||
|
- alert: HostOutOfMemory
|
||||||
|
expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Host out of memory on {{ $labels.instance }}"
|
||||||
|
description: "Node memory is filling up (< 10% left)"
|
||||||
|
|
||||||
|
- alert: HostOutOfDiskSpace
|
||||||
|
expr: (node_filesystem_avail_bytes / node_filesystem_size_bytes * 100) < 10
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Host out of disk space on {{ $labels.instance }}"
|
||||||
|
description: "Disk is almost full (< 10% left)"
|
||||||
|
|
||||||
|
- name: container_alerts
|
||||||
|
rules:
|
||||||
|
- alert: ContainerHighCPU
|
||||||
|
expr: (sum(rate(container_cpu_usage_seconds_total{name!=""}[3m])) by (name) * 100) > 80
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High CPU usage in container {{ $labels.name }}"
|
||||||
|
description: "Container {{ $labels.name }} CPU usage is > 80% for 5 minutes"
|
||||||
|
|
||||||
|
- alert: ContainerHighMemory
|
||||||
|
expr: (container_memory_usage_bytes{name!=""} / container_spec_memory_limit_bytes{name!=""} * 100) > 80
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High memory usage in container {{ $labels.name }}"
|
||||||
|
description: "Container {{ $labels.name }} memory usage is > 80% for 5 minutes"
|
||||||
|
|
||||||
|
- alert: ContainerDown
|
||||||
|
expr: absent(container_last_seen{name!=""})
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Container {{ $labels.name }} down"
|
||||||
|
description: "Container {{ $labels.name }} has been down for more than 1 minute"
|
||||||
|
|
||||||
|
- alert: ContainerRestarting
|
||||||
|
expr: changes(container_start_time_seconds{name!=""}[15m]) > 2
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Container {{ $labels.name }} restarting frequently"
|
||||||
|
description: "Container {{ $labels.name }} has restarted more than 2 times in the last 15 minutes"
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
global:
|
||||||
|
scrape_interval: 15s
|
||||||
|
evaluation_interval: 15s
|
||||||
|
|
||||||
|
# Rule files
|
||||||
|
rule_files:
|
||||||
|
- 'alerts.yml'
|
||||||
|
|
||||||
|
alerting:
|
||||||
|
alertmanagers:
|
||||||
|
- static_configs:
|
||||||
|
- targets:
|
||||||
|
# We're not setting up Alertmanager in this config
|
||||||
|
# but you can add it later if needed
|
||||||
|
|
||||||
|
scrape_configs:
|
||||||
|
# Prometheus itself
|
||||||
|
- job_name: 'prometheus'
|
||||||
|
static_configs:
|
||||||
|
- targets: ['localhost:9090']
|
||||||
|
|
||||||
|
# Container metrics via cAdvisor
|
||||||
|
- job_name: 'cadvisor'
|
||||||
|
static_configs:
|
||||||
|
- targets: ['cadvisor:8080']
|
||||||
|
metrics_path: /metrics
|
||||||
|
scheme: http
|
||||||
|
|
||||||
|
# Host metrics via node-exporter
|
||||||
|
- job_name: 'node-exporter'
|
||||||
|
static_configs:
|
||||||
|
- targets: ['node-exporter:9100']
|
||||||
|
|
||||||
|
# Loki metrics
|
||||||
|
- job_name: 'loki'
|
||||||
|
static_configs:
|
||||||
|
- targets: ['loki:3100']
|
||||||
|
|
||||||
|
# Grafana metrics
|
||||||
|
- job_name: 'grafana'
|
||||||
|
static_configs:
|
||||||
|
- targets: ['grafana:3000']
|
||||||
|
|
||||||
|
# Alloy metrics
|
||||||
|
- job_name: 'alloy'
|
||||||
|
static_configs:
|
||||||
|
- targets: ['alloy:12345'] # Default metrics port for Alloy
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
services:
|
||||||
|
# Grafana for visualization
|
||||||
|
grafana:
|
||||||
|
image: grafana/grafana:latest
|
||||||
|
container_name: grafana
|
||||||
|
ports:
|
||||||
|
- "3000:3000"
|
||||||
|
environment:
|
||||||
|
- GF_SECURITY_ADMIN_USER=admin
|
||||||
|
- GF_SECURITY_ADMIN_PASSWORD=password # Change this in production
|
||||||
|
- GF_USERS_ALLOW_SIGN_UP=false
|
||||||
|
volumes:
|
||||||
|
- grafana-data:/var/lib/grafana
|
||||||
|
- ./config/provisioning/:/etc/grafana/provisioning
|
||||||
|
depends_on:
|
||||||
|
- loki
|
||||||
|
- prometheus
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- monitoring
|
||||||
|
|
||||||
|
loki:
|
||||||
|
image: grafana/loki:latest
|
||||||
|
container_name: loki
|
||||||
|
ports:
|
||||||
|
- "3100:3100"
|
||||||
|
volumes:
|
||||||
|
- ./config/loki.yml:/etc/loki/local-config.yaml
|
||||||
|
- loki-data:/var/loki
|
||||||
|
command: -config.file=/etc/loki/local-config.yaml
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- monitoring
|
||||||
|
|
||||||
|
alloy:
|
||||||
|
image: grafana/alloy:latest
|
||||||
|
container_name: grafana-alloy
|
||||||
|
ports:
|
||||||
|
- "12345:12345"
|
||||||
|
volumes:
|
||||||
|
- ./config/alloy.conf:/etc/alloy/config.alloy
|
||||||
|
- alloy-data:/var/lib/alloy/data
|
||||||
|
command:
|
||||||
|
- run
|
||||||
|
- --server.http.listen-addr=0.0.0.0:12345
|
||||||
|
- --storage.path=/var/lib/alloy/data
|
||||||
|
- /etc/alloy/config.alloy
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- monitoring
|
||||||
|
|
||||||
|
# Prometheus for metrics collection
|
||||||
|
prometheus:
|
||||||
|
image: prom/prometheus:latest
|
||||||
|
container_name: prometheus
|
||||||
|
ports:
|
||||||
|
- "9090:9090"
|
||||||
|
volumes:
|
||||||
|
- ./config/prometheus.yml:/etc/prometheus/prometheus.yml
|
||||||
|
- ./config/prometheus-alerts.yml:/etc/prometheus/alerts.yml
|
||||||
|
- prometheus-data:/prometheus
|
||||||
|
command:
|
||||||
|
- '--config.file=/etc/prometheus/prometheus.yml'
|
||||||
|
- '--storage.tsdb.path=/prometheus'
|
||||||
|
- '--web.console.libraries=/etc/prometheus/console_libraries'
|
||||||
|
- '--web.console.templates=/etc/prometheus/consoles'
|
||||||
|
- '--web.enable-lifecycle'
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- monitoring
|
||||||
|
|
||||||
|
# cAdvisor for container metrics
|
||||||
|
cadvisor:
|
||||||
|
image: gcr.io/cadvisor/cadvisor:v0.49.1
|
||||||
|
container_name: cadvisor
|
||||||
|
volumes:
|
||||||
|
- /:/rootfs:ro
|
||||||
|
- /var/run:/var/run:ro
|
||||||
|
- /sys:/sys:ro
|
||||||
|
- /var/lib/docker/:/var/lib/docker:ro
|
||||||
|
- /dev/disk/:/dev/disk:ro
|
||||||
|
ports:
|
||||||
|
- "8080:8080"
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- monitoring
|
||||||
|
privileged: true
|
||||||
|
devices:
|
||||||
|
- /dev/kmsg:/dev/kmsg
|
||||||
|
|
||||||
|
# Node Exporter for host metrics
|
||||||
|
node-exporter:
|
||||||
|
image: prom/node-exporter:latest
|
||||||
|
container_name: node-exporter
|
||||||
|
volumes:
|
||||||
|
- /proc:/host/proc:ro
|
||||||
|
- /sys:/host/sys:ro
|
||||||
|
- /:/rootfs:ro
|
||||||
|
command:
|
||||||
|
- '--path.procfs=/host/proc'
|
||||||
|
- '--path.sysfs=/host/sys'
|
||||||
|
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
|
||||||
|
ports:
|
||||||
|
- "9100:9100"
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- monitoring
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
grafana-data:
|
||||||
|
loki-data:
|
||||||
|
loki-wal-data:
|
||||||
|
prometheus-data:
|
||||||
|
alloy-data:
|
||||||
|
driver: local
|
||||||
|
|
||||||
|
networks:
|
||||||
|
monitoring:
|
||||||
|
driver: bridge
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
providers:
|
||||||
|
- name: 'Node Exporter Full'
|
||||||
|
orgId: 1860
|
||||||
|
folder: ''
|
||||||
|
type: file
|
||||||
|
disableDeletion: false
|
||||||
|
updateIntervalSeconds: 10
|
||||||
|
allowUiUpdates: false
|
||||||
|
options:
|
||||||
|
path: /etc/grafana/provisioning/
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
datasources:
|
||||||
|
- name: Prometheus
|
||||||
|
type: prometheus
|
||||||
|
access: proxy
|
||||||
|
url: http://prometheus:9090
|
||||||
|
isDefault: true
|
||||||
|
version: 1
|
||||||
|
editable: true
|
||||||
|
|
||||||
|
- name: Loki
|
||||||
|
type: loki
|
||||||
|
access: proxy
|
||||||
|
url: http://loki:3100
|
||||||
|
version: 1
|
||||||
|
editable: true
|
||||||
|
jsonData:
|
||||||
|
maxLines: 1000
|
||||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user