initial commit
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
logging {
|
||||
level = "info"
|
||||
format = "logfmt"
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
|
||||
common:
|
||||
path_prefix: /var/loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /var/loki/chunks
|
||||
rules_directory: /var/loki/rules
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2020-10-24
|
||||
store: boltdb-shipper
|
||||
object_store: filesystem
|
||||
schema: v11
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
storage_config:
|
||||
boltdb_shipper:
|
||||
active_index_directory: /var/loki/index
|
||||
cache_location: /var/loki/cache
|
||||
cache_ttl: 24h
|
||||
filesystem:
|
||||
directory: /var/loki/chunks
|
||||
|
||||
ingester:
|
||||
wal:
|
||||
dir: /var/loki/wal
|
||||
lifecycler:
|
||||
ring:
|
||||
replication_factor: 1
|
||||
|
||||
limits_config:
|
||||
reject_old_samples: true
|
||||
reject_old_samples_max_age: 168h
|
||||
max_entries_limit_per_query: 5000
|
||||
allow_structured_metadata: false
|
||||
|
||||
compactor:
|
||||
working_directory: /var/loki/compactor
|
||||
@@ -0,0 +1,67 @@
|
||||
groups:
|
||||
- name: host_alerts
|
||||
rules:
|
||||
- alert: HostHighCPULoad
|
||||
expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High CPU load on {{ $labels.instance }}"
|
||||
description: "CPU load is > 80% for 5 minutes"
|
||||
|
||||
- alert: HostOutOfMemory
|
||||
expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Host out of memory on {{ $labels.instance }}"
|
||||
description: "Node memory is filling up (< 10% left)"
|
||||
|
||||
- alert: HostOutOfDiskSpace
|
||||
expr: (node_filesystem_avail_bytes / node_filesystem_size_bytes * 100) < 10
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Host out of disk space on {{ $labels.instance }}"
|
||||
description: "Disk is almost full (< 10% left)"
|
||||
|
||||
- name: container_alerts
|
||||
rules:
|
||||
- alert: ContainerHighCPU
|
||||
expr: (sum(rate(container_cpu_usage_seconds_total{name!=""}[3m])) by (name) * 100) > 80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High CPU usage in container {{ $labels.name }}"
|
||||
description: "Container {{ $labels.name }} CPU usage is > 80% for 5 minutes"
|
||||
|
||||
- alert: ContainerHighMemory
|
||||
expr: (container_memory_usage_bytes{name!=""} / container_spec_memory_limit_bytes{name!=""} * 100) > 80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High memory usage in container {{ $labels.name }}"
|
||||
description: "Container {{ $labels.name }} memory usage is > 80% for 5 minutes"
|
||||
|
||||
- alert: ContainerDown
|
||||
expr: absent(container_last_seen{name!=""})
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Container {{ $labels.name }} down"
|
||||
description: "Container {{ $labels.name }} has been down for more than 1 minute"
|
||||
|
||||
- alert: ContainerRestarting
|
||||
expr: changes(container_start_time_seconds{name!=""}[15m]) > 2
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Container {{ $labels.name }} restarting frequently"
|
||||
description: "Container {{ $labels.name }} has restarted more than 2 times in the last 15 minutes"
|
||||
@@ -0,0 +1,47 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
# Rule files
|
||||
rule_files:
|
||||
- 'alerts.yml'
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets:
|
||||
# We're not setting up Alertmanager in this config
|
||||
# but you can add it later if needed
|
||||
|
||||
scrape_configs:
|
||||
# Prometheus itself
|
||||
- job_name: 'prometheus'
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
|
||||
# Container metrics via cAdvisor
|
||||
- job_name: 'cadvisor'
|
||||
static_configs:
|
||||
- targets: ['cadvisor:8080']
|
||||
metrics_path: /metrics
|
||||
scheme: http
|
||||
|
||||
# Host metrics via node-exporter
|
||||
- job_name: 'node-exporter'
|
||||
static_configs:
|
||||
- targets: ['node-exporter:9100']
|
||||
|
||||
# Loki metrics
|
||||
- job_name: 'loki'
|
||||
static_configs:
|
||||
- targets: ['loki:3100']
|
||||
|
||||
# Grafana metrics
|
||||
- job_name: 'grafana'
|
||||
static_configs:
|
||||
- targets: ['grafana:3000']
|
||||
|
||||
# Alloy metrics
|
||||
- job_name: 'alloy'
|
||||
static_configs:
|
||||
- targets: ['alloy:12345'] # Default metrics port for Alloy
|
||||
@@ -0,0 +1,119 @@
|
||||
services:
|
||||
# Grafana for visualization
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
container_name: grafana
|
||||
ports:
|
||||
- "3000:3000"
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_USER=admin
|
||||
- GF_SECURITY_ADMIN_PASSWORD=password # Change this in production
|
||||
- GF_USERS_ALLOW_SIGN_UP=false
|
||||
volumes:
|
||||
- grafana-data:/var/lib/grafana
|
||||
- ./config/provisioning/:/etc/grafana/provisioning
|
||||
depends_on:
|
||||
- loki
|
||||
- prometheus
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- monitoring
|
||||
|
||||
loki:
|
||||
image: grafana/loki:latest
|
||||
container_name: loki
|
||||
ports:
|
||||
- "3100:3100"
|
||||
volumes:
|
||||
- ./config/loki.yml:/etc/loki/local-config.yaml
|
||||
- loki-data:/var/loki
|
||||
command: -config.file=/etc/loki/local-config.yaml
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- monitoring
|
||||
|
||||
alloy:
|
||||
image: grafana/alloy:latest
|
||||
container_name: grafana-alloy
|
||||
ports:
|
||||
- "12345:12345"
|
||||
volumes:
|
||||
- ./config/alloy.conf:/etc/alloy/config.alloy
|
||||
- alloy-data:/var/lib/alloy/data
|
||||
command:
|
||||
- run
|
||||
- --server.http.listen-addr=0.0.0.0:12345
|
||||
- --storage.path=/var/lib/alloy/data
|
||||
- /etc/alloy/config.alloy
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- monitoring
|
||||
|
||||
# Prometheus for metrics collection
|
||||
prometheus:
|
||||
image: prom/prometheus:latest
|
||||
container_name: prometheus
|
||||
ports:
|
||||
- "9090:9090"
|
||||
volumes:
|
||||
- ./config/prometheus.yml:/etc/prometheus/prometheus.yml
|
||||
- ./config/prometheus-alerts.yml:/etc/prometheus/alerts.yml
|
||||
- prometheus-data:/prometheus
|
||||
command:
|
||||
- '--config.file=/etc/prometheus/prometheus.yml'
|
||||
- '--storage.tsdb.path=/prometheus'
|
||||
- '--web.console.libraries=/etc/prometheus/console_libraries'
|
||||
- '--web.console.templates=/etc/prometheus/consoles'
|
||||
- '--web.enable-lifecycle'
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- monitoring
|
||||
|
||||
# cAdvisor for container metrics
|
||||
cadvisor:
|
||||
image: gcr.io/cadvisor/cadvisor:v0.49.1
|
||||
container_name: cadvisor
|
||||
volumes:
|
||||
- /:/rootfs:ro
|
||||
- /var/run:/var/run:ro
|
||||
- /sys:/sys:ro
|
||||
- /var/lib/docker/:/var/lib/docker:ro
|
||||
- /dev/disk/:/dev/disk:ro
|
||||
ports:
|
||||
- "8080:8080"
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- monitoring
|
||||
privileged: true
|
||||
devices:
|
||||
- /dev/kmsg:/dev/kmsg
|
||||
|
||||
# Node Exporter for host metrics
|
||||
node-exporter:
|
||||
image: prom/node-exporter:latest
|
||||
container_name: node-exporter
|
||||
volumes:
|
||||
- /proc:/host/proc:ro
|
||||
- /sys:/host/sys:ro
|
||||
- /:/rootfs:ro
|
||||
command:
|
||||
- '--path.procfs=/host/proc'
|
||||
- '--path.sysfs=/host/sys'
|
||||
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
|
||||
ports:
|
||||
- "9100:9100"
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- monitoring
|
||||
|
||||
volumes:
|
||||
grafana-data:
|
||||
loki-data:
|
||||
loki-wal-data:
|
||||
prometheus-data:
|
||||
alloy-data:
|
||||
driver: local
|
||||
|
||||
networks:
|
||||
monitoring:
|
||||
driver: bridge
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: 'Node Exporter Full'
|
||||
orgId: 1860
|
||||
folder: ''
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 10
|
||||
allowUiUpdates: false
|
||||
options:
|
||||
path: /etc/grafana/provisioning/
|
||||
@@ -0,0 +1,19 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
version: 1
|
||||
editable: true
|
||||
|
||||
- name: Loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
version: 1
|
||||
editable: true
|
||||
jsonData:
|
||||
maxLines: 1000
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user