initial commit

This commit is contained in:
Bartal Læarsson
2025-03-08 23:09:54 +00:00
commit 54b6198f14
8 changed files with 14433 additions and 0 deletions
+4
View File
@@ -0,0 +1,4 @@
logging {
level = "info"
format = "logfmt"
}
+48
View File
@@ -0,0 +1,48 @@
auth_enabled: false
server:
http_listen_port: 3100
common:
path_prefix: /var/loki
storage:
filesystem:
chunks_directory: /var/loki/chunks
rules_directory: /var/loki/rules
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: 2020-10-24
store: boltdb-shipper
object_store: filesystem
schema: v11
index:
prefix: index_
period: 24h
storage_config:
boltdb_shipper:
active_index_directory: /var/loki/index
cache_location: /var/loki/cache
cache_ttl: 24h
filesystem:
directory: /var/loki/chunks
ingester:
wal:
dir: /var/loki/wal
lifecycler:
ring:
replication_factor: 1
limits_config:
reject_old_samples: true
reject_old_samples_max_age: 168h
max_entries_limit_per_query: 5000
allow_structured_metadata: false
compactor:
working_directory: /var/loki/compactor
+67
View File
@@ -0,0 +1,67 @@
groups:
- name: host_alerts
rules:
- alert: HostHighCPULoad
expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
for: 5m
labels:
severity: warning
annotations:
summary: "High CPU load on {{ $labels.instance }}"
description: "CPU load is > 80% for 5 minutes"
- alert: HostOutOfMemory
expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
for: 5m
labels:
severity: warning
annotations:
summary: "Host out of memory on {{ $labels.instance }}"
description: "Node memory is filling up (< 10% left)"
- alert: HostOutOfDiskSpace
expr: (node_filesystem_avail_bytes / node_filesystem_size_bytes * 100) < 10
for: 5m
labels:
severity: warning
annotations:
summary: "Host out of disk space on {{ $labels.instance }}"
description: "Disk is almost full (< 10% left)"
- name: container_alerts
rules:
- alert: ContainerHighCPU
expr: (sum(rate(container_cpu_usage_seconds_total{name!=""}[3m])) by (name) * 100) > 80
for: 5m
labels:
severity: warning
annotations:
summary: "High CPU usage in container {{ $labels.name }}"
description: "Container {{ $labels.name }} CPU usage is > 80% for 5 minutes"
- alert: ContainerHighMemory
expr: (container_memory_usage_bytes{name!=""} / container_spec_memory_limit_bytes{name!=""} * 100) > 80
for: 5m
labels:
severity: warning
annotations:
summary: "High memory usage in container {{ $labels.name }}"
description: "Container {{ $labels.name }} memory usage is > 80% for 5 minutes"
- alert: ContainerDown
expr: absent(container_last_seen{name!=""})
for: 1m
labels:
severity: critical
annotations:
summary: "Container {{ $labels.name }} down"
description: "Container {{ $labels.name }} has been down for more than 1 minute"
- alert: ContainerRestarting
expr: changes(container_start_time_seconds{name!=""}[15m]) > 2
for: 5m
labels:
severity: warning
annotations:
summary: "Container {{ $labels.name }} restarting frequently"
description: "Container {{ $labels.name }} has restarted more than 2 times in the last 15 minutes"
+47
View File
@@ -0,0 +1,47 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
# Rule files
rule_files:
- 'alerts.yml'
alerting:
alertmanagers:
- static_configs:
- targets:
# We're not setting up Alertmanager in this config
# but you can add it later if needed
scrape_configs:
# Prometheus itself
- job_name: 'prometheus'
static_configs:
- targets: ['localhost:9090']
# Container metrics via cAdvisor
- job_name: 'cadvisor'
static_configs:
- targets: ['cadvisor:8080']
metrics_path: /metrics
scheme: http
# Host metrics via node-exporter
- job_name: 'node-exporter'
static_configs:
- targets: ['node-exporter:9100']
# Loki metrics
- job_name: 'loki'
static_configs:
- targets: ['loki:3100']
# Grafana metrics
- job_name: 'grafana'
static_configs:
- targets: ['grafana:3000']
# Alloy metrics
- job_name: 'alloy'
static_configs:
- targets: ['alloy:12345'] # Default metrics port for Alloy
+119
View File
@@ -0,0 +1,119 @@
services:
# Grafana for visualization
grafana:
image: grafana/grafana:latest
container_name: grafana
ports:
- "3000:3000"
environment:
- GF_SECURITY_ADMIN_USER=admin
- GF_SECURITY_ADMIN_PASSWORD=password # Change this in production
- GF_USERS_ALLOW_SIGN_UP=false
volumes:
- grafana-data:/var/lib/grafana
- ./config/provisioning/:/etc/grafana/provisioning
depends_on:
- loki
- prometheus
restart: unless-stopped
networks:
- monitoring
loki:
image: grafana/loki:latest
container_name: loki
ports:
- "3100:3100"
volumes:
- ./config/loki.yml:/etc/loki/local-config.yaml
- loki-data:/var/loki
command: -config.file=/etc/loki/local-config.yaml
restart: unless-stopped
networks:
- monitoring
alloy:
image: grafana/alloy:latest
container_name: grafana-alloy
ports:
- "12345:12345"
volumes:
- ./config/alloy.conf:/etc/alloy/config.alloy
- alloy-data:/var/lib/alloy/data
command:
- run
- --server.http.listen-addr=0.0.0.0:12345
- --storage.path=/var/lib/alloy/data
- /etc/alloy/config.alloy
restart: unless-stopped
networks:
- monitoring
# Prometheus for metrics collection
prometheus:
image: prom/prometheus:latest
container_name: prometheus
ports:
- "9090:9090"
volumes:
- ./config/prometheus.yml:/etc/prometheus/prometheus.yml
- ./config/prometheus-alerts.yml:/etc/prometheus/alerts.yml
- prometheus-data:/prometheus
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--web.console.libraries=/etc/prometheus/console_libraries'
- '--web.console.templates=/etc/prometheus/consoles'
- '--web.enable-lifecycle'
restart: unless-stopped
networks:
- monitoring
# cAdvisor for container metrics
cadvisor:
image: gcr.io/cadvisor/cadvisor:v0.49.1
container_name: cadvisor
volumes:
- /:/rootfs:ro
- /var/run:/var/run:ro
- /sys:/sys:ro
- /var/lib/docker/:/var/lib/docker:ro
- /dev/disk/:/dev/disk:ro
ports:
- "8080:8080"
restart: unless-stopped
networks:
- monitoring
privileged: true
devices:
- /dev/kmsg:/dev/kmsg
# Node Exporter for host metrics
node-exporter:
image: prom/node-exporter:latest
container_name: node-exporter
volumes:
- /proc:/host/proc:ro
- /sys:/host/sys:ro
- /:/rootfs:ro
command:
- '--path.procfs=/host/proc'
- '--path.sysfs=/host/sys'
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
ports:
- "9100:9100"
restart: unless-stopped
networks:
- monitoring
volumes:
grafana-data:
loki-data:
loki-wal-data:
prometheus-data:
alloy-data:
driver: local
networks:
monitoring:
driver: bridge
+12
View File
@@ -0,0 +1,12 @@
apiVersion: 1
providers:
- name: 'Node Exporter Full'
orgId: 1860
folder: ''
type: file
disableDeletion: false
updateIntervalSeconds: 10
allowUiUpdates: false
options:
path: /etc/grafana/provisioning/
+19
View File
@@ -0,0 +1,19 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
version: 1
editable: true
- name: Loki
type: loki
access: proxy
url: http://loki:3100
version: 1
editable: true
jsonData:
maxLines: 1000
File diff suppressed because it is too large Load Diff