remove node exporter and add alloy for metrics

This commit is contained in:
Your Name
2025-03-19 10:06:48 +00:00
parent 0046ff0f07
commit 32b4d8ba43
4 changed files with 45 additions and 134 deletions
-4
View File
@@ -1,4 +0,0 @@
logging {
level = "info"
format = "logfmt"
}
+26 -59
View File
@@ -1,71 +1,38 @@
// File log collection for application.log prometheus.exporter.unix "host_metrics" {
local.file_match "application_logs" { // Enable specific collectors that would provide the missing metrics
path_targets = [{"__path__" = "/var/log/application.log"}] enable_collectors = [
} "cpu", "pressure", "netdev", "netstat", "diskstats",
"meminfo", "loadavg", "softnet", "stat", "thermal_zone",
]
loki.source.file "app_logs" { // Set paths for accessing host metrics when running in a container
targets = local.file_match.application_logs.targets procfs_path = "/host/proc"
forward_to = [loki.process.application_logs.receiver] sysfs_path = "/host/sys"
} rootfs_path = "/rootfs"
// Process application logs // Configure filesystem collector
loki.process "application_logs" { filesystem {
// Extract log levels with regex mount_points_exclude = "^/(sys|proc|dev|host|etc)($$|/)"
stage.regex {
expression = "(?i)(?P<level>error|warn|info|debug)"
} }
// Add extracted level as a label // Configure netdev collector
stage.labels { netdev {
values = { // Include all network devices for network traffic metrics
level = "", device_exclude = ""
}
}
// Add a source label to distinguish from container logs
stage.static_labels {
values = {
source = "file",
}
}
forward_to = [loki.write.loki_gateway.receiver]
}
// Container log collection - KEEPING THIS AS IT WORKS
discovery.docker "containers" {
host = "unix:///var/run/docker.sock"
}
discovery.relabel "containers" {
targets = discovery.docker.containers.targets
rule {
source_labels = ["__meta_docker_container_name"]
target_label = "container"
}
rule {
source_labels = ["__meta_docker_container_id"]
target_label = "container_id"
}
rule {
source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
target_label = "service"
}
rule {
replacement = "container"
target_label = "source"
} }
} }
loki.source.docker "container_log_collector" { // Make sure we scrape the host metrics
host = "unix:///var/run/docker.sock" prometheus.scrape "host_metrics" {
targets = discovery.relabel.containers.output targets = prometheus.exporter.unix.host_metrics.targets
forward_to = [loki.write.loki_gateway.receiver] forward_to = [prometheus.remote_write.prometheus.receiver]
scrape_interval = "15s"
scrape_timeout = "10s"
job_name = "node"
} }
// Send all logs to Loki prometheus.remote_write "prometheus" {
loki.write "loki_gateway" {
endpoint { endpoint {
url = "http://loki:3100/loki/api/v1/push" url = "http://prometheus:9090/api/v1/write"
} }
} }
+10 -29
View File
@@ -20,32 +20,6 @@ scrape_configs:
static_configs: static_configs:
- targets: ['prometheus:9090'] - targets: ['prometheus:9090']
# Container metrics via cAdvisor
- job_name: 'ktpv-orch_cadvisor'
sample_limit: 50000
static_configs:
- targets: ['cadvisor:8080']
metrics_path: /metrics
scheme: http
- job_name: 'ktpv-apps01_cadvisor'
sample_limit: 50000
static_configs:
- targets: ['10.249.84.41:9050']
metrics_path: /metrics
scheme: http
# Metrics via node-exporter
- job_name: 'ktpv-orch_node-exporter'
sample_limit: 50000
static_configs:
- targets: ['node-exporter:9100']
- job_name: 'ktpv-apps01_node-exporter'
sample_limit: 50000
static_configs:
- targets: ['10.249.84.41:9060']
# Loki metrics # Loki metrics
- job_name: 'loki' - job_name: 'loki'
sample_limit: 50000 sample_limit: 50000
@@ -58,8 +32,15 @@ scrape_configs:
static_configs: static_configs:
- targets: ['grafana:3000'] - targets: ['grafana:3000']
# Alloy metrics # Replace node-exporter job with Alloy
- job_name: 'alloy' - job_name: 'ktpv-orch_node-exporter'
sample_limit: 50000 sample_limit: 50000
static_configs: static_configs:
- targets: ['10.249.84.41:9030'] - targets: ['alloy:12345'] # Scrape from Alloy's metrics endpoint
metrics_path: /metrics
- job_name: 'ktpv-apps01_node-exporter'
sample_limit: 50000
static_configs:
- targets: ['10.249.84.41:12345'] # Adjust IP as needed
metrics_path: /metrics
+7 -40
View File
@@ -84,60 +84,27 @@ services:
max_attempts: 3 max_attempts: 3
window: 120s window: 120s
# cAdvisor for container metrics
cadvisor:
image: gcr.io/cadvisor/cadvisor:v0.49.1
container_name: cadvisor
volumes:
- /:/rootfs:ro
- /var/run:/var/run:ro
- /sys:/sys:ro
- /var/lib/docker/:/var/lib/docker:ro
- /dev/disk/:/dev/disk:ro
ports:
- "9050:8080"
restart: unless-stopped
networks:
- monitoring
privileged: true
devices:
- /dev/kmsg:/dev/kmsg
# Node Exporter for host metrics
node-exporter:
image: prom/node-exporter:latest
container_name: node-exporter
volumes:
- /proc:/host/proc:ro
- /sys:/host/sys:ro
- /:/rootfs:ro
command:
- '--path.procfs=/host/proc'
- '--path.sysfs=/host/sys'
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
ports:
- "9060:9100"
restart: unless-stopped
networks:
- monitoring
alloy: alloy:
image: grafana/alloy:latest image: grafana/alloy:latest
container_name: alloy container_name: alloy
volumes: volumes:
- ./config/config.alloy:/etc/alloy/config.alloy - ./config/config.alloy:/etc/alloy/config.alloy
- /home/container_user/logs:/var/log:ro # Read-only mount of host logs - /proc:/host/proc:ro # CRITICAL - must be mounted at /host/proc
- /var/run/docker.sock:/var/run/docker.sock - /sys:/host/sys:ro # CRITICAL - must be mounted at /host/sys
- /:/rootfs:ro # CRITICAL - must be mounted at /rootfs
command: command:
- run - run
- --server.http.listen-addr=0.0.0.0:12345 - --server.http.listen-addr=0.0.0.0:12345
- --storage.path=/var/lib/alloy/data - --storage.path=/var/lib/alloy/data
- /etc/alloy/config.alloy - /etc/alloy/config.alloy
ports: ports:
- "12345:12345" # Expose the Alloy UI - "12345:12345"
restart: unless-stopped privileged: true # CRITICAL for accessing system metrics
pid: "host" # CRITICAL for accessing process metrics
networks: networks:
- monitoring - monitoring
volumes: volumes:
grafana-data: grafana-data:
prometheus-data: prometheus-data: