remove node exporter and add alloy for metrics

This commit is contained in:
Your Name
2025-03-19 10:06:48 +00:00
parent 0046ff0f07
commit 32b4d8ba43
4 changed files with 45 additions and 134 deletions
-4
View File
@@ -1,4 +0,0 @@
logging {
level = "info"
format = "logfmt"
}
+28 -61
View File
@@ -1,71 +1,38 @@
// File log collection for application.log
local.file_match "application_logs" {
path_targets = [{"__path__" = "/var/log/application.log"}]
prometheus.exporter.unix "host_metrics" {
// Enable specific collectors that would provide the missing metrics
enable_collectors = [
"cpu", "pressure", "netdev", "netstat", "diskstats",
"meminfo", "loadavg", "softnet", "stat", "thermal_zone",
]
// Set paths for accessing host metrics when running in a container
procfs_path = "/host/proc"
sysfs_path = "/host/sys"
rootfs_path = "/rootfs"
// Configure filesystem collector
filesystem {
mount_points_exclude = "^/(sys|proc|dev|host|etc)($$|/)"
}
loki.source.file "app_logs" {
targets = local.file_match.application_logs.targets
forward_to = [loki.process.application_logs.receiver]
}
// Process application logs
loki.process "application_logs" {
// Extract log levels with regex
stage.regex {
expression = "(?i)(?P<level>error|warn|info|debug)"
}
// Add extracted level as a label
stage.labels {
values = {
level = "",
// Configure netdev collector
netdev {
// Include all network devices for network traffic metrics
device_exclude = ""
}
}
// Add a source label to distinguish from container logs
stage.static_labels {
values = {
source = "file",
}
// Make sure we scrape the host metrics
prometheus.scrape "host_metrics" {
targets = prometheus.exporter.unix.host_metrics.targets
forward_to = [prometheus.remote_write.prometheus.receiver]
scrape_interval = "15s"
scrape_timeout = "10s"
job_name = "node"
}
forward_to = [loki.write.loki_gateway.receiver]
}
// Container log collection - KEEPING THIS AS IT WORKS
discovery.docker "containers" {
host = "unix:///var/run/docker.sock"
}
discovery.relabel "containers" {
targets = discovery.docker.containers.targets
rule {
source_labels = ["__meta_docker_container_name"]
target_label = "container"
}
rule {
source_labels = ["__meta_docker_container_id"]
target_label = "container_id"
}
rule {
source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
target_label = "service"
}
rule {
replacement = "container"
target_label = "source"
}
}
loki.source.docker "container_log_collector" {
host = "unix:///var/run/docker.sock"
targets = discovery.relabel.containers.output
forward_to = [loki.write.loki_gateway.receiver]
}
// Send all logs to Loki
loki.write "loki_gateway" {
prometheus.remote_write "prometheus" {
endpoint {
url = "http://loki:3100/loki/api/v1/push"
url = "http://prometheus:9090/api/v1/write"
}
}
+10 -29
View File
@@ -20,32 +20,6 @@ scrape_configs:
static_configs:
- targets: ['prometheus:9090']
# Container metrics via cAdvisor
- job_name: 'ktpv-orch_cadvisor'
sample_limit: 50000
static_configs:
- targets: ['cadvisor:8080']
metrics_path: /metrics
scheme: http
- job_name: 'ktpv-apps01_cadvisor'
sample_limit: 50000
static_configs:
- targets: ['10.249.84.41:9050']
metrics_path: /metrics
scheme: http
# Metrics via node-exporter
- job_name: 'ktpv-orch_node-exporter'
sample_limit: 50000
static_configs:
- targets: ['node-exporter:9100']
- job_name: 'ktpv-apps01_node-exporter'
sample_limit: 50000
static_configs:
- targets: ['10.249.84.41:9060']
# Loki metrics
- job_name: 'loki'
sample_limit: 50000
@@ -58,8 +32,15 @@ scrape_configs:
static_configs:
- targets: ['grafana:3000']
# Alloy metrics
- job_name: 'alloy'
# Replace node-exporter job with Alloy
- job_name: 'ktpv-orch_node-exporter'
sample_limit: 50000
static_configs:
- targets: ['10.249.84.41:9030']
- targets: ['alloy:12345'] # Scrape from Alloy's metrics endpoint
metrics_path: /metrics
- job_name: 'ktpv-apps01_node-exporter'
sample_limit: 50000
static_configs:
- targets: ['10.249.84.41:12345'] # Adjust IP as needed
metrics_path: /metrics
+7 -40
View File
@@ -84,60 +84,27 @@ services:
max_attempts: 3
window: 120s
# cAdvisor for container metrics
cadvisor:
image: gcr.io/cadvisor/cadvisor:v0.49.1
container_name: cadvisor
volumes:
- /:/rootfs:ro
- /var/run:/var/run:ro
- /sys:/sys:ro
- /var/lib/docker/:/var/lib/docker:ro
- /dev/disk/:/dev/disk:ro
ports:
- "9050:8080"
restart: unless-stopped
networks:
- monitoring
privileged: true
devices:
- /dev/kmsg:/dev/kmsg
# Node Exporter for host metrics
node-exporter:
image: prom/node-exporter:latest
container_name: node-exporter
volumes:
- /proc:/host/proc:ro
- /sys:/host/sys:ro
- /:/rootfs:ro
command:
- '--path.procfs=/host/proc'
- '--path.sysfs=/host/sys'
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
ports:
- "9060:9100"
restart: unless-stopped
networks:
- monitoring
alloy:
image: grafana/alloy:latest
container_name: alloy
volumes:
- ./config/config.alloy:/etc/alloy/config.alloy
- /home/container_user/logs:/var/log:ro # Read-only mount of host logs
- /var/run/docker.sock:/var/run/docker.sock
- /proc:/host/proc:ro # CRITICAL - must be mounted at /host/proc
- /sys:/host/sys:ro # CRITICAL - must be mounted at /host/sys
- /:/rootfs:ro # CRITICAL - must be mounted at /rootfs
command:
- run
- --server.http.listen-addr=0.0.0.0:12345
- --storage.path=/var/lib/alloy/data
- /etc/alloy/config.alloy
ports:
- "12345:12345" # Expose the Alloy UI
restart: unless-stopped
- "12345:12345"
privileged: true # CRITICAL for accessing system metrics
pid: "host" # CRITICAL for accessing process metrics
networks:
- monitoring
volumes:
grafana-data:
prometheus-data: