remove node exporter and add alloy for metrics
This commit is contained in:
@@ -1,4 +0,0 @@
|
||||
logging {
|
||||
level = "info"
|
||||
format = "logfmt"
|
||||
}
|
||||
+28
-61
@@ -1,71 +1,38 @@
|
||||
// File log collection for application.log
|
||||
local.file_match "application_logs" {
|
||||
path_targets = [{"__path__" = "/var/log/application.log"}]
|
||||
}
|
||||
|
||||
loki.source.file "app_logs" {
|
||||
targets = local.file_match.application_logs.targets
|
||||
forward_to = [loki.process.application_logs.receiver]
|
||||
}
|
||||
|
||||
// Process application logs
|
||||
loki.process "application_logs" {
|
||||
// Extract log levels with regex
|
||||
stage.regex {
|
||||
expression = "(?i)(?P<level>error|warn|info|debug)"
|
||||
prometheus.exporter.unix "host_metrics" {
|
||||
// Enable specific collectors that would provide the missing metrics
|
||||
enable_collectors = [
|
||||
"cpu", "pressure", "netdev", "netstat", "diskstats",
|
||||
"meminfo", "loadavg", "softnet", "stat", "thermal_zone",
|
||||
]
|
||||
|
||||
// Set paths for accessing host metrics when running in a container
|
||||
procfs_path = "/host/proc"
|
||||
sysfs_path = "/host/sys"
|
||||
rootfs_path = "/rootfs"
|
||||
|
||||
// Configure filesystem collector
|
||||
filesystem {
|
||||
mount_points_exclude = "^/(sys|proc|dev|host|etc)($$|/)"
|
||||
}
|
||||
|
||||
// Add extracted level as a label
|
||||
stage.labels {
|
||||
values = {
|
||||
level = "",
|
||||
}
|
||||
}
|
||||
|
||||
// Add a source label to distinguish from container logs
|
||||
stage.static_labels {
|
||||
values = {
|
||||
source = "file",
|
||||
}
|
||||
}
|
||||
|
||||
forward_to = [loki.write.loki_gateway.receiver]
|
||||
}
|
||||
|
||||
// Container log collection - KEEPING THIS AS IT WORKS
|
||||
discovery.docker "containers" {
|
||||
host = "unix:///var/run/docker.sock"
|
||||
}
|
||||
|
||||
discovery.relabel "containers" {
|
||||
targets = discovery.docker.containers.targets
|
||||
rule {
|
||||
source_labels = ["__meta_docker_container_name"]
|
||||
target_label = "container"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_docker_container_id"]
|
||||
target_label = "container_id"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
|
||||
target_label = "service"
|
||||
}
|
||||
rule {
|
||||
replacement = "container"
|
||||
target_label = "source"
|
||||
// Configure netdev collector
|
||||
netdev {
|
||||
// Include all network devices for network traffic metrics
|
||||
device_exclude = ""
|
||||
}
|
||||
}
|
||||
|
||||
loki.source.docker "container_log_collector" {
|
||||
host = "unix:///var/run/docker.sock"
|
||||
targets = discovery.relabel.containers.output
|
||||
forward_to = [loki.write.loki_gateway.receiver]
|
||||
// Make sure we scrape the host metrics
|
||||
prometheus.scrape "host_metrics" {
|
||||
targets = prometheus.exporter.unix.host_metrics.targets
|
||||
forward_to = [prometheus.remote_write.prometheus.receiver]
|
||||
scrape_interval = "15s"
|
||||
scrape_timeout = "10s"
|
||||
job_name = "node"
|
||||
}
|
||||
|
||||
// Send all logs to Loki
|
||||
loki.write "loki_gateway" {
|
||||
prometheus.remote_write "prometheus" {
|
||||
endpoint {
|
||||
url = "http://loki:3100/loki/api/v1/push"
|
||||
url = "http://prometheus:9090/api/v1/write"
|
||||
}
|
||||
}
|
||||
|
||||
+10
-29
@@ -20,32 +20,6 @@ scrape_configs:
|
||||
static_configs:
|
||||
- targets: ['prometheus:9090']
|
||||
|
||||
# Container metrics via cAdvisor
|
||||
- job_name: 'ktpv-orch_cadvisor'
|
||||
sample_limit: 50000
|
||||
static_configs:
|
||||
- targets: ['cadvisor:8080']
|
||||
metrics_path: /metrics
|
||||
scheme: http
|
||||
|
||||
- job_name: 'ktpv-apps01_cadvisor'
|
||||
sample_limit: 50000
|
||||
static_configs:
|
||||
- targets: ['10.249.84.41:9050']
|
||||
metrics_path: /metrics
|
||||
scheme: http
|
||||
|
||||
# Metrics via node-exporter
|
||||
- job_name: 'ktpv-orch_node-exporter'
|
||||
sample_limit: 50000
|
||||
static_configs:
|
||||
- targets: ['node-exporter:9100']
|
||||
|
||||
- job_name: 'ktpv-apps01_node-exporter'
|
||||
sample_limit: 50000
|
||||
static_configs:
|
||||
- targets: ['10.249.84.41:9060']
|
||||
|
||||
# Loki metrics
|
||||
- job_name: 'loki'
|
||||
sample_limit: 50000
|
||||
@@ -58,8 +32,15 @@ scrape_configs:
|
||||
static_configs:
|
||||
- targets: ['grafana:3000']
|
||||
|
||||
# Alloy metrics
|
||||
- job_name: 'alloy'
|
||||
# Replace node-exporter job with Alloy
|
||||
- job_name: 'ktpv-orch_node-exporter'
|
||||
sample_limit: 50000
|
||||
static_configs:
|
||||
- targets: ['10.249.84.41:9030']
|
||||
- targets: ['alloy:12345'] # Scrape from Alloy's metrics endpoint
|
||||
metrics_path: /metrics
|
||||
|
||||
- job_name: 'ktpv-apps01_node-exporter'
|
||||
sample_limit: 50000
|
||||
static_configs:
|
||||
- targets: ['10.249.84.41:12345'] # Adjust IP as needed
|
||||
metrics_path: /metrics
|
||||
|
||||
+7
-40
@@ -84,60 +84,27 @@ services:
|
||||
max_attempts: 3
|
||||
window: 120s
|
||||
|
||||
# cAdvisor for container metrics
|
||||
cadvisor:
|
||||
image: gcr.io/cadvisor/cadvisor:v0.49.1
|
||||
container_name: cadvisor
|
||||
volumes:
|
||||
- /:/rootfs:ro
|
||||
- /var/run:/var/run:ro
|
||||
- /sys:/sys:ro
|
||||
- /var/lib/docker/:/var/lib/docker:ro
|
||||
- /dev/disk/:/dev/disk:ro
|
||||
ports:
|
||||
- "9050:8080"
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- monitoring
|
||||
privileged: true
|
||||
devices:
|
||||
- /dev/kmsg:/dev/kmsg
|
||||
|
||||
# Node Exporter for host metrics
|
||||
node-exporter:
|
||||
image: prom/node-exporter:latest
|
||||
container_name: node-exporter
|
||||
volumes:
|
||||
- /proc:/host/proc:ro
|
||||
- /sys:/host/sys:ro
|
||||
- /:/rootfs:ro
|
||||
command:
|
||||
- '--path.procfs=/host/proc'
|
||||
- '--path.sysfs=/host/sys'
|
||||
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
|
||||
ports:
|
||||
- "9060:9100"
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- monitoring
|
||||
alloy:
|
||||
image: grafana/alloy:latest
|
||||
container_name: alloy
|
||||
volumes:
|
||||
- ./config/config.alloy:/etc/alloy/config.alloy
|
||||
- /home/container_user/logs:/var/log:ro # Read-only mount of host logs
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- /proc:/host/proc:ro # CRITICAL - must be mounted at /host/proc
|
||||
- /sys:/host/sys:ro # CRITICAL - must be mounted at /host/sys
|
||||
- /:/rootfs:ro # CRITICAL - must be mounted at /rootfs
|
||||
command:
|
||||
- run
|
||||
- --server.http.listen-addr=0.0.0.0:12345
|
||||
- --storage.path=/var/lib/alloy/data
|
||||
- /etc/alloy/config.alloy
|
||||
ports:
|
||||
- "12345:12345" # Expose the Alloy UI
|
||||
restart: unless-stopped
|
||||
- "12345:12345"
|
||||
privileged: true # CRITICAL for accessing system metrics
|
||||
pid: "host" # CRITICAL for accessing process metrics
|
||||
networks:
|
||||
- monitoring
|
||||
|
||||
|
||||
volumes:
|
||||
grafana-data:
|
||||
prometheus-data:
|
||||
|
||||
Reference in New Issue
Block a user