From 32b4d8ba437c415296ea1cb0b9bdef810e9df79b Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 19 Mar 2025 10:06:48 +0000 Subject: [PATCH] remove node exporter and add alloy for metrics --- config/alloy.conf | 4 -- config/config.alloy | 89 ++++++++++++++----------------------------- config/prometheus.yml | 39 +++++-------------- docker-compose.yml | 47 ++++------------------- 4 files changed, 45 insertions(+), 134 deletions(-) delete mode 100644 config/alloy.conf diff --git a/config/alloy.conf b/config/alloy.conf deleted file mode 100644 index 2dc79e3..0000000 --- a/config/alloy.conf +++ /dev/null @@ -1,4 +0,0 @@ -logging { - level = "info" - format = "logfmt" -} diff --git a/config/config.alloy b/config/config.alloy index a747b6d..06d7024 100644 --- a/config/config.alloy +++ b/config/config.alloy @@ -1,71 +1,38 @@ -// File log collection for application.log -local.file_match "application_logs" { - path_targets = [{"__path__" = "/var/log/application.log"}] -} - -loki.source.file "app_logs" { - targets = local.file_match.application_logs.targets - forward_to = [loki.process.application_logs.receiver] -} - -// Process application logs -loki.process "application_logs" { - // Extract log levels with regex - stage.regex { - expression = "(?i)(?Perror|warn|info|debug)" +prometheus.exporter.unix "host_metrics" { + // Enable specific collectors that would provide the missing metrics + enable_collectors = [ + "cpu", "pressure", "netdev", "netstat", "diskstats", + "meminfo", "loadavg", "softnet", "stat", "thermal_zone", + ] + + // Set paths for accessing host metrics when running in a container + procfs_path = "/host/proc" + sysfs_path = "/host/sys" + rootfs_path = "/rootfs" + + // Configure filesystem collector + filesystem { + mount_points_exclude = "^/(sys|proc|dev|host|etc)($$|/)" } - // Add extracted level as a label - stage.labels { - values = { - level = "", - } - } - - // Add a source label to distinguish from container logs - stage.static_labels { - values = { - source = "file", - } - } - - forward_to = [loki.write.loki_gateway.receiver] -} - -// Container log collection - KEEPING THIS AS IT WORKS -discovery.docker "containers" { - host = "unix:///var/run/docker.sock" -} - -discovery.relabel "containers" { - targets = discovery.docker.containers.targets - rule { - source_labels = ["__meta_docker_container_name"] - target_label = "container" - } - rule { - source_labels = ["__meta_docker_container_id"] - target_label = "container_id" - } - rule { - source_labels = ["__meta_docker_container_label_com_docker_compose_service"] - target_label = "service" - } - rule { - replacement = "container" - target_label = "source" + // Configure netdev collector + netdev { + // Include all network devices for network traffic metrics + device_exclude = "" } } -loki.source.docker "container_log_collector" { - host = "unix:///var/run/docker.sock" - targets = discovery.relabel.containers.output - forward_to = [loki.write.loki_gateway.receiver] +// Make sure we scrape the host metrics +prometheus.scrape "host_metrics" { + targets = prometheus.exporter.unix.host_metrics.targets + forward_to = [prometheus.remote_write.prometheus.receiver] + scrape_interval = "15s" + scrape_timeout = "10s" + job_name = "node" } -// Send all logs to Loki -loki.write "loki_gateway" { +prometheus.remote_write "prometheus" { endpoint { - url = "http://loki:3100/loki/api/v1/push" + url = "http://prometheus:9090/api/v1/write" } } diff --git a/config/prometheus.yml b/config/prometheus.yml index 6cf032d..fde2624 100644 --- a/config/prometheus.yml +++ b/config/prometheus.yml @@ -20,32 +20,6 @@ scrape_configs: static_configs: - targets: ['prometheus:9090'] - # Container metrics via cAdvisor - - job_name: 'ktpv-orch_cadvisor' - sample_limit: 50000 - static_configs: - - targets: ['cadvisor:8080'] - metrics_path: /metrics - scheme: http - - - job_name: 'ktpv-apps01_cadvisor' - sample_limit: 50000 - static_configs: - - targets: ['10.249.84.41:9050'] - metrics_path: /metrics - scheme: http - - # Metrics via node-exporter - - job_name: 'ktpv-orch_node-exporter' - sample_limit: 50000 - static_configs: - - targets: ['node-exporter:9100'] - - - job_name: 'ktpv-apps01_node-exporter' - sample_limit: 50000 - static_configs: - - targets: ['10.249.84.41:9060'] - # Loki metrics - job_name: 'loki' sample_limit: 50000 @@ -58,8 +32,15 @@ scrape_configs: static_configs: - targets: ['grafana:3000'] - # Alloy metrics - - job_name: 'alloy' + # Replace node-exporter job with Alloy + - job_name: 'ktpv-orch_node-exporter' sample_limit: 50000 static_configs: - - targets: ['10.249.84.41:9030'] + - targets: ['alloy:12345'] # Scrape from Alloy's metrics endpoint + metrics_path: /metrics + + - job_name: 'ktpv-apps01_node-exporter' + sample_limit: 50000 + static_configs: + - targets: ['10.249.84.41:12345'] # Adjust IP as needed + metrics_path: /metrics diff --git a/docker-compose.yml b/docker-compose.yml index e8a1dc6..9b0cd15 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,60 +84,27 @@ services: max_attempts: 3 window: 120s - # cAdvisor for container metrics - cadvisor: - image: gcr.io/cadvisor/cadvisor:v0.49.1 - container_name: cadvisor - volumes: - - /:/rootfs:ro - - /var/run:/var/run:ro - - /sys:/sys:ro - - /var/lib/docker/:/var/lib/docker:ro - - /dev/disk/:/dev/disk:ro - ports: - - "9050:8080" - restart: unless-stopped - networks: - - monitoring - privileged: true - devices: - - /dev/kmsg:/dev/kmsg - - # Node Exporter for host metrics - node-exporter: - image: prom/node-exporter:latest - container_name: node-exporter - volumes: - - /proc:/host/proc:ro - - /sys:/host/sys:ro - - /:/rootfs:ro - command: - - '--path.procfs=/host/proc' - - '--path.sysfs=/host/sys' - - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' - ports: - - "9060:9100" - restart: unless-stopped - networks: - - monitoring alloy: image: grafana/alloy:latest container_name: alloy volumes: - ./config/config.alloy:/etc/alloy/config.alloy - - /home/container_user/logs:/var/log:ro # Read-only mount of host logs - - /var/run/docker.sock:/var/run/docker.sock + - /proc:/host/proc:ro # CRITICAL - must be mounted at /host/proc + - /sys:/host/sys:ro # CRITICAL - must be mounted at /host/sys + - /:/rootfs:ro # CRITICAL - must be mounted at /rootfs command: - run - --server.http.listen-addr=0.0.0.0:12345 - --storage.path=/var/lib/alloy/data - /etc/alloy/config.alloy ports: - - "12345:12345" # Expose the Alloy UI - restart: unless-stopped + - "12345:12345" + privileged: true # CRITICAL for accessing system metrics + pid: "host" # CRITICAL for accessing process metrics networks: - monitoring + volumes: grafana-data: prometheus-data: