remove node exporter and add alloy for metrics
This commit is contained in:
@@ -1,4 +0,0 @@
|
|||||||
logging {
|
|
||||||
level = "info"
|
|
||||||
format = "logfmt"
|
|
||||||
}
|
|
||||||
+26
-59
@@ -1,71 +1,38 @@
|
|||||||
// File log collection for application.log
|
prometheus.exporter.unix "host_metrics" {
|
||||||
local.file_match "application_logs" {
|
// Enable specific collectors that would provide the missing metrics
|
||||||
path_targets = [{"__path__" = "/var/log/application.log"}]
|
enable_collectors = [
|
||||||
}
|
"cpu", "pressure", "netdev", "netstat", "diskstats",
|
||||||
|
"meminfo", "loadavg", "softnet", "stat", "thermal_zone",
|
||||||
|
]
|
||||||
|
|
||||||
loki.source.file "app_logs" {
|
// Set paths for accessing host metrics when running in a container
|
||||||
targets = local.file_match.application_logs.targets
|
procfs_path = "/host/proc"
|
||||||
forward_to = [loki.process.application_logs.receiver]
|
sysfs_path = "/host/sys"
|
||||||
}
|
rootfs_path = "/rootfs"
|
||||||
|
|
||||||
// Process application logs
|
// Configure filesystem collector
|
||||||
loki.process "application_logs" {
|
filesystem {
|
||||||
// Extract log levels with regex
|
mount_points_exclude = "^/(sys|proc|dev|host|etc)($$|/)"
|
||||||
stage.regex {
|
|
||||||
expression = "(?i)(?P<level>error|warn|info|debug)"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add extracted level as a label
|
// Configure netdev collector
|
||||||
stage.labels {
|
netdev {
|
||||||
values = {
|
// Include all network devices for network traffic metrics
|
||||||
level = "",
|
device_exclude = ""
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add a source label to distinguish from container logs
|
|
||||||
stage.static_labels {
|
|
||||||
values = {
|
|
||||||
source = "file",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
forward_to = [loki.write.loki_gateway.receiver]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Container log collection - KEEPING THIS AS IT WORKS
|
|
||||||
discovery.docker "containers" {
|
|
||||||
host = "unix:///var/run/docker.sock"
|
|
||||||
}
|
|
||||||
|
|
||||||
discovery.relabel "containers" {
|
|
||||||
targets = discovery.docker.containers.targets
|
|
||||||
rule {
|
|
||||||
source_labels = ["__meta_docker_container_name"]
|
|
||||||
target_label = "container"
|
|
||||||
}
|
|
||||||
rule {
|
|
||||||
source_labels = ["__meta_docker_container_id"]
|
|
||||||
target_label = "container_id"
|
|
||||||
}
|
|
||||||
rule {
|
|
||||||
source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
|
|
||||||
target_label = "service"
|
|
||||||
}
|
|
||||||
rule {
|
|
||||||
replacement = "container"
|
|
||||||
target_label = "source"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
loki.source.docker "container_log_collector" {
|
// Make sure we scrape the host metrics
|
||||||
host = "unix:///var/run/docker.sock"
|
prometheus.scrape "host_metrics" {
|
||||||
targets = discovery.relabel.containers.output
|
targets = prometheus.exporter.unix.host_metrics.targets
|
||||||
forward_to = [loki.write.loki_gateway.receiver]
|
forward_to = [prometheus.remote_write.prometheus.receiver]
|
||||||
|
scrape_interval = "15s"
|
||||||
|
scrape_timeout = "10s"
|
||||||
|
job_name = "node"
|
||||||
}
|
}
|
||||||
|
|
||||||
// Send all logs to Loki
|
prometheus.remote_write "prometheus" {
|
||||||
loki.write "loki_gateway" {
|
|
||||||
endpoint {
|
endpoint {
|
||||||
url = "http://loki:3100/loki/api/v1/push"
|
url = "http://prometheus:9090/api/v1/write"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+10
-29
@@ -20,32 +20,6 @@ scrape_configs:
|
|||||||
static_configs:
|
static_configs:
|
||||||
- targets: ['prometheus:9090']
|
- targets: ['prometheus:9090']
|
||||||
|
|
||||||
# Container metrics via cAdvisor
|
|
||||||
- job_name: 'ktpv-orch_cadvisor'
|
|
||||||
sample_limit: 50000
|
|
||||||
static_configs:
|
|
||||||
- targets: ['cadvisor:8080']
|
|
||||||
metrics_path: /metrics
|
|
||||||
scheme: http
|
|
||||||
|
|
||||||
- job_name: 'ktpv-apps01_cadvisor'
|
|
||||||
sample_limit: 50000
|
|
||||||
static_configs:
|
|
||||||
- targets: ['10.249.84.41:9050']
|
|
||||||
metrics_path: /metrics
|
|
||||||
scheme: http
|
|
||||||
|
|
||||||
# Metrics via node-exporter
|
|
||||||
- job_name: 'ktpv-orch_node-exporter'
|
|
||||||
sample_limit: 50000
|
|
||||||
static_configs:
|
|
||||||
- targets: ['node-exporter:9100']
|
|
||||||
|
|
||||||
- job_name: 'ktpv-apps01_node-exporter'
|
|
||||||
sample_limit: 50000
|
|
||||||
static_configs:
|
|
||||||
- targets: ['10.249.84.41:9060']
|
|
||||||
|
|
||||||
# Loki metrics
|
# Loki metrics
|
||||||
- job_name: 'loki'
|
- job_name: 'loki'
|
||||||
sample_limit: 50000
|
sample_limit: 50000
|
||||||
@@ -58,8 +32,15 @@ scrape_configs:
|
|||||||
static_configs:
|
static_configs:
|
||||||
- targets: ['grafana:3000']
|
- targets: ['grafana:3000']
|
||||||
|
|
||||||
# Alloy metrics
|
# Replace node-exporter job with Alloy
|
||||||
- job_name: 'alloy'
|
- job_name: 'ktpv-orch_node-exporter'
|
||||||
sample_limit: 50000
|
sample_limit: 50000
|
||||||
static_configs:
|
static_configs:
|
||||||
- targets: ['10.249.84.41:9030']
|
- targets: ['alloy:12345'] # Scrape from Alloy's metrics endpoint
|
||||||
|
metrics_path: /metrics
|
||||||
|
|
||||||
|
- job_name: 'ktpv-apps01_node-exporter'
|
||||||
|
sample_limit: 50000
|
||||||
|
static_configs:
|
||||||
|
- targets: ['10.249.84.41:12345'] # Adjust IP as needed
|
||||||
|
metrics_path: /metrics
|
||||||
|
|||||||
+7
-40
@@ -84,60 +84,27 @@ services:
|
|||||||
max_attempts: 3
|
max_attempts: 3
|
||||||
window: 120s
|
window: 120s
|
||||||
|
|
||||||
# cAdvisor for container metrics
|
|
||||||
cadvisor:
|
|
||||||
image: gcr.io/cadvisor/cadvisor:v0.49.1
|
|
||||||
container_name: cadvisor
|
|
||||||
volumes:
|
|
||||||
- /:/rootfs:ro
|
|
||||||
- /var/run:/var/run:ro
|
|
||||||
- /sys:/sys:ro
|
|
||||||
- /var/lib/docker/:/var/lib/docker:ro
|
|
||||||
- /dev/disk/:/dev/disk:ro
|
|
||||||
ports:
|
|
||||||
- "9050:8080"
|
|
||||||
restart: unless-stopped
|
|
||||||
networks:
|
|
||||||
- monitoring
|
|
||||||
privileged: true
|
|
||||||
devices:
|
|
||||||
- /dev/kmsg:/dev/kmsg
|
|
||||||
|
|
||||||
# Node Exporter for host metrics
|
|
||||||
node-exporter:
|
|
||||||
image: prom/node-exporter:latest
|
|
||||||
container_name: node-exporter
|
|
||||||
volumes:
|
|
||||||
- /proc:/host/proc:ro
|
|
||||||
- /sys:/host/sys:ro
|
|
||||||
- /:/rootfs:ro
|
|
||||||
command:
|
|
||||||
- '--path.procfs=/host/proc'
|
|
||||||
- '--path.sysfs=/host/sys'
|
|
||||||
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
|
|
||||||
ports:
|
|
||||||
- "9060:9100"
|
|
||||||
restart: unless-stopped
|
|
||||||
networks:
|
|
||||||
- monitoring
|
|
||||||
alloy:
|
alloy:
|
||||||
image: grafana/alloy:latest
|
image: grafana/alloy:latest
|
||||||
container_name: alloy
|
container_name: alloy
|
||||||
volumes:
|
volumes:
|
||||||
- ./config/config.alloy:/etc/alloy/config.alloy
|
- ./config/config.alloy:/etc/alloy/config.alloy
|
||||||
- /home/container_user/logs:/var/log:ro # Read-only mount of host logs
|
- /proc:/host/proc:ro # CRITICAL - must be mounted at /host/proc
|
||||||
- /var/run/docker.sock:/var/run/docker.sock
|
- /sys:/host/sys:ro # CRITICAL - must be mounted at /host/sys
|
||||||
|
- /:/rootfs:ro # CRITICAL - must be mounted at /rootfs
|
||||||
command:
|
command:
|
||||||
- run
|
- run
|
||||||
- --server.http.listen-addr=0.0.0.0:12345
|
- --server.http.listen-addr=0.0.0.0:12345
|
||||||
- --storage.path=/var/lib/alloy/data
|
- --storage.path=/var/lib/alloy/data
|
||||||
- /etc/alloy/config.alloy
|
- /etc/alloy/config.alloy
|
||||||
ports:
|
ports:
|
||||||
- "12345:12345" # Expose the Alloy UI
|
- "12345:12345"
|
||||||
restart: unless-stopped
|
privileged: true # CRITICAL for accessing system metrics
|
||||||
|
pid: "host" # CRITICAL for accessing process metrics
|
||||||
networks:
|
networks:
|
||||||
- monitoring
|
- monitoring
|
||||||
|
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
grafana-data:
|
grafana-data:
|
||||||
prometheus-data:
|
prometheus-data:
|
||||||
|
|||||||
Reference in New Issue
Block a user