Server Monitoring Tools: Grafana, Prometheus and Netdata

Effectively monitoring your server infrastructure is essential for early detection of performance issues and preventing downtime. In this guide, we will set up a comprehensive server monitoring system using the Prometheus, Grafana, and Netdata trio.

What is Prometheus?

Prometheus is an open-source metric collection and monitoring system supported by the Cloud Native Computing Foundation (CNCF). With its pull-based architecture, it regularly scrapes metrics from target systems via HTTP and stores them in a time-series database.

Key features of Prometheus:

PromQL: Powerful query language for metric analysis
Pull-Based Model: Scrapes metrics from targets over HTTP
Service Discovery: Automatic target discovery (Kubernetes, Consul, DNS)
Alert Manager: Condition-based alert management
Multi-dimensional Data: Flexible label-based data model
Federation: Hierarchical linking of multiple Prometheus servers

Node Exporter Installation

Node Exporter is a Prometheus exporter that collects hardware and OS-level metrics from Linux servers.

Installation with Systemd

hljs bash

# Download Node Exporter
cd /tmp
curl -LO https://github.com/prometheus/node_exporter/releases/download/v1.7.0/node_exporter-1.7.0.linux-amd64.tar.gz
tar xzf node_exporter-1.7.0.linux-amd64.tar.gz

# Move binary
sudo mv node_exporter-1.7.0.linux-amd64/node_exporter /usr/local/bin/

# Create user
sudo useradd --no-create-home --shell /bin/false node_exporter

# Systemd service file
sudo tee /etc/systemd/system/node_exporter.service > /dev/null <<EOF
[Unit]
Description=Node Exporter
Wants=network-online.target
After=network-online.target

[Service]
User=node_exporter
Group=node_exporter
Type=simple
ExecStart=/usr/local/bin/node_exporter \\
  --collector.systemd \\
  --collector.processes \\
  --web.listen-address=:9100

[Install]
WantedBy=multi-user.target
EOF

# Start the service
sudo systemctl daemon-reload
sudo systemctl enable --now node_exporter
sudo systemctl status node_exporter

Node Exporter with Docker

hljs yaml

# docker-compose.yml
version: '3.8'
services:
  node-exporter:
    image: prom/node-exporter:latest
    container_name: node-exporter
    restart: unless-stopped
    ports:
      - "9100:9100"
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.sysfs=/host/sys'
      - '--path.rootfs=/rootfs'
      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'

Verify metrics:

hljs bash

curl http://localhost:9100/metrics | head -50

Prometheus Installation

Prometheus Stack with Docker Compose

hljs yaml

# docker-compose.yml
version: '3.8'
services:
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    restart: unless-stopped
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
      - ./prometheus/alert_rules.yml:/etc/prometheus/alert_rules.yml
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention.time=30d'
      - '--web.enable-lifecycle'

volumes:
  prometheus_data:

Prometheus Configuration

hljs yaml

# prometheus/prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s
  scrape_timeout: 10s

alerting:
  alertmanagers:
    - static_configs:
        - targets:
            - alertmanager:9093

rule_files:
  - "alert_rules.yml"

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'node-exporter'
    static_configs:
      - targets:
          - 'node-exporter:9100'
          - '192.168.1.10:9100'
          - '192.168.1.11:9100'
        labels:
          env: 'production'

  - job_name: 'docker'
    static_configs:
      - targets: ['cadvisor:8080']

Alert Rules

Detect critical conditions with Prometheus alert rules:

hljs yaml

# prometheus/alert_rules.yml
groups:
  - name: server_alerts
    rules:
      - alert: HighCPUUsage
        expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High CPU usage ({{ $labels.instance }})"
          description: "CPU usage is {{ $value | printf \"%.1f\" }}% - above 85% for 5 minutes."

      - alert: HighMemoryUsage
        expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "High memory usage ({{ $labels.instance }})"
          description: "Memory usage is {{ $value | printf \"%.1f\" }}% - above 90%."

      - alert: DiskAlmostFull
        expr: (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"})) * 100 > 85
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Disk almost full ({{ $labels.instance }})"
          description: "{{ $labels.mountpoint }} disk usage is {{ $value | printf \"%.1f\" }}%"

      - alert: InstanceDown
        expr: up == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Instance unreachable ({{ $labels.instance }})"
          description: "{{ $labels.job }} target has been unreachable for 2 minutes."

      - alert: HighDiskIO
        expr: rate(node_disk_io_time_seconds_total[5m]) > 0.9
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "High Disk I/O ({{ $labels.instance }})"
          description: "Disk I/O usage is {{ $value | printf \"%.1f\" }}% - high for 10 minutes."

Alertmanager Setup

hljs yaml

# alertmanager/alertmanager.yml
global:
  resolve_timeout: 5m

route:
  group_by: ['alertname', 'severity']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: 'telegram'
  routes:
    - match:
        severity: critical
      receiver: 'telegram'
      repeat_interval: 1h

receivers:
  - name: 'telegram'
    telegram_configs:
      - bot_token: 'YOUR_BOT_TOKEN'
        chat_id: YOUR_CHAT_ID
        parse_mode: 'HTML'
        message: |
          <b>{{ .Status | toUpper }}</b>
          <b>Alert:</b> {{ .CommonLabels.alertname }}
          <b>Severity:</b> {{ .CommonLabels.severity }}
          <b>Instance:</b> {{ .CommonLabels.instance }}
          <b>Description:</b> {{ .CommonAnnotations.description }}

Grafana Installation

Grafana with Docker Compose

hljs yaml

# docker-compose.yml (add to prometheus stack)
services:
  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    restart: unless-stopped
    ports:
      - "3000:3000"
    volumes:
      - grafana_data:/var/lib/grafana
      - ./grafana/provisioning:/etc/grafana/provisioning
    environment:
      - GF_SECURITY_ADMIN_USER=admin
      - GF_SECURITY_ADMIN_PASSWORD=StrongPassword123!
      - GF_USERS_ALLOW_SIGN_UP=false
      - GF_SERVER_ROOT_URL=https://grafana.example.com
      - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-piechart-panel

volumes:
  grafana_data:

Grafana Data Source Provisioning

hljs yaml

# grafana/provisioning/datasources/prometheus.yml
apiVersion: 1
datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    editable: true

Popular Grafana Dashboards

You can import ready-made dashboards from Grafana.com:

# Node Exporter Full Dashboard
Dashboard ID: 1860
Data Source: Prometheus

# Docker Container Monitoring
Dashboard ID: 893

# Prometheus Stats
Dashboard ID: 2

# Alertmanager Dashboard
Dashboard ID: 9578

To import: Grafana → Dashboards → Import → Enter Dashboard ID → Load → Select Prometheus data source → Import.

Custom PromQL Queries

hljs promql

# CPU usage percentage
100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# RAM usage percentage
(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100

# Disk usage percentage
(1 - (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"})) * 100

# Network traffic (bytes/second)
rate(node_network_receive_bytes_total{device="eth0"}[5m])
rate(node_network_transmit_bytes_total{device="eth0"}[5m])

# Disk I/O (read/write bytes/second)
rate(node_disk_read_bytes_total[5m])
rate(node_disk_written_bytes_total[5m])

# System uptime
node_time_seconds - node_boot_time_seconds

# Open file descriptors
node_filefd_allocated

Netdata Installation

Netdata is a real-time server monitoring tool. Installation is extremely easy and provides detailed metrics instantly.

One-Command Installation

hljs bash

# Automatic installation script
bash <(curl -Ss https://my-netdata.io/kickstart.sh)

# After installation
sudo systemctl status netdata

Web interface: http://SERVER_IP:19999

Netdata with Docker

hljs yaml

# docker-compose.yml
version: '3.8'
services:
  netdata:
    image: netdata/netdata:latest
    container_name: netdata
    restart: unless-stopped
    ports:
      - "19999:19999"
    cap_add:
      - SYS_PTRACE
      - SYS_ADMIN
    security_opt:
      - apparmor:unconfined
    volumes:
      - netdataconfig:/etc/netdata
      - netdatalib:/var/lib/netdata
      - netdatacache:/var/cache/netdata
      - /etc/passwd:/host/etc/passwd:ro
      - /etc/group:/host/etc/group:ro
      - /etc/localtime:/etc/localtime:ro
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /etc/os-release:/host/etc/os-release:ro
      - /var/log:/host/var/log:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro

volumes:
  netdataconfig:
  netdatalib:
  netdatacache:

Netdata Configuration

hljs bash

# Edit main configuration file
sudo /etc/netdata/edit-config netdata.conf

hljs ini

# /etc/netdata/netdata.conf
[global]
  hostname = server-01
  history = 3996
  update every = 1
  memory mode = dbengine
  page cache size = 64
  dbengine multihost disk space = 2048

[web]
  bind to = 0.0.0.0
  default port = 19999
  allow connections from = localhost 192.168.1.*

Netdata Alarm Configuration

hljs bash

sudo /etc/netdata/edit-config health.d/cpu.conf

hljs yaml

alarm: cpu_usage_high
    on: system.cpu
    lookup: average -5m percentage foreach user,system
    units: %
    every: 1m
    warn: $this > 80
    crit: $this > 95
    info: CPU usage averaged $this% over the last 5 minutes
    to: sysadmin

Complete Monitoring Stack (Docker Compose)

Combine all components in a single compose file:

hljs yaml

# docker-compose.monitoring.yml
version: '3.8'
services:
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    restart: unless-stopped
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus:/etc/prometheus
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.retention.time=30d'
    networks:
      - monitoring

  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    restart: unless-stopped
    ports:
      - "3000:3000"
    volumes:
      - grafana_data:/var/lib/grafana
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=StrongPassword123!
    depends_on:
      - prometheus
    networks:
      - monitoring

  node-exporter:
    image: prom/node-exporter:latest
    container_name: node-exporter
    restart: unless-stopped
    ports:
      - "9100:9100"
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.sysfs=/host/sys'
      - '--path.rootfs=/rootfs'
    networks:
      - monitoring

  alertmanager:
    image: prom/alertmanager:latest
    container_name: alertmanager
    restart: unless-stopped
    ports:
      - "9093:9093"
    volumes:
      - ./alertmanager:/etc/alertmanager
    networks:
      - monitoring

  cadvisor:
    image: gcr.io/cadvisor/cadvisor:latest
    container_name: cadvisor
    restart: unless-stopped
    ports:
      - "8080:8080"
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
    networks:
      - monitoring

networks:
  monitoring:
    driver: bridge

volumes:
  prometheus_data:
  grafana_data:

hljs bash

docker compose -f docker-compose.monitoring.yml up -d

You can monitor your entire REXE server infrastructure from a single dashboard by setting up the Prometheus + Grafana + Netdata stack. Alert rules enable proactive issue detection before they impact your services.

Server Monitoring Tools: Grafana, Prometheus and Netdata

Table of Contents