Log Management: Preventing Disk Full with logrotate

# Main system log directory
ls -lh /var/log/

# Important log files:
# /var/log/syslog          — General system messages (Debian/Ubuntu)
# /var/log/messages        — General system messages (RHEL/CentOS)
# /var/log/auth.log        — Authentication logs (Debian/Ubuntu)
# /var/log/secure          — Authentication logs (RHEL/CentOS)
# /var/log/kern.log        — Kernel messages
# /var/log/dmesg           — Boot messages
# /var/log/cron            — Cron job logs
# /var/log/mail.log        — Mail server logs
# /var/log/dpkg.log        — Package installation logs

# Web server logs
/var/log/nginx/access.log
/var/log/nginx/error.log
/var/log/apache2/access.log
/var/log/apache2/error.log

# Database logs
/var/log/mysql/error.log
/var/log/postgresql/

# Real-time log monitoring
tail -f /var/log/syslog
tail -f /var/log/nginx/access.log

# Last 100 lines
tail -n 100 /var/log/auth.log

# Search for specific keyword
grep "ERROR" /var/log/nginx/error.log
grep "Failed password" /var/log/auth.log

# Search in date range
grep "Jan 15" /var/log/syslog

# Search across multiple files
grep -r "error" /var/log/nginx/

# Find largest files under /var/log
du -sh /var/log/* | sort -rh | head -20

# Find large files across the system (100MB+)
find / -type f -size +100M -exec ls -lh {} \; 2>/dev/null

# Files larger than 50MB under /var/log
find /var/log -type f -size +50M -exec ls -lh {} \;

# Check disk usage
df -h

# View directory sizes
du -sh /var/log/
du -sh /var/log/* | sort -rh

# Total size of all log files
du -sh /var/log/

# Journald disk usage
journalctl --disk-usage

# Nginx log sizes
ls -lh /var/log/nginx/

# logrotate typically runs daily via cron
cat /etc/cron.daily/logrotate

# Run manually
sudo logrotate /etc/logrotate.conf

# With a specific configuration file
sudo logrotate /etc/logrotate.d/nginx

# Simulation mode (no actual operations)
sudo logrotate --debug /etc/logrotate.conf

# Force rotation (even if time hasn't elapsed)
sudo logrotate --force /etc/logrotate.d/nginx

# Main configuration file
sudo nano /etc/logrotate.conf

# Default settings
weekly          # Rotate weekly
rotate 4        # Keep 4 old versions
create          # Create new file after rotation
compress        # Compress old logs
delaycompress   # Don't compress the most recently rotated log
include /etc/logrotate.d  # Include application configs

# logrotate configuration for Nginx
sudo nano /etc/logrotate.d/nginx

/var/log/nginx/*.log {
    daily           # Rotate daily
    missingok       # Don't error if file is missing
    rotate 14       # Keep 14 days of logs
    compress        # Compress
    delaycompress   # Don't compress latest (nginx may still be writing)
    notifempty      # Don't rotate empty files
    create 0640 www-data adm  # New file permissions
    sharedscripts   # Run scripts once for all files
    postrotate
        # Tell nginx to use new log file
        if [ -f /var/run/nginx.pid ]; then
            kill -USR1 `cat /var/run/nginx.pid`
        fi
    endscript
}

# logrotate configuration for your own application
sudo nano /etc/logrotate.d/myapp

/var/log/myapp/*.log {
    daily
    rotate 30
    compress
    delaycompress
    missingok
    notifempty
    create 0644 www-data www-data
    dateext         # Use date extension (myapp.log-20240115.gz)
    dateformat -%Y%m%d
    postrotate
        systemctl reload myapp 2>/dev/null || true
    endscript
}

# Check current journal size
journalctl --disk-usage

# Clean up old logs (older than 2 weeks)
sudo journalctl --vacuum-time=2weeks

# Clean up with size limit (delete more than 500MB)
sudo journalctl --vacuum-size=500M

# Keep specific number of files
sudo journalctl --vacuum-files=5

# Edit journald configuration file
sudo nano /etc/systemd/journald.conf

[Journal]
# Maximum disk usage
SystemMaxUse=500M

# Minimum free disk space
SystemKeepFree=1G

# Maximum file size
SystemMaxFileSize=50M

# Log retention period
MaxRetentionSec=2weeks

# Compression
Compress=yes

# Persistent storage (disk)
Storage=persistent

# Apply configuration
sudo systemctl restart systemd-journald

# Analyze Nginx error logs
grep -c "error" /var/log/nginx/error.log  # Error count
grep "error" /var/log/nginx/error.log | tail -20  # Recent errors

# Detect SSH brute force attempts
grep "Failed password" /var/log/auth.log | awk '{print $11}' | sort | uniq -c | sort -rn | head -10

# Find top requesting IPs
awk '{print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head -10

# Count HTTP status codes
awk '{print $9}' /var/log/nginx/access.log | sort | uniq -c | sort -rn

# Filter errors from systemd journal
journalctl -p err --since "1 hour ago"

# Quick response when disk is full

# 1. Check disk status
df -h

# 2. Find largest files
du -sh /var/log/* | sort -rh | head -10

# 3. Clean old journal logs
sudo journalctl --vacuum-time=3days

# 4. Compress uncompressed old logs
find /var/log -name "*.log.*" ! -name "*.gz" -exec gzip {} \;

# 5. Force logrotate
sudo logrotate --force /etc/logrotate.conf

# 6. Clean package cache
sudo apt clean  # Ubuntu/Debian
sudo dnf clean all  # RHEL/CentOS