bot_detector.hl

// Script for detecting unwanted bots on our sites and blocking their IPs.
// Usage: ./bot-detector.sh <LOG_FILE_PATH>
// The script is triggered by a cronjob every 10 minutes.

fn main (args: any[]) {
    if $args.length() < 1 {
        throw """Path to log file missing.
Usage: bot-detector.sh <logfile>
       bot-detector.sh /var/log/nginx/access.log
"""
    }

    let logfile = args[0]
    try {
        test -r $logfile
    } catch {
        throw "File not found or not readable: $logfile"
    }

    let start = num.parse(${ date +%s })

    // Get server IP address for excluding.
    let server_ip = $(hostname -i)

    // We want to check the previous hour and the current hour.
    let timeframes = ["1 hour ago", "now"]
    loop $timeframe in $timeframes {
        if $timeframe == "1 hour ago" {
            echo "Checking the previous hour..."
        } else {
            echo "Checking the current hour..."
        }

        let hour_timestamp = ${date "+%d/%b/%Y:%H" -d "{timeframe}"}
        // Get the top 20 IP addresses that accessed job pages for the given hour
        // timestamp. Includes a count per IP address in the format: "<count> <ip>".
        // Only check GET requests to certain job-related paths.
        // Ignore requests from well-behaved bots that send a bot user agent.
        // Never block requests from Google in the user agent.
        // Exclude requests to /files/ paths.
        // Check the top 20 results.
        let ip_log = ${grep -e "{hour_timestamp}" "{logfile}" | \
            grep \
                -e "GET /job/" \
                -e "GET /jobs/" \
                -e "GET /stelle/" \
                -e "GET /stellen/" \
                -e "GET /stellenangebot/" \
                -e "GET /de/stelle/" \
                -e "GET /de/stellen/" \
                -e "GET /de/stellenangebot/" | \
            grep -i -v "bot" | \
            grep -v "Google" | \
            grep -v /files/ | \
            awk '\{print \$1}' | sort | uniq -c | sort -nr | \
            grep -v "{{&server_ip}}" | \
            head -n 20}

        loop $line in lines(ip_log) {
            let parts = $line.split(" ")
            let count = num.parse(parts[0])
            // Skip IP addresses that sent less than 1000 requests.
            if count < 1000 {
                continue
            }

            let ip = parts[1]
            try {
                grep $ip "/etc/ipblocklist.txt"
                echo "IP address $ip is already blocked."
                continue
            } catch { }

            try {
                grep $ip "/etc/ipexcludedlist.txt"
                echo "IP address $ip is allow-listed and will not be blocked."
                continue
            } catch { }

            echo "Blocking IP address: $ip ($count requests)"
            echo $ip >> f"/etc/ipblocklist.txt"
            echo "$(date) | IP addess $ip added to the block list, RPH=$count" >> f"/var/log/bot-detector.log"
        }
    }
    let end = num.parse($ date +%s $?)?
    let duration = $end - $start
    echo "Execution time: $duration seconds"
}