#!/usr/bin/env bash

set -euo pipefail

URL_FILE="${1:-urls.txt}"
OUTPUT_DIR="${2:-archives}"

mkdir -p "$OUTPUT_DIR"

while IFS= read -r url || [[ -n "$url" ]]; do

    # Skip empty lines
    [[ -z "$url" ]] && continue

    # Skip comments
    [[ "$url" =~ ^# ]] && continue

    echo
    echo "Processing:"
    echo "$url"

    # Create safe filename
    filename=$(echo "$url" \
        | sed 's|https\?://||' \
        | sed 's|[^a-zA-Z0-9]|_|g')

    output="$OUTPUT_DIR/${filename}.html"

    echo "Saving:"
    echo "$output"

    monolith \
        --isolate \
        --user-agent "Mozilla/5.0" \
        "$url" \
        > "$output"

    echo "Compressing with Brotli..."

    brotli -f -q 11 "$output"

    echo "Done."

done < "$URL_FILE"

echo
echo "All URLs processed."