hoardom/scripts/auto-violator.sh

#!/usr/bin/env bash
# auto-vibrator.sh — piss slop that barely works <3
#
# Usage:
#   ./scripts/auto-violator.sh              # fetch all, show summary
#   ./scripts/auto-violator.sh porkbun      # porkbun only
#   ./scripts/auto-violator.sh inwx         # inwx only
#   ./scripts/auto-violator.sh --raw        # output raw TLD lists (one per line)
#   ./scripts/auto-violator.sh --toml       # output TOML-ready arrays
#   ./scripts/auto-violator.sh --diff       # compare against current Lists.toml
#   ./scripts/auto-violator.sh --template   # generate Lists.toml into violator-workdir/
#   ./scripts/auto-violator.sh --probe      # probe unknown TLDs for WHOIS servers
#
# Outputs:
#   violator-workdir/cache/       — cached API responses
#   violator-workdir/pdom.txt     — purchasable TLDs with known WHOIS/RDAP servers
#   violator-workdir/sdom.txt     — TLDs where no WHOIS server could be found
#   violator-workdir/Lists.toml   — generated Lists.toml (never overwrites project root)
#
# Config: scripts/violator.conf
#
# Notes : yea parts of this is ai slop, didnt make it myself oooo scary,
#         but most of the rust i did myself just didnt feel like doing
#         this at 4am and it somewhat works
# Correction : The initial porkbun fetching was mostly me but porkbun
#              lacked many domains so yea

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
LISTS_TOML="$PROJECT_DIR/Lists.toml"
WORK_DIR="$SCRIPT_DIR/violator-workdir"
CACHE_DIR="$WORK_DIR/cache"
CONF_FILE="$SCRIPT_DIR/violator.conf"
PDOM_FILE="$WORK_DIR/pdom.txt"
SDOM_FILE="$WORK_DIR/sdom.txt"
OUTPUT_TOML="$WORK_DIR/Lists.toml"

mkdir -p "$CACHE_DIR"

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
BOLD='\033[1m'
NC='\033[0m'

# ═══════════════════════════════════════════════════════════════════════════
# Config parser
# ═══════════════════════════════════════════════════════════════════════════

# Read a value from violator.conf
# Usage: conf_get section key [default]
conf_get() {
    local section="$1" key="$2" default="${3:-}"
    if [[ ! -f "$CONF_FILE" ]]; then
        echo "$default"
        return
    fi
    awk -v section="$section" -v key="$key" -v default="$default" '
        /^\[/ { in_section = ($0 == "[" section "]") ; next }
        in_section && /^[[:space:]]*#/ { next }
        in_section && /^[[:space:]]*$/ { next }
        in_section {
            # key = value
            split($0, kv, "=")
            gsub(/^[[:space:]]+|[[:space:]]+$/, "", kv[1])
            gsub(/^[[:space:]]+|[[:space:]]+$/, "", kv[2])
            if (kv[1] == key) { print kv[2]; found=1; exit }
        }
        END { if (!found) print default }
    ' "$CONF_FILE"
}

# Read all keys from a section (returns "key value" lines)
conf_section_keys() {
    local section="$1"
    if [[ ! -f "$CONF_FILE" ]]; then
        return
    fi
    awk -v section="$section" '
        /^\[/ { in_section = ($0 == "[" section "]") ; next }
        in_section && /^[[:space:]]*#/ { next }
        in_section && /^[[:space:]]*$/ { next }
        in_section {
            split($0, kv, "=")
            gsub(/^[[:space:]]+|[[:space:]]+$/, "", kv[1])
            gsub(/^[[:space:]]+|[[:space:]]+$/, "", kv[2])
            if (kv[1] != "" && kv[2] != "") print kv[1], kv[2]
        }
    ' "$CONF_FILE"
}

# Read a bare-value section (values are just words, possibly multiline)
conf_section_values() {
    local section="$1"
    if [[ ! -f "$CONF_FILE" ]]; then
        return
    fi
    awk -v section="$section" '
        /^\[/ { in_section = ($0 == "[" section "]") ; next }
        in_section && /^[[:space:]]*#/ { next }
        in_section && /^[[:space:]]*$/ { next }
        in_section { print }
    ' "$CONF_FILE"
}

# Read the tlds field from a [list.NAME] section (may be multiline, indented continuation)
conf_list_tlds() {
    local list_name="$1"
    local section="list.${list_name}"
    if [[ ! -f "$CONF_FILE" ]]; then
        return
    fi
    awk -v section="$section" '
        /^\[/ { in_section = ($0 == "[" section "]"); next }
        in_section && /^[[:space:]]*#/ { next }
        in_section && /^[[:space:]]*$/ { next }
        in_section {
            split($0, kv, "=")
            gsub(/^[[:space:]]+|[[:space:]]+$/, "", kv[1])
            if (kv[1] == "tlds") { in_tlds=1; gsub(/^[^=]*=/, ""); print; next }
            if (in_tlds && /^[[:space:]]/) { print; next }
            in_tlds=0
        }
    ' "$CONF_FILE" | tr '\n' ' ' | sed 's/[[:space:]]\+/ /g; s/^ //; s/ $//'
}

# Get all [list.*] section names
conf_list_names() {
    if [[ ! -f "$CONF_FILE" ]]; then
        echo "standard decent swiss country two three four long all"
        return
    fi
    grep -oE '^\[list\.[a-z0-9_]+\]' "$CONF_FILE" | sed 's/\[list\.//;s/\]//'
}

# Load skip TLDs from config
load_skip_tlds() {
    local skip
    skip=$(conf_section_values "skip_tlds" | tr '\n' ' ')
    if [[ -z "$skip" ]]; then
        skip="bl bq eh mf gb bv sj kp hm"
    fi
    echo "$skip"
}

# Load whois overrides from config
load_whois_overrides() {
    conf_section_keys "whois_overrides"
}

# ═══════════════════════════════════════════════════════════════════════════
# Fetchers
# ═══════════════════════════════════════════════════════════════════════════

# ─── Porkbun ────────────────────────────────────────────────────────────────
fetch_porkbun() {
    local cache="$CACHE_DIR/porkbun.json"
    local max_age=86400

    if [[ -f "$cache" ]]; then
        local age=$(( $(date +%s) - $(stat -f%m "$cache" 2>/dev/null || stat -c%Y "$cache" 2>/dev/null || echo 0) ))
        if (( age < max_age )); then
            echo "$cache"
            return 0
        fi
    fi

    echo -e "${CYAN}Fetching Porkbun pricing API...${NC}" >&2
    if curl -sf -X POST "https://api.porkbun.com/api/json/v3/pricing/get" \
        -H "Content-Type: application/json" \
        -d '{}' \
        -o "$cache" 2>/dev/null; then
        echo "$cache"
    else
        echo -e "${RED}Failed to fetch Porkbun data${NC}" >&2
        return 1
    fi
}

parse_porkbun() {
    local json_file="$1"
    if command -v jq &>/dev/null; then
        jq -r '.pricing // {} | keys[]' "$json_file" 2>/dev/null | sort -u
    else
        grep -o '"[a-z][a-z0-9.-]*":{' "$json_file" | sed 's/"//g; s/:{//' | sort -u
    fi
}

# ─── INWX ───────────────────────────────────────────────────────────────────
fetch_inwx() {
    local cache="$CACHE_DIR/inwx.csv"
    local max_age=86400

    if [[ -f "$cache" ]]; then
        local age=$(( $(date +%s) - $(stat -f%m "$cache" 2>/dev/null || stat -c%Y "$cache" 2>/dev/null || echo 0) ))
        if (( age < max_age )); then
            echo "$cache"
            return 0
        fi
    fi

    echo -e "${CYAN}Fetching INWX pricelist CSV...${NC}" >&2
    if curl -sf "https://www.inwx.ch/en/domain/pricelist/vat/1/file/csv" \
        -o "$cache" 2>/dev/null; then
        echo "$cache"
    else
        echo -e "${YELLOW}Could not fetch INWX${NC}" >&2
        return 1
    fi
}

parse_inwx() {
    local csv_file="$1"
    sed 's/;.*//' "$csv_file" | tr -d '"' | grep -E '^[a-z][a-z0-9]*$' | sort -u
}

# ─── OVH ────────────────────────────────────────────────────────────────────
fetch_ovh() {
    local cache="$CACHE_DIR/ovh.json"
    local max_age=86400

    if [[ -f "$cache" ]]; then
        local age=$(( $(date +%s) - $(stat -f%m "$cache" 2>/dev/null || stat -c%Y "$cache" 2>/dev/null || echo 0) ))
        if (( age < max_age )); then
            echo "$cache"
            return 0
        fi
    fi

    echo -e "${CYAN}Fetching OVH domain extensions...${NC}" >&2
    if curl -sf "https://www.ovh.com/engine/apiv6/domain/extensions" \
        -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)" \
        -o "$cache" 2>/dev/null; then
        echo "$cache"
    else
        echo -e "${YELLOW}Could not fetch OVH extensions${NC}" >&2
        return 1
    fi
}

parse_ovh() {
    local json_file="$1"
    if command -v jq &>/dev/null; then
        jq -r '.[]' "$json_file" 2>/dev/null | grep -vE '\.' | sort -u
    else
        grep -oE '"[a-z]{2,20}"' "$json_file" | tr -d '"' | grep -vE '\.' | sort -u
    fi
}

# ─── DomainOffer.net ────────────────────────────────────────────────────────
fetch_domainoffer() {
    local cache="$CACHE_DIR/domainoffer.html"
    local max_age=86400

    if [[ -f "$cache" ]]; then
        local age=$(( $(date +%s) - $(stat -f%m "$cache" 2>/dev/null || stat -c%Y "$cache" 2>/dev/null || echo 0) ))
        if (( age < max_age )); then
            echo "$cache"
            return 0
        fi
    fi

    echo -e "${CYAN}Fetching DomainOffer.net price compare...${NC}" >&2
    if curl -sf "https://domainoffer.net/price-compare" \
        -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)" \
        -o "$cache" 2>/dev/null; then
        echo "$cache"
    else
        echo -e "${YELLOW}Could not fetch DomainOffer.net${NC}" >&2
        return 1
    fi
}

parse_domainoffer() {
    local html_file="$1"
    local parsed_cache="$CACHE_DIR/domainoffer-tlds.txt"

    # Return cached parsed list if newer than HTML
    if [[ -f "$parsed_cache" && "$parsed_cache" -nt "$html_file" ]]; then
        cat "$parsed_cache"
        return
    fi

    local result=""
    if command -v python3 &>/dev/null; then
        result=$(python3 -c "
import re, json, sys
with open('$html_file') as f:
    html = f.read()
m = re.search(r'var domainData = (\[.*?\]);', html, re.DOTALL)
if not m:
    sys.exit(0)
for entry in json.loads(m.group(1)):
    tld = entry[0].lstrip('.').lower() if isinstance(entry, list) and entry else ''
    if '.' not in tld and re.match(r'^[a-z][a-z0-9]*$', tld):
        print(tld)
" | sort -u)
    else
        result=$(grep -oE '\["[a-z][a-z0-9]*",' "$html_file" | sed 's/\["//;s/",//' | sort -u)
    fi

    # Cache the parsed result
    echo "$result" > "$parsed_cache"
    echo "$result"
}

# ─── tld-list.com ───────────────────────────────────────────────────────────
fetch_tldlist() {
    local cache="$CACHE_DIR/tldlist-basic.txt"
    local max_age=86400

    if [[ -f "$cache" ]]; then
        local age=$(( $(date +%s) - $(stat -f%m "$cache" 2>/dev/null || stat -c%Y "$cache" 2>/dev/null || echo 0) ))
        if (( age < max_age )); then
            echo "$cache"
            return 0
        fi
    fi

    echo -e "${CYAN}Fetching tld-list.com basic list...${NC}" >&2
    if curl -sf "https://tld-list.com/df/tld-list-basic.csv" \
        -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)" \
        -o "$cache" 2>/dev/null; then
        echo "$cache"
    else
        echo -e "${YELLOW}Could not fetch tld-list.com${NC}" >&2
        return 1
    fi
}

parse_tldlist() {
    local file="$1"
    tr -d '\r' < "$file" | grep -E '^[a-z][a-z0-9]*$' | sort -u
}

# ─── IANA root zone ─────────────────────────────────────────────────────────
fetch_iana() {
    local cache="$CACHE_DIR/iana-tlds.txt"
    local max_age=604800

    if [[ -f "$cache" ]]; then
        local age=$(( $(date +%s) - $(stat -f%m "$cache" 2>/dev/null || stat -c%Y "$cache" 2>/dev/null || echo 0) ))
        if (( age < max_age )); then
            echo "$cache"
            return 0
        fi
    fi

    echo -e "${CYAN}Fetching IANA TLD list...${NC}" >&2
    if curl -sf "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" -o "$cache" 2>/dev/null; then
        echo "$cache"
    else
        echo -e "${RED}Failed to fetch IANA list${NC}" >&2
        return 1
    fi
}

parse_iana() {
    local file="$1"
    tail -n +2 "$file" | tr '[:upper:]' '[:lower:]' | sort -u
}

parse_iana_cctlds() {
    local file="$1"
    tail -n +2 "$file" | tr '[:upper:]' '[:lower:]' | grep -E '^[a-z]{2}$' | sort -u
}

# ─── RDAP bootstrap ─────────────────────────────────────────────────────────
fetch_rdap() {
    local cache="$CACHE_DIR/rdap-dns.json"
    local max_age=86400

    if [[ -f "$cache" ]]; then
        local age=$(( $(date +%s) - $(stat -f%m "$cache" 2>/dev/null || stat -c%Y "$cache" 2>/dev/null || echo 0) ))
        if (( age < max_age )); then
            echo "$cache"
            return 0
        fi
    fi

    echo -e "${CYAN}Fetching RDAP bootstrap...${NC}" >&2
    if curl -sf "https://data.iana.org/rdap/dns.json" -o "$cache" 2>/dev/null; then
        echo "$cache"
    else
        echo -e "${RED}Failed to fetch RDAP bootstrap${NC}" >&2
        return 1
    fi
}

parse_rdap_tlds() {
    local json_file="$1"
    if command -v jq &>/dev/null; then
        jq -r '.services[][] | .[]' "$json_file" 2>/dev/null | grep -v '^http' | tr '[:upper:]' '[:lower:]' | sort -u
    else
        grep -oE '"[a-z]{2,20}"' "$json_file" | tr -d '"' | sort -u
    fi
}

# ─── WHOIS server list ──────────────────────────────────────────────────────
fetch_whois_servers() {
    local cache="$CACHE_DIR/tld_serv_list.txt"
    local max_age=604800

    if [[ -f "$cache" ]]; then
        local age=$(( $(date +%s) - $(stat -f%m "$cache" 2>/dev/null || stat -c%Y "$cache" 2>/dev/null || echo 0) ))
        if (( age < max_age )); then
            echo "$cache"
            return 0
        fi
    fi

    echo -e "${CYAN}Fetching WHOIS server list...${NC}" >&2
    if curl -sf "https://raw.githubusercontent.com/rfc1036/whois/next/tld_serv_list" -o "$cache" 2>/dev/null; then
        echo "$cache"
    else
        echo -e "${YELLOW}Could not fetch WHOIS server list${NC}" >&2
        return 1
    fi
}

get_whois_server() {
    local tld="$1"
    local serv_file="$2"
    local line
    line=$(grep -E "^\\.${tld}[[:space:]]" "$serv_file" 2>/dev/null | head -1)
    if [[ -z "$line" ]]; then
        echo ""
        return
    fi
    local server
    server=$(echo "$line" | awk '{
        for (i=NF; i>=2; i--) {
            if ($i ~ /^[a-z0-9].*\.[a-z]/) { print $i; exit }
        }
    }')
    if [[ "$server" == "NONE" || "$server" == "ARPA" || -z "$server" || "$server" == http* ]]; then
        echo ""
    else
        echo "$server"
    fi
}

get_iana_whois_server() {
    local tld="$1"
    curl -s "https://www.iana.org/domains/root/db/${tld}.html" 2>/dev/null \
        | sed -n 's/.*WHOIS Server:<\/b> *\([^ <]*\).*/\1/p' \
        | head -1
}

# ═══════════════════════════════════════════════════════════════════════════
# WHOIS Probe — try common server patterns for unknown TLDs
# ═══════════════════════════════════════════════════════════════════════════

probe_whois_server() {
    local tld="$1"
    local timeout_s="$2"
    local patterns_str="$3"

    # Split patterns on comma
    IFS=',' read -ra patterns <<< "$patterns_str"
    for pattern in "${patterns[@]}"; do
        pattern=$(echo "$pattern" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
        local server="${pattern//\{\}/$tld}"
        # Perl socket connect — Timeout covers both DNS resolution and TCP connect
        # unlike nc -w which only covers TCP and lets DNS hang forever on macOS
        if perl -e 'use IO::Socket::INET; exit(0) if IO::Socket::INET->new(PeerAddr=>$ARGV[0],PeerPort=>43,Timeout=>$ARGV[1]); exit(1)' "$server" "$timeout_s" 2>/dev/null; then
            echo "$server"
            return 0
        fi
    done
    return 1
}

# Worker function for parallel probing — writes result to a file and prints live
_probe_worker() {
    local tld="$1" timeout_s="$2" patterns="$3" result_dir="$4" idx="$5" total="$6"
    local server
    if server=$(probe_whois_server "$tld" "$timeout_s" "$patterns"); then
        echo "${tld}:${server}" > "${result_dir}/${tld}.found"
        echo -e "  [${idx}/${total}] ${GREEN}✓${NC} ${tld} → ${server}" >&2
    else
        touch "${result_dir}/${tld}.miss"
        echo -e "  [${idx}/${total}] ${RED}✗${NC} ${tld}" >&2
    fi
}

run_whois_probes() {
    local sdom_file="$1"
    local pdom_file="$2"

    local probe_enabled
    probe_enabled=$(conf_get "whois_probe" "enabled" "true")
    if [[ "$probe_enabled" != "true" ]]; then
        echo -e "  ${YELLOW}WHOIS probing disabled in config${NC}" >&2
        return
    fi

    local timeout_s
    timeout_s=$(conf_get "whois_probe" "timeout" "2")
    local patterns
    patterns=$(conf_get "whois_probe" "patterns" "whois.nic.{}, whois.{}, whois.registry.{}")
    local max_jobs
    max_jobs=$(conf_get "whois_probe" "parallel" "10")

    if [[ ! -f "$sdom_file" ]] || [[ ! -s "$sdom_file" ]]; then
        return
    fi

    # Count patterns for info
    local pattern_count=0
    IFS=',' read -ra _pats <<< "$patterns"
    pattern_count=${#_pats[@]}
    unset _pats

    local total
    total=$(wc -l < "$sdom_file" | tr -d ' ')
    echo -e "  ${CYAN}Probing ${total} TLDs (${pattern_count} patterns, ${timeout_s}s timeout, ${max_jobs} parallel)${NC}" >&2

    # Use workdir for temp results so they're visible
    local result_dir="$WORK_DIR/probe-tmp"
    rm -rf "$result_dir"
    mkdir -p "$result_dir"

    # Read all TLDs into array
    local -a tld_list=()
    while IFS= read -r tld; do
        [[ -n "$tld" ]] && tld_list+=("$tld")
    done < "$sdom_file"

    # Launch all jobs with max_jobs concurrency
    # Each worker prints its own result immediately to stderr
    local running=0 idx=0
    for tld in "${tld_list[@]}"; do
        ((idx++)) || true
        _probe_worker "$tld" "$timeout_s" "$patterns" "$result_dir" "$idx" "$total" &
        ((running++)) || true

        # Throttle: when we hit max_jobs, wait for any one to finish
        if (( running >= max_jobs )); then
            wait -n 2>/dev/null || wait
            ((running--)) || true
        fi
    done

    # Wait for remaining
    wait

    # Collect results: append found to pdom, rebuild sdom
    local found=0
    local new_sdom=""
    for tld in "${tld_list[@]}"; do
        if [[ -f "${result_dir}/${tld}.found" ]]; then
            cat "${result_dir}/${tld}.found" >> "$pdom_file"
            ((found++)) || true
        else
            new_sdom+="${tld}"$'\n'
        fi
    done

    # Cleanup
    rm -rf "$result_dir"

    # Rewrite sdom with remaining unknowns
    echo -n "$new_sdom" | sort -u > "$sdom_file"

    if (( found > 0 )); then
        sort -u -o "$pdom_file" "$pdom_file"
        echo -e "  ${GREEN}Probing complete: found ${found}/${total} WHOIS servers${NC}" >&2
    else
        echo -e "  ${YELLOW}Probing complete: no new servers (0/${total})${NC}" >&2
    fi
}

# ═══════════════════════════════════════════════════════════════════════════
# Helpers
# ═══════════════════════════════════════════════════════════════════════════

parse_current_lists() {
    local list_name="${1:-all}"
    if [[ ! -f "$LISTS_TOML" ]]; then
        echo -e "${RED}No Lists.toml found at $LISTS_TOML${NC}" >&2
        return 1
    fi
    awk -v list="$list_name" '
        $0 ~ "^"list" *= *\\[" { found=1; next }
        found && /^\]/ { exit }
        found && /^[[:space:]]*\[/ { exit }
        found {
            gsub(/["\t,]/, " ")
            n = split($0, parts, " ")
            for (i=1; i<=n; i++) {
                if (parts[i] != "") {
                    sub(/:.*/, "", parts[i])
                    print parts[i]
                }
            }
        }
    ' "$LISTS_TOML" | sort -u
}

to_toml_array() {
    local tlds=()
    while IFS= read -r tld; do
        [[ -z "$tld" ]] && continue
        tlds+=("$tld")
    done
    local line='\t'
    local first=true
    for tld in "${tlds[@]}"; do
        local entry="\"$tld\""
        if $first; then
            line+="$entry"
            first=false
        else
            local test_line="$line, $entry"
            if (( ${#test_line} > 78 )); then
                echo -e "$line,"
                line="\t$entry"
            else
                line+=", $entry"
            fi
        fi
    done
    [[ -n "$line" ]] && echo -e "$line,"
}

filter_cctlds() {
    grep -E '^[a-z]{2}$'
}

filter_short_tlds() {
    grep -E '^[a-z]{2,6}$'
}

SKIP_TLDS=""

filter_skip() {
    if [[ -z "$SKIP_TLDS" ]]; then
        cat
        return
    fi
    local skip_pattern
    # trim whitespace and convert spaces to regex alternation
    skip_pattern=$(echo "$SKIP_TLDS" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | tr -s ' ' '|')
    if [[ -z "$skip_pattern" ]]; then
        cat
        return
    fi
    grep -vE "^($skip_pattern)$"
}

# ═══════════════════════════════════════════════════════════════════════════
# Template generation (Lists.toml)
# ═══════════════════════════════════════════════════════════════════════════

generate_template() {
    local all_registrar_tlds="$1"
    local rdap_tlds="$2"
    local source_summary="$3"

    # Fetch WHOIS server list
    local whois_serv_file=""
    if whois_serv_file=$(fetch_whois_servers 2>/dev/null); then
        true
    fi

    # Load manual whois overrides from config
    local -A manual_whois=()
    while IFS=' ' read -r tld server; do
        [[ -z "$tld" ]] && continue
        manual_whois["$tld"]="$server"
    done < <(load_whois_overrides)

    local buyable_tlds
    buyable_tlds=$(echo "$all_registrar_tlds" | filter_skip | sort -u)

    local buyable_count
    buyable_count=$(echo "$buyable_tlds" | grep -c . || echo 0)

    # Build annotated TLD list: "tld" or "tld:whois_server"
    local annotated_all=()
    local annotated_cc=()
    local rdap_hit=0 whois_hit=0 bare_hit=0
    local pdom_entries=()
    local sdom_entries=()

    while IFS= read -r tld; do
        [[ -z "$tld" ]] && continue
        local entry=""

        # Check manual override first
        if [[ -n "${manual_whois[$tld]:-}" ]]; then
            entry="${tld}:${manual_whois[$tld]}"
            ((whois_hit++)) || true
            pdom_entries+=("$entry")
        elif echo "$rdap_tlds" | grep -qx "$tld" 2>/dev/null; then
            entry="$tld"
            ((rdap_hit++)) || true
            pdom_entries+=("$tld")
        else
            local server=""
            if [[ -n "$whois_serv_file" ]]; then
                server=$(get_whois_server "$tld" "$whois_serv_file")
            fi
            if [[ -n "$server" ]]; then
                entry="${tld}:${server}"
                ((whois_hit++)) || true
                pdom_entries+=("$entry")
            else
                entry="$tld"
                ((bare_hit++)) || true
                sdom_entries+=("$tld")
            fi
        fi
        annotated_all+=("$entry")
        local base_tld="${tld%%:*}"
        if [[ "$base_tld" =~ ^[a-z]{2}$ ]]; then
            annotated_cc+=("$entry")
        fi
    done <<< "$buyable_tlds"

    # Write pdom.txt and sdom.txt
    printf '%s\n' "${pdom_entries[@]}" | sort -u > "$PDOM_FILE"
    printf '%s\n' "${sdom_entries[@]}" | sort -u > "$SDOM_FILE"

    echo -e "${CYAN}Building template...${NC}" >&2
    echo -e "  ${GREEN}${rdap_hit}${NC} TLDs with RDAP (direct lookup)" >&2
    echo -e "  ${YELLOW}${whois_hit}${NC} TLDs with WHOIS override" >&2
    echo -e "  ${RED}${bare_hit}${NC} TLDs with no known server (will probe)" >&2
    echo -e "  ${CYAN}pdom.txt:${NC} $(wc -l < "$PDOM_FILE" | tr -d ' ') entries" >&2
    echo -e "  ${CYAN}sdom.txt:${NC} $(wc -l < "$SDOM_FILE" | tr -d ' ') entries" >&2
    echo "" >&2

    # Run WHOIS probe on sdom entries if enabled
    if [[ "$DO_PROBE" == true ]]; then
        run_whois_probes "$SDOM_FILE" "$PDOM_FILE"
        # Reload pdom into annotated_all (probed ones now have servers)
        if [[ -s "$PDOM_FILE" ]]; then
            local probed_tlds
            probed_tlds=$(cat "$PDOM_FILE")
            # Rebuild annotated arrays with probed data
            annotated_all=()
            annotated_cc=()
            while IFS= read -r entry; do
                [[ -z "$entry" ]] && continue
                annotated_all+=("$entry")
                local base="${entry%%:*}"
                if [[ "$base" =~ ^[a-z]{2}$ ]]; then
                    annotated_cc+=("$entry")
                fi
            done <<< "$probed_tlds"
            # Add remaining sdom entries (no server found) unless --strict
            if [[ -s "$SDOM_FILE" && "$STRICT" != true ]]; then
                while IFS= read -r tld; do
                    [[ -z "$tld" ]] && continue
                    annotated_all+=("$tld")
                    if [[ "$tld" =~ ^[a-z]{2}$ ]]; then
                        annotated_cc+=("$tld")
                    fi
                done < "$SDOM_FILE"
            fi
        fi
    fi

    # --strict without --probe: filter out sdom entries from annotated arrays
    if [[ "$STRICT" == true && "$DO_PROBE" != true && -s "$SDOM_FILE" ]]; then
        local -A sdom_set=()
        while IFS= read -r tld; do
            [[ -n "$tld" ]] && sdom_set[$tld]=1
        done < "$SDOM_FILE"
        local filtered_all=() filtered_cc=()
        for ann in "${annotated_all[@]}"; do
            local base="${ann%%:*}"
            if [[ -z "${sdom_set[$base]:-}" ]]; then
                filtered_all+=("$ann")
                if [[ "$base" =~ ^[a-z]{2}$ ]]; then
                    filtered_cc+=("$ann")
                fi
            fi
        done
        local stripped=$(( ${#annotated_all[@]} - ${#filtered_all[@]} ))
        annotated_all=("${filtered_all[@]}")
        annotated_cc=("${filtered_cc[@]}")
        echo -e "  ${YELLOW}--strict:${NC} removed $stripped TLDs with no working server" >&2
    fi

    # ── Build list output from config ──────────────────────────────────

    # Annotate a curated tld list with whois overrides
    annotate_tld() {
        local bare_tld="$1"
        for ann in "${annotated_all[@]}"; do
            local ann_base="${ann%%:*}"
            if [[ "$ann_base" == "$bare_tld" ]]; then
                echo "$ann"
                return
            fi
        done
        echo "$bare_tld"
    }

    filter_annotated_by_length() {
        local min="$1" max="$2"
        for ann in "${annotated_all[@]}"; do
            local base="${ann%%:*}"
            local len=${#base}
            if (( len >= min && len <= max )); then
                echo "$ann"
            fi
        done
    }

    filter_annotated_cctlds() {
        for ann in "${annotated_all[@]}"; do
            local base="${ann%%:*}"
            if [[ "$base" =~ ^[a-z]{2}$ ]]; then
                echo "$ann"
            fi
        done
    }

    # ── Output Lists.toml ──────────────────────────────────────────────
    local date_str
    date_str=$(date +%Y-%m-%d)

    # Build description comments from config
    local list_descriptions=""
    while IFS= read -r name; do
        local desc
        desc=$(conf_get "list.${name}" "description" "$name")
        list_descriptions+="#   ${name}$(printf '%*s' $((10 - ${#name})) '')— ${desc}"$'\n'
    done < <(conf_list_names)

    cat <<HEADER
# Lists.toml — Built-in TLD lists for hoardom
# Auto-generated on ${date_str} from ${source_summary}
#
# Format:
#   "tld"              — TLD has RDAP support, lookup works directly
#   "tld:whois.server" — No RDAP: use this WHOIS server for fallback
#
# ${buyable_count} purchasable TLDs (handshake/sub-TLDs excluded)
# ${rdap_hit} have RDAP, ${whois_hit} need WHOIS override, ${bare_hit} will auto-probe
#
# Lists:
${list_descriptions}
HEADER

    # Generate each list from config
    while IFS= read -r name; do
        local type
        type=$(conf_get "list.${name}" "type" "curated")

        case "$type" in
            curated)
                local tlds_str
                tlds_str=$(conf_list_tlds "$name")
                echo "${name} = ["
                for bare_tld in $tlds_str; do
                    annotate_tld "$bare_tld"
                done | to_toml_array
                echo "]"
                echo ""
                ;;
            filter)
                local min_len max_len country_only
                min_len=$(conf_get "list.${name}" "min_length" "2")
                max_len=$(conf_get "list.${name}" "max_length" "99")
                country_only=$(conf_get "list.${name}" "country_only" "false")

                echo "${name} = ["
                if [[ "$country_only" == "true" ]]; then
                    filter_annotated_cctlds | to_toml_array
                else
                    filter_annotated_by_length "$min_len" "$max_len" | to_toml_array
                fi
                echo "]"
                echo ""
                ;;
            all)
                echo "${name} = ["
                printf '%s\n' "${annotated_all[@]}" | to_toml_array
                echo "]"
                echo ""
                ;;
        esac
    done < <(conf_list_names)
}

# ═══════════════════════════════════════════════════════════════════════════
# Main
# ═══════════════════════════════════════════════════════════════════════════

main() {
    local mode="summary"
    local source="all"
    local all_sources=false
    DO_PROBE=false
    STRICT=false

    # Load skip TLDs from config
    SKIP_TLDS=$(load_skip_tlds)

    for arg in "$@"; do
        case "$arg" in
            --raw)       mode="raw" ;;
            --toml)      mode="toml" ;;
            --diff)      mode="diff" ;;
            --template)  mode="template" ;;
            --probe)     DO_PROBE=true ;;
            --strict)    STRICT=true ;;
            --all-sources) all_sources=true ;;
            porkbun)     source="porkbun" ;;
            inwx)        source="inwx" ;;
            ovh)         source="ovh" ;;
            domainoffer) source="domainoffer" ;;
            iana)        source="iana" ;;
            rdap)        source="rdap" ;;
            tldlist)     source="tldlist" ;;
            --help|-h)
                echo "Usage: $0 [source] [--raw|--toml|--diff|--template] [--probe] [--strict] [--all-sources]"
                echo ""
                echo "Sources: porkbun, ovh, inwx, domainoffer, iana, rdap, tldlist"
                echo ""
                echo "Flags:"
                echo "  --raw          Output raw TLD list (one per line)"
                echo "  --toml         Output TOML-ready arrays"
                echo "  --diff         Compare against current Lists.toml"
                echo "  --template     Generate Lists.toml into violator-workdir/"
                echo "  --probe        Probe unknown TLDs for WHOIS servers"
                echo "  --strict       Remove TLDs with no working server from output"
                echo "  --all-sources  Include tld-list.com for extra coverage"
                echo ""
                echo "Config:  $CONF_FILE"
                echo "Workdir: $WORK_DIR"
                exit 0 ;;
        esac
    done

    local porkbun_tlds="" inwx_tlds="" ovh_tlds="" domainoffer_tlds="" iana_tlds="" rdap_tlds="" tldlist_tlds=""
    local porkbun_count=0 inwx_count=0 ovh_count=0 domainoffer_count=0 iana_count=0 rdap_count=0 tldlist_count=0

    # Template mode needs all sources
    if [[ "$mode" == "template" ]]; then
        source="all"
    fi

    # ── Fetch from selected sources ──

    if [[ "$source" == "all" || "$source" == "porkbun" ]]; then
        if porkbun_file=$(fetch_porkbun); then
            porkbun_tlds=$(parse_porkbun "$porkbun_file")
            porkbun_count=$(echo "$porkbun_tlds" | grep -c . || true)
        fi
    fi

    if [[ "$source" == "all" || "$source" == "ovh" ]]; then
        if ovh_file=$(fetch_ovh); then
            ovh_tlds=$(parse_ovh "$ovh_file")
            ovh_count=$(echo "$ovh_tlds" | grep -c . || true)
        fi
    fi

    if [[ "$source" == "all" || "$source" == "inwx" ]]; then
        if inwx_file=$(fetch_inwx 2>/dev/null); then
            inwx_tlds=$(parse_inwx "$inwx_file")
            inwx_count=$(echo "$inwx_tlds" | grep -c . || true)
        fi
    fi

    if [[ "$source" == "all" || "$source" == "domainoffer" ]]; then
        if domainoffer_file=$(fetch_domainoffer 2>/dev/null); then
            domainoffer_tlds=$(parse_domainoffer "$domainoffer_file")
            domainoffer_count=$(echo "$domainoffer_tlds" | grep -c . || true)
        fi
    fi

    if [[ "$source" == "all" || "$source" == "iana" ]]; then
        if iana_file=$(fetch_iana); then
            iana_tlds=$(parse_iana "$iana_file")
            iana_count=$(echo "$iana_tlds" | grep -c . || true)
        fi
    fi

    if [[ "$source" == "all" || "$source" == "rdap" ]]; then
        if rdap_file=$(fetch_rdap); then
            rdap_tlds=$(parse_rdap_tlds "$rdap_file")
            rdap_count=$(echo "$rdap_tlds" | grep -c . || true)
        fi
    fi

    if [[ "$all_sources" == true || "$source" == "tldlist" ]]; then
        if tldlist_file=$(fetch_tldlist); then
            tldlist_tlds=$(parse_tldlist "$tldlist_file")
            tldlist_count=$(echo "$tldlist_tlds" | grep -c . || true)
        fi
    fi

    # ── Filter porkbun: no handshake, no sub-TLDs ──
    local porkbun_filtered=""
    if [[ -n "$porkbun_tlds" ]]; then
        local porkbun_file="$CACHE_DIR/porkbun.json"
        if command -v jq &>/dev/null && [[ -f "$porkbun_file" ]]; then
            porkbun_filtered=$(jq -r '
                .pricing // {} | to_entries[] |
                select(.key | contains(".") | not) |
                select(.value.specialType // "" | test("handshake") | not) |
                .key
            ' "$porkbun_file" 2>/dev/null | sort -u)
        else
            porkbun_filtered=$(echo "$porkbun_tlds" | grep -v '\.' | sort -u)
        fi
    fi

    # ── Merge all registrar TLDs ──
    local registrar_tlds
    registrar_tlds=$(echo -e "${porkbun_filtered}\n${ovh_tlds}\n${inwx_tlds}\n${domainoffer_tlds}" | grep -E '^[a-z]' | sort -u | filter_skip)

    if [[ "$all_sources" == true && -n "$tldlist_tlds" ]]; then
        local tldlist_extra
        tldlist_extra=$(comm -23 <(echo "$tldlist_tlds") <(echo "$registrar_tlds") 2>/dev/null || true)
        local extra_count
        extra_count=$(echo "$tldlist_extra" | grep -c . || echo 0)
        echo -e "  ${YELLOW}tld-list.com:${NC} $extra_count TLDs with no registrar pricing (excluded)" >&2
    fi

    local all_tlds="$registrar_tlds"

    # Also include IANA ccTLDs with RDAP/WHOIS
    if [[ -n "$iana_tlds" ]]; then
        local iana_cctlds
        iana_cctlds=$(echo "$iana_tlds" | filter_cctlds | filter_skip)
        local whois_serv_file_extra=""
        if [[ -f "$CACHE_DIR/tld_serv_list.txt" ]]; then
            whois_serv_file_extra="$CACHE_DIR/tld_serv_list.txt"
        elif whois_serv_file_extra=$(fetch_whois_servers 2>/dev/null); then
            true
        fi
        local iana_extra=0
        while IFS= read -r cctld; do
            [[ -z "$cctld" ]] && continue
            if echo "$registrar_tlds" | grep -qx "$cctld" 2>/dev/null; then
                continue
            fi
            if echo "$rdap_tlds" | grep -qx "$cctld" 2>/dev/null; then
                all_tlds=$(echo -e "${all_tlds}\n${cctld}")
                ((iana_extra++)) || true
                continue
            fi
            if [[ -n "$whois_serv_file_extra" ]]; then
                local srv
                srv=$(get_whois_server "$cctld" "$whois_serv_file_extra")
                if [[ -n "$srv" ]]; then
                    all_tlds=$(echo -e "${all_tlds}\n${cctld}")
                    ((iana_extra++)) || true
                fi
            fi
        done <<< "$iana_cctlds"
        all_tlds=$(echo "$all_tlds" | sort -u)
        if (( iana_extra > 0 )); then
            echo -e "  ${CYAN}IANA adds${NC}    $iana_extra ccTLDs with RDAP/WHOIS not at any registrar" >&2
        fi
    fi
    local all_cctlds
    all_cctlds=$(echo "$all_tlds" | filter_cctlds)

    # Always generate pdom.txt / sdom.txt (even outside template mode)
    generate_pdom_sdom() {
        local whois_serv_file=""
        if whois_serv_file=$(fetch_whois_servers 2>/dev/null); then true; fi

        local -A manual_whois=()
        while IFS=' ' read -r tld server; do
            [[ -z "$tld" ]] && continue
            manual_whois["$tld"]="$server"
        done < <(load_whois_overrides)

        local pdom_list=() sdom_list=()
        while IFS= read -r tld; do
            [[ -z "$tld" ]] && continue
            if [[ -n "${manual_whois[$tld]:-}" ]]; then
                pdom_list+=("${tld}:${manual_whois[$tld]}")
            elif echo "$rdap_tlds" | grep -qx "$tld" 2>/dev/null; then
                pdom_list+=("$tld")
            else
                local server=""
                if [[ -n "$whois_serv_file" ]]; then
                    server=$(get_whois_server "$tld" "$whois_serv_file")
                fi
                if [[ -n "$server" ]]; then
                    pdom_list+=("${tld}:${server}")
                else
                    sdom_list+=("$tld")
                fi
            fi
        done <<< "$all_tlds"

        printf '%s\n' "${pdom_list[@]}" | sort -u > "$PDOM_FILE"
        if (( ${#sdom_list[@]} > 0 )); then
            printf '%s\n' "${sdom_list[@]}" | sort -u > "$SDOM_FILE"
        else
            > "$SDOM_FILE"
        fi

        # Run probes on sdom if enabled
        if [[ "$DO_PROBE" == true ]]; then
            run_whois_probes "$SDOM_FILE" "$PDOM_FILE"
        fi
    }

    # Build source summary
    local sources_used=()
    [[ $porkbun_count -gt 0 ]] && sources_used+=("Porkbun")
    [[ $ovh_count -gt 0 ]] && sources_used+=("OVH")
    [[ $inwx_count -gt 0 ]] && sources_used+=("INWX")
    [[ $domainoffer_count -gt 0 ]] && sources_used+=("DomainOffer")
    local source_summary joined
    joined=$(printf " + %s" "${sources_used[@]}")
    joined="${joined:3}"
    source_summary="${joined} + RDAP bootstrap + WHOIS server list"

    case "$mode" in
        raw)
            generate_pdom_sdom
            echo "$all_tlds"
            ;;
        toml)
            generate_pdom_sdom
            echo -e "${BOLD}# Purchasable TLDs from all registrars ($(echo "$all_tlds" | wc -l | tr -d ' ') total)${NC}"
            echo "all_registrars = ["
            echo "$all_tlds" | to_toml_array
            echo "]"
            echo ""
            echo "# Country-code TLDs (purchasable)"
            echo "cctlds = ["
            echo "$all_cctlds" | to_toml_array
            echo "]"
            ;;
        diff)
            generate_pdom_sdom
            echo -e "${BOLD}Comparing registrar data vs current Lists.toml${NC}"
            echo ""
            local current_all current_country
            current_all=$(parse_current_lists "all")
            current_country=$(parse_current_lists "country")

            if [[ -n "$all_tlds" ]]; then
                local missing_from_all
                missing_from_all=$(comm -23 <(echo "$all_tlds" | filter_short_tlds | sort) <(echo "$current_all" | sort) 2>/dev/null || true)
                if [[ -n "$missing_from_all" ]]; then
                    local mc
                    mc=$(echo "$missing_from_all" | wc -l | tr -d ' ')
                    echo -e "${YELLOW}TLDs at registrars but NOT in our 'all' list ($mc):${NC}"
                    echo "$missing_from_all" | tr '\n' ' '
                    echo "" && echo ""
                fi

                local missing_cc
                missing_cc=$(comm -23 <(echo "$all_cctlds" | sort) <(echo "$current_country" | sort) 2>/dev/null || true)
                if [[ -n "$missing_cc" ]]; then
                    local mcc
                    mcc=$(echo "$missing_cc" | wc -l | tr -d ' ')
                    echo -e "${YELLOW}ccTLDs at registrars but NOT in 'country' list ($mcc):${NC}"
                    echo "$missing_cc" | tr '\n' ' '
                    echo "" && echo ""
                fi

                local extra
                extra=$(comm -13 <(echo "$all_tlds" | sort) <(echo "$current_all" | sort) 2>/dev/null || true)
                if [[ -n "$extra" ]]; then
                    local ec
                    ec=$(echo "$extra" | wc -l | tr -d ' ')
                    echo -e "${CYAN}TLDs in our 'all' list but NOT at any registrar ($ec):${NC}"
                    echo "$extra" | tr '\n' ' '
                    echo "" && echo ""
                fi
            fi

            if [[ -n "$rdap_tlds" && -n "$current_all" ]]; then
                local no_rdap
                no_rdap=$(comm -23 <(echo "$current_all" | sort) <(echo "$rdap_tlds" | sort) 2>/dev/null || true)
                if [[ -n "$no_rdap" ]]; then
                    local nrc
                    nrc=$(echo "$no_rdap" | wc -l | tr -d ' ')
                    echo -e "${RED}TLDs in our lists with NO RDAP server ($nrc) — need WHOIS fallback:${NC}"
                    echo "$no_rdap" | tr '\n' ' '
                    echo ""
                fi
            fi
            ;;
        template)
            generate_template "$registrar_tlds" "$rdap_tlds" "$source_summary" > "$OUTPUT_TOML"
            echo -e "  ${GREEN}Lists.toml written to:${NC} ${OUTPUT_TOML}" >&2
            echo -e "  ${GREEN}pdom.txt:${NC} $(wc -l < "$PDOM_FILE" | tr -d ' ') purchasable TLDs with servers" >&2
            echo -e "  ${GREEN}sdom.txt:${NC} $(wc -l < "$SDOM_FILE" | tr -d ' ') TLDs with no known server" >&2
            ;;
        summary)
            generate_pdom_sdom
            echo -e "${BOLD}═══ TLD Source Summary ═══${NC}"
            echo ""
            [[ $porkbun_count -gt 0 ]]      && echo -e "  ${GREEN}Porkbun${NC}      $(echo "$porkbun_filtered" | grep -c . || echo 0) TLDs ($(echo "$porkbun_filtered" | filter_cctlds | wc -l | tr -d ' ') ccTLDs)"
            [[ $ovh_count -gt 0 ]]          && echo -e "  ${GREEN}OVH${NC}          $ovh_count TLDs ($(echo "$ovh_tlds" | filter_cctlds | wc -l | tr -d ' ') ccTLDs)"
            [[ $inwx_count -gt 0 ]]         && echo -e "  ${GREEN}INWX${NC}         $inwx_count TLDs ($(echo "$inwx_tlds" | filter_cctlds | wc -l | tr -d ' ') ccTLDs)"
            [[ $domainoffer_count -gt 0 ]]  && echo -e "  ${GREEN}DomainOffer${NC}  $domainoffer_count TLDs ($(echo "$domainoffer_tlds" | filter_cctlds | wc -l | tr -d ' ') ccTLDs)"
            [[ $tldlist_count -gt 0 ]]      && echo -e "  ${GREEN}tld-list.com${NC} $tldlist_count TLDs (community registry)"
            [[ $iana_count -gt 0 ]]         && echo -e "  ${GREEN}IANA${NC}         $iana_count TLDs"
            [[ $rdap_count -gt 0 ]]         && echo -e "  ${GREEN}RDAP${NC}         $rdap_count TLDs with lookup servers"
            echo ""

            if [[ $porkbun_count -gt 0 && $ovh_count -gt 0 ]]; then
                local ovh_unique inwx_unique domainoffer_unique
                ovh_unique=$(comm -23 <(echo "$ovh_tlds" | sort) <(echo "$porkbun_filtered" | sort) | wc -l | tr -d ' ')
                echo -e "  ${CYAN}OVH adds${NC}     $ovh_unique TLDs not on Porkbun"
                if [[ $inwx_count -gt 0 ]]; then
                    inwx_unique=$(comm -23 <(echo "$inwx_tlds" | sort) <(echo -e "${porkbun_filtered}\n${ovh_tlds}" | sort -u) | wc -l | tr -d ' ')
                    echo -e "  ${CYAN}INWX adds${NC}    $inwx_unique TLDs not on Porkbun/OVH"
                fi
                if [[ $domainoffer_count -gt 0 ]]; then
                    domainoffer_unique=$(comm -23 <(echo "$domainoffer_tlds" | sort) <(echo -e "${porkbun_filtered}\n${ovh_tlds}\n${inwx_tlds}" | sort -u) | wc -l | tr -d ' ')
                    echo -e "  ${CYAN}DomainOffer adds${NC} $domainoffer_unique TLDs not on Porkbun/OVH/INWX"
                fi
                echo ""
            fi

            echo -e "  ${BOLD}Merged purchasable:${NC} $(echo "$all_tlds" | wc -l | tr -d ' ') TLDs"
            echo -e "  ${BOLD}Merged ccTLDs:${NC}      $(echo "$all_cctlds" | wc -l | tr -d ' ')"
            echo -e "  ${BOLD}pdom.txt:${NC}           $(wc -l < "$PDOM_FILE" | tr -d ' ') with known servers"
            echo -e "  ${BOLD}sdom.txt:${NC}           $(wc -l < "$SDOM_FILE" | tr -d ' ') with no known server"
            echo ""
            echo -e "  Workdir: ${CYAN}$WORK_DIR${NC}"
            echo -e "  Config:  ${CYAN}$CONF_FILE${NC}"
            echo -e "  Use ${BOLD}--diff${NC} to compare against Lists.toml"
            echo -e "  Use ${BOLD}--toml${NC} to output TOML-ready arrays"
            echo -e "  Use ${BOLD}--template${NC} to generate Lists.toml into workdir"
            echo -e "  Use ${BOLD}--probe${NC} to probe unknown TLDs for WHOIS servers"
            echo -e "  Use ${BOLD}--all-sources${NC} to also fetch tld-list.com"
            echo -e "  Use ${BOLD}--raw${NC}  for raw TLD list (one per line)"
            ;;
    esac
}

main "$@"