bbrkn/scripts/generate-configs.sh

#!/usr/bin/env bash
set -euo pipefail

# ==============================
# Конфигурация через переменные окружения
# ==============================
INPUT_FILE="${DOMAINS_FILE:-domains.txt}"

IPSET_CONF="${IPSET_CONF:-/tmp/91-ipset-bbrkn.conf}"
RESOLVE_CONF="${RESOLVE_CONF:-/tmp/92-resolve-bbrkn.conf}"

API_URL="${CHROME_SERVER:-http://127.0.0.1:3000}/domains?domain="
DNS_SERVER="${DNS_SERVER:-8.8.8.8}"

# Debug knobs
DEBUG="${DEBUG:-0}"
DEBUG_LOG="${DEBUG_LOG:-/tmp/generate-configs.$$.debug.log}"

DRY_RUN=false
if [[ "${1:-}" == "--dry-run" ]]; then
  DRY_RUN=true
fi

log()  { printf '%s\n' "$*"; }
dbg()  { if [ "$DEBUG" != "0" ]; then printf '[DEBUG] %s\n' "$*" | tee -a "$DEBUG_LOG"; fi }
err()  { printf '[ERROR] %s\n' "$*" | tee -a "$DEBUG_LOG" >&2; }

for cmd in curl jq; do
    if ! command -v "$cmd" >/dev/null 2>&1; then err "$cmd is required"; exit 2; fi
done

if [ "$DEBUG" != "0" ]; then : > "$DEBUG_LOG"; dbg "Debugging enabled"; fi

log "Starting generate-configs.sh"
dbg "ENV: INPUT_FILE=$INPUT_FILE IPSET_CONF=$IPSET_CONF RESOLVE_CONF=$RESOLVE_CONF API_URL=$API_URL DNS_SERVER=$DNS_SERVER DRY_RUN=$DRY_RUN"

if ! $DRY_RUN; then
  : > "$IPSET_CONF"
  : > "$RESOLVE_CONF"
fi

declare -A DOM_ROLE
declare -A EXPANDED
declare -A SOURCES
declare -A ERRORS
declare -A VALID_SITES

total_lines=0
normalized_ok=0
normalized_skip=0
api_success=0
api_error=0
related_total=0

normalize_domain() {
  local raw="$1"
  raw="$(printf '%s' "$raw" | sed -E 's/#.*$//' | awk '{$1=$1};1')"
  [ -z "$raw" ] && return 1
  raw="$(printf '%s' "$raw" | tr '[:upper:]' '[:lower:]')"
  raw="$(printf '%s' "$raw" | sed -E 's/^\*\.\s*//; s/^\.+//; s/\.+$//; s/\.+/./g')"
  if ! printf '%s' "$raw" | grep -Eq '^[a-z0-9-]+(\.[a-z0-9-]+)+$'; then return 1; fi
  if ! printf '%s' "$raw" | grep -Eq '\.[a-z0-9-]{2,}$'; then return 2; fi
  printf '%s' "$raw"
  return 0
}

query_api() {
  local domain="$1"
  local max_attempts=5
  local attempt=0
  local max_sleep=8
  local tmpfile body http_code sleep_time

  while :; do
    attempt=$((attempt+1))
    dbg "  -> API attempt #$attempt for $domain"

    tmpfile=$(mktemp)
    # curl с отдельным файлом для тела, HTTP-код через -w
    http_code=$(curl -sS --compressed \
        -m 10 --connect-timeout 5 \
        -H 'Accept: application/json' \
        -w '%{http_code}' \
        -o "$tmpfile" \
        "${API_URL}${domain}" 2>>"$DEBUG_LOG" || true)

    body=$(cat "$tmpfile")
    rm -f "$tmpfile"

    preview="$(printf '%s' "$body" | tr '\n' ' ' | cut -c1-400)"
    dbg "  -> HTTP=$http_code, preview=${preview}"

    if [ "$http_code" = "200" ] && jq -e . >/dev/null 2>&1 <<<"$body"; then
      echo "$body"
      return 0
    fi

    if [ "$attempt" -ge "$max_attempts" ]; then
      ERRORS["$domain"]="http_${http_code}_or_invalid_json"
      dbg "  -> Failed after $attempt attempts, preview=${preview}"
      return 1
    fi

    sleep_time=$((2 ** (attempt-1)))
    [ "$sleep_time" -gt "$max_sleep" ] && sleep_time=$max_sleep
    dbg "  -> Retry after ${sleep_time}s..."
    sleep "$sleep_time"
  done
}

if [ ! -f "$INPUT_FILE" ]; then
  err "Input file not found: $INPUT_FILE"
  exit 3
fi

raw_total_lines=$(wc -l < "$INPUT_FILE" | tr -d ' ')
dbg "Raw input lines: $raw_total_lines"

lineno=0
while IFS= read -r line || [ -n "$line" ]; do
  lineno=$((lineno+1))
  total_lines=$((total_lines+1))
  dbg "Processing line #$lineno: '$line'"

  dom_norm="$(normalize_domain "$line" || true)"
  if [ -z "$dom_norm" ]; then
    normalized_skip=$((normalized_skip+1))
    dbg "  -> SKIP (normalization failed)"
    continue
  fi
  normalized_ok=$((normalized_ok+1))
  dbg "  -> NORMALIZED: $dom_norm"

  resp="$(query_api "$dom_norm" || true)"
  if [ -z "$resp" ]; then
    dbg "  -> No valid response for $dom_norm, skipping."
    continue
  fi

  if jq -e 'has("error")' <<<"$resp" >/dev/null; then
    err_msg="$(jq -r '.error' <<<"$resp")"
    dbg "  -> API error: $err_msg"

    if grep -Eq "ERR_NAME_NOT_RESOLVED|Timeout" <<<"$err_msg"; then
      DOM_ROLE["$dom_norm"]="dead"
      ERRORS["$dom_norm"]="$err_msg"
      continue
    fi

    if grep -Eq "ERR_CERT_COMMON_NAME_INVALID|ERR_CONNECTION_REFUSED" <<<"$err_msg"; then
      DOM_ROLE["$dom_norm"]="service"
      SOURCES["$dom_norm"]="base"
      EXPANDED["$dom_norm"]=1
      ERRORS["$dom_norm"]="$err_msg"
      continue
    fi

    DOM_ROLE["$dom_norm"]="unknown"
    SOURCES["$dom_norm"]="base"
    EXPANDED["$dom_norm"]=1
    ERRORS["$dom_norm"]="$err_msg"
    continue
  fi

  api_success=$((api_success+1))
  DOM_ROLE["$dom_norm"]="site"
  SOURCES["$dom_norm"]="base"
  EXPANDED["$dom_norm"]=1
  VALID_SITES["$dom_norm"]=1

  mapfile -t subs < <(jq -r '.relatedDomains[]? // empty' <<<"$resp")
  dbg "  -> API returned ${#subs[@]} related domains"
  for s in "${subs[@]}"; do
    nd="$(normalize_domain "$s" || true)"
    [ -z "$nd" ] && continue
    EXPANDED["$nd"]=1
    [ -z "${SOURCES[$nd]:-}" ] && SOURCES["$nd"]="related"
    related_total=$((related_total+1))
    dbg "    - RELATED ADD: $nd"
  done

done < "$INPUT_FILE"

mapfile -t ALL_DOMAINS < <(printf "%s\n" "${!EXPANDED[@]}" | sort -u)

if ! $DRY_RUN; then
  for d in "${ALL_DOMAINS[@]}"; do
    printf 'ipset=/%s/bbrkn\n' "$d" >> "$IPSET_CONF"
    printf 'server=/%s/%s\n' "$d" "$DNS_SERVER" >> "$RESOLVE_CONF"
  done
fi

echo
echo "===== DEBUG REPORT ====="
echo "Input file: $INPUT_FILE"
echo "Raw input lines: $raw_total_lines"
echo "Processed lines: $total_lines"
echo "Normalized OK: $normalized_ok"
echo "Normalized skipped: $normalized_skip"
echo
echo "API success (sites): $api_success"
echo "API error/ignored: $api_error"
echo "Related domains added: $related_total"
echo "Final unique domains: ${#ALL_DOMAINS[@]}"
echo
echo "---- VALID BASE SITES ----"
printf '%s\n' "${!VALID_SITES[@]}" | sort
echo "===== END DEBUG REPORT ====="

if [ "$DEBUG" != "0" ]; then
  echo "Detailed debug log: $DEBUG_LOG"
fi