#!/usr/bin/env bash set -euo pipefail # ============================== # Конфигурация через переменные окружения # ============================== INPUT_FILE="${DOMAINS_FILE:-domains.txt}" IPSET_CONF="${IPSET_CONF:-/tmp/91-ipset-bbrkn.conf}" RESOLVE_CONF="${RESOLVE_CONF:-/tmp/92-resolve-bbrkn.conf}" API_URL="${CHROME_SERVER:-http://127.0.0.1:3000}/domains?domain=" DNS_SERVER="${DNS_SERVER:-8.8.8.8}" # Debug knobs DEBUG="${DEBUG:-0}" DEBUG_LOG="${DEBUG_LOG:-/tmp/generate-configs.debug.log}" DRY_RUN=false if [[ "${1:-}" == "--dry-run" ]]; then DRY_RUN=true fi # Helpers log() { printf '%s\n' "$*"; } dbg() { if [ "$DEBUG" != "0" ]; then printf '[DEBUG] %s\n' "$*" | tee -a "$DEBUG_LOG"; fi } err() { printf '[ERROR] %s\n' "$*" | tee -a "$DEBUG_LOG" >&2; } if ! command -v curl >/dev/null 2>&1; then err "curl is required"; exit 2; fi if ! command -v jq >/dev/null 2>&1; then err "jq is required"; exit 2; fi if [ "$DEBUG" != "0" ]; then : > "$DEBUG_LOG"; dbg "Debugging enabled"; fi log "Starting generate-configs.sh" dbg "ENV: INPUT_FILE=$INPUT_FILE IPSET_CONF=$IPSET_CONF RESOLVE_CONF=$RESOLVE_CONF API_URL=$API_URL DNS_SERVER=$DNS_SERVER DRY_RUN=$DRY_RUN" if ! $DRY_RUN; then : > "$IPSET_CONF" : > "$RESOLVE_CONF" fi declare -A DOM_ROLE declare -A EXPANDED declare -A SOURCES declare -A ERRORS declare -A VALID_SITES total_lines=0 normalized_ok=0 normalized_skip=0 api_success=0 api_error=0 related_total=0 normalize_domain() { local raw="$1" raw="$(printf '%s' "$raw" | sed -E 's/#.*$//' | awk '{$1=$1};1')" [ -z "$raw" ] && return 1 raw="$(printf '%s' "$raw" | tr '[:upper:]' '[:lower:]')" raw="$(printf '%s' "$raw" | sed -E 's/^\*\.\s*//; s/^\.+//; s/\.+$//; s/\.+/./g')" if ! printf '%s' "$raw" | grep -Eq '^[a-z0-9-]+(\.[a-z0-9-]+)+$'; then return 1; fi if ! printf '%s' "$raw" | grep -Eq '\.[a-z0-9-]{2,}$'; then return 2; fi printf '%s' "$raw" return 0 } fetch_json() { local domain="$1" local max_attempts=5 local attempt=0 local max_sleep=8 local resp http_code json sleep_time preview while :; do attempt=$((attempt+1)) dbg " -> API attempt #$attempt for $domain" resp=$(curl -sS --compressed \ -m 10 --connect-timeout 5 \ --retry 2 --retry-connrefused \ -H 'Accept: application/json' \ -w '%{http_code}' \ "${API_URL}${domain}" 2>>"$DEBUG_LOG") || true http_code="${resp: -3}" # последние 3 символа json="${resp:0:-3}" # тело без http_code preview="$(printf '%s' "$json" | tr '\n' ' ' | cut -c1-400)" dbg " -> HTTP=$http_code, preview=${preview}" if [ "$http_code" = "200" ] && jq -e . >/dev/null 2>&1 <<<"$json"; then echo "$json" return 0 fi if [ "$attempt" -ge "$max_attempts" ]; then ERRORS["$domain"]="http_${http_code}_or_invalid_json" dbg " -> Failed after $attempt attempts, preview=${preview}" return 1 fi sleep_time=$((2 ** (attempt-1))) [ "$sleep_time" -gt "$max_sleep" ] && sleep_time=$max_sleep dbg " -> Retry after ${sleep_time}s..." sleep "$sleep_time" done } if [ ! -f "$INPUT_FILE" ]; then err "Input file not found: $INPUT_FILE" exit 3 fi raw_total_lines=$(wc -l < "$INPUT_FILE" | tr -d ' ') dbg "Raw input lines: $raw_total_lines" lineno=0 while IFS= read -r line || [ -n "$line" ]; do lineno=$((lineno+1)) total_lines=$((total_lines+1)) dbg "Processing line #$lineno: '$line'" dom_norm="$(normalize_domain "$line" || true)" if [ -z "$dom_norm" ]; then normalized_skip=$((normalized_skip+1)) dbg " -> SKIP (normalization failed)" continue fi normalized_ok=$((normalized_ok+1)) dbg " -> NORMALIZED: $dom_norm" json="$(fetch_json "$dom_norm" || true)" if [ -z "$json" ]; then api_error=$((api_error+1)) dbg " -> API returned empty or invalid JSON for $dom_norm" continue fi if jq -e 'has("error")' <<<"$json" >/dev/null; then err_msg="$(jq -r '.error' <<<"$json")" dbg " -> API error: $err_msg" if grep -Eq "ERR_NAME_NOT_RESOLVED|Timeout" <<<"$err_msg"; then continue fi DOM_ROLE["$dom_norm"]="service" SOURCES["$dom_norm"]="base" EXPANDED["$dom_norm"]=1 ERRORS["$dom_norm"]="$err_msg" continue fi # valid JSON -> site api_success=$((api_success+1)) DOM_ROLE["$dom_norm"]="site" SOURCES["$dom_norm"]="base" EXPANDED["$dom_norm"]=1 VALID_SITES["$dom_norm"]=1 mapfile -t subs < <(jq -r '.relatedDomains[]? // empty' <<<"$json") dbg " -> API returned ${#subs[@]} related domains" for s in "${subs[@]}"; do nd="$(normalize_domain "$s" || true)" [ -z "$nd" ] && continue EXPANDED["$nd"]=1 [ -z "${SOURCES[$nd]:-}" ] && SOURCES["$nd"]="related" related_total=$((related_total+1)) dbg " - RELATED ADD: $nd" done done < "$INPUT_FILE" mapfile -t ALL_DOMAINS < <(printf "%s\n" "${!EXPANDED[@]}" | sort -u) if ! $DRY_RUN; then for d in "${ALL_DOMAINS[@]}"; do printf 'ipset=/%s/bbrkn\n' "$d" >> "$IPSET_CONF" printf 'server=/%s/%s\n' "$d" "$DNS_SERVER" >> "$RESOLVE_CONF" done fi echo echo "===== DEBUG REPORT =====" echo "Input file: $INPUT_FILE" echo "Raw input lines: $raw_total_lines" echo "Processed lines: $total_lines" echo "Normalized OK: $normalized_ok" echo "Normalized skipped: $normalized_skip" echo echo "API success (sites): $api_success" echo "API error/ignored: $api_error" echo "Related domains added: $related_total" echo "Final unique domains: ${#ALL_DOMAINS[@]}" echo echo "---- VALID BASE SITES ----" printf '%s\n' "${!VALID_SITES[@]}" | sort echo "===== END DEBUG REPORT =====" if [ "$DEBUG" != "0" ]; then echo "Detailed debug log: $DEBUG_LOG" fi