218 lines
6.2 KiB
Bash
218 lines
6.2 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# ==============================
|
|
# Конфигурация через переменные окружения
|
|
# ==============================
|
|
INPUT_FILE="${DOMAINS_FILE:-domains.txt}"
|
|
|
|
IPSET_CONF="${IPSET_CONF:-/tmp/91-ipset-bbrkn.conf}"
|
|
RESOLVE_CONF="${RESOLVE_CONF:-/tmp/92-resolve-bbrkn.conf}"
|
|
|
|
API_URL="${CHROME_SERVER:-http://127.0.0.1:3000}/domains?domain="
|
|
DNS_SERVER="${DNS_SERVER:-8.8.8.8}"
|
|
|
|
# Debug knobs
|
|
DEBUG="${DEBUG:-0}"
|
|
DEBUG_LOG="${DEBUG_LOG:-/tmp/generate-configs.debug.log}"
|
|
|
|
DRY_RUN=false
|
|
if [[ "${1:-}" == "--dry-run" ]]; then
|
|
DRY_RUN=true
|
|
fi
|
|
|
|
# Helpers
|
|
log() { printf '%s\n' "$*"; }
|
|
dbg() { if [ "$DEBUG" != "0" ]; then printf '[DEBUG] %s\n' "$*" | tee -a "$DEBUG_LOG"; fi }
|
|
err() { printf '[ERROR] %s\n' "$*" | tee -a "$DEBUG_LOG" >&2; }
|
|
|
|
if ! command -v curl >/dev/null 2>&1; then err "curl is required"; exit 2; fi
|
|
if ! command -v jq >/dev/null 2>&1; then err "jq is required"; exit 2; fi
|
|
|
|
if [ "$DEBUG" != "0" ]; then : > "$DEBUG_LOG"; dbg "Debugging enabled"; fi
|
|
|
|
log "Starting generate-configs.sh"
|
|
dbg "ENV: INPUT_FILE=$INPUT_FILE IPSET_CONF=$IPSET_CONF RESOLVE_CONF=$RESOLVE_CONF API_URL=$API_URL DNS_SERVER=$DNS_SERVER DRY_RUN=$DRY_RUN"
|
|
|
|
if ! $DRY_RUN; then
|
|
: > "$IPSET_CONF"
|
|
: > "$RESOLVE_CONF"
|
|
fi
|
|
|
|
declare -A DOM_ROLE
|
|
declare -A EXPANDED
|
|
declare -A SOURCES
|
|
declare -A ERRORS
|
|
declare -A VALID_SITES # только живые site-домены
|
|
|
|
total_lines=0
|
|
normalized_ok=0
|
|
normalized_skip=0
|
|
api_success=0
|
|
api_error=0
|
|
related_total=0
|
|
|
|
normalize_domain() {
|
|
local raw="$1"
|
|
raw="$(printf '%s' "$raw" | sed -E 's/#.*$//' | awk '{$1=$1};1')"
|
|
[ -z "$raw" ] && return 1
|
|
raw="$(printf '%s' "$raw" | tr '[:upper:]' '[:lower:]')"
|
|
raw="$(printf '%s' "$raw" | sed -E 's/^\*\.\s*//; s/^\.+//; s/\.+$//; s/\.+/./g')"
|
|
if ! printf '%s' "$raw" | grep -Eq '^[a-z0-9-]+(\.[a-z0-9-]+)+$'; then return 1; fi
|
|
if ! printf '%s' "$raw" | grep -Eq '\.[a-z0-9-]{2,}$'; then return 2; fi
|
|
printf '%s' "$raw"
|
|
return 0
|
|
}
|
|
|
|
if [ ! -f "$INPUT_FILE" ]; then
|
|
err "Input file not found: $INPUT_FILE"
|
|
exit 3
|
|
fi
|
|
|
|
raw_total_lines=$(wc -l < "$INPUT_FILE" | tr -d ' ')
|
|
dbg "Raw input lines: $raw_total_lines"
|
|
|
|
lineno=0
|
|
while IFS= read -r line || [ -n "$line" ]; do
|
|
lineno=$((lineno+1))
|
|
total_lines=$((total_lines+1))
|
|
dbg "Processing line #$lineno: '$line'"
|
|
|
|
dom_norm="$(normalize_domain "$line" || true)"
|
|
if [ -z "$dom_norm" ]; then
|
|
normalized_skip=$((normalized_skip+1))
|
|
dbg " -> SKIP (normalization failed)"
|
|
continue
|
|
fi
|
|
normalized_ok=$((normalized_ok+1))
|
|
dbg " -> NORMALIZED: $dom_norm"
|
|
|
|
dbg " -> Querying API: ${API_URL}${dom_norm}"
|
|
|
|
# параметры retry
|
|
max_attempts=5
|
|
attempt=0
|
|
resp=""
|
|
http_code=0
|
|
content_type=""
|
|
|
|
while :; do
|
|
attempt=$((attempt+1))
|
|
raw="$(curl -sS --compressed \
|
|
-m 10 --connect-timeout 5 \
|
|
--retry 3 --retry-delay 1 \
|
|
-H 'Accept: application/json' \
|
|
-w '\n%{http_code}\n%{content_type}' \
|
|
"${API_URL}${dom_norm}" 2>>"$DEBUG_LOG" || true)"
|
|
|
|
http_code="$(printf '%s' "$raw" | tail -n1)"
|
|
content_type="$(printf '%s' "$raw" | tail -n2 | head -n1)"
|
|
resp="$(printf '%s' "$raw" | sed '$d' | sed '$d')"
|
|
|
|
preview="$(printf '%s' "$resp" | tr '\n' ' ' | cut -c1-400)"
|
|
dbg " -> Attempt #${attempt}: HTTP=${http_code}, Content-Type=${content_type}, preview=${preview}"
|
|
|
|
if [ "$http_code" = "200" ] && printf '%s' "$content_type" | grep -qEi 'application/(json|javascript)|^text/json'; then
|
|
break
|
|
fi
|
|
|
|
if jq -e . >/dev/null 2>&1 <<<"$resp"; then
|
|
dbg " -> Body is valid JSON despite HTTP=${http_code}, accepting."
|
|
break
|
|
fi
|
|
|
|
if [ "$attempt" -ge "$max_attempts" ]; then
|
|
api_error=$((api_error+1))
|
|
ERRORS["$dom_norm"]="http_${http_code}_or_nonjson"
|
|
dbg " -> Failed after ${attempt} attempts: HTTP=${http_code}, preview=${preview}"
|
|
resp=""
|
|
break
|
|
fi
|
|
|
|
# backoff: 1s, 2s, 4s, 8s ...
|
|
sleep_time=$((2 ** (attempt-1)))
|
|
dbg " -> Retry after ${sleep_time}s..."
|
|
sleep "$sleep_time"
|
|
done
|
|
|
|
if [ -z "$resp" ] || ! jq -e . >/dev/null 2>&1 <<<"$resp"; then
|
|
dbg " -> non-JSON or empty response, skipping domain: $dom_norm"
|
|
continue
|
|
fi
|
|
|
|
if jq -e 'has("error")' <<<"$resp" >/dev/null; then
|
|
err_msg="$(jq -r '.error' <<<"$resp")"
|
|
dbg " -> API error: $err_msg"
|
|
|
|
if grep -Eq "ERR_NAME_NOT_RESOLVED|Timeout" <<<"$err_msg"; then
|
|
dbg " -> Dead domain, completely skipped"
|
|
continue
|
|
fi
|
|
|
|
if grep -Eq "ERR_CERT_COMMON_NAME_INVALID|ERR_CONNECTION_REFUSED" <<<"$err_msg"; then
|
|
dbg " -> Service domain, keep only base"
|
|
DOM_ROLE["$dom_norm"]="service"
|
|
SOURCES["$dom_norm"]="base"
|
|
EXPANDED["$dom_norm"]=1
|
|
ERRORS["$dom_norm"]="$err_msg"
|
|
continue
|
|
fi
|
|
|
|
DOM_ROLE["$dom_norm"]="service"
|
|
SOURCES["$dom_norm"]="base"
|
|
EXPANDED["$dom_norm"]=1
|
|
ERRORS["$dom_norm"]="$err_msg"
|
|
continue
|
|
fi
|
|
|
|
api_success=$((api_success+1))
|
|
DOM_ROLE["$dom_norm"]="site"
|
|
SOURCES["$dom_norm"]="base"
|
|
EXPANDED["$dom_norm"]=1
|
|
VALID_SITES["$dom_norm"]=1
|
|
|
|
mapfile -t subs < <(jq -r '.relatedDomains[]? // empty' <<<"$resp")
|
|
dbg " -> API returned ${#subs[@]} related domains"
|
|
for s in "${subs[@]}"; do
|
|
nd="$(normalize_domain "$s" || true)"
|
|
if [ -z "$nd" ]; then
|
|
dbg " - RELATED SKIP: '$s'"
|
|
continue
|
|
fi
|
|
EXPANDED["$nd"]=1
|
|
[ -z "${SOURCES[$nd]:-}" ] && SOURCES["$nd"]="related"
|
|
related_total=$((related_total+1))
|
|
dbg " - RELATED ADD: $nd"
|
|
done
|
|
|
|
done < "$INPUT_FILE"
|
|
|
|
mapfile -t ALL_DOMAINS < <(printf "%s\n" "${!EXPANDED[@]}" | sort -u)
|
|
|
|
if ! $DRY_RUN; then
|
|
for d in "${ALL_DOMAINS[@]}"; do
|
|
printf 'ipset=/%s/bbrkn\n' "$d" >> "$IPSET_CONF"
|
|
printf 'server=/%s/%s\n' "$d" "$DNS_SERVER" >> "$RESOLVE_CONF"
|
|
done
|
|
fi
|
|
|
|
echo
|
|
echo "===== DEBUG REPORT ====="
|
|
echo "Input file: $INPUT_FILE"
|
|
echo "Raw input lines: $raw_total_lines"
|
|
echo "Processed lines: $total_lines"
|
|
echo "Normalized OK: $normalized_ok"
|
|
echo "Normalized skipped: $normalized_skip"
|
|
echo
|
|
echo "API success (sites): $api_success"
|
|
echo "API error/ignored: $api_error"
|
|
echo "Related domains added: $related_total"
|
|
echo "Final unique domains: ${#ALL_DOMAINS[@]}"
|
|
echo
|
|
echo "---- VALID BASE SITES ----"
|
|
printf '%s\n' "${!VALID_SITES[@]}" | sort
|
|
echo "===== END DEBUG REPORT ====="
|
|
|
|
if [ "$DEBUG" != "0" ]; then
|
|
echo "Detailed debug log: $DEBUG_LOG"
|
|
fi
|