fca3ae254a
All checks were successful
Deploy DNS Configuration / deploy (push) Successful in 2m15s
ADD DOMAIN: cloud.hetzner.com
227 lines
7.2 KiB
Bash
227 lines
7.2 KiB
Bash
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
|
||
# ==============================
|
||
# Конфигурация через переменные окружения
|
||
# ==============================
|
||
INPUT_FILE="${DOMAINS_FILE:-domains.txt}"
|
||
|
||
IPSET_CONF="${IPSET_CONF:-/tmp/91-ipset-bbrkn.conf}"
|
||
RESOLVE_CONF="${RESOLVE_CONF:-/tmp/92-resolve-bbrkn.conf}"
|
||
|
||
API_URL="${CHROME_SERVER:-http://127.0.0.1:3000}/domains?domain="
|
||
DNS_SERVER="${DNS_SERVER:-8.8.8.8}"
|
||
|
||
# Debug knobs
|
||
DEBUG="${DEBUG:-0}"
|
||
DEBUG_LOG="${DEBUG_LOG:-/tmp/generate-configs.debug.log}"
|
||
|
||
DRY_RUN=false
|
||
if [[ "${1:-}" == "--dry-run" ]]; then
|
||
DRY_RUN=true
|
||
fi
|
||
|
||
# Helpers
|
||
log() { printf '%s\n' "$*"; }
|
||
dbg() { if [ "$DEBUG" != "0" ]; then printf '[DEBUG] %s\n' "$*" | tee -a "$DEBUG_LOG"; fi }
|
||
err() { printf '[ERROR] %s\n' "$*" | tee -a "$DEBUG_LOG" >&2; }
|
||
|
||
if ! command -v curl >/dev/null 2>&1; then err "curl is required"; exit 2; fi
|
||
if ! command -v jq >/dev/null 2>&1; then err "jq is required"; exit 2; fi
|
||
|
||
if [ "$DEBUG" != "0" ]; then : > "$DEBUG_LOG"; dbg "Debugging enabled"; fi
|
||
|
||
log "Starting generate-configs.sh"
|
||
dbg "ENV: INPUT_FILE=$INPUT_FILE IPSET_CONF=$IPSET_CONF RESOLVE_CONF=$RESOLVE_CONF API_URL=$API_URL DNS_SERVER=$DNS_SERVER DRY_RUN=$DRY_RUN"
|
||
|
||
if ! $DRY_RUN; then
|
||
: > "$IPSET_CONF"
|
||
: > "$RESOLVE_CONF"
|
||
fi
|
||
|
||
declare -A DOM_ROLE
|
||
declare -A EXPANDED
|
||
declare -A SOURCES
|
||
declare -A ERRORS
|
||
declare -A VALID_SITES # только живые site-домены
|
||
|
||
total_lines=0
|
||
normalized_ok=0
|
||
normalized_skip=0
|
||
api_success=0
|
||
api_error=0
|
||
related_total=0
|
||
|
||
normalize_domain() {
|
||
local raw="$1"
|
||
raw="$(printf '%s' "$raw" | sed -E 's/#.*$//' | awk '{$1=$1};1')"
|
||
[ -z "$raw" ] && return 1
|
||
raw="$(printf '%s' "$raw" | tr '[:upper:]' '[:lower:]')"
|
||
raw="$(printf '%s' "$raw" | sed -E 's/^\*\.\s*//; s/^\.+//; s/\.+$//; s/\.+/./g')"
|
||
if ! printf '%s' "$raw" | grep -Eq '^[a-z0-9-]+(\.[a-z0-9-]+)+$'; then return 1; fi
|
||
if ! printf '%s' "$raw" | grep -Eq '\.[a-z0-9-]{2,}$'; then return 2; fi
|
||
printf '%s' "$raw"
|
||
return 0
|
||
}
|
||
|
||
if [ ! -f "$INPUT_FILE" ]; then
|
||
err "Input file not found: $INPUT_FILE"
|
||
exit 3
|
||
fi
|
||
|
||
raw_total_lines=$(wc -l < "$INPUT_FILE" | tr -d ' ')
|
||
dbg "Raw input lines: $raw_total_lines"
|
||
|
||
lineno=0
|
||
while IFS= read -r line || [ -n "$line" ]; do
|
||
lineno=$((lineno+1))
|
||
total_lines=$((total_lines+1))
|
||
dbg "Processing line #$lineno: '$line'"
|
||
|
||
dom_norm="$(normalize_domain "$line" || true)"
|
||
if [ -z "$dom_norm" ]; then
|
||
normalized_skip=$((normalized_skip+1))
|
||
dbg " -> SKIP (normalization failed)"
|
||
continue
|
||
fi
|
||
normalized_ok=$((normalized_ok+1))
|
||
dbg " -> NORMALIZED: $dom_norm"
|
||
|
||
dbg " -> Querying API: ${API_URL}${dom_norm}"
|
||
|
||
# параметры retry
|
||
max_attempts=3
|
||
attempt=0
|
||
resp=""
|
||
http_code=0
|
||
content_type=""
|
||
|
||
while :; do
|
||
attempt=$((attempt+1))
|
||
# делаем запрос, записываем тело и код отдельно
|
||
# --compressed: поддержка gzip
|
||
# -sS: показывать ошибки curl в debug (если DEBUG enabled, они попадут в DEBUG_LOG)
|
||
# -m: общий timeout
|
||
# --connect-timeout: timeout на установление соединения
|
||
raw="$(curl -sS --compressed -m 20 --connect-timeout 8 -H 'Accept: application/json' -w '\n%{http_code}\n%{content_type}' "${API_URL}${dom_norm}" 2>>"$DEBUG_LOG" )" || true
|
||
|
||
# последний \n%{http_code}\n%{content_type} — отделяем
|
||
http_code="$(printf '%s' "$raw" | tail -n1)"
|
||
content_type="$(printf '%s' "$raw" | tail -n2 | head -n1)"
|
||
resp="$(printf '%s' "$raw" | sed '$d' | sed '$d')" # всё кроме двух последних строк
|
||
|
||
preview="$(printf '%s' "$resp" | tr '\n' ' ' | cut -c1-400)"
|
||
dbg " -> Attempt #${attempt}: HTTP=${http_code}, Content-Type=${content_type}, preview=${preview}"
|
||
|
||
# если получили 200 и content-type похож на json — выходим
|
||
if [ "$http_code" = "200" ] && printf '%s' "$content_type" | grep -qEi 'application/(json|javascript)|^text/json'; then
|
||
break
|
||
fi
|
||
|
||
# если тело выглядит как json (на всякий случай), тоже принимаем
|
||
if jq -e . >/dev/null 2>&1 <<<"$resp"; then
|
||
dbg " -> Body is valid JSON despite HTTP=${http_code}, accepting."
|
||
break
|
||
fi
|
||
|
||
# если исчерпали попытки — помечаем как ошибка
|
||
if [ "$attempt" -ge "$max_attempts" ]; then
|
||
api_error=$((api_error+1))
|
||
ERRORS["$dom_norm"]="http_${http_code}_or_nonjson"
|
||
dbg " -> Failed after ${attempt} attempts: HTTP=${http_code}, preview=${preview}"
|
||
resp="" # явный маркер
|
||
break
|
||
fi
|
||
|
||
# backoff: 0.5s, 1s, 2s ...
|
||
sleep_time=$(awk "BEGIN {printf \"%.0f\", 0.5 * (2 ^ ($attempt - 1))}")
|
||
dbg " -> Retry after ${sleep_time}s..."
|
||
sleep "$sleep_time"
|
||
done
|
||
|
||
# если нет валидного JSON — пропускаем, с логом
|
||
if [ -z "$resp" ] || ! jq -e . >/dev/null 2>&1 <<<"$resp"; then
|
||
dbg " -> non-JSON or empty response, skipping domain: $dom_norm"
|
||
continue
|
||
fi
|
||
|
||
|
||
# error handling
|
||
if jq -e 'has("error")' <<<"$resp" >/dev/null; then
|
||
err_msg="$(jq -r '.error' <<<"$resp")"
|
||
dbg " -> API error: $err_msg"
|
||
|
||
if grep -Eq "ERR_NAME_NOT_RESOLVED|Timeout" <<<"$err_msg"; then
|
||
dbg " -> Dead domain, completely skipped"
|
||
continue
|
||
fi
|
||
|
||
if grep -Eq "ERR_CERT_COMMON_NAME_INVALID|ERR_CONNECTION_REFUSED" <<<"$err_msg"; then
|
||
dbg " -> Service domain, keep only base"
|
||
DOM_ROLE["$dom_norm"]="service"
|
||
SOURCES["$dom_norm"]="base"
|
||
EXPANDED["$dom_norm"]=1
|
||
ERRORS["$dom_norm"]="$err_msg"
|
||
continue
|
||
fi
|
||
|
||
# default case: treat as service
|
||
DOM_ROLE["$dom_norm"]="service"
|
||
SOURCES["$dom_norm"]="base"
|
||
EXPANDED["$dom_norm"]=1
|
||
ERRORS["$dom_norm"]="$err_msg"
|
||
continue
|
||
fi
|
||
|
||
# valid JSON without error -> must be site
|
||
api_success=$((api_success+1))
|
||
DOM_ROLE["$dom_norm"]="site"
|
||
SOURCES["$dom_norm"]="base"
|
||
EXPANDED["$dom_norm"]=1
|
||
VALID_SITES["$dom_norm"]=1
|
||
|
||
mapfile -t subs < <(jq -r '.relatedDomains[]? // empty' <<<"$resp")
|
||
dbg " -> API returned ${#subs[@]} related domains"
|
||
for s in "${subs[@]}"; do
|
||
nd="$(normalize_domain "$s" || true)"
|
||
if [ -z "$nd" ]; then
|
||
dbg " - RELATED SKIP: '$s'"
|
||
continue
|
||
fi
|
||
EXPANDED["$nd"]=1
|
||
[ -z "${SOURCES[$nd]:-}" ] && SOURCES["$nd"]="related"
|
||
related_total=$((related_total+1))
|
||
dbg " - RELATED ADD: $nd"
|
||
done
|
||
|
||
done < "$INPUT_FILE"
|
||
|
||
mapfile -t ALL_DOMAINS < <(printf "%s\n" "${!EXPANDED[@]}" | sort -u)
|
||
|
||
if ! $DRY_RUN; then
|
||
for d in "${ALL_DOMAINS[@]}"; do
|
||
printf 'ipset=/%s/bbrkn\n' "$d" >> "$IPSET_CONF"
|
||
printf 'server=/%s/%s\n' "$d" "$DNS_SERVER" >> "$RESOLVE_CONF"
|
||
done
|
||
fi
|
||
|
||
echo
|
||
echo "===== DEBUG REPORT ====="
|
||
echo "Input file: $INPUT_FILE"
|
||
echo "Raw input lines: $raw_total_lines"
|
||
echo "Processed lines: $total_lines"
|
||
echo "Normalized OK: $normalized_ok"
|
||
echo "Normalized skipped: $normalized_skip"
|
||
echo
|
||
echo "API success (sites): $api_success"
|
||
echo "API error/ignored: $api_error"
|
||
echo "Related domains added: $related_total"
|
||
echo "Final unique domains: ${#ALL_DOMAINS[@]}"
|
||
echo
|
||
echo "---- VALID BASE SITES ----"
|
||
printf '%s\n' "${!VALID_SITES[@]}" | sort
|
||
echo "===== END DEBUG REPORT ====="
|
||
|
||
if [ "$DEBUG" != "0" ]; then
|
||
echo "Detailed debug log: $DEBUG_LOG"
|
||
fi
|