Merge branch 'main' of https://git.0x25.icu/0x25/bbrkn

scripts/generate-configs.sh

@@ -13,7 +13,7 @@ API_URL="${CHROME_SERVER:-http://127.0.0.1:3000}/domains?domain="
 DNS_SERVER="${DNS_SERVER:-8.8.8.8}"
 
 # Debug knobs
-DEBUG="${DEBUG:-0}"                    # set to 1 to enable verbose debug
+DEBUG="${DEBUG:-0}"
 DEBUG_LOG="${DEBUG_LOG:-/tmp/generate-configs.debug.log}"
 
 DRY_RUN=false
@@ -26,38 +26,25 @@ log()  { printf '%s\n' "$*"; }
 dbg()  { if [ "$DEBUG" != "0" ]; then printf '[DEBUG] %s\n' "$*" | tee -a "$DEBUG_LOG"; fi }
 err()  { printf '[ERROR] %s\n' "$*" | tee -a "$DEBUG_LOG" >&2; }
 
-# sanity
+if ! command -v curl >/dev/null 2>&1; then err "curl is required"; exit 2; fi
-if ! command -v curl >/dev/null 2>&1; then
+if ! command -v jq >/dev/null 2>&1; then err "jq is required"; exit 2; fi
-  err "curl is required but not found in PATH"
-  exit 2
-fi
-if ! command -v jq >/dev/null 2>&1; then
-  err "jq is required but not found in PATH"
-  exit 2
-fi
 
-# reset debug log if debugging enabled
+if [ "$DEBUG" != "0" ]; then : > "$DEBUG_LOG"; dbg "Debugging enabled"; fi
-if [ "$DEBUG" != "0" ]; then
-  : > "$DEBUG_LOG"
-  dbg "Debugging enabled. Log: $DEBUG_LOG"
-fi
 
 log "Starting generate-configs.sh"
 dbg "ENV: INPUT_FILE=$INPUT_FILE IPSET_CONF=$IPSET_CONF RESOLVE_CONF=$RESOLVE_CONF API_URL=$API_URL DNS_SERVER=$DNS_SERVER DRY_RUN=$DRY_RUN"
 
-# clear outputs (only if not dry-run)
 if ! $DRY_RUN; then
-  dbg "Clearing output files: $IPSET_CONF , $RESOLVE_CONF"
   : > "$IPSET_CONF"
   : > "$RESOLVE_CONF"
 fi
 
-declare -A DOM_ROLE    # 'site' or 'service' for base domains
+declare -A DOM_ROLE
-declare -A EXPANDED    # unique set of resulting domains
+declare -A EXPANDED
-declare -A SOURCES     # source: base or related
+declare -A SOURCES
-declare -A ERRORS      # API error messages per base domain (if any)
+declare -A ERRORS
+declare -A VALID_SITES   # только живые site-домены
 
-# Counters
 total_lines=0
 normalized_ok=0
 normalized_skip=0
@@ -65,31 +52,18 @@ api_success=0
 api_error=0
 related_total=0
 
-# Normalizer with explicit reasons on failure
 normalize_domain() {
   local raw="$1"
-  # trim + remove inline comments
   raw="$(printf '%s' "$raw" | sed -E 's/#.*$//' | awk '{$1=$1};1')"
-  [ -z "$raw" ] && { printf ''; return 1; }
+  [ -z "$raw" ] && return 1
-  # lowercase
   raw="$(printf '%s' "$raw" | tr '[:upper:]' '[:lower:]')"
-  # remove leading "*." and leading/trailing dots, compress multiple dots
   raw="$(printf '%s' "$raw" | sed -E 's/^\*\.\s*//; s/^\.+//; s/\.+$//; s/\.+/./g')"
-
+  if ! printf '%s' "$raw" | grep -Eq '^[a-z0-9-]+(\.[a-z0-9-]+)+$'; then return 1; fi
-  # syntactic checks: буквы, цифры, дефис, минимум одна точка
+  if ! printf '%s' "$raw" | grep -Eq '\.[a-z0-9-]{2,}$'; then return 2; fi
-  if ! printf '%s' "$raw" | grep -Eq '^[a-z0-9-]+(\.[a-z0-9-]+)+$'; then
-    return 1
-  fi
-  # TLD check: >=2 символов, буквы/цифры/дефис
-  if ! printf '%s' "$raw" | grep -Eq '\.[a-z0-9-]{2,}$'; then
-    return 2
-  fi
-
   printf '%s' "$raw"
   return 0
 }
 
-# read input
 if [ ! -f "$INPUT_FILE" ]; then
   err "Input file not found: $INPUT_FILE"
   exit 3
@@ -102,86 +76,88 @@ lineno=0
 while IFS= read -r line || [ -n "$line" ]; do
   lineno=$((lineno+1))
   total_lines=$((total_lines+1))
-  dbg "Processing line #$lineno: '$(printf "%s" "$line")'"
+  dbg "Processing line #$lineno: '$line'"
 
   dom_norm="$(normalize_domain "$line" || true)"
   if [ -z "$dom_norm" ]; then
     normalized_skip=$((normalized_skip+1))
-    dbg "  -> SKIP (normalization failed) - original: '$line'"
+    dbg "  -> SKIP (normalization failed)"
     continue
   fi
   normalized_ok=$((normalized_ok+1))
   dbg "  -> NORMALIZED: $dom_norm"
 
-  # query Chromium-service
   dbg "  -> Querying API: ${API_URL}${dom_norm}"
-  resp="$(curl -s --max-time 15 "${API_URL}${dom_norm}" 2>&1 || true)"
+  resp="$(curl -s --max-time 15 "${API_URL}${dom_norm}" || true)"
   preview="$(printf '%s' "$resp" | tr '\n' ' ' | cut -c1-400)"
   dbg "  -> API response preview: ${preview}"
 
-  # check JSON validity
   if ! jq -e . >/dev/null 2>&1 <<<"$resp"; then
     api_error=$((api_error+1))
     ERRORS["$dom_norm"]="non-json-response"
-    DOM_ROLE["$dom_norm"]="service"
+    dbg "  -> non-JSON, skipping"
-    SOURCES["$dom_norm"]="base"
-    EXPANDED["$dom_norm"]=1
-    dbg "  -> API returned non-JSON. Marked as 'service' and included base only."
     continue
   fi
 
-  # valid json: check for .relatedDomains array
+  # error handling
-  has_related=$(jq -r 'if has("relatedDomains") then "1" else "0" end' <<<"$resp" 2>/dev/null || echo "0")
+  if jq -e 'has("error")' <<<"$resp" >/dev/null; then
-  if [ "$has_related" = "1" ]; then
+    err_msg="$(jq -r '.error' <<<"$resp")"
-    api_success=$((api_success+1))
+    dbg "  -> API error: $err_msg"
-    DOM_ROLE["$dom_norm"]="site"
-    SOURCES["$dom_norm"]="base"
-    EXPANDED["$dom_norm"]=1
 
-    # extract related domains
+    if grep -Eq "ERR_NAME_NOT_RESOLVED|Timeout" <<<"$err_msg"; then
-    mapfile -t subs < <(jq -r '.relatedDomains[]? // empty' <<<"$resp")
+      dbg "    -> Dead domain, completely skipped"
-    dbg "  -> API returned ${#subs[@]} related domains"
+      continue
+    fi
 
-    for s in "${subs[@]}"; do
+    if grep -Eq "ERR_CERT_COMMON_NAME_INVALID|ERR_CONNECTION_REFUSED" <<<"$err_msg"; then
-      nd="$(normalize_domain "$s" || true)"
+      dbg "    -> Service domain, keep only base"
-      if [ -z "$nd" ]; then
+      DOM_ROLE["$dom_norm"]="service"
-        dbg "    - RELATED SKIP (normalize failed): '$s'"
+      SOURCES["$dom_norm"]="base"
-        continue
+      EXPANDED["$dom_norm"]=1
-      fi
+      ERRORS["$dom_norm"]="$err_msg"
-      EXPANDED["$nd"]=1
+      continue
-      [ -z "${SOURCES[$nd]:-}" ] && SOURCES["$nd"]="related"
+    fi
-      related_total=$((related_total+1))
+
-      dbg "    - RELATED ADD: $nd"
+    # default case: treat as service
-    done
-  else
-    api_error=$((api_error+1))
-    err_msg="$(jq -r '.error // empty' <<<"$resp" | sed 's/[\r\n]/ /g')"
-    ERRORS["$dom_norm"]="${err_msg:-unknown}"
     DOM_ROLE["$dom_norm"]="service"
     SOURCES["$dom_norm"]="base"
     EXPANDED["$dom_norm"]=1
-    dbg "  -> API returned no 'relatedDomains' array; error='${ERRORS[$dom_norm]}' ; marked as 'service'"
+    ERRORS["$dom_norm"]="$err_msg"
+    continue
   fi
 
+  # valid JSON without error -> must be site
+  api_success=$((api_success+1))
+  DOM_ROLE["$dom_norm"]="site"
+  SOURCES["$dom_norm"]="base"
+  EXPANDED["$dom_norm"]=1
+  VALID_SITES["$dom_norm"]=1
+
+  mapfile -t subs < <(jq -r '.relatedDomains[]? // empty' <<<"$resp")
+  dbg "  -> API returned ${#subs[@]} related domains"
+  for s in "${subs[@]}"; do
+    nd="$(normalize_domain "$s" || true)"
+    if [ -z "$nd" ]; then
+      dbg "    - RELATED SKIP: '$s'"
+      continue
+    fi
+    EXPANDED["$nd"]=1
+    [ -z "${SOURCES[$nd]:-}" ] && SOURCES["$nd"]="related"
+    related_total=$((related_total+1))
+    dbg "    - RELATED ADD: $nd"
+  done
+
 done < "$INPUT_FILE"
 
-# sort & unique final list
 mapfile -t ALL_DOMAINS < <(printf "%s\n" "${!EXPANDED[@]}" | sort -u)
-total_unique=${#ALL_DOMAINS[@]}
 
-# write files (if not dry-run)
 if ! $DRY_RUN; then
-  dbg "Writing outputs to: $IPSET_CONF and $RESOLVE_CONF"
   for d in "${ALL_DOMAINS[@]}"; do
     printf 'ipset=/%s/bbrkn\n' "$d" >> "$IPSET_CONF"
     printf 'server=/%s/%s\n' "$d" "$DNS_SERVER" >> "$RESOLVE_CONF"
   done
-  dbg "Wrote $(wc -l < "$IPSET_CONF" 2>/dev/null || echo 0) ipset lines and $(wc -l < "$RESOLVE_CONF" 2>/dev/null || echo 0) server lines"
-else
-  dbg "Dry-run: not writing output files"
 fi
 
-# Final debug report
 echo
 echo "===== DEBUG REPORT ====="
 echo "Input file: $INPUT_FILE"
@@ -190,34 +166,15 @@ echo "Processed lines: $total_lines"
 echo "Normalized OK: $normalized_ok"
 echo "Normalized skipped: $normalized_skip"
 echo
-echo "API calls succeeded (returned relatedDomains): $api_success"
+echo "API success (sites): $api_success"
-echo "API calls failed / returned error: $api_error"
+echo "API error/ignored: $api_error"
-echo "Related domains collected (raw count): $related_total"
+echo "Related domains added: $related_total"
-echo "Final unique domains (base + related): $total_unique"
+echo "Final unique domains: ${#ALL_DOMAINS[@]}"
 echo
-
+echo "---- VALID BASE SITES ----"
-echo "---- base domains (from input) ----"
+printf '%s\n' "${!VALID_SITES[@]}" | sort
-for base in "${!DOM_ROLE[@]}"; do
-  if [ "${DOM_ROLE[$base]}" = "site" ] || [ "${DOM_ROLE[$base]}" = "service" ]; then
-    echo "$base - ${DOM_ROLE[$base]} - ${ERRORS[$base]:-ok}"
-  fi
-done | sort
-
-echo
-echo "---- sample of final unique domains (first 200) ----"
-printf '%s\n' "${ALL_DOMAINS[@]}" | head -n 200
-
-echo
-echo "---- API errors (if any) ----"
-for k in "${!ERRORS[@]}"; do
-  if [ "${ERRORS[$k]}" != "ok" ]; then
-    echo "$k -> ${ERRORS[$k]}"
-  fi
-done | sort
-
 echo "===== END DEBUG REPORT ====="
 
 if [ "$DEBUG" != "0" ]; then
-  echo
+  echo "Detailed debug log: $DEBUG_LOG"
-  echo "Detailed debug log appended to: $DEBUG_LOG"
 fi