Split curl output to body and header

scripts/generate-configs.sh

@@ -5,8 +5,10 @@ set -euo pipefail
 # Конфигурация через переменные окружения
 # ==============================
 INPUT_FILE="${DOMAINS_FILE:-domains.txt}"
+
 IPSET_CONF="${IPSET_CONF:-/tmp/91-ipset-bbrkn.conf}"
 RESOLVE_CONF="${RESOLVE_CONF:-/tmp/92-resolve-bbrkn.conf}"
+
 API_URL="${CHROME_SERVER:-http://127.0.0.1:3000}/domains?domain="
 DNS_SERVER="${DNS_SERVER:-8.8.8.8}"
 
@@ -62,48 +64,6 @@ normalize_domain() {
   return 0
 }
 
-fetch_json() {
-  local domain="$1"
-  local max_attempts=5
-  local attempt=0
-  local max_sleep=8
-  local resp http_code json sleep_time preview
-
-  while :; do
-    attempt=$((attempt+1))
-    dbg "  -> API attempt #$attempt for $domain"
-
-    resp=$(curl -sS --compressed \
-      -m 10 --connect-timeout 5 \
-      --retry 2 --retry-connrefused \
-      -H 'Accept: application/json' \
-      -w '%{http_code}' \
-      "${API_URL}${domain}" 2>>"$DEBUG_LOG") || true
-
-    http_code="${resp: -3}"       # последние 3 символа
-    json="${resp:0:-3}"           # тело без http_code
-
-    preview="$(printf '%s' "$json" | tr '\n' ' ' | cut -c1-400)"
-    dbg "  -> HTTP=$http_code, preview=${preview}"
-
-    if [ "$http_code" = "200" ] && jq -e . >/dev/null 2>&1 <<<"$json"; then
-      echo "$json"
-      return 0
-    fi
-
-    if [ "$attempt" -ge "$max_attempts" ]; then
-      ERRORS["$domain"]="http_${http_code}_or_invalid_json"
-      dbg "  -> Failed after $attempt attempts, preview=${preview}"
-      return 1
-    fi
-
-    sleep_time=$((2 ** (attempt-1)))
-    [ "$sleep_time" -gt "$max_sleep" ] && sleep_time=$max_sleep
-    dbg "  -> Retry after ${sleep_time}s..."
-    sleep "$sleep_time"
-  done
-}
-
 if [ ! -f "$INPUT_FILE" ]; then
   err "Input file not found: $INPUT_FILE"
   exit 3
@@ -127,21 +87,56 @@ while IFS= read -r line || [ -n "$line" ]; do
   normalized_ok=$((normalized_ok+1))
   dbg "  -> NORMALIZED: $dom_norm"
 
-  json="$(fetch_json "$dom_norm" || true)"
+  max_attempts=5
-  if [ -z "$json" ]; then
+  attempt=0
+  resp=""
+  http_code=0
+
+  while :; do
+    attempt=$((attempt+1))
+    tmp_body=$(mktemp)
+    tmp_header=$(mktemp)
+
+    dbg "  -> API attempt #$attempt for $dom_norm"
+    curl -sS --compressed -m 10 --connect-timeout 5 --retry 2 --retry-connrefused \
+         -H 'Accept: application/json' \
+         -D "$tmp_header" -o "$tmp_body" "${API_URL}${dom_norm}" 2>>"$DEBUG_LOG" || true
+
+    http_code=$(awk '/^HTTP/{code=$2} END{print code}' "$tmp_header")
+    resp=$(cat "$tmp_body")
+    rm -f "$tmp_body" "$tmp_header"
+
+    preview="$(printf '%s' "$resp" | tr '\n' ' ' | cut -c1-400)"
+    dbg "  -> HTTP=$http_code, preview=$preview"
+
+    if [ "$http_code" = "200" ] && jq -e . >/dev/null 2>&1 <<<"$resp"; then
+      break
+    fi
+
+    if [ "$attempt" -ge "$max_attempts" ]; then
       api_error=$((api_error+1))
-    dbg "  -> API returned empty or invalid JSON for $dom_norm"
+      ERRORS["$dom_norm"]="http_${http_code}_or_nonjson"
+      dbg "  -> Failed after ${attempt} attempts: HTTP=${http_code}, preview=${preview}"
+      resp=""
+      break
+    fi
+
+    sleep_time=$(( 1 << (attempt - 1) ))
+    dbg "  -> Retry after ${sleep_time}s..."
+    sleep "$sleep_time"
+  done
+
+  if [ -z "$resp" ] || ! jq -e . >/dev/null 2>&1 <<<"$resp"; then
+    dbg "  -> non-JSON or empty response, skipping domain: $dom_norm"
     continue
   fi
 
-  if jq -e 'has("error")' <<<"$json" >/dev/null; then
+  if jq -e 'has("error")' <<<"$resp" >/dev/null; then
-    err_msg="$(jq -r '.error' <<<"$json")"
+    err_msg="$(jq -r '.error' <<<"$resp")"
     dbg "  -> API error: $err_msg"
-
     if grep -Eq "ERR_NAME_NOT_RESOLVED|Timeout" <<<"$err_msg"; then
       continue
     fi
-
     DOM_ROLE["$dom_norm"]="service"
     SOURCES["$dom_norm"]="base"
     EXPANDED["$dom_norm"]=1
@@ -149,14 +144,13 @@ while IFS= read -r line || [ -n "$line" ]; do
     continue
   fi
 
-  # valid JSON -> site
   api_success=$((api_success+1))
   DOM_ROLE["$dom_norm"]="site"
   SOURCES["$dom_norm"]="base"
   EXPANDED["$dom_norm"]=1
   VALID_SITES["$dom_norm"]=1
 
-  mapfile -t subs < <(jq -r '.relatedDomains[]? // empty' <<<"$json")
+  mapfile -t subs < <(jq -r '.relatedDomains[]? // empty' <<<"$resp")
   dbg "  -> API returned ${#subs[@]} related domains"
   for s in "${subs[@]}"; do
     nd="$(normalize_domain "$s" || true)"
@@ -164,9 +158,7 @@ while IFS= read -r line || [ -n "$line" ]; do
     EXPANDED["$nd"]=1
     [ -z "${SOURCES[$nd]:-}" ] && SOURCES["$nd"]="related"
     related_total=$((related_total+1))
-    dbg "    - RELATED ADD: $nd"
   done
-
 done < "$INPUT_FILE"
 
 mapfile -t ALL_DOMAINS < <(printf "%s\n" "${!EXPANDED[@]}" | sort -u)