From a206ec509d39144e7bea83a7454fc3a03023edfc Mon Sep 17 00:00:00 2001 From: Kirill Kodanev Date: Mon, 15 Sep 2025 20:51:34 +0300 Subject: [PATCH] Use tmpfiles for JSON --- scripts/generate-configs.sh | 46 +++++++++++++++---------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/scripts/generate-configs.sh b/scripts/generate-configs.sh index fdebee8..47a2d9a 100644 --- a/scripts/generate-configs.sh +++ b/scripts/generate-configs.sh @@ -52,6 +52,14 @@ api_success=0 api_error=0 related_total=0 +TEMP_FILES=() +cleanup_tmp() { + for f in "${TEMP_FILES[@]}"; do + [ -f "$f" ] && rm -f "$f" + done +} +trap cleanup_tmp EXIT + normalize_domain() { local raw="$1" raw="$(printf '%s' "$raw" | sed -E 's/#.*$//' | awk '{$1=$1};1')" @@ -76,22 +84,22 @@ query_api() { dbg " -> API attempt #$attempt for $domain" tmpfile=$(mktemp) - # curl с отдельным файлом для тела, HTTP-код через -w + TEMP_FILES+=("$tmpfile") http_code=$(curl -sS --compressed \ -m 10 --connect-timeout 5 \ + --retry 2 --retry-connrefused \ -H 'Accept: application/json' \ -w '%{http_code}' \ -o "$tmpfile" \ "${API_URL}${domain}" 2>>"$DEBUG_LOG" || true) body=$(cat "$tmpfile") - rm -f "$tmpfile" preview="$(printf '%s' "$body" | tr '\n' ' ' | cut -c1-400)" dbg " -> HTTP=$http_code, preview=${preview}" - if [ "$http_code" = "200" ] && jq -e . >/dev/null 2>&1 <<<"$body"; then - echo "$body" + if [ "$http_code" = "200" ] && jq -e . "$tmpfile" >/dev/null 2>&1; then + echo "$tmpfile" # возвращаем путь к tmp json файлу return 0 fi @@ -131,44 +139,28 @@ while IFS= read -r line || [ -n "$line" ]; do normalized_ok=$((normalized_ok+1)) dbg " -> NORMALIZED: $dom_norm" - resp="$(query_api "$dom_norm" || true)" - if [ -z "$resp" ]; then + tmpjson=$(query_api "$dom_norm" || true) + if [ -z "$tmpjson" ]; then dbg " -> No valid response for $dom_norm, skipping." continue fi - if jq -e 'has("error")' <<<"$resp" >/dev/null; then - err_msg="$(jq -r '.error' <<<"$resp")" + if jq -e 'has("error")' "$tmpjson" >/dev/null 2>&1; then + err_msg=$(jq -r '.error' "$tmpjson") dbg " -> API error: $err_msg" - - if grep -Eq "ERR_NAME_NOT_RESOLVED|Timeout" <<<"$err_msg"; then - DOM_ROLE["$dom_norm"]="dead" - ERRORS["$dom_norm"]="$err_msg" - continue - fi - - if grep -Eq "ERR_CERT_COMMON_NAME_INVALID|ERR_CONNECTION_REFUSED" <<<"$err_msg"; then - DOM_ROLE["$dom_norm"]="service" - SOURCES["$dom_norm"]="base" - EXPANDED["$dom_norm"]=1 - ERRORS["$dom_norm"]="$err_msg" - continue - fi - - DOM_ROLE["$dom_norm"]="unknown" - SOURCES["$dom_norm"]="base" - EXPANDED["$dom_norm"]=1 + DOM_ROLE["$dom_norm"]="error" ERRORS["$dom_norm"]="$err_msg" continue fi + # валидный сайт api_success=$((api_success+1)) DOM_ROLE["$dom_norm"]="site" SOURCES["$dom_norm"]="base" EXPANDED["$dom_norm"]=1 VALID_SITES["$dom_norm"]=1 - mapfile -t subs < <(jq -r '.relatedDomains[]? // empty' <<<"$resp") + mapfile -t subs < <(jq -r '.relatedDomains[]? // empty' "$tmpjson") dbg " -> API returned ${#subs[@]} related domains" for s in "${subs[@]}"; do nd="$(normalize_domain "$s" || true)"