Use tmpfiles for JSON
Some checks failed
Deploy DNS Configuration / deploy (push) Has been cancelled
Some checks failed
Deploy DNS Configuration / deploy (push) Has been cancelled
This commit is contained in:
parent
568701d1cb
commit
a206ec509d
1 changed files with 19 additions and 27 deletions
|
|
@ -52,6 +52,14 @@ api_success=0
|
|||
api_error=0
|
||||
related_total=0
|
||||
|
||||
TEMP_FILES=()
|
||||
cleanup_tmp() {
|
||||
for f in "${TEMP_FILES[@]}"; do
|
||||
[ -f "$f" ] && rm -f "$f"
|
||||
done
|
||||
}
|
||||
trap cleanup_tmp EXIT
|
||||
|
||||
normalize_domain() {
|
||||
local raw="$1"
|
||||
raw="$(printf '%s' "$raw" | sed -E 's/#.*$//' | awk '{$1=$1};1')"
|
||||
|
|
@ -76,22 +84,22 @@ query_api() {
|
|||
dbg " -> API attempt #$attempt for $domain"
|
||||
|
||||
tmpfile=$(mktemp)
|
||||
# curl с отдельным файлом для тела, HTTP-код через -w
|
||||
TEMP_FILES+=("$tmpfile")
|
||||
http_code=$(curl -sS --compressed \
|
||||
-m 10 --connect-timeout 5 \
|
||||
--retry 2 --retry-connrefused \
|
||||
-H 'Accept: application/json' \
|
||||
-w '%{http_code}' \
|
||||
-o "$tmpfile" \
|
||||
"${API_URL}${domain}" 2>>"$DEBUG_LOG" || true)
|
||||
|
||||
body=$(cat "$tmpfile")
|
||||
rm -f "$tmpfile"
|
||||
|
||||
preview="$(printf '%s' "$body" | tr '\n' ' ' | cut -c1-400)"
|
||||
dbg " -> HTTP=$http_code, preview=${preview}"
|
||||
|
||||
if [ "$http_code" = "200" ] && jq -e . >/dev/null 2>&1 <<<"$body"; then
|
||||
echo "$body"
|
||||
if [ "$http_code" = "200" ] && jq -e . "$tmpfile" >/dev/null 2>&1; then
|
||||
echo "$tmpfile" # возвращаем путь к tmp json файлу
|
||||
return 0
|
||||
fi
|
||||
|
||||
|
|
@ -131,44 +139,28 @@ while IFS= read -r line || [ -n "$line" ]; do
|
|||
normalized_ok=$((normalized_ok+1))
|
||||
dbg " -> NORMALIZED: $dom_norm"
|
||||
|
||||
resp="$(query_api "$dom_norm" || true)"
|
||||
if [ -z "$resp" ]; then
|
||||
tmpjson=$(query_api "$dom_norm" || true)
|
||||
if [ -z "$tmpjson" ]; then
|
||||
dbg " -> No valid response for $dom_norm, skipping."
|
||||
continue
|
||||
fi
|
||||
|
||||
if jq -e 'has("error")' <<<"$resp" >/dev/null; then
|
||||
err_msg="$(jq -r '.error' <<<"$resp")"
|
||||
if jq -e 'has("error")' "$tmpjson" >/dev/null 2>&1; then
|
||||
err_msg=$(jq -r '.error' "$tmpjson")
|
||||
dbg " -> API error: $err_msg"
|
||||
|
||||
if grep -Eq "ERR_NAME_NOT_RESOLVED|Timeout" <<<"$err_msg"; then
|
||||
DOM_ROLE["$dom_norm"]="dead"
|
||||
ERRORS["$dom_norm"]="$err_msg"
|
||||
continue
|
||||
fi
|
||||
|
||||
if grep -Eq "ERR_CERT_COMMON_NAME_INVALID|ERR_CONNECTION_REFUSED" <<<"$err_msg"; then
|
||||
DOM_ROLE["$dom_norm"]="service"
|
||||
SOURCES["$dom_norm"]="base"
|
||||
EXPANDED["$dom_norm"]=1
|
||||
ERRORS["$dom_norm"]="$err_msg"
|
||||
continue
|
||||
fi
|
||||
|
||||
DOM_ROLE["$dom_norm"]="unknown"
|
||||
SOURCES["$dom_norm"]="base"
|
||||
EXPANDED["$dom_norm"]=1
|
||||
DOM_ROLE["$dom_norm"]="error"
|
||||
ERRORS["$dom_norm"]="$err_msg"
|
||||
continue
|
||||
fi
|
||||
|
||||
# валидный сайт
|
||||
api_success=$((api_success+1))
|
||||
DOM_ROLE["$dom_norm"]="site"
|
||||
SOURCES["$dom_norm"]="base"
|
||||
EXPANDED["$dom_norm"]=1
|
||||
VALID_SITES["$dom_norm"]=1
|
||||
|
||||
mapfile -t subs < <(jq -r '.relatedDomains[]? // empty' <<<"$resp")
|
||||
mapfile -t subs < <(jq -r '.relatedDomains[]? // empty' "$tmpjson")
|
||||
dbg " -> API returned ${#subs[@]} related domains"
|
||||
for s in "${subs[@]}"; do
|
||||
nd="$(normalize_domain "$s" || true)"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue