goodvin
8598a1a48f
All checks were successful
Deploy DNS Configuration / deploy (push) Successful in 1h15m4s
dnsmasq adds IPs to an nft set only on a fresh upstream reply; a cache hit never refreshes the set. On Archie the :5350 instance also wedged: TCP child processes first, then the main process, logged 'nftset ... netlink: Error: cache initialization failed: Protocol error' on every add and stopped filling bbrkn_v4/v6 until restarted. warm-nftset.sh now counts those errors for the main PID during the run (journalctl _PID=) and, if any, restarts the service and warms again; exit 1 if it still fails. The state dir defaults to /var/lib/bbrkn so the root cron job and the deploy share one lock (LOCK_WAIT lets deploy wait). deploy-to-gateway.sh runs the warmer right after flushing the sets, since clients keep old IPs cached and would bypass the tunnel until the next fresh resolve. A warm failure only warns; it does not roll back. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01DR7zpSsdX28T4Kn6v4jPSS
254 lines
10 KiB
Bash
254 lines
10 KiB
Bash
#!/bin/bash
|
||
set -euo pipefail
|
||
|
||
# ==============================
|
||
# Конфигурация через переменные окружения
|
||
# ==============================
|
||
# Сгенерированные конфиги (из generate-configs.sh)
|
||
NFTSET_CONF="${NFTSET_CONF:-/tmp/90-nftset.conf}"
|
||
RESOLVE_CONF="${RESOLVE_CONF:-/tmp/92-resolve-bbrkn.conf}"
|
||
|
||
# Два таргета — два инстанса dnsmasq:
|
||
# NFTSET_CONF → host dnsmasq (:5350), грузит /etc/dnsmasq.d, наполняет nft-сеты
|
||
# RESOLVE_CONF → pihole (docker), делегирует bbrkn-домены инстансу :5350
|
||
NFTSET_TARGET_DIR="${NFTSET_TARGET_DIR:-/etc/dnsmasq.d}"
|
||
RESOLVE_TARGET_DIR="${RESOLVE_TARGET_DIR:-${TARGET_DIR:-/opt/appdata/pihole/etc/dnsmasq.d}}"
|
||
|
||
# Бэкапы — ВНЕ conf-dir: dnsmasq (и pihole) грузит из dnsmasq.d все файлы,
|
||
# кроме исключённых расширений, так что *.backup.* там стал бы живым конфигом.
|
||
BACKUP_DIR="${BACKUP_DIR:-/var/backups/bbrkn}"
|
||
BACKUP_KEEP="${BACKUP_KEEP:-10}" # сколько последних копий хранить на каждый файл
|
||
|
||
DOCKER_CONTAINER="${DOCKER_CONTAINER:-pihole}"
|
||
# systemd-сервис host-инстанса dnsmasq (:5350)
|
||
HOST_DNSMASQ_SVC="${HOST_DNSMASQ_SVC:-dnsmasq}"
|
||
|
||
# nft-сеты для сброса после деплоя (иначе висят до timeout 24h).
|
||
# Формат: "family table set" — перечисляются через ';' в NFT_SETS.
|
||
NFT_SETS="${NFT_SETS:-inet filter bbrkn_v4;inet filter bbrkn_v6}"
|
||
|
||
# --- Health check ---
|
||
DNS_LISTEN_ADDR="${DNS_LISTEN_ADDR:-127.0.0.1}" # pihole :53 (клиентский)
|
||
DNS_CHECK_DOMAIN="${DNS_CHECK_DOMAIN:-google.com}"
|
||
# Домен, который ЕСТЬ в bbrkn-сете — для end-to-end проверки nftset-захвата
|
||
NFTSET_CHECK_DOMAIN="${NFTSET_CHECK_DOMAIN:-chatgpt.com}"
|
||
RESOLVER_ADDR="${RESOLVER_ADDR:-127.0.0.1}" # host dnsmasq (:5350)
|
||
RESOLVER_PORT="${RESOLVER_PORT:-5350}"
|
||
NFT_CHECK_SET="${NFT_CHECK_SET:-inet filter bbrkn_v4}"
|
||
|
||
# Имена/пути назначения (параметрическая подстановка, без basename)
|
||
nftset_name="${NFTSET_CONF##*/}"
|
||
resolve_name="${RESOLVE_CONF##*/}"
|
||
nftset_dest="$NFTSET_TARGET_DIR/$nftset_name"
|
||
resolve_dest="$RESOLVE_TARGET_DIR/$resolve_name"
|
||
|
||
# Пути к бэкапам (для rollback)
|
||
NFTSET_BACKUP=""
|
||
RESOLVE_BACKUP=""
|
||
|
||
echo "Deploying nftset config → $nftset_dest (host dnsmasq :$RESOLVER_PORT)"
|
||
echo "Deploying resolve config → $resolve_dest ($DOCKER_CONTAINER)"
|
||
|
||
# ==============================
|
||
# Проверка входных файлов
|
||
# ==============================
|
||
if [ ! -f "$NFTSET_CONF" ] || [ ! -f "$RESOLVE_CONF" ]; then
|
||
echo "Error: one or both generated config files not found!"
|
||
echo " NFTSET_CONF=$NFTSET_CONF RESOLVE_CONF=$RESOLVE_CONF"
|
||
exit 1
|
||
fi
|
||
|
||
# ==============================
|
||
# Функция отката: восстанавливает бэкапы и перезапускает оба инстанса
|
||
# ==============================
|
||
rollback() {
|
||
echo "[ROLLBACK] Restoring previous configuration..."
|
||
local restored=false
|
||
if [ -n "$NFTSET_BACKUP" ] && [ -f "$NFTSET_BACKUP" ]; then
|
||
cp "$NFTSET_BACKUP" "$nftset_dest"
|
||
echo "[ROLLBACK] Restored $nftset_dest"
|
||
restored=true
|
||
fi
|
||
if [ -n "$RESOLVE_BACKUP" ] && [ -f "$RESOLVE_BACKUP" ]; then
|
||
cp "$RESOLVE_BACKUP" "$resolve_dest"
|
||
echo "[ROLLBACK] Restored $resolve_dest"
|
||
restored=true
|
||
fi
|
||
if $restored; then
|
||
echo "[ROLLBACK] Restarting host dnsmasq ($HOST_DNSMASQ_SVC) and $DOCKER_CONTAINER..."
|
||
systemctl restart "$HOST_DNSMASQ_SVC" 2>/dev/null || true
|
||
docker restart "$DOCKER_CONTAINER" 2>/dev/null || true
|
||
else
|
||
echo "[ROLLBACK] No backups found, cannot restore"
|
||
fi
|
||
echo "[ROLLBACK] Done"
|
||
}
|
||
|
||
# ==============================
|
||
# Бэкап существующих файлов
|
||
# ==============================
|
||
# Оставляет BACKUP_KEEP самых свежих копий <name>.backup.<ts>
|
||
# (ts вида YYYYmmdd-HHMMSS сортируется лексикографически).
|
||
prune_backups() { # <name>
|
||
local name="$1" old
|
||
find "$BACKUP_DIR" -maxdepth 1 -type f -name "${name}.backup.*" -printf '%f\n' \
|
||
| sort -r | tail -n +"$((BACKUP_KEEP + 1))" \
|
||
| while read -r old; do
|
||
rm -f -- "$BACKUP_DIR/$old"
|
||
echo " pruned old backup $old"
|
||
done
|
||
}
|
||
|
||
mkdir -p "$BACKUP_DIR"
|
||
ts=$(date +%Y%m%d-%H%M%S)
|
||
if [ -f "$nftset_dest" ]; then
|
||
NFTSET_BACKUP="$BACKUP_DIR/${nftset_name}.backup.${ts}"
|
||
cp "$nftset_dest" "$NFTSET_BACKUP"
|
||
echo "Backed up $nftset_dest → $NFTSET_BACKUP"
|
||
prune_backups "$nftset_name"
|
||
fi
|
||
if [ -f "$resolve_dest" ]; then
|
||
RESOLVE_BACKUP="$BACKUP_DIR/${resolve_name}.backup.${ts}"
|
||
cp "$resolve_dest" "$RESOLVE_BACKUP"
|
||
echo "Backed up $resolve_dest → $RESOLVE_BACKUP"
|
||
prune_backups "$resolve_name"
|
||
fi
|
||
|
||
# ==============================
|
||
# Копирование новых файлов
|
||
# ==============================
|
||
cp "$NFTSET_CONF" "$nftset_dest"
|
||
cp "$RESOLVE_CONF" "$resolve_dest"
|
||
|
||
# ==============================
|
||
# Перезапуск обоих инстансов.
|
||
# ВАЖНО: dnsmasq по SIGHUP НЕ перечитывает директивы конфига (nftset=/server=),
|
||
# только hosts-файлы — новые домены подхватываются лишь полным рестартом.
|
||
# ==============================
|
||
echo "Restarting host dnsmasq ($HOST_DNSMASQ_SVC)..."
|
||
systemctl restart "$HOST_DNSMASQ_SVC"
|
||
|
||
echo "Restarting $DOCKER_CONTAINER container..."
|
||
docker restart "$DOCKER_CONTAINER"
|
||
|
||
# --- Ждём host dnsmasq (:5350) ---
|
||
echo "Waiting for host dnsmasq on :$RESOLVER_PORT (up to 15s)..."
|
||
host_ready=false
|
||
for i in $(seq 1 15); do
|
||
if ss -uln 2>/dev/null | grep -q ":$RESOLVER_PORT "; then
|
||
host_ready=true
|
||
break
|
||
fi
|
||
sleep 1
|
||
done
|
||
if ! $host_ready; then
|
||
echo "Error: host dnsmasq not listening on :$RESOLVER_PORT after 15s"
|
||
rollback
|
||
exit 1
|
||
fi
|
||
echo "host dnsmasq is listening on :$RESOLVER_PORT"
|
||
|
||
# --- Ждём pihole ---
|
||
echo "Waiting for $DOCKER_CONTAINER to be ready (up to 30s)..."
|
||
ready=false
|
||
for i in $(seq 1 30); do
|
||
status=$(docker inspect "$DOCKER_CONTAINER" --format '{{.State.Status}}' 2>/dev/null || echo "missing")
|
||
if [ "$status" = "running" ]; then
|
||
ready=true
|
||
break
|
||
fi
|
||
sleep 1
|
||
done
|
||
if ! $ready; then
|
||
echo "Error: $DOCKER_CONTAINER failed to start after 30s (status: ${status:-unknown})"
|
||
rollback
|
||
exit 1
|
||
fi
|
||
echo "$DOCKER_CONTAINER is running"
|
||
|
||
# ==============================
|
||
# Сброс nft-сетов (чистим устаревшие записи удалённых доменов; timeout 24h иначе)
|
||
# ==============================
|
||
echo "Flushing nft sets..."
|
||
IFS=';' read -ra _sets <<< "$NFT_SETS"
|
||
for s in "${_sets[@]}"; do
|
||
[ -z "$s" ] && continue
|
||
# shellcheck disable=SC2086
|
||
if nft flush set $s 2>/dev/null; then
|
||
echo " flushed set $s"
|
||
else
|
||
echo " WARN: could not flush set $s"
|
||
fi
|
||
done
|
||
|
||
# ==============================
|
||
# Health check 1: базовый DNS через pihole :53
|
||
# ==============================
|
||
dns_query() { # <addr> <port> <domain>
|
||
local addr="$1" port="$2" dom="$3"
|
||
if command -v dig >/dev/null 2>&1; then
|
||
dig +short +time=3 +tries=2 -p "$port" "@$addr" "$dom" 2>/dev/null
|
||
elif command -v nslookup >/dev/null 2>&1; then
|
||
nslookup -port="$port" "$dom" "$addr" 2>/dev/null | awk '/^Address: /{print $2}'
|
||
fi
|
||
}
|
||
|
||
if ! command -v dig >/dev/null 2>&1 && ! command -v nslookup >/dev/null 2>&1; then
|
||
echo "Warning: neither 'dig' nor 'nslookup' found, skipping DNS health checks"
|
||
else
|
||
echo "DNS health check: $DNS_CHECK_DOMAIN via $DNS_LISTEN_ADDR:53 (up to 15 attempts)..."
|
||
dns_ok=false
|
||
for i in $(seq 1 15); do
|
||
if [ -n "$(dns_query "$DNS_LISTEN_ADDR" 53 "$DNS_CHECK_DOMAIN")" ]; then
|
||
dns_ok=true
|
||
echo " passed (attempt $i)"
|
||
break
|
||
fi
|
||
echo " attempt $i failed, retry in 1s..."
|
||
sleep 1
|
||
done
|
||
if ! $dns_ok; then
|
||
echo "Error: DNS health check failed — pihole not resolving on $DNS_LISTEN_ADDR:53"
|
||
rollback
|
||
exit 1
|
||
fi
|
||
|
||
# ==============================
|
||
# Health check 2: end-to-end nftset-захват.
|
||
# Резолвим bbrkn-домен через :5350 и убеждаемся, что nft-сет наполнился.
|
||
# ==============================
|
||
echo "nftset capture check: $NFTSET_CHECK_DOMAIN via $RESOLVER_ADDR:$RESOLVER_PORT..."
|
||
dns_query "$RESOLVER_ADDR" "$RESOLVER_PORT" "$NFTSET_CHECK_DOMAIN" >/dev/null
|
||
sleep 1
|
||
# shellcheck disable=SC2086
|
||
set_count=$(nft list set $NFT_CHECK_SET 2>/dev/null | grep -cE '([0-9]{1,3}\.){3}[0-9]{1,3}' || echo 0)
|
||
if [ "${set_count:-0}" -gt 0 ]; then
|
||
echo " passed — set '$NFT_CHECK_SET' populated ($set_count element line(s))"
|
||
else
|
||
echo "Error: nftset '$NFT_CHECK_SET' still empty after resolving $NFTSET_CHECK_DOMAIN"
|
||
echo " host dnsmasq likely not loading $nftset_dest"
|
||
rollback
|
||
exit 1
|
||
fi
|
||
fi
|
||
|
||
# ==============================
|
||
# Прогрев nft-сетов: после flush сеты пусты, а у клиентов в кэше (Edge,
|
||
# Windows) остаются старые IP — без прогрева их трафик идёт мимо туннеля до
|
||
# первого свежего резолва. Ошибка прогрева деплой не откатывает: конфиги уже
|
||
# проверены, а cron-прогрев повторит попытку.
|
||
# ==============================
|
||
WARM_SCRIPT="${WARM_SCRIPT:-$(dirname "$0")/warm-nftset.sh}"
|
||
if [ -f "$WARM_SCRIPT" ]; then
|
||
echo "Warming nft sets ($WARM_SCRIPT)..."
|
||
if ! NFTSET_CONF="$nftset_dest" HOST_DNSMASQ_SVC="$HOST_DNSMASQ_SVC" \
|
||
LOCK_WAIT="${WARM_LOCK_WAIT:-300}" bash "$WARM_SCRIPT"; then
|
||
echo " WARN: warm failed — sets will refill on client queries / hourly cron"
|
||
fi
|
||
else
|
||
echo " WARN: $WARM_SCRIPT not found, skipping warm"
|
||
fi
|
||
|
||
echo "Deployment completed successfully"
|
||
DOMAIN_COUNT=$(grep -c '^nftset=' "$nftset_dest" || echo "0")
|
||
echo "Applied nftset configuration for $DOMAIN_COUNT domain directives"
|