490 lines
12 KiB
Bash
Executable File
490 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# Validate Source0 URL reachability in RPM spec files.
|
|
# - Scans current directory pattern: ./*/*.spec
|
|
# - Extracts Source0 from each spec
|
|
# - Expands common macros (%{version}, %{name}, %{_name}, %{?dist}, and %global/%define vars)
|
|
# - Strips URL fragment (#/filename) before HTTP request
|
|
# - Uses curl HEAD first, then falls back to GET(range) for servers that block HEAD
|
|
|
|
DRY_RUN=0
|
|
JOBS="${JOBS:-4}"
|
|
WORKER_MODE=0
|
|
WORKER_SPEC=""
|
|
TIMEOUT="${TIMEOUT:-15}"
|
|
RETRIES="${RETRIES:-3}"
|
|
RETRY_DELAY="${RETRY_DELAY:-1}"
|
|
USER_AGENT="spec-source0-check/1.0"
|
|
|
|
usage() {
|
|
local script_name
|
|
script_name="$(basename "$0")"
|
|
cat <<USAGE
|
|
Usage:
|
|
${script_name} [--dry-run] [--jobs N]
|
|
|
|
Options:
|
|
--dry-run only print expanded Source0 URL, do not perform HTTP checks
|
|
--jobs, -j number of parallel workers (default: 4, set 1 for serial)
|
|
USAGE
|
|
}
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--dry-run)
|
|
DRY_RUN=1
|
|
shift
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
--jobs|-j)
|
|
if [[ $# -lt 2 ]]; then
|
|
echo "[fatal] missing value for $1" >&2
|
|
usage >&2
|
|
exit 2
|
|
fi
|
|
JOBS="$2"
|
|
shift 2
|
|
;;
|
|
--worker)
|
|
if [[ $# -lt 2 ]]; then
|
|
echo "[fatal] missing spec path for --worker" >&2
|
|
exit 2
|
|
fi
|
|
WORKER_MODE=1
|
|
WORKER_SPEC="$2"
|
|
shift 2
|
|
;;
|
|
*)
|
|
echo "[fatal] unexpected argument: $1" >&2
|
|
usage >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [[ ! "$JOBS" =~ ^[1-9][0-9]*$ ]]; then
|
|
echo "[fatal] jobs must be a positive integer: $JOBS" >&2
|
|
exit 2
|
|
fi
|
|
if [[ ! "$RETRIES" =~ ^[1-9][0-9]*$ ]]; then
|
|
echo "[fatal] retries must be a positive integer: $RETRIES" >&2
|
|
exit 2
|
|
fi
|
|
if [[ ! "$RETRY_DELAY" =~ ^[0-9]+$ ]]; then
|
|
echo "[fatal] retry delay must be a non-negative integer: $RETRY_DELAY" >&2
|
|
exit 2
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" -eq 0 ]] && ! command -v curl >/dev/null 2>&1; then
|
|
echo "[fatal] curl not found" >&2
|
|
exit 2
|
|
fi
|
|
if ! command -v grep >/dev/null 2>&1; then
|
|
echo "[fatal] grep not found" >&2
|
|
exit 2
|
|
fi
|
|
if ! command -v xargs >/dev/null 2>&1; then
|
|
echo "[fatal] xargs not found" >&2
|
|
exit 2
|
|
fi
|
|
|
|
trim() {
|
|
local s="$1"
|
|
s="${s#"${s%%[![:space:]]*}"}"
|
|
s="${s%"${s##*[![:space:]]}"}"
|
|
printf '%s' "$s"
|
|
}
|
|
|
|
parse_spec_var() {
|
|
local spec="$1"
|
|
local key="$2"
|
|
awk -v k="$key" '
|
|
BEGIN { IGNORECASE=1 }
|
|
/^[[:space:]]*#/ { next }
|
|
{
|
|
line=$0
|
|
sub(/^[[:space:]]+/, "", line)
|
|
if (tolower(line) ~ "^" tolower(k) "[[:space:]]*:") {
|
|
sub(/^[^:]*:[[:space:]]*/, "", line)
|
|
print line
|
|
exit
|
|
}
|
|
}
|
|
' "$spec"
|
|
}
|
|
|
|
load_macros_from_spec() {
|
|
local spec="$1"
|
|
awk '
|
|
BEGIN { IGNORECASE=1 }
|
|
/^[[:space:]]*#/ { next }
|
|
{
|
|
line=$0
|
|
sub(/^[[:space:]]+/, "", line)
|
|
if (tolower(line) ~ /^%(global|define)[[:space:]]+/) {
|
|
n=split(line, a, /[[:space:]]+/)
|
|
if (n >= 3) {
|
|
key=a[2]
|
|
val=""
|
|
for (i=3; i<=n; i++) {
|
|
val = val (i==3 ? "" : " ") a[i]
|
|
}
|
|
print key "=" val
|
|
}
|
|
}
|
|
}
|
|
' "$spec"
|
|
}
|
|
|
|
expand_macros() {
|
|
local text="$1"
|
|
local name="$2"
|
|
local version="$3"
|
|
local dist="$4"
|
|
|
|
text="${text//\%\{name\}/$name}"
|
|
text="${text//\%\{version\}/$version}"
|
|
text="${text//\%\{_name\}/$name}"
|
|
text="${text//\%\{\?dist\}/$dist}"
|
|
|
|
local kv key val
|
|
while IFS= read -r kv; do
|
|
key="${kv%%=*}"
|
|
val="${kv#*=}"
|
|
[[ -z "$key" ]] && continue
|
|
text="${text//\%\{$key\}/$val}"
|
|
done < <(printf '%s\n' "${MACROS:-}")
|
|
|
|
printf '%s' "$text"
|
|
}
|
|
|
|
extract_source0() {
|
|
local spec="$1"
|
|
awk '
|
|
BEGIN { IGNORECASE=1 }
|
|
/^[[:space:]]*#/ { next }
|
|
{
|
|
line=$0
|
|
sub(/^[[:space:]]+/, "", line)
|
|
if (tolower(line) ~ /^source0[[:space:]]*:/) {
|
|
sub(/^[^:]*:[[:space:]]*/, "", line)
|
|
print line
|
|
exit
|
|
}
|
|
}
|
|
' "$spec"
|
|
}
|
|
|
|
http_check() {
|
|
local url="$1"
|
|
local code
|
|
|
|
# HEAD first
|
|
code="$(curl_http_code_with_retry "$url" -I || true)"
|
|
if [[ "$code" =~ ^[23][0-9][0-9]$ ]]; then
|
|
return 0
|
|
fi
|
|
|
|
# Fallback: partial GET for servers not supporting HEAD
|
|
code="$(curl_http_code_with_retry "$url" -r 0-0 || true)"
|
|
[[ "$code" =~ ^[23][0-9][0-9]$ ]]
|
|
}
|
|
|
|
curl_http_code_with_retry() {
|
|
local url="$1"
|
|
shift
|
|
local attempt=1
|
|
local code
|
|
local rc
|
|
|
|
while [[ "$attempt" -le "$RETRIES" ]]; do
|
|
code="$(curl -A "$USER_AGENT" -L -sS -o /dev/null -w '%{http_code}' \
|
|
--connect-timeout "$TIMEOUT" --max-time "$((TIMEOUT * 2))" "$@" "$url" 2>/dev/null)"
|
|
rc=$?
|
|
if [[ "$rc" -eq 0 && "$code" =~ ^[0-9][0-9][0-9]$ ]]; then
|
|
# Most 4xx are permanent for current request context; do not retry.
|
|
# Retry only on 429 (rate limit) and 5xx (server-side/transient).
|
|
if [[ "$code" =~ ^4[0-9][0-9]$ && "$code" != "429" ]]; then
|
|
printf '%s' "$code"
|
|
return 0
|
|
fi
|
|
if [[ "$code" == "429" || "$code" =~ ^5[0-9][0-9]$ ]]; then
|
|
if [[ "$attempt" -lt "$RETRIES" ]]; then
|
|
if [[ "$RETRY_DELAY" -gt 0 ]]; then
|
|
sleep "$RETRY_DELAY"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
continue
|
|
fi
|
|
fi
|
|
printf '%s' "$code"
|
|
return 0
|
|
fi
|
|
|
|
# Network-level failure (timeout, DNS, connect reset, etc.): retry.
|
|
if [[ "$attempt" -lt "$RETRIES" && "$RETRY_DELAY" -gt 0 ]]; then
|
|
sleep "$RETRY_DELAY"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
printf '000'
|
|
return 1
|
|
}
|
|
|
|
rewrite_source0() {
|
|
local spec="$1"
|
|
local new_source0="$2"
|
|
local tmp_file
|
|
|
|
tmp_file="$(mktemp)"
|
|
if ! awk -v replacement="$new_source0" '
|
|
BEGIN { done=0 }
|
|
{
|
|
if (!done) {
|
|
line=$0
|
|
t=$0
|
|
sub(/^[[:space:]]+/, "", t)
|
|
if (tolower(t) ~ /^source0[[:space:]]*:/) {
|
|
pos=index(line, ":")
|
|
if (pos > 0) {
|
|
prefix=substr(line, 1, pos)
|
|
rest=substr(line, pos + 1)
|
|
ws=""
|
|
if (match(rest, /^[[:space:]]+/)) {
|
|
ws=substr(rest, RSTART, RLENGTH)
|
|
}
|
|
print prefix ws replacement
|
|
done=1
|
|
next
|
|
}
|
|
}
|
|
}
|
|
print
|
|
}
|
|
END {
|
|
if (!done) {
|
|
exit 3
|
|
}
|
|
}
|
|
' "$spec" > "$tmp_file"; then
|
|
rm -f "$tmp_file"
|
|
return 1
|
|
fi
|
|
|
|
if ! mv "$tmp_file" "$spec"; then
|
|
rm -f "$tmp_file"
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
spec_files=()
|
|
while IFS= read -r spec_file; do
|
|
spec_files+=("$spec_file")
|
|
done < <(find . -type f -name '*.spec' -print | grep -E '^\./[^/]+/[^/]+\.spec$' | sort)
|
|
|
|
process_spec() {
|
|
local spec="$1"
|
|
local r_ok=0
|
|
local r_fail=0
|
|
local r_no_source0=0
|
|
local r_skip_non_http=0
|
|
local r_dry=0
|
|
local r_archive_fix=0
|
|
local r_fix_write_err=0
|
|
local r_fixed_spec=""
|
|
local r_failed_spec=""
|
|
|
|
source0_raw="$(extract_source0 "$spec" || true)"
|
|
source0_raw="$(trim "$source0_raw")"
|
|
|
|
if [[ -z "$source0_raw" ]]; then
|
|
r_no_source0=1
|
|
echo "[skip] $spec | no Source0"
|
|
printf '__RESULT__|%d|%d|%d|%d|%d|%d|%d|%s|%s\n' \
|
|
"$r_ok" "$r_fail" "$r_no_source0" "$r_skip_non_http" "$r_dry" "$r_archive_fix" "$r_fix_write_err" "$r_fixed_spec" "$r_failed_spec"
|
|
return 0
|
|
fi
|
|
|
|
name="$(trim "$(parse_spec_var "$spec" "Name" || true)")"
|
|
version="$(trim "$(parse_spec_var "$spec" "Version" || true)")"
|
|
dist=""
|
|
MACROS="$(load_macros_from_spec "$spec" || true)"
|
|
expanded="$(expand_macros "$source0_raw" "$name" "$version" "$dist")"
|
|
expanded="$(trim "$expanded")"
|
|
|
|
check_url="${expanded%%#*}"
|
|
retry_url=""
|
|
if [[ "$check_url" == *"/archive/"* && "$check_url" != *"/archive/refs/tags/"* ]]; then
|
|
retry_url="${check_url/\/archive\//\/archive\/refs\/tags\/}"
|
|
fi
|
|
echo "[url] $spec"
|
|
echo " expanded=$expanded"
|
|
echo " check=$check_url"
|
|
if [[ -n "$retry_url" ]]; then
|
|
echo " retry=$retry_url"
|
|
fi
|
|
|
|
if [[ ! "$check_url" =~ ^https?:// ]]; then
|
|
r_skip_non_http=1
|
|
echo "[skip] $spec | non-http Source0"
|
|
echo " Source0=$source0_raw"
|
|
echo " expanded=$expanded"
|
|
printf '__RESULT__|%d|%d|%d|%d|%d|%d|%d|%s|%s\n' \
|
|
"$r_ok" "$r_fail" "$r_no_source0" "$r_skip_non_http" "$r_dry" "$r_archive_fix" "$r_fix_write_err" "$r_fixed_spec" "$r_failed_spec"
|
|
return 0
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" -eq 1 ]]; then
|
|
r_dry=1
|
|
echo "[dry] $spec | skip http check"
|
|
printf '__RESULT__|%d|%d|%d|%d|%d|%d|%d|%s|%s\n' \
|
|
"$r_ok" "$r_fail" "$r_no_source0" "$r_skip_non_http" "$r_dry" "$r_archive_fix" "$r_fix_write_err" "$r_fixed_spec" "$r_failed_spec"
|
|
return 0
|
|
fi
|
|
|
|
if http_check "$check_url"; then
|
|
r_ok=1
|
|
echo "[ok] $spec | $check_url"
|
|
else
|
|
if [[ -n "$retry_url" ]] && http_check "$retry_url"; then
|
|
r_ok=1
|
|
r_archive_fix=1
|
|
echo "[ok] $spec | retry with /archive/refs/tags/"
|
|
echo " final=$retry_url"
|
|
|
|
new_source0="${source0_raw/\/archive\//\/archive\/refs\/tags\/}"
|
|
if [[ "$new_source0" != "$source0_raw" ]]; then
|
|
if rewrite_source0 "$spec" "$new_source0"; then
|
|
r_fixed_spec="$spec"
|
|
echo "[fix] $spec | Source0 updated"
|
|
else
|
|
r_fix_write_err=1
|
|
echo "[warn] $spec | Source0 check passed via retry, but write-back failed" >&2
|
|
fi
|
|
fi
|
|
else
|
|
r_fail=1
|
|
r_failed_spec="$spec"
|
|
echo "[fail] $spec | $check_url"
|
|
fi
|
|
fi
|
|
|
|
printf '__RESULT__|%d|%d|%d|%d|%d|%d|%d|%s|%s\n' \
|
|
"$r_ok" "$r_fail" "$r_no_source0" "$r_skip_non_http" "$r_dry" "$r_archive_fix" "$r_fix_write_err" "$r_fixed_spec" "$r_failed_spec"
|
|
}
|
|
|
|
aggregate_result_line() {
|
|
local line="$1"
|
|
local tag ok_i fail_i no_i skip_non_http_i dry_i arch_i werr_i fixed_spec failed_spec
|
|
IFS='|' read -r tag ok_i fail_i no_i skip_non_http_i dry_i arch_i werr_i fixed_spec failed_spec <<< "$line"
|
|
[[ "$tag" != "__RESULT__" ]] && return 1
|
|
ok_i="${ok_i:-0}"
|
|
fail_i="${fail_i:-0}"
|
|
no_i="${no_i:-0}"
|
|
skip_non_http_i="${skip_non_http_i:-0}"
|
|
dry_i="${dry_i:-0}"
|
|
arch_i="${arch_i:-0}"
|
|
werr_i="${werr_i:-0}"
|
|
ok=$((ok + ok_i))
|
|
fail=$((fail + fail_i))
|
|
no_source0=$((no_source0 + no_i))
|
|
skip_non_http=$((skip_non_http + skip_non_http_i))
|
|
dry_run_only=$((dry_run_only + dry_i))
|
|
archive_refs_tags_ok=$((archive_refs_tags_ok + arch_i))
|
|
archive_fix_write_fail=$((archive_fix_write_fail + werr_i))
|
|
if [[ "$arch_i" -eq 1 && -n "$fixed_spec" ]]; then
|
|
archive_fixed_specs+=("$fixed_spec")
|
|
fi
|
|
if [[ "$fail_i" -eq 1 && -n "$failed_spec" ]]; then
|
|
failed_specs+=("$failed_spec")
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
if [[ "$WORKER_MODE" -eq 1 ]]; then
|
|
process_spec "$WORKER_SPEC"
|
|
exit 0
|
|
fi
|
|
|
|
if [[ ${#spec_files[@]} -eq 0 ]]; then
|
|
echo "[info] no ./*/*.spec files found under current directory"
|
|
exit 0
|
|
fi
|
|
|
|
total="${#spec_files[@]}"
|
|
ok=0
|
|
fail=0
|
|
no_source0=0
|
|
skip_non_http=0
|
|
dry_run_only=0
|
|
archive_refs_tags_ok=0
|
|
archive_fix_write_fail=0
|
|
declare -a archive_fixed_specs=()
|
|
declare -a failed_specs=()
|
|
|
|
if [[ "$JOBS" -le 1 ]]; then
|
|
for spec in "${spec_files[@]}"; do
|
|
while IFS= read -r line; do
|
|
if ! aggregate_result_line "$line"; then
|
|
echo "$line"
|
|
fi
|
|
done < <(process_spec "$spec")
|
|
done
|
|
else
|
|
tmp_output="$(mktemp)"
|
|
worker_cmd=("$0")
|
|
if [[ "$DRY_RUN" -eq 1 ]]; then
|
|
worker_cmd+=("--dry-run")
|
|
fi
|
|
worker_cmd+=("--worker")
|
|
|
|
printf '%s\0' "${spec_files[@]}" \
|
|
| xargs -0 -n1 -P "$JOBS" "${worker_cmd[@]}" > "$tmp_output"
|
|
|
|
while IFS= read -r line; do
|
|
if ! aggregate_result_line "$line"; then
|
|
echo "$line"
|
|
fi
|
|
done < "$tmp_output"
|
|
rm -f "$tmp_output"
|
|
fi
|
|
|
|
echo
|
|
echo "===== summary ====="
|
|
echo "spec total: $total"
|
|
echo "ok: $ok"
|
|
echo "fail: $fail"
|
|
echo "no Source0: $no_source0"
|
|
echo "skip non-http:$skip_non_http"
|
|
echo "archive fix: $archive_refs_tags_ok"
|
|
echo "fix write err:$archive_fix_write_fail"
|
|
if [[ "$DRY_RUN" -eq 1 ]]; then
|
|
echo "dry-run urls: $dry_run_only"
|
|
fi
|
|
|
|
if [[ ${#archive_fixed_specs[@]} -gt 0 ]]; then
|
|
echo
|
|
echo "archive fixed specs:"
|
|
for spec in "${archive_fixed_specs[@]}"; do
|
|
echo " $spec"
|
|
done
|
|
fi
|
|
|
|
if [[ ${#failed_specs[@]} -gt 0 ]]; then
|
|
echo
|
|
echo "failed specs:"
|
|
for spec in "${failed_specs[@]}"; do
|
|
echo " $spec"
|
|
done
|
|
fi
|
|
|
|
if [[ $fail -gt 0 ]]; then
|
|
exit 1
|
|
fi
|