fix: harden ingest.sh against partial downloads and tar size limits

arxiv pdf: download to temp file then move, so a failed download no
longer poisons re-entry with a corrupt partial file. e-print tar:
--max-size is not a GNU tar option and silently did nothing; extract
first then audit with du, failing if >50M.
This commit is contained in:
dela
2026-08-21 14:57:27 +08:00
parent e80d5c9067
commit fc3c69bc23
3 changed files with 18 additions and 6 deletions
+16 -4
View File
@@ -72,8 +72,15 @@ pages: $pages"
mkdir -p "$WORK/source"
pdf="$WORK/source/paper.pdf"
if [[ ! -f "$pdf" ]]; then
curl -fsSL "https://arxiv.org/pdf/${id}.pdf" -o "$pdf" \
|| curl -fsSL "https://export.arxiv.org/pdf/${id}.pdf" -o "$pdf"
pdf_tmp=$(mktemp --suffix=.pdf)
if curl -fsSL "https://arxiv.org/pdf/${id}.pdf" -o "$pdf_tmp" \
|| curl -fsSL "https://export.arxiv.org/pdf/${id}.pdf" -o "$pdf_tmp"; then
mv "$pdf_tmp" "$pdf"
else
rm -f "$pdf_tmp"
echo "error: failed to download arxiv pdf $id" >&2
exit 1
fi
fi
pdftotext -layout "$pdf" "$WORK/source/paper.txt" || true
pdftoppm -png -r 120 "$pdf" "$WORK/source/pages/pg"
@@ -92,8 +99,13 @@ PY
tmp=$(mktemp)
if curl -fsSL "https://arxiv.org/e-print/${id}" -o "$tmp"; then
mkdir -p "$WORK/source/eprint"
tar -xf "$tmp" -C "$WORK/source/eprint" --max-size=50M 2>/dev/null \
|| tar -xf "$tmp" -C "$WORK/source/eprint" || true
tar -xf "$tmp" -C "$WORK/source/eprint" 2>/dev/null || true
extracted_size=$(du -sb "$WORK/source/eprint" | awk '{print $1}')
if (( extracted_size > 52428800 )); then
rm -rf "$WORK/source/eprint"
echo "error: e-print exceeds 50M limit ($extracted_size bytes)" >&2
exit 1
fi
fi
rm -f "$tmp"
fi