fix: harden ingest.sh against partial downloads and tar size limits
arxiv pdf: download to temp file then move, so a failed download no longer poisons re-entry with a corrupt partial file. e-print tar: --max-size is not a GNU tar option and silently did nothing; extract first then audit with du, failing if >50M.
This commit is contained in:
+16
-4
@@ -72,8 +72,15 @@ pages: $pages"
|
||||
mkdir -p "$WORK/source"
|
||||
pdf="$WORK/source/paper.pdf"
|
||||
if [[ ! -f "$pdf" ]]; then
|
||||
curl -fsSL "https://arxiv.org/pdf/${id}.pdf" -o "$pdf" \
|
||||
|| curl -fsSL "https://export.arxiv.org/pdf/${id}.pdf" -o "$pdf"
|
||||
pdf_tmp=$(mktemp --suffix=.pdf)
|
||||
if curl -fsSL "https://arxiv.org/pdf/${id}.pdf" -o "$pdf_tmp" \
|
||||
|| curl -fsSL "https://export.arxiv.org/pdf/${id}.pdf" -o "$pdf_tmp"; then
|
||||
mv "$pdf_tmp" "$pdf"
|
||||
else
|
||||
rm -f "$pdf_tmp"
|
||||
echo "error: failed to download arxiv pdf $id" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
pdftotext -layout "$pdf" "$WORK/source/paper.txt" || true
|
||||
pdftoppm -png -r 120 "$pdf" "$WORK/source/pages/pg"
|
||||
@@ -92,8 +99,13 @@ PY
|
||||
tmp=$(mktemp)
|
||||
if curl -fsSL "https://arxiv.org/e-print/${id}" -o "$tmp"; then
|
||||
mkdir -p "$WORK/source/eprint"
|
||||
tar -xf "$tmp" -C "$WORK/source/eprint" --max-size=50M 2>/dev/null \
|
||||
|| tar -xf "$tmp" -C "$WORK/source/eprint" || true
|
||||
tar -xf "$tmp" -C "$WORK/source/eprint" 2>/dev/null || true
|
||||
extracted_size=$(du -sb "$WORK/source/eprint" | awk '{print $1}')
|
||||
if (( extracted_size > 52428800 )); then
|
||||
rm -rf "$WORK/source/eprint"
|
||||
echo "error: e-print exceeds 50M limit ($extracted_size bytes)" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
rm -f "$tmp"
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user