fix: harden ingest.sh against partial downloads and tar size limits
arxiv pdf: download to temp file then move, so a failed download no longer poisons re-entry with a corrupt partial file. e-print tar: --max-size is not a GNU tar option and silently did nothing; extract first then audit with du, failing if >50M.
This commit is contained in:
+16
-4
@@ -72,8 +72,15 @@ pages: $pages"
|
|||||||
mkdir -p "$WORK/source"
|
mkdir -p "$WORK/source"
|
||||||
pdf="$WORK/source/paper.pdf"
|
pdf="$WORK/source/paper.pdf"
|
||||||
if [[ ! -f "$pdf" ]]; then
|
if [[ ! -f "$pdf" ]]; then
|
||||||
curl -fsSL "https://arxiv.org/pdf/${id}.pdf" -o "$pdf" \
|
pdf_tmp=$(mktemp --suffix=.pdf)
|
||||||
|| curl -fsSL "https://export.arxiv.org/pdf/${id}.pdf" -o "$pdf"
|
if curl -fsSL "https://arxiv.org/pdf/${id}.pdf" -o "$pdf_tmp" \
|
||||||
|
|| curl -fsSL "https://export.arxiv.org/pdf/${id}.pdf" -o "$pdf_tmp"; then
|
||||||
|
mv "$pdf_tmp" "$pdf"
|
||||||
|
else
|
||||||
|
rm -f "$pdf_tmp"
|
||||||
|
echo "error: failed to download arxiv pdf $id" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
pdftotext -layout "$pdf" "$WORK/source/paper.txt" || true
|
pdftotext -layout "$pdf" "$WORK/source/paper.txt" || true
|
||||||
pdftoppm -png -r 120 "$pdf" "$WORK/source/pages/pg"
|
pdftoppm -png -r 120 "$pdf" "$WORK/source/pages/pg"
|
||||||
@@ -92,8 +99,13 @@ PY
|
|||||||
tmp=$(mktemp)
|
tmp=$(mktemp)
|
||||||
if curl -fsSL "https://arxiv.org/e-print/${id}" -o "$tmp"; then
|
if curl -fsSL "https://arxiv.org/e-print/${id}" -o "$tmp"; then
|
||||||
mkdir -p "$WORK/source/eprint"
|
mkdir -p "$WORK/source/eprint"
|
||||||
tar -xf "$tmp" -C "$WORK/source/eprint" --max-size=50M 2>/dev/null \
|
tar -xf "$tmp" -C "$WORK/source/eprint" 2>/dev/null || true
|
||||||
|| tar -xf "$tmp" -C "$WORK/source/eprint" || true
|
extracted_size=$(du -sb "$WORK/source/eprint" | awk '{print $1}')
|
||||||
|
if (( extracted_size > 52428800 )); then
|
||||||
|
rm -rf "$WORK/source/eprint"
|
||||||
|
echo "error: e-print exceeds 50M limit ($extracted_size bytes)" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
rm -f "$tmp"
|
rm -f "$tmp"
|
||||||
fi
|
fi
|
||||||
|
|||||||
+1
-1
Submodule superderive updated: 45db61f3e2...f2ccd6fb08
+1
-1
Submodule supertensor updated: 8ada8e57c5...7210f890ca
Reference in New Issue
Block a user