From fc3c69bc236d97ded9bf243d9e07786b3db3ced2 Mon Sep 17 00:00:00 2001
From: dela <delahayecarry@outlook.com>
Date: Fri, 21 Aug 2026 14:57:27 +0800
Subject: [PATCH] fix: harden ingest.sh against partial downloads and tar size
 limits

arxiv pdf: download to temp file then move, so a failed download no
longer poisons re-entry with a corrupt partial file. e-print tar:
--max-size is not a GNU tar option and silently did nothing; extract
first then audit with du, failing if >50M.
---
 scripts/ingest.sh | 20 ++++++++++++++++----
 superderive       |  2 +-
 supertensor       |  2 +-
 3 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/scripts/ingest.sh b/scripts/ingest.sh
index f859ed2..b49d1fe 100755
--- a/scripts/ingest.sh
+++ b/scripts/ingest.sh
@@ -72,8 +72,15 @@ pages: $pages"
     mkdir -p "$WORK/source"
     pdf="$WORK/source/paper.pdf"
     if [[ ! -f "$pdf" ]]; then
-      curl -fsSL "https://arxiv.org/pdf/${id}.pdf" -o "$pdf" \
-        || curl -fsSL "https://export.arxiv.org/pdf/${id}.pdf" -o "$pdf"
+      pdf_tmp=$(mktemp --suffix=.pdf)
+      if curl -fsSL "https://arxiv.org/pdf/${id}.pdf" -o "$pdf_tmp" \
+        || curl -fsSL "https://export.arxiv.org/pdf/${id}.pdf" -o "$pdf_tmp"; then
+        mv "$pdf_tmp" "$pdf"
+      else
+        rm -f "$pdf_tmp"
+        echo "error: failed to download arxiv pdf $id" >&2
+        exit 1
+      fi
     fi
     pdftotext -layout "$pdf" "$WORK/source/paper.txt" || true
     pdftoppm -png -r 120 "$pdf" "$WORK/source/pages/pg"
@@ -92,8 +99,13 @@ PY
       tmp=$(mktemp)
       if curl -fsSL "https://arxiv.org/e-print/${id}" -o "$tmp"; then
         mkdir -p "$WORK/source/eprint"
-        tar -xf "$tmp" -C "$WORK/source/eprint" --max-size=50M 2>/dev/null \
-          || tar -xf "$tmp" -C "$WORK/source/eprint" || true
+        tar -xf "$tmp" -C "$WORK/source/eprint" 2>/dev/null || true
+        extracted_size=$(du -sb "$WORK/source/eprint" | awk '{print $1}')
+        if (( extracted_size > 52428800 )); then
+          rm -rf "$WORK/source/eprint"
+          echo "error: e-print exceeds 50M limit ($extracted_size bytes)" >&2
+          exit 1
+        fi
       fi
       rm -f "$tmp"
     fi
diff --git a/superderive b/superderive
index 45db61f..f2ccd6f 160000
--- a/superderive
+++ b/superderive
@@ -1 +1 @@
-Subproject commit 45db61f3e2de53bf0c0c990fe709b3a51aa4cdc5
+Subproject commit f2ccd6fb088f545572f513c158666e4b2a0ce4c6
diff --git a/supertensor b/supertensor
index 8ada8e5..7210f89 160000
--- a/supertensor
+++ b/supertensor
@@ -1 +1 @@
-Subproject commit 8ada8e57c5f5989d54667842338f12487f0ab64a
+Subproject commit 7210f890caefffc6e08e5d53d177da96e3c6f858