First Commit

traffic/archive.go

@@ -0,0 +1,237 @@
+package traffic
+
+import (
+	"crypto/sha256"
+	"database/sql"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// ArchiveSnapshot is the portable traffic subset embedded in one task archive.
+// Large content-addressed bodies are copied alongside this manifest rather than
+// base64-encoded, preserving deduplication and allowing the tar writer to stream.
+type ArchiveSnapshot struct {
+	Version   int               `json:"version"`
+	Exchanges []ArchiveExchange `json:"exchanges"`
+	Blobs     []string          `json:"blobs"`
+}
+
+type ArchiveExchange struct {
+	ID          string `json:"id"`
+	TS          int64  `json:"ts"`
+	Host        string `json:"host"`
+	Method      string `json:"method"`
+	URLTemplate string `json:"url_template"`
+	URL         string `json:"url"`
+	Status      int    `json:"status"`
+	ContentType string `json:"content_type"`
+	ReqLen      int    `json:"req_len"`
+	RespLen     int    `json:"resp_len"`
+	ReqHead     string `json:"req_head"`
+	ReqBody     []byte `json:"req_body,omitempty"`
+	ReqBlob     string `json:"req_blob,omitempty"`
+	RespHead    string `json:"resp_head"`
+	RespBody    []byte `json:"resp_body,omitempty"`
+	RespBlob    string `json:"resp_blob,omitempty"`
+}
+
+// ExportHosts writes an exact-host snapshot to dir. It holds the traffic writer
+// lock while reading SQLite and blobs, so each body and its index row come from
+// one consistent point in time.
+func (t *Traffic) ExportHosts(hosts []string, dir string) (int64, error) {
+	if t == nil || len(hosts) == 0 {
+		return 0, nil
+	}
+	if err := os.MkdirAll(filepath.Join(dir, "blobs"), 0o700); err != nil {
+		return 0, err
+	}
+	t.wmu.Lock()
+	defer t.wmu.Unlock()
+	unique := uniqueArchiveHosts(hosts)
+	placeholders := make([]string, len(unique))
+	args := make([]any, len(unique))
+	for i, host := range unique {
+		placeholders[i] = "?"
+		args[i] = host
+	}
+	rows, err := t.db.Query(`SELECT id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path
+FROM exchanges WHERE host IN (`+strings.Join(placeholders, ",")+`) ORDER BY ts,id`, args...)
+	if err != nil {
+		return 0, err
+	}
+	defer rows.Close()
+	snapshot := ArchiveSnapshot{Version: 1}
+	blobs := map[string]struct{}{}
+	for rows.Next() {
+		var item ArchiveExchange
+		var legacyPath string
+		if err := rows.Scan(&item.ID, &item.TS, &item.Host, &item.Method, &item.URLTemplate,
+			&item.URL, &item.Status, &item.ContentType, &item.ReqLen, &item.RespLen, &legacyPath); err != nil {
+			return 0, err
+		}
+		var reqBlob, respBlob sql.NullString
+		err := t.db.QueryRow(`SELECT req_head,req_body,req_blob,resp_head,resp_body,resp_blob
+FROM exchange_bodies WHERE id=?`, item.ID).Scan(&item.ReqHead, &item.ReqBody, &reqBlob, &item.RespHead, &item.RespBody, &respBlob)
+		if errors.Is(err, sql.ErrNoRows) && strings.TrimSpace(legacyPath) != "" {
+			req, readErr := os.ReadFile(filepath.Join(t.dir, legacyPath, "request.http"))
+			if readErr != nil && !os.IsNotExist(readErr) {
+				return 0, readErr
+			}
+			resp, readErr := os.ReadFile(filepath.Join(t.dir, legacyPath, "response.http"))
+			if readErr != nil && !os.IsNotExist(readErr) {
+				return 0, readErr
+			}
+			item.ReqHead, item.RespHead = string(req), string(resp)
+		} else if err != nil {
+			return 0, err
+		}
+		item.ReqBlob, item.RespBlob = reqBlob.String, respBlob.String
+		for _, hash := range []string{item.ReqBlob, item.RespBlob} {
+			if hash != "" {
+				blobs[hash] = struct{}{}
+			}
+		}
+		snapshot.Exchanges = append(snapshot.Exchanges, item)
+	}
+	if err := rows.Err(); err != nil {
+		return 0, err
+	}
+	for hash := range blobs {
+		source, err := t.blobPath(hash)
+		if err != nil {
+			return 0, err
+		}
+		data, err := os.ReadFile(source)
+		if err != nil {
+			return 0, err
+		}
+		if err := os.WriteFile(filepath.Join(dir, "blobs", hash+".bin"), data, 0o600); err != nil {
+			return 0, err
+		}
+		snapshot.Blobs = append(snapshot.Blobs, hash)
+	}
+	sort.Strings(snapshot.Blobs)
+	raw, err := json.Marshal(snapshot)
+	if err != nil {
+		return 0, err
+	}
+	if err := os.WriteFile(filepath.Join(dir, "traffic.json"), raw, 0o600); err != nil {
+		return 0, err
+	}
+	return int64(len(snapshot.Exchanges)), nil
+}
+
+// ImportArchive imports only missing exchange IDs. Current hot rows always win,
+// and repeated restore attempts are safe after a partial external failure.
+func (t *Traffic) ImportArchive(dir string) (int64, error) {
+	if t == nil {
+		return 0, nil
+	}
+	raw, err := os.ReadFile(filepath.Join(dir, "traffic.json"))
+	if os.IsNotExist(err) {
+		return 0, nil
+	}
+	if err != nil {
+		return 0, err
+	}
+	var snapshot ArchiveSnapshot
+	if err := json.Unmarshal(raw, &snapshot); err != nil {
+		return 0, err
+	}
+	if snapshot.Version != 1 {
+		return 0, fmt.Errorf("unsupported traffic archive version %d", snapshot.Version)
+	}
+	t.wmu.Lock()
+	defer t.wmu.Unlock()
+	for _, hash := range snapshot.Blobs {
+		if !blobHashRe.MatchString(hash) {
+			return 0, fmt.Errorf("invalid archived traffic blob %q", hash)
+		}
+		data, err := os.ReadFile(filepath.Join(dir, "blobs", hash+".bin"))
+		if err != nil {
+			return 0, err
+		}
+		sum := sha256.Sum256(data)
+		if hex.EncodeToString(sum[:]) != hash {
+			return 0, fmt.Errorf("traffic blob checksum mismatch: %s", hash)
+		}
+		destination := filepath.Join(t.dir, "_blobs", "sha256", hash[:2], hash+".bin")
+		if _, err := os.Stat(destination); os.IsNotExist(err) {
+			if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil {
+				return 0, err
+			}
+			if err := os.WriteFile(destination, data, 0o644); err != nil {
+				return 0, err
+			}
+		}
+	}
+	tx, err := t.db.Begin()
+	if err != nil {
+		return 0, err
+	}
+	defer tx.Rollback() //nolint:errcheck
+	var imported int64
+	for _, item := range snapshot.Exchanges {
+		res, err := tx.Exec(`INSERT OR IGNORE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
+VALUES(?,?,?,?,?,?,?,?,?,?,'')`, item.ID, item.TS, item.Host, item.Method, item.URLTemplate,
+			item.URL, item.Status, item.ContentType, item.ReqLen, item.RespLen)
+		if err != nil {
+			return imported, err
+		}
+		n, _ := res.RowsAffected()
+		if n == 0 {
+			continue
+		}
+		if _, err := tx.Exec(`INSERT INTO exchange_bodies(id,req_head,req_body,req_blob,resp_head,resp_body,resp_blob)
+VALUES(?,?,?,?,?,?,?)`, item.ID, item.ReqHead, item.ReqBody, nullIfEmpty(item.ReqBlob),
+			item.RespHead, item.RespBody, nullIfEmpty(item.RespBlob)); err != nil {
+			return imported, err
+		}
+		for _, hash := range []string{item.ReqBlob, item.RespBlob} {
+			if hash != "" {
+				if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, hash, item.ID); err != nil {
+					return imported, err
+				}
+			}
+		}
+		if t.fts {
+			var rowID int64
+			if err := tx.QueryRow(`SELECT rowid FROM exchanges WHERE id=?`, item.ID).Scan(&rowID); err != nil {
+				return imported, err
+			}
+			content := strings.Join([]string{item.URL, item.ReqHead, string(item.ReqBody), item.RespHead, string(item.RespBody)}, "\n")
+			if _, err := tx.Exec(`INSERT INTO ex_fts(rowid,content) VALUES(?,?)`, rowID, content); err != nil {
+				return imported, err
+			}
+		}
+		imported++
+	}
+	if err := tx.Commit(); err != nil {
+		return imported, err
+	}
+	return imported, nil
+}
+
+func uniqueArchiveHosts(hosts []string) []string {
+	seen := make(map[string]struct{}, len(hosts))
+	out := make([]string, 0, len(hosts))
+	for _, host := range hosts {
+		host = strings.ToLower(strings.TrimSpace(host))
+		if host == "" {
+			continue
+		}
+		if _, exists := seen[host]; exists {
+			continue
+		}
+		seen[host] = struct{}{}
+		out = append(out, host)
+	}
+	sort.Strings(out)
+	return out
+}

traffic/archive_test.go

@@ -0,0 +1,158 @@
+package traffic
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestTrafficArchiveRoundTripAndCurrentRowWins(t *testing.T) {
+	trafficDir := t.TempDir()
+	tr, err := Open(trafficDir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tr.Close()
+	const (
+		id   = "archive-exchange-1"
+		host = "archive.example"
+	)
+	requestHead := "POST /v1/test HTTP/1.1\nHost: archive.example"
+	responseHead := "HTTP/1.1 200 OK\nContent-Type: application/json"
+	if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
+VALUES(?,?,?,?,?,?,?,?,?,?,?)`, id, 1, host, "POST", "/v1/test", "https://archive.example/v1/test", 200,
+		"application/json", 7, 11, ""); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := tr.DB().Exec(`INSERT INTO exchange_bodies(id,req_head,req_body,resp_head,resp_body)
+VALUES(?,?,?,?,?)`, id, requestHead, []byte("payload"), responseHead, []byte(`{"ok":true}`)); err != nil {
+		t.Fatal(err)
+	}
+
+	archiveDir := filepath.Join(t.TempDir(), "traffic")
+	count, err := tr.ExportHosts([]string{host, host}, archiveDir)
+	if err != nil || count != 1 {
+		t.Fatalf("ExportHosts count=%d err=%v", count, err)
+	}
+	if _, err := os.Stat(filepath.Join(archiveDir, "traffic.json")); err != nil {
+		t.Fatal(err)
+	}
+	stage, err := tr.StageDeleteHostsExact([]string{host})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if stage.Deleted() != 1 {
+		t.Fatalf("staged deleted=%d, want 1", stage.Deleted())
+	}
+	if err := stage.Commit(); err != nil {
+		t.Fatal(err)
+	}
+	if _, _, err := tr.Get(id); err == nil {
+		t.Fatal("deleted exchange remained readable")
+	}
+	imported, err := tr.ImportArchive(archiveDir)
+	if err != nil || imported != 1 {
+		t.Fatalf("ImportArchive imported=%d err=%v", imported, err)
+	}
+	req, resp, err := tr.Get(id)
+	if err != nil || !strings.Contains(req, "payload") || !strings.Contains(resp, `{"ok":true}`) {
+		t.Fatalf("restored exchange req=%q resp=%q err=%v", req, resp, err)
+	}
+	if _, err := tr.DB().Exec(`UPDATE exchange_bodies SET resp_body=? WHERE id=?`, []byte(`{"current":true}`), id); err != nil {
+		t.Fatal(err)
+	}
+	imported, err = tr.ImportArchive(archiveDir)
+	if err != nil || imported != 0 {
+		t.Fatalf("idempotent import imported=%d err=%v", imported, err)
+	}
+	_, resp, err = tr.Get(id)
+	if err != nil || !strings.Contains(resp, `{"current":true}`) || strings.Contains(resp, `{"ok":true}`) {
+		t.Fatalf("archive overwrote current exchange resp=%q err=%v", resp, err)
+	}
+}
+
+func TestRecoverArchiveHostDeleteStageRollsBackBeforePostgresCommit(t *testing.T) {
+	trafficDir := t.TempDir()
+	tr, err := Open(trafficDir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tr.Close()
+	const host = "rollback-archive.example"
+	insertArchiveRecoveryExchange(t, tr, "rollback-exchange", host)
+	legacyDir := filepath.Join(trafficDir, sanitize(host))
+	if err := os.MkdirAll(legacyDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(legacyDir, "request.http"), []byte("request"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	stage, err := tr.StageDeleteHostsExactForArchive([]string{host}, 17, 42)
+	if err != nil {
+		t.Fatal(err)
+	}
+	simulateTrafficStageProcessExit(t, stage)
+	if err := tr.RecoverHostDeleteStages(func(id, taskID int64) (bool, error) {
+		if id != 17 || taskID != 42 {
+			t.Fatalf("archive id=%d task=%d, want 17/42", id, taskID)
+		}
+		return false, nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(filepath.Join(legacyDir, "request.http")); err != nil {
+		t.Fatalf("legacy traffic tree not restored: %v", err)
+	}
+	var count int
+	if err := tr.DB().QueryRow(`SELECT count(*) FROM exchanges WHERE host=?`, host).Scan(&count); err != nil || count != 1 {
+		t.Fatalf("exchange count=%d err=%v, want 1", count, err)
+	}
+}
+
+func TestRecoverArchiveHostDeleteStageCompletesAfterPostgresCommit(t *testing.T) {
+	trafficDir := t.TempDir()
+	tr, err := Open(trafficDir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tr.Close()
+	const host = "committed-archive.example"
+	insertArchiveRecoveryExchange(t, tr, "committed-exchange", host)
+	legacyDir := filepath.Join(trafficDir, sanitize(host))
+	if err := os.MkdirAll(legacyDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	stage, err := tr.StageDeleteHostsExactForArchive([]string{host}, 18, 43)
+	if err != nil {
+		t.Fatal(err)
+	}
+	simulateTrafficStageProcessExit(t, stage)
+	if err := tr.RecoverHostDeleteStages(func(id, taskID int64) (bool, error) { return id == 18 && taskID == 43, nil }); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(legacyDir); !os.IsNotExist(err) {
+		t.Fatalf("committed legacy traffic tree remains: %v", err)
+	}
+	var count int
+	if err := tr.DB().QueryRow(`SELECT count(*) FROM exchanges WHERE host=?`, host).Scan(&count); err != nil || count != 0 {
+		t.Fatalf("exchange count=%d err=%v, want 0", count, err)
+	}
+}
+
+func insertArchiveRecoveryExchange(t *testing.T, tr *Traffic, id, host string) {
+	t.Helper()
+	if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
+VALUES(?,?,?,?,?,?,?,?,?,?,?)`, id, 1, host, "GET", "/", "https://"+host+"/", 200, "text/plain", 0, 0, ""); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func simulateTrafficStageProcessExit(t *testing.T, stage *HostDeleteStage) {
+	t.Helper()
+	if err := stage.tx.Rollback(); err != nil {
+		t.Fatal(err)
+	}
+	stage.done = true
+	stage.traffic.wmu.Unlock()
+}

traffic/evidence.go

@@ -0,0 +1,126 @@
+package traffic
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// EvidenceExchange exposes complete captured bodies only during the callback.
+// Readers are closed before releasing the writer lock, so host deletion cannot
+// remove a blob between its index lookup and its evidence copy.
+type EvidenceExchange struct {
+	ID, URL, Method, ContentType string
+	TS                           int64
+	Status                       int
+	ReqHead, RespHead            string
+	ReqLen, RespLen              int64
+	ReqHash, RespHash            string
+	Request, Response            io.Reader
+}
+
+func (t *Traffic) ReadEvidence(ctx context.Context, ids []string, consume func(EvidenceExchange) error) error {
+	if t == nil {
+		return errors.New("流量录制存储不可用")
+	}
+	t.wmu.Lock()
+	defer t.wmu.Unlock()
+	for _, id := range ids {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		if err := t.readEvidence(id, consume); err != nil {
+			return fmt.Errorf("流量 %s: %w", id, err)
+		}
+	}
+	return nil
+}
+
+func (t *Traffic) readEvidence(id string, consume func(EvidenceExchange) error) error {
+	e := EvidenceExchange{ID: id}
+	var legacy string
+	if err := t.db.QueryRow(`SELECT ts,url,method,status,content_type,req_len,resp_len,path FROM exchanges WHERE id=?`, id).
+		Scan(&e.TS, &e.URL, &e.Method, &e.Status, &e.ContentType, &e.ReqLen, &e.RespLen, &legacy); err != nil {
+		return err
+	}
+	var req, resp []byte
+	var reqBlob, respBlob sql.NullString
+	err := t.db.QueryRow(`SELECT req_head,req_body,req_blob,resp_head,resp_body,resp_blob FROM exchange_bodies WHERE id=?`, id).
+		Scan(&e.ReqHead, &req, &reqBlob, &e.RespHead, &resp, &respBlob)
+	if errors.Is(err, sql.ErrNoRows) && legacy != "" {
+		if !filepath.IsLocal(legacy) {
+			return errors.New("旧流量路径无效")
+		}
+		r, err := os.Open(filepath.Join(t.dir, legacy, "request.http"))
+		if err != nil {
+			return err
+		}
+		defer r.Close()
+		p, err := os.Open(filepath.Join(t.dir, legacy, "response.http"))
+		if err != nil {
+			return err
+		}
+		defer p.Close()
+		e.ReqHead, e.Request, err = splitLegacyEvidence(r)
+		if err != nil {
+			return err
+		}
+		e.RespHead, e.Response, err = splitLegacyEvidence(p)
+		if err != nil {
+			return err
+		}
+		return consume(e)
+	}
+	if err != nil {
+		return err
+	}
+	e.Request = bytes.NewReader(req)
+	e.Response = bytes.NewReader(resp)
+	if reqBlob.Valid && reqBlob.String != "" {
+		f, _, err := t.Blob(reqBlob.String)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		e.Request = f
+		e.ReqHash = reqBlob.String
+	}
+	if respBlob.Valid && respBlob.String != "" {
+		f, _, err := t.Blob(respBlob.String)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		e.Response = f
+		e.RespHash = respBlob.String
+	}
+	return consume(e)
+}
+
+func splitLegacyEvidence(r io.Reader) (string, io.Reader, error) {
+	b := bufio.NewReader(r)
+	var head strings.Builder
+	for {
+		line, err := b.ReadString('\n')
+		if line == "\n" || line == "\r\n" {
+			return head.String(), b, nil
+		}
+		head.WriteString(line)
+		if head.Len() > 1<<20 {
+			return "", nil, errors.New("旧流量报文头过大")
+		}
+		if errors.Is(err, io.EOF) {
+			return head.String(), b, nil
+		}
+		if err != nil {
+			return "", nil, err
+		}
+	}
+}

traffic/passthrough_test.go

@@ -0,0 +1,68 @@
+package traffic
+
+import (
+	"errors"
+	"net/url"
+	"testing"
+
+	mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
+)
+
+func TestHostOnly(t *testing.T) {
+	cases := map[string]string{
+		"example.com:443": "example.com",
+		"example.com":     "example.com",
+		"10.0.0.1:8080":   "10.0.0.1",
+	}
+	for in, want := range cases {
+		if got := hostOnly(in); got != want {
+			t.Errorf("hostOnly(%q)=%q want %q", in, got, want)
+		}
+	}
+}
+
+func TestProxyCausedErr(t *testing.T) {
+	proxy := []string{
+		"protocol error: received DATA on a HEAD request",
+		"http2: server sent GOAWAY",
+		"malformed HTTP response",
+	}
+	target := []string{ // target-side failures must NOT trigger passthrough
+		"dial tcp 1.2.3.4:443: connect: connection refused",
+		"read: connection reset by peer",
+		"context deadline exceeded",
+	}
+	for _, s := range proxy {
+		if !proxyCausedErr(errors.New(s)) {
+			t.Errorf("expected proxy-caused: %q", s)
+		}
+	}
+	for _, s := range target {
+		if proxyCausedErr(errors.New(s)) {
+			t.Errorf("expected NOT proxy-caused: %q", s)
+		}
+	}
+}
+
+func TestMaybePassthroughFlagsHostOnce(t *testing.T) {
+	tr := &Traffic{}
+	f := &mproxy.Flow{Request: &mproxy.Request{URL: &url.URL{Host: "target.test:443"}}}
+
+	// Target-caused error → do NOT flag (keep MITM + recording).
+	tr.maybePassthrough(f, errors.New("connection refused"))
+	if _, ok := tr.pass.Load("target.test"); ok {
+		t.Fatal("target-caused error must not flag passthrough")
+	}
+
+	// Proxy-caused error → flag the host for transparent passthrough.
+	tr.maybePassthrough(f, errors.New("protocol error: received DATA on a HEAD request"))
+	if _, ok := tr.pass.Load("target.test"); !ok {
+		t.Fatal("proxy-caused error must flag passthrough")
+	}
+
+	// The shouldIntercept rule uses hostOnly(req.Host); the CONNECT host carries a
+	// port, so it must resolve to the same flagged key → intercept=false (tunnel).
+	if _, tunnel := tr.pass.Load(hostOnly("target.test:443")); !tunnel {
+		t.Fatal("flagged host must be recognized for the CONNECT form with port")
+	}
+}

traffic/proxy_test.go

@@ -0,0 +1,80 @@
+package traffic
+
+import "testing"
+
+func TestValidateProxyURL(t *testing.T) {
+	ok := []string{
+		"http://127.0.0.1:8080",
+		"https://proxy.example.com:3128",
+		"socks5://10.0.0.1:1080",
+		"socks5://user:pass@10.0.0.1:1080",
+	}
+	for _, raw := range ok {
+		if _, err := ValidateProxyURL(raw); err != nil {
+			t.Errorf("ValidateProxyURL(%q) unexpected error: %v", raw, err)
+		}
+	}
+	bad := []string{
+		"127.0.0.1:8080",         // no scheme
+		"ftp://host:21",          // unsupported scheme
+		"http://",                // no host
+		"socks4://10.0.0.1:1080", // unsupported scheme
+	}
+	for _, raw := range bad {
+		if _, err := ValidateProxyURL(raw); err == nil {
+			t.Errorf("ValidateProxyURL(%q) expected error, got nil", raw)
+		}
+	}
+}
+
+func TestSetUpstreamProxyStoreClear(t *testing.T) {
+	tr := &Traffic{}
+	if got := tr.upstream.Load(); got != nil {
+		t.Fatalf("initial upstream = %v, want nil", got)
+	}
+	if err := tr.SetUpstreamProxy("socks5://user:pass@10.0.0.1:1080"); err != nil {
+		t.Fatalf("SetUpstreamProxy: %v", err)
+	}
+	u := tr.upstream.Load()
+	if u == nil || u.Scheme != "socks5" || u.Host != "10.0.0.1:1080" {
+		t.Fatalf("stored upstream = %v, want socks5://10.0.0.1:1080", u)
+	}
+	if pw, _ := u.User.Password(); u.User.Username() != "user" || pw != "pass" {
+		t.Fatalf("stored upstream lost credentials: %v", u)
+	}
+	// Empty clears back to direct.
+	if err := tr.SetUpstreamProxy("  "); err != nil {
+		t.Fatalf("SetUpstreamProxy(clear): %v", err)
+	}
+	if got := tr.upstream.Load(); got != nil {
+		t.Fatalf("after clear upstream = %v, want nil", got)
+	}
+	// Invalid value is rejected and does not mutate current state.
+	if err := tr.SetUpstreamProxy("nope://x"); err == nil {
+		t.Fatal("SetUpstreamProxy(invalid) expected error")
+	}
+	if got := tr.upstream.Load(); got != nil {
+		t.Fatalf("invalid set mutated upstream to %v, want nil", got)
+	}
+}
+
+func TestProxyAddr(t *testing.T) {
+	cases := []struct {
+		addr string
+		want string
+	}{
+		// Bare :port means "bind all interfaces" — the legacy default. The URL
+		// agents consume must still point at loopback so they reach the local proxy.
+		{":8788", "http://127.0.0.1:8788"},
+		// Explicit loopback — the current default since #129 (open proxy exposure).
+		{"127.0.0.1:8788", "http://127.0.0.1:8788"},
+		// Explicit all-interface bind is still supported (remote capture via SSH).
+		{"0.0.0.0:8788", "http://0.0.0.0:8788"},
+	}
+	for _, c := range cases {
+		got := (&Traffic{addr: c.addr}).ProxyAddr()
+		if got != c.want {
+			t.Errorf("ProxyAddr(%q) = %q, want %q", c.addr, got, c.want)
+		}
+	}
+}

traffic/reclaim_test.go

@@ -0,0 +1,257 @@
+package traffic
+
+import (
+	"database/sql"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// bulkRecord fills the index with inline bodies — the ones that actually make
+// index.sqlite grow. Binary content type keeps them out of the full-text index so
+// the test stays fast; the FTS side is covered by TestReclaimMergesFTSTombstones.
+func bulkRecord(tr *Traffic, host string, n, size int) {
+	body := []byte(strings.Repeat("A", size))
+	for i := 0; i < n; i++ {
+		tr.record(newFlow(host, "GET", fmt.Sprintf("/blob/%d", i), nil, body,
+			withRespType("application/octet-stream")))
+	}
+}
+
+func TestNewIndexEnablesIncrementalVacuum(t *testing.T) {
+	tr, _ := openTraffic(t)
+	if !tr.incrementalVacuum {
+		t.Fatal("新建索引库未启用增量回收")
+	}
+	var mode int
+	if err := tr.DB().QueryRow(`PRAGMA auto_vacuum`).Scan(&mode); err != nil {
+		t.Fatal(err)
+	}
+	if mode != autoVacuumIncremental {
+		t.Fatalf("auto_vacuum=%d，应为 %d", mode, autoVacuumIncremental)
+	}
+}
+
+// TestDeleteReclaimsIndexSpace is the regression: deleting traffic used to leave
+// index.sqlite at its high-water mark forever, because SQLite only chains freed
+// pages onto its freelist and nothing ever returned them to the filesystem.
+func TestDeleteReclaimsIndexSpace(t *testing.T) {
+	tr, _ := openTraffic(t)
+	const host = "bulk.example.com"
+	// 30 × 200KB stays under maxInlineBody, so every body lands in the database
+	// itself rather than the blob store — that is where the growth was invisible.
+	bulkRecord(tr, host, 30, 200*1024)
+	grown := tr.indexBytes()
+	if grown < 5<<20 {
+		t.Fatalf("索引只有 %d 字节，样本不足以验证回收", grown)
+	}
+
+	if n, err := tr.DeleteHostsExact([]string{host}); err != nil || n != 30 {
+		t.Fatalf("DeleteHostsExact=(%d,%v)，应为 (30,nil)", n, err)
+	}
+	tr.reaping.Wait() // 回收在后台分块进行
+
+	after := tr.indexBytes()
+	if after > grown/4 {
+		t.Fatalf("删除后索引仍占 %d 字节（删除前 %d），空间没有还给文件系统", after, grown)
+	}
+	// A handful of pages incremental_vacuum could not move to the end of the file
+	// is a normal residual; the ~1500 that the deletion freed must be gone.
+	var free int
+	if err := tr.DB().QueryRow(`PRAGMA freelist_count`).Scan(&free); err != nil {
+		t.Fatal(err)
+	}
+	if free > 64 {
+		t.Fatalf("仍有 %d 个空闲页未回收", free)
+	}
+}
+
+// TestReclaimMergesFTSTombstones covers the second half of the leak: ex_fts is a
+// contentless_delete index, so a DELETE only writes tombstones. Without a merge
+// the index keeps growing on every deletion — deleting traffic made it bigger.
+func TestReclaimMergesFTSTombstones(t *testing.T) {
+	tr, _ := openTraffic(t)
+	if !tr.fts {
+		t.Skip("驱动未启用 FTS5")
+	}
+	// Deleted in batches, which is what leaves tombstones spread over many
+	// segments rather than emptying the index in one shot.
+	for round := 0; round < 4; round++ {
+		host := fmt.Sprintf("fts%d.example.com", round)
+		for i := 0; i < 20; i++ {
+			tr.record(newFlow(host, "GET", fmt.Sprintf("/p/%d", i), nil,
+				[]byte(strings.Repeat("secret token 中文正文 padding ", 200))))
+		}
+		if _, err := tr.DeleteHostsExact([]string{host}); err != nil {
+			t.Fatal(err)
+		}
+		tr.reaping.Wait()
+	}
+
+	var exchanges, segments int
+	if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchanges`).Scan(&exchanges); err != nil {
+		t.Fatal(err)
+	}
+	if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM ex_fts_data`).Scan(&segments); err != nil {
+		t.Fatal(err)
+	}
+	if exchanges != 0 {
+		t.Fatalf("还剩 %d 条流量", exchanges)
+	}
+	// A fully merged, empty contentless index keeps only its structure rows.
+	if segments > 8 {
+		t.Fatalf("全文索引残留 %d 行段数据，tombstone 未被合并回收", segments)
+	}
+}
+
+// TestReclaimOnLegacyIndexIsHarmless covers installs created before
+// auto_vacuum=incremental became the default: incremental_vacuum is a silent
+// no-op there, so reclamation must report the situation and finish rather than
+// spin or fail. Only a full compaction can convert such a file.
+func TestReclaimOnLegacyIndexIsHarmless(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(dir, "_index"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	// Create the tables first, with auto_vacuum left at its default 0 — exactly the
+	// shape Open used to leave behind.
+	legacy, err := sql.Open("sqlite", filepath.Join(dir, "_index", "index.sqlite"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := legacy.Exec(indexSchema); err != nil {
+		t.Fatal(err)
+	}
+	if err := legacy.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	tr, err := Open(dir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { tr.Close() })
+	if tr.incrementalVacuum {
+		t.Fatal("旧库不应报告已启用增量回收")
+	}
+
+	const host = "legacy.example.com"
+	bulkRecord(tr, host, 8, 200*1024)
+	if n, err := tr.DeleteHostsExact([]string{host}); err != nil || n != 8 {
+		t.Fatalf("DeleteHostsExact=(%d,%v)，应为 (8,nil)", n, err)
+	}
+	tr.reaping.Wait() // 必须收敛，不能卡在预算里
+
+	// The freelist stays populated: that is the whole reason a compaction entry
+	// point is needed for pre-existing databases.
+	var free int
+	if err := tr.DB().QueryRow(`PRAGMA freelist_count`).Scan(&free); err != nil {
+		t.Fatal(err)
+	}
+	if free == 0 {
+		t.Fatal("旧库居然回收了空闲页，说明测试没有真的构造出旧库")
+	}
+}
+
+// TestDeleteAllPurgesAndCompacts covers the page's clear-everything action: it
+// must leave nothing behind — including host directories the index no longer
+// knows about — and it must hand the index space back, since an emptied index is
+// the one moment a full rewrite is cheap.
+func TestDeleteAllPurgesAndCompacts(t *testing.T) {
+	tr, dir := openTraffic(t)
+	bulkRecord(tr, "a.example.com", 10, 200*1024)
+	bulkRecord(tr, "b.example.com", 10, 200*1024)
+	// A text body so the full-text index has real content, and a spilled one so a
+	// blob exists to collect.
+	tr.record(newFlow("c.example.com", "GET", "/page", nil, []byte(strings.Repeat("secret-token ", 500))))
+	tr.record(newFlow("c.example.com", "GET", "/big", nil,
+		[]byte(strings.Repeat("B", maxInlineBody+1024)), withRespType("application/sql")))
+	// An orphaned legacy directory: no index row points at it, so only a
+	// clear-everything should take it.
+	orphan := filepath.Join(dir, "orphan.example.com")
+	if err := os.MkdirAll(orphan, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	grown := tr.indexBytes()
+	if grown < 5<<20 {
+		t.Fatalf("索引只有 %d 字节，样本不足", grown)
+	}
+
+	deleted, reclaimed, err := tr.DeleteAll()
+	if err != nil {
+		t.Fatalf("DeleteAll: %v", err)
+	}
+	if deleted != 22 {
+		t.Fatalf("deleted=%d，应为 22", deleted)
+	}
+	tr.reaping.Wait()
+
+	if reclaimed < grown/2 {
+		t.Fatalf("只回收了 %d 字节（删除前索引 %d）", reclaimed, grown)
+	}
+	if after := tr.indexBytes(); after > grown/8 {
+		t.Fatalf("清空后索引仍占 %d 字节（删除前 %d）", after, grown)
+	}
+	for _, q := range []string{
+		`SELECT COUNT(*) FROM exchanges`,
+		`SELECT COUNT(*) FROM exchange_bodies`,
+		`SELECT COUNT(*) FROM blob_refs`,
+	} {
+		var c int
+		if err := tr.DB().QueryRow(q).Scan(&c); err != nil {
+			t.Fatal(err)
+		}
+		if c != 0 {
+			t.Fatalf("%s = %d，应为 0", q, c)
+		}
+	}
+	if _, err := os.Stat(orphan); !os.IsNotExist(err) {
+		t.Fatalf("孤立的历史 host 目录未被清理：%v", err)
+	}
+	// Recording must keep working against the freshly rewritten file.
+	tr.record(newFlow("d.example.com", "GET", "/after", nil, []byte("清空后仍可录制")))
+	if n, err := tr.Count(); err != nil || n != 1 {
+		t.Fatalf("清空后 Count=(%d,%v)，应为 (1,nil)", n, err)
+	}
+}
+
+// TestDeleteAllConvertsLegacyIndex is why the purge compacts rather than just
+// deleting: auto_vacuum cannot be switched on after the fact except through a
+// VACUUM, and an emptied index is the cheapest place to pay for one. After this,
+// ordinary deletions reclaim space on their own.
+func TestDeleteAllConvertsLegacyIndex(t *testing.T) {
+	dir := t.TempDir()
+	old := openLegacyIndex(t, dir)
+	if err := old.Close(); err != nil {
+		t.Fatal(err)
+	}
+	tr, err := Open(dir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { tr.Close() })
+	if tr.incrementalVacuum {
+		t.Fatal("旧库不应报告已启用增量回收")
+	}
+
+	bulkRecord(tr, "legacy.example.com", 10, 200*1024)
+	if _, _, err := tr.DeleteAll(); err != nil {
+		t.Fatalf("DeleteAll: %v", err)
+	}
+	if !tr.incrementalVacuum {
+		t.Fatal("清空后旧库未被转换为增量回收模式")
+	}
+
+	// The converted database now reclaims on an ordinary host deletion.
+	bulkRecord(tr, "again.example.com", 10, 200*1024)
+	grown := tr.indexBytes()
+	if _, err := tr.DeleteHostsExact([]string{"again.example.com"}); err != nil {
+		t.Fatal(err)
+	}
+	tr.reaping.Wait()
+	if after := tr.indexBytes(); after > grown/4 {
+		t.Fatalf("转换后普通删除仍未回收：%d 字节（删除前 %d）", after, grown)
+	}
+}

traffic/traffic_store_test.go

@@ -0,0 +1,528 @@
+package traffic
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+
+	mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
+)
+
+// flowOpt tweaks the synthetic flow built by newFlow.
+type flowOpt func(*mproxy.Flow)
+
+func withRespType(ct string) flowOpt {
+	return func(f *mproxy.Flow) { f.Response.Header.Set("Content-Type", ct) }
+}
+
+// newFlow builds the minimal flow record() needs: a request with a URL, method
+// and body, plus a response with a status and body.
+func newFlow(host, method, path string, reqBody, respBody []byte, opts ...flowOpt) *mproxy.Flow {
+	u, err := url.Parse("http://" + host + path)
+	if err != nil {
+		panic(err)
+	}
+	f := &mproxy.Flow{
+		Request: &mproxy.Request{
+			Method: method,
+			URL:    u,
+			Proto:  "HTTP/1.1",
+			Header: http.Header{"Content-Type": []string{"application/json"}},
+			Body:   reqBody,
+		},
+		Response: &mproxy.Response{
+			StatusCode: 200,
+			Header:     http.Header{"Content-Type": []string{"application/json"}},
+			Body:       respBody,
+		},
+	}
+	for _, o := range opts {
+		o(f)
+	}
+	return f
+}
+
+func openTraffic(t *testing.T) (*Traffic, string) {
+	t.Helper()
+	dir := t.TempDir()
+	tr, err := Open(dir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { tr.Close() })
+	return tr, dir
+}
+
+func onlyExchangeID(t *testing.T, tr *Traffic) string {
+	t.Helper()
+	var id string
+	if err := tr.DB().QueryRow(`SELECT id FROM exchanges`).Scan(&id); err != nil {
+		t.Fatalf("读取 exchange id: %v", err)
+	}
+	return id
+}
+
+// TestRecordKeepsBodiesInIndex is the core of the storage change: a recorded
+// exchange produces no per-request directory at all, and its bodies are served
+// back out of SQLite.
+func TestRecordKeepsBodiesInIndex(t *testing.T) {
+	tr, dir := openTraffic(t)
+
+	tr.record(newFlow("api.example.com", "POST", "/v1/login",
+		[]byte(`{"user":"admin","password":"P@ssw0rd"}`),
+		[]byte(`{"token":"abc123","note":"内网测试账号"}`)))
+
+	// The URL-mirroring tree is gone: no host directory, no nested path segments.
+	if _, err := os.Stat(filepath.Join(dir, "api.example.com")); !os.IsNotExist(err) {
+		t.Fatalf("record 仍在磁盘上创建 host 目录（stat err=%v）", err)
+	}
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, e := range entries {
+		if !strings.HasPrefix(e.Name(), "_") {
+			t.Fatalf("data 目录下出现非内部目录 %q，说明仍在写文件树", e.Name())
+		}
+	}
+
+	id := onlyExchangeID(t, tr)
+	req, resp, err := tr.Get(id)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{"POST /v1/login HTTP/1.1", "Host: api.example.com", `"password":"P@ssw0rd"`} {
+		if !strings.Contains(req, want) {
+			t.Fatalf("请求原文缺少 %q，实际：\n%s", want, req)
+		}
+	}
+	for _, want := range []string{"HTTP 200", `"token":"abc123"`, "内网测试账号"} {
+		if !strings.Contains(resp, want) {
+			t.Fatalf("响应原文缺少 %q，实际：\n%s", want, resp)
+		}
+	}
+}
+
+// TestFullTextSearchMatchesBodies covers what the trigram index buys over the
+// previous URL-only search: arbitrary substrings and CJK, across request and
+// response bodies.
+func TestFullTextSearchMatchesBodies(t *testing.T) {
+	tr, _ := openTraffic(t)
+	if !tr.fts {
+		t.Skip("驱动未启用 FTS5")
+	}
+	const host = "api.example.com"
+	tr.record(newFlow(host, "POST", "/v1/login",
+		[]byte(`{"user":"admin","password":"P@ssw0rd"}`),
+		[]byte(`{"token":"abc123","note":"内网测试账号"}`)))
+	tr.record(newFlow(host, "GET", "/v1/health", nil, []byte(`{"status":"ok"}`)))
+
+	hits := func(term string) int {
+		t.Helper()
+		rows, err := tr.query(host, "", term, 0, 10)
+		if err != nil {
+			t.Fatalf("按正文搜索 %q 出错：%v", term, err)
+		}
+		return len(rows)
+	}
+	if n := hits("password"); n != 1 {
+		t.Fatalf("搜 password 命中 %d 条，应为 1", n)
+	}
+	// Substring inside a token — the default unicode61 tokenizer cannot do this.
+	if n := hits("ssw0r"); n != 1 {
+		t.Fatalf("搜子串 ssw0r 命中 %d 条，应为 1", n)
+	}
+	if n := hits("内网测试"); n != 1 {
+		t.Fatalf("搜中文命中 %d 条，应为 1", n)
+	}
+	if n := hits("nonexistent-marker"); n != 0 {
+		t.Fatalf("无关关键词命中 %d 条，应为 0", n)
+	}
+
+	// Too-short terms are reported, not silently treated as "no match".
+	if _, err := tr.query(host, "", "ab", 0, 10); err == nil {
+		t.Fatal("两字符正文关键词应返回明确错误")
+	}
+}
+
+// TestLargeBodySpillsButStaysSearchable is the case that motivated indexing from
+// memory: the body lives in the blob store, only a preview is inline, and the
+// part past the preview is still findable.
+func TestLargeBodySpillsButStaysSearchable(t *testing.T) {
+	tr, dir := openTraffic(t)
+	if !tr.fts {
+		t.Skip("驱动未启用 FTS5")
+	}
+	const host = "dump.example.com"
+	const marker = "DB_PASSWORD=hunter2"
+	// Marker sits far past blobPreview, so only the full-text index can find it.
+	big := []byte(strings.Repeat("-- MySQL dump\n", maxInlineBody/14+2000) + marker)
+	if len(big) <= maxInlineBody+blobPreview {
+		t.Fatalf("测试数据不够大：%d 字节", len(big))
+	}
+	tr.record(newFlow(host, "GET", "/backup.sql", nil, big, withRespType("application/sql")))
+
+	// Stored under a single bucket level, named by hash.
+	var hash string
+	if err := tr.DB().QueryRow(`SELECT resp_blob FROM exchange_bodies`).Scan(&hash); err != nil {
+		t.Fatal(err)
+	}
+	if len(hash) != 64 {
+		t.Fatalf("resp_blob=%q，应为 64 位 sha256", hash)
+	}
+	blob := filepath.Join(dir, "_blobs", "sha256", hash[:2], hash+".bin")
+	st, err := os.Stat(blob)
+	if err != nil {
+		t.Fatalf("blob 未落盘到单层桶 %s：%v", blob, err)
+	}
+	if st.Size() != int64(len(big)) {
+		t.Fatalf("blob 大小 %d，应为 %d", st.Size(), len(big))
+	}
+
+	// The reference is registered, which is what GC consults.
+	var refs int
+	if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM blob_refs WHERE hash=?`, hash).Scan(&refs); err != nil {
+		t.Fatal(err)
+	}
+	if refs != 1 {
+		t.Fatalf("blob_refs 行数 %d，应为 1", refs)
+	}
+
+	// Inline: a readable preview plus the pointer, not the whole body.
+	_, resp, err := tr.Get(onlyExchangeID(t, tr))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(resp, "-- MySQL dump") {
+		t.Fatalf("响应缺少头部预览：\n%s", clip(resp, 300))
+	}
+	if !strings.Contains(resp, "@blob sha256:"+hash) {
+		t.Fatalf("响应缺少 blob 指针：\n%s", clip(resp, 300))
+	}
+	if strings.Contains(resp, marker) {
+		t.Fatal("预览不应包含超出 blobPreview 的内容")
+	}
+	if len(resp) > blobPreview*2 {
+		t.Fatalf("内联内容 %d 字节，远超预览上限", len(resp))
+	}
+
+	// Searchable despite living on disk — the index was fed from memory.
+	rows, err := tr.query(host, "", marker, 0, 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rows) != 1 {
+		t.Fatalf("超大正文中的关键词命中 %d 条，应为 1", len(rows))
+	}
+
+	// And retrievable in pages.
+	data, total, err := tr.BlobRange(hash, int64(len(big)-len(marker)), 100)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if total != int64(len(big)) {
+		t.Fatalf("BlobRange total=%d，应为 %d", total, len(big))
+	}
+	if string(data) != marker {
+		t.Fatalf("BlobRange 读到 %q，应为 %q", data, marker)
+	}
+	if _, _, err := tr.BlobRange("../../etc/passwd", 0, 10); err == nil {
+		t.Fatal("非法 hash 应被拒绝")
+	}
+}
+
+// TestBinaryBodyStaysOutOfIndex keeps the index spend on things worth searching:
+// binary payloads contribute nothing but a type tag.
+func TestBinaryBodyStaysOutOfIndex(t *testing.T) {
+	tr, _ := openTraffic(t)
+	if !tr.fts {
+		t.Skip("驱动未启用 FTS5")
+	}
+	const host = "cdn.example.com"
+	const marker = "SECRETINIMAGE"
+	png := append([]byte{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00},
+		[]byte(strings.Repeat("x", maxInlineBody)+marker)...)
+	tr.record(newFlow(host, "GET", "/logo.png", nil, png, withRespType("image/png")))
+
+	rows, err := tr.query(host, "", marker, 0, 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rows) != 0 {
+		t.Fatalf("二进制正文不应进入全文索引，却命中 %d 条", len(rows))
+	}
+	_, resp, err := tr.Get(onlyExchangeID(t, tr))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(resp, "[binary image/png") || !strings.Contains(resp, "magic=89504e47") {
+		t.Fatalf("二进制正文应展示类型与魔数，实际：\n%s", clip(resp, 300))
+	}
+}
+
+// TestGetFallsBackToLegacyTree keeps pre-migration captures readable: their rows
+// carry a path and their bodies are still .http files on disk.
+func TestGetFallsBackToLegacyTree(t *testing.T) {
+	tr, dir := openTraffic(t)
+	const host = "old.example.com"
+	const id = "1-0001"
+	rel := filepath.Join(host, "GET", id)
+	exDir := filepath.Join(dir, rel)
+	if err := os.MkdirAll(exDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(exDir, "request.http"), []byte("GET / HTTP/1.1\nHost: old.example.com\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(exDir, "response.http"), []byte("HTTP 200\n\nlegacy body"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
+VALUES(?,?,?,?,?,?,?,?,?,?,?)`, id, 1, host, "GET", "/", "http://"+host+"/", 200, "text/html", 0, 11, rel); err != nil {
+		t.Fatal(err)
+	}
+
+	req, resp, err := tr.Get(id)
+	if err != nil {
+		t.Fatalf("历史记录应仍可读取：%v", err)
+	}
+	if !strings.Contains(req, "Host: old.example.com") {
+		t.Fatalf("历史请求原文错误：%q", req)
+	}
+	if !strings.Contains(resp, "legacy body") {
+		t.Fatalf("历史响应原文错误：%q", resp)
+	}
+}
+
+// TestGCCollectsBlobsAndEmptyBuckets covers both halves of the collector: the
+// reference lookup now comes from blob_refs, and emptied buckets are removed
+// instead of accumulating forever.
+func TestGCCollectsBlobsAndEmptyBuckets(t *testing.T) {
+	tr, dir := openTraffic(t)
+	const host = "dump.example.com"
+	big := []byte(strings.Repeat("A", maxInlineBody+1024))
+	tr.record(newFlow(host, "GET", "/big.bin", nil, big, withRespType("application/sql")))
+
+	var hash string
+	if err := tr.DB().QueryRow(`SELECT resp_blob FROM exchange_bodies`).Scan(&hash); err != nil {
+		t.Fatal(err)
+	}
+	bucket := filepath.Join(dir, "_blobs", "sha256", hash[:2])
+	if _, err := os.Stat(filepath.Join(bucket, hash+".bin")); err != nil {
+		t.Fatal(err)
+	}
+
+	if n, err := tr.DeleteHostsExact([]string{host}); err != nil || n != 1 {
+		t.Fatalf("DeleteHostsExact=(%d,%v)，应为 (1,nil)", n, err)
+	}
+	if _, err := os.Stat(filepath.Join(bucket, hash+".bin")); !os.IsNotExist(err) {
+		t.Fatalf("失去引用的 blob 未被回收：%v", err)
+	}
+	if _, err := os.Stat(bucket); !os.IsNotExist(err) {
+		t.Fatalf("空桶目录未被清理：%v", err)
+	}
+	// Bodies and full-text rows go with the exchange.
+	for _, q := range []string{
+		`SELECT COUNT(*) FROM exchange_bodies`,
+		`SELECT COUNT(*) FROM blob_refs`,
+	} {
+		var c int
+		if err := tr.DB().QueryRow(q).Scan(&c); err != nil {
+			t.Fatal(err)
+		}
+		if c != 0 {
+			t.Fatalf("%s = %d，应为 0", q, c)
+		}
+	}
+	if tr.fts {
+		var c int
+		if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM ex_fts WHERE ex_fts MATCH ?`, ftsQuote("AAAA")).Scan(&c); err != nil {
+			t.Fatal(err)
+		}
+		if c != 0 {
+			t.Fatalf("全文索引残留 %d 条", c)
+		}
+	}
+}
+
+// TestPageSearchesBodies checks the UI-facing search box picks up the full-text
+// index too, not just metadata columns.
+func TestPageSearchesBodies(t *testing.T) {
+	tr, _ := openTraffic(t)
+	if !tr.fts {
+		t.Skip("驱动未启用 FTS5")
+	}
+	tr.record(newFlow("api.example.com", "POST", "/v1/login", nil, []byte(`{"error":"invalid credentials"}`)))
+	tr.record(newFlow("api.example.com", "GET", "/v1/health", nil, []byte(`{"status":"ok"}`)))
+
+	rows, total, err := tr.Page(PageQuery{Query: "invalid credentials", RespMin: -1, RespMax: -1}, 0, 100)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if total != 1 || len(rows) != 1 {
+		t.Fatalf("正文关键词命中 total=%d rows=%d，应为 1/1", total, len(rows))
+	}
+	// Metadata matching still works alongside it.
+	if _, total, err := tr.Page(PageQuery{Query: "health", RespMin: -1, RespMax: -1}, 0, 100); err != nil || total != 1 {
+		t.Fatalf("URL 关键词 total=%d err=%v，应为 1", total, err)
+	}
+}
+
+// TestPageFiltersAndSort covers the issue #177 additions: status-class/exact
+// filtering, response-size bounds, path (url_template) filtering, and
+// server-side sorting by resp_len.
+func TestPageFiltersAndSort(t *testing.T) {
+	tr, _ := openTraffic(t)
+	status := func(code int) flowOpt { return func(f *mproxy.Flow) { f.Response.StatusCode = code } }
+	// Three exchanges with distinct status codes and response sizes.
+	tr.record(newFlow("api.example.com", "GET", "/api/users", nil, make([]byte, 10), status(200)))
+	tr.record(newFlow("api.example.com", "GET", "/api/admin", nil, make([]byte, 100), status(404)))
+	tr.record(newFlow("api.example.com", "GET", "/api/users/1", nil, make([]byte, 50), status(500)))
+
+	// Status class band.
+	if rows, _, err := tr.Page(PageQuery{Status: "4xx", RespMin: -1, RespMax: -1}, 0, 100); err != nil || len(rows) != 1 || rows[0].Status != 404 {
+		t.Fatalf("status=4xx 应命中 1 条 404，得 %d 条 err=%v", len(rows), err)
+	}
+	// Exact status.
+	if rows, _, err := tr.Page(PageQuery{Status: "500", RespMin: -1, RespMax: -1}, 0, 100); err != nil || len(rows) != 1 || rows[0].Status != 500 {
+		t.Fatalf("status=500 应命中 1 条，得 %d 条 err=%v", len(rows), err)
+	}
+	// Response-size lower bound (>=60 keeps only the 100-byte row).
+	if rows, _, err := tr.Page(PageQuery{RespMin: 60, RespMax: -1}, 0, 100); err != nil || len(rows) != 1 || rows[0].RespLen != 100 {
+		t.Fatalf("resp_min=60 应命中 1 条 100B，得 %d 条 err=%v", len(rows), err)
+	}
+	// Path (url_template) filter narrows to the /api/admin exchange.
+	if rows, _, err := tr.Page(PageQuery{Path: "/api/admin", RespMin: -1, RespMax: -1}, 0, 100); err != nil || len(rows) != 1 || rows[0].Status != 404 {
+		t.Fatalf("path=/api/admin 应命中 1 条，得 %d 条 err=%v", len(rows), err)
+	}
+	// Sort by response length, ascending then descending.
+	asc, _, err := tr.Page(PageQuery{RespMin: -1, RespMax: -1, Sort: "resp_len", Order: "asc"}, 0, 100)
+	if err != nil || len(asc) != 3 {
+		t.Fatalf("resp_len asc 应返回 3 条，得 %d 条 err=%v", len(asc), err)
+	}
+	if asc[0].RespLen != 10 || asc[1].RespLen != 50 || asc[2].RespLen != 100 {
+		t.Fatalf("resp_len asc 顺序错误：%d,%d,%d", asc[0].RespLen, asc[1].RespLen, asc[2].RespLen)
+	}
+	desc, _, err := tr.Page(PageQuery{RespMin: -1, RespMax: -1, Sort: "resp_len", Order: "desc"}, 0, 100)
+	if err != nil || len(desc) != 3 || desc[0].RespLen != 100 || desc[2].RespLen != 10 {
+		t.Fatalf("resp_len desc 顺序错误 err=%v", err)
+	}
+}
+
+func TestQueryHostPortAndURLForms(t *testing.T) {
+	tr, _ := openTraffic(t)
+	tr.record(newFlow("api.example.com:8082", "GET", "/admin", nil, []byte("8082")))
+	tr.record(newFlow("api.example.com:8088", "GET", "/admin", nil, []byte("8088")))
+	tr.record(newFlow("[2001:db8::1]:8443", "GET", "/admin", nil, []byte("8443")))
+
+	for _, tc := range []struct {
+		name string
+		host string
+		want int
+	}{
+		{name: "bare host", host: "api.example.com", want: 2},
+		{name: "host and port", host: "api.example.com:8082", want: 1},
+		{name: "full URL", host: "http://api.example.com:8088/admin", want: 1},
+		{name: "IPv6 host and port", host: "[2001:db8::1]:8443", want: 1},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			rows, err := tr.query(tc.host, "", "", 0, 10)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(rows) != tc.want {
+				t.Fatalf("query(%q) returned %d rows, want %d", tc.host, len(rows), tc.want)
+			}
+		})
+	}
+}
+
+func TestNormalizeSearchHost(t *testing.T) {
+	for _, tc := range []struct {
+		raw, host, port string
+	}{
+		{raw: "API.Example.com", host: "api.example.com"},
+		{raw: "api.example.com:8088", host: "api.example.com", port: "8088"},
+		{raw: "https://[2001:db8::1]:8443/path", host: "2001:db8::1", port: "8443"},
+		{raw: "[2001:db8::1]", host: "2001:db8::1"},
+	} {
+		host, port, err := normalizeSearchHost(tc.raw)
+		if err != nil {
+			t.Fatalf("normalizeSearchHost(%q): %v", tc.raw, err)
+		}
+		if host != tc.host || port != tc.port {
+			t.Fatalf("normalizeSearchHost(%q)=(%q,%q), want (%q,%q)", tc.raw, host, port, tc.host, tc.port)
+		}
+	}
+}
+
+// TestTruncateUTF8 guards the preview cut: never split a multi-byte rune.
+func TestTruncateUTF8(t *testing.T) {
+	s := "内网测试账号"
+	for n := 0; n <= len(s); n++ {
+		got := truncateUTF8([]byte(s), n)
+		if !strings.HasPrefix(s, got) {
+			t.Fatalf("n=%d 截断结果 %q 不是原串前缀", n, got)
+		}
+		if len(got) > n {
+			t.Fatalf("n=%d 截断后 %d 字节，超出上限", n, len(got))
+		}
+	}
+	if got := truncateUTF8([]byte("abc"), 10); got != "abc" {
+		t.Fatalf("短于上限时应原样返回，得到 %q", got)
+	}
+}
+
+// TestIsBinaryBody documents the classification: declared binary types, and the
+// NUL backstop for anything mislabeled.
+func TestIsBinaryBody(t *testing.T) {
+	cases := []struct {
+		ct   string
+		body string
+		want bool
+	}{
+		{"application/json", `{"a":1}`, false},
+		{"text/html; charset=utf-8", "<html>", false},
+		{"application/sql", "-- dump", false},
+		{"", "plain text", false},
+		{"image/png", "whatever", true},
+		{"application/zip", "PK", true},
+		{"APPLICATION/PDF", "%PDF", true},
+		{"text/plain", "has\x00nul", true},
+	}
+	for _, c := range cases {
+		if got := isBinaryBody(c.ct, []byte(c.body)); got != c.want {
+			t.Errorf("isBinaryBody(%q, %q)=%v，应为 %v", c.ct, c.body, got, c.want)
+		}
+	}
+}
+
+// TestRecordConcurrent exercises the write path under contention: ids stay
+// unique and every exchange lands in all three tables.
+func TestRecordConcurrent(t *testing.T) {
+	tr, _ := openTraffic(t)
+	const n = 50
+	var wg sync.WaitGroup
+	for i := range n {
+		wg.Go(func() {
+			tr.record(newFlow("api.example.com", "GET", fmt.Sprintf("/item/%d", i),
+				nil, fmt.Appendf(nil, `{"id":%d}`, i)))
+		})
+	}
+	wg.Wait()
+	var exchanges, bodies int
+	if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchanges`).Scan(&exchanges); err != nil {
+		t.Fatal(err)
+	}
+	if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchange_bodies`).Scan(&bodies); err != nil {
+		t.Fatal(err)
+	}
+	if exchanges != n || bodies != n {
+		t.Fatalf("并发写入后 exchanges=%d bodies=%d，应各为 %d", exchanges, bodies, n)
+	}
+}

traffic/traffic_test.go

@@ -0,0 +1,410 @@
+package traffic
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
+)
+
+func TestRequestHeaderLinesIncludesHost(t *testing.T) {
+	req := &mproxy.Request{
+		URL:    &url.URL{Host: "target.example:8443"},
+		Header: http.Header{"Accept": []string{"application/json"}},
+	}
+	got := requestHeaderLines(req)
+	if !strings.Contains(got, "Host: target.example:8443\n") {
+		t.Fatalf("request headers missing Host: %q", got)
+	}
+	if !strings.Contains(got, "Accept: application/json\n") {
+		t.Fatalf("request headers missing regular header: %q", got)
+	}
+}
+
+// TestDeleteHost verifies the delete contract: rows for hosts containing the
+// substring are removed together with their file trees, non-matching hosts are
+// untouched, and the count is right.
+func TestDeleteHost(t *testing.T) {
+	dir := t.TempDir()
+	tr, err := Open(dir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tr.Close()
+
+	// Seed two hosts' index rows + trees directly (record() needs a live Flow).
+	for i, h := range []string{"a.example.com", "b.example.com"} {
+		id := fmt.Sprintf("1-%04d", i+1)
+		exDir := filepath.Join(dir, h, "GET", id)
+		if err := os.MkdirAll(exDir, 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(filepath.Join(exDir, "meta.json"), []byte(fmt.Sprintf(`{"id":%q,"host":%q}`, id, h)), 0o644); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
+VALUES(?,?,?,?,?,?,?,?,?,?,?)`,
+			id, i+1, h, "GET", "/", "http://"+h+"/", 200, "text/html", 0, 0, h+"/GET/"+id); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// Substring: "a.example" matches a.example.com only, leaves b.example.com.
+	n, err := tr.DeleteHost("a.example")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != 1 {
+		t.Fatalf("deleted=%d, want 1", n)
+	}
+	// Tree removed for the target, intact for the other host.
+	if _, err := os.Stat(filepath.Join(dir, "a.example.com")); !os.IsNotExist(err) {
+		t.Fatalf("a.example.com tree still exists (stat err=%v)", err)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "b.example.com")); err != nil {
+		t.Fatalf("b.example.com tree removed: %v", err)
+	}
+	// Index reduced to the other host's single row.
+	var c int
+	if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchanges`).Scan(&c); err != nil {
+		t.Fatal(err)
+	}
+	if c != 1 {
+		t.Fatalf("rows=%d, want 1", c)
+	}
+	// A substring matching nothing is a no-op, not an error.
+	n, err = tr.DeleteHost("nope.example")
+	if err != nil || n != 0 {
+		t.Fatalf("DeleteHost(missing)=%d, err=%v; want 0, nil", n, err)
+	}
+	// A broader substring sweeps the remaining host too.
+	if n, err = tr.DeleteHost("example.com"); err != nil || n != 1 {
+		t.Fatalf("DeleteHost(example.com)=%d, err=%v; want 1, nil", n, err)
+	}
+	c = 0
+	if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchanges`).Scan(&c); err == nil && c != 0 {
+		t.Fatalf("rows=%d, want 0 after full sweep", c)
+	}
+}
+
+// TestHosts verifies the target picker contract: distinct hosts with counts,
+// most recent activity first.
+func TestHosts(t *testing.T) {
+	dir := t.TempDir()
+	tr, err := Open(dir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tr.Close()
+	for i, row := range []struct {
+		host string
+		ts   int64
+	}{{"old.example.com", 1}, {"new.example.com", 3}, {"old.example.com", 2}} {
+		id := fmt.Sprintf("1-%04d", i+1)
+		if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
+VALUES(?,?,?,?,?,?,?,?,?,?,?)`,
+			id, row.ts, row.host, "GET", "/", "http://"+row.host+"/", 200, "text/html", 0, 0, row.host+"/GET/"+id); err != nil {
+			t.Fatal(err)
+		}
+	}
+	hosts, err := tr.Hosts()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(hosts) != 2 {
+		t.Fatalf("hosts=%d, want 2", len(hosts))
+	}
+	// newest activity (ts=3) first
+	if hosts[0].Host != "new.example.com" || hosts[0].Count != 1 {
+		t.Fatalf("hosts[0]=%+v, want new.example.com/1", hosts[0])
+	}
+	if hosts[1].Host != "old.example.com" || hosts[1].Count != 2 {
+		t.Fatalf("hosts[1]=%+v, want old.example.com/2", hosts[1])
+	}
+}
+
+// TestDeleteHostsExact verifies the batch delete: exact host match only — a
+// host whose name contains another as a substring is untouched — duplicates in
+// the batch are harmless, and the per-host trees are removed.
+func TestDeleteHostsExact(t *testing.T) {
+	dir := t.TempDir()
+	tr, err := Open(dir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tr.Close()
+	seed := func(id, h string) {
+		exDir := filepath.Join(dir, h, "GET", id)
+		if err := os.MkdirAll(exDir, 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
+VALUES(?,?,?,?,?,?,?,?,?,?,?)`,
+			id, 1, h, "GET", "/", "http://"+h+"/", 200, "text/html", 0, 0, h+"/GET/"+id); err != nil {
+			t.Fatal(err)
+		}
+	}
+	// "api.example.com" is a substring of "api.example.com.cn".
+	seed("1-0001", "api.example.com")
+	seed("1-0002", "api.example.com.cn")
+	seed("1-0003", "shop.example.com")
+
+	// Duplicate entry in the batch must not double-delete or error.
+	n, err := tr.DeleteHostsExact([]string{"api.example.com", "api.example.com", "shop.example.com"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != 2 {
+		t.Fatalf("deleted=%d, want 2", n)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "api.example.com")); !os.IsNotExist(err) {
+		t.Fatalf("api.example.com tree still exists: %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "shop.example.com")); !os.IsNotExist(err) {
+		t.Fatalf("shop.example.com tree still exists: %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "api.example.com.cn")); err != nil {
+		t.Fatalf("api.example.com.cn removed by an exact delete that shouldn't match: %v", err)
+	}
+	var c int
+	if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchanges`).Scan(&c); err != nil {
+		t.Fatal(err)
+	}
+	if c != 1 {
+		t.Fatalf("rows=%d, want 1 (api.example.com.cn only)", c)
+	}
+}
+
+func TestDeleteHostsExactReportsTreeRemovalFailure(t *testing.T) {
+	dir := t.TempDir()
+	tr, err := Open(dir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tr.Close()
+
+	const host = "api.example.com"
+	if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
+VALUES(?,?,?,?,?,?,?,?,?,?,?)`,
+		"1-0001", 1, host, "GET", "/", "http://"+host+"/", 200, "text/html", 0, 0, host+"/GET/1-0001"); err != nil {
+		t.Fatal(err)
+	}
+
+	notDir := filepath.Join(dir, "not-a-directory")
+	if err := os.WriteFile(notDir, []byte("x"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	tr.dir = notDir
+
+	n, err := tr.DeleteHostsExact([]string{host})
+	if err == nil {
+		t.Fatal("DeleteHostsExact returned nil after traffic tree removal failed")
+	}
+	if n != 0 {
+		t.Fatalf("deleted=%d, want 0 after atomic rollback", n)
+	}
+	var count int
+	if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchanges WHERE host=?`, host).Scan(&count); err != nil || count != 1 {
+		t.Fatalf("rolled-back index count=%d err=%v, want 1", count, err)
+	}
+}
+
+func TestDeleteHostsExactRollsBackWholeIndexBatch(t *testing.T) {
+	dir := t.TempDir()
+	tr, err := Open(dir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tr.Close()
+
+	for i, host := range []string{"a.example.com", "b.example.com"} {
+		id := fmt.Sprintf("1-%04d", i+1)
+		if err := os.MkdirAll(filepath.Join(dir, host, "GET", id), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
+VALUES(?,?,?,?,?,?,?,?,?,?,?)`, id, 1, host, "GET", "/", "http://"+host+"/", 200, "text/html", 0, 0, host+"/GET/"+id); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if _, err := tr.DB().Exec(`CREATE TRIGGER fail_second_host BEFORE DELETE ON exchanges
+WHEN OLD.host='b.example.com' BEGIN SELECT RAISE(ABORT, 'forced delete failure'); END`); err != nil {
+		t.Fatal(err)
+	}
+
+	if n, err := tr.DeleteHostsExact([]string{"a.example.com", "b.example.com"}); err == nil || n != 0 {
+		t.Fatalf("DeleteHostsExact failure=(%d,%v), want (0,error)", n, err)
+	}
+	var count int
+	if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchanges`).Scan(&count); err != nil || count != 2 {
+		t.Fatalf("rolled-back index count=%d err=%v, want 2", count, err)
+	}
+	for _, host := range []string{"a.example.com", "b.example.com"} {
+		if _, err := os.Stat(filepath.Join(dir, host)); err != nil {
+			t.Fatalf("tree %s changed despite index rollback: %v", host, err)
+		}
+	}
+}
+
+func TestStageDeleteHostsExactRollbackRestoresIndexAndTree(t *testing.T) {
+	dir := t.TempDir()
+	tr, err := Open(dir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tr.Close()
+
+	const host = "rollback.example.com"
+	const id = "1-0001"
+	tree := filepath.Join(dir, host, "GET", id)
+	if err := os.MkdirAll(tree, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	marker := filepath.Join(tree, "request.http")
+	if err := os.WriteFile(marker, []byte("original"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
+VALUES(?,?,?,?,?,?,?,?,?,?,?)`, id, 1, host, "GET", "/", "http://"+host+"/", 200, "text/html", 0, 0, host+"/GET/"+id); err != nil {
+		t.Fatal(err)
+	}
+
+	stage, err := tr.StageDeleteHostsExact([]string{host})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if stage.Deleted() != 1 {
+		t.Fatalf("staged deleted=%d, want 1", stage.Deleted())
+	}
+	if _, err := os.Stat(filepath.Join(dir, host)); !os.IsNotExist(err) {
+		t.Fatalf("host tree was not staged: %v", err)
+	}
+	if err := stage.Rollback(); err != nil {
+		t.Fatal(err)
+	}
+	if got, err := os.ReadFile(marker); err != nil || string(got) != "original" {
+		t.Fatalf("restored tree content=%q err=%v", got, err)
+	}
+	var count int
+	if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchanges WHERE host=?`, host).Scan(&count); err != nil || count != 1 {
+		t.Fatalf("restored index count=%d err=%v, want 1", count, err)
+	}
+}
+
+func TestStageDeleteHostsExactRollbackReportsRestoreFailure(t *testing.T) {
+	dir := t.TempDir()
+	tr, err := Open(dir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tr.Close()
+
+	const host = "conflict.example.com"
+	hostDir := filepath.Join(dir, host)
+	if err := os.MkdirAll(hostDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	stage, err := tr.StageDeleteHostsExact([]string{host})
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Simulate an out-of-band conflicting destination. Rollback must surface the
+	// failed rename instead of claiming the external data was restored.
+	if err := os.WriteFile(hostDir, []byte("conflict"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := stage.Rollback(); err == nil || !strings.Contains(err.Error(), "restore") {
+		t.Fatalf("rollback err=%v, want restore failure", err)
+	}
+	if _, err := os.Stat(stage.stageDir); err != nil {
+		t.Fatalf("staging was removed after failed restore: %v", err)
+	}
+}
+
+// TestDeleteHostGCBlobs verifies blob garbage collection: after a host's trees
+// are removed, blobs referenced by no remaining exchange are deleted, while
+// blobs still referenced (including shared ones) survive.
+func TestDeleteHostGCBlobs(t *testing.T) {
+	dir := t.TempDir()
+	tr, err := Open(dir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tr.Close()
+
+	// Two distinct blobs + one shared blob (referenced by two hosts).
+	blobA := filepath.Join(dir, "_blobs", "sha256", "aa", "aa", strings.Repeat("a", 64)+".bin")
+	blobB := filepath.Join(dir, "_blobs", "sha256", "bb", "bb", strings.Repeat("b", 64)+".bin")
+	blobC := filepath.Join(dir, "_blobs", "sha256", "cc", "cc", strings.Repeat("c", 64)+".bin")
+	for _, b := range []string{blobA, blobB, blobC} {
+		if err := os.MkdirAll(filepath.Dir(b), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(b, []byte("x"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	seed := func(id, h, ref string) {
+		exDir := filepath.Join(dir, h, "GET", id)
+		if err := os.MkdirAll(exDir, 0o755); err != nil {
+			t.Fatal(err)
+		}
+		body := "no blob"
+		if ref != "" {
+			body = "@blob sha256:" + ref + " (len=1)"
+		}
+		if err := os.WriteFile(filepath.Join(exDir, "request.http"), []byte("GET / HTTP/1.1\n\n"+body), 0o644); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(filepath.Join(exDir, "response.http"), []byte("HTTP 200 OK\n"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
+VALUES(?,?,?,?,?,?,?,?,?,?,?)`,
+			id, 1, h, "GET", "/", "http://"+h+"/", 200, "text/html", 0, 0, h+"/GET/"+id); err != nil {
+			t.Fatal(err)
+		}
+	}
+	ha := strings.Repeat("a", 64)
+	hb := strings.Repeat("b", 64)
+	hc := strings.Repeat("c", 64)
+	seed("1-0001", "a.example.com", ha) // sole reference to blobA
+	seed("1-0002", "b.example.com", hb) // sole reference to blobB
+	seed("1-0003", "c.example.com", hc) // shares blobC with d
+	seed("1-0004", "d.example.com", hc)
+
+	// Delete a: blobA orphaned → removed; blobB/blobC still referenced → kept.
+	if n, err := tr.DeleteHost("a.example"); err != nil || n != 1 {
+		t.Fatalf("DeleteHost(a.example)=%d, err=%v; want 1, nil", n, err)
+	}
+	if _, err := os.Stat(blobA); !os.IsNotExist(err) {
+		t.Fatalf("orphaned blobA still exists: %v", err)
+	}
+	if _, err := os.Stat(blobB); err != nil {
+		t.Fatalf("referenced blobB removed: %v", err)
+	}
+	if _, err := os.Stat(blobC); err != nil {
+		t.Fatalf("shared blobC removed while d still references it: %v", err)
+	}
+
+	// Delete c (shares blobC with d): blobC must survive.
+	if n, err := tr.DeleteHost("c.example"); err != nil || n != 1 {
+		t.Fatalf("DeleteHost(c.example)=%d, err=%v; want 1, nil", n, err)
+	}
+	if _, err := os.Stat(blobC); err != nil {
+		t.Fatalf("shared blobC removed after deleting one sharer: %v", err)
+	}
+
+	// Delete d: last reference gone → blobC collected.
+	if n, err := tr.DeleteHost("d.example"); err != nil || n != 1 {
+		t.Fatalf("DeleteHost(d.example)=%d, err=%v; want 1, nil", n, err)
+	}
+	if _, err := os.Stat(blobC); !os.IsNotExist(err) {
+		t.Fatalf("blobC still exists after last reference removed: %v", err)
+	}
+}

traffic/upgrade_test.go

@@ -0,0 +1,164 @@
+package traffic
+
+import (
+	"database/sql"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// openLegacyIndex builds the index exactly as the pre-reclamation Open did: a
+// plain-path DSN, pragmas via the pool, and auto_vacuum left at its default 0.
+func openLegacyIndex(t *testing.T, dir string) *sql.DB {
+	t.Helper()
+	if err := os.MkdirAll(filepath.Join(dir, "_index"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	old, err := sql.Open("sqlite", filepath.Join(dir, "_index", "index.sqlite"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, p := range []string{"PRAGMA journal_mode=WAL", "PRAGMA busy_timeout=5000"} {
+		if _, err := old.Exec(p); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if _, err := old.Exec(indexSchema); err != nil {
+		t.Fatal(err)
+	}
+	return old
+}
+
+// TestUpgradeFromOldInstall guards the upgrade path. Open now names the database
+// through a file: URI so per-connection pragmas can ride in the DSN, and a
+// driver that did not treat that as a URI would quietly open a file literally
+// named "file:/…" — an empty index, with every recorded exchange apparently
+// gone. The assertions below are what prove that does not happen.
+func TestUpgradeFromOldInstall(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "_index", "index.sqlite")
+	old := openLegacyIndex(t, dir)
+	if _, err := old.Exec(ftsSchema); err != nil {
+		t.Fatal(err)
+	}
+	// 三条历史流量，含一条 legacy path<>'' 的行
+	for i, row := range [][]any{
+		{"1700000000-0001", "old.example.com", ""},
+		{"1700000000-0002", "old.example.com", ""},
+		{"1700000000-0003", "legacy.example.com", "legacy.example.com/GET/x"},
+	} {
+		if _, err := old.Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
+VALUES(?,?,?,'GET','/x','http://x/x',200,'text/html',0,9,?)`, row[0], 1700000000+i, row[1], row[2]); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := old.Exec(`INSERT INTO exchange_bodies(id,req_head,req_body,resp_head,resp_body)
+VALUES(?,'GET /x','','HTTP 200','老数据正文')`, row[0]); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := old.Exec(`INSERT INTO ex_fts(rowid,content) VALUES(?,?)`, i+1, "老数据正文 secret-token"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := old.Close(); err != nil {
+		t.Fatal(err)
+	}
+	stat, err := os.Stat(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// ---- 新版本接管
+	tr, err := Open(dir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("新版本无法打开旧库: %v", err)
+	}
+	defer tr.Close()
+
+	// 1. 必须是同一个文件，不能悄悄开了个新空库
+	if st2, err := os.Stat(path); err != nil || st2.Size() == 0 {
+		t.Fatalf("原索引文件异常: size=%v err=%v", st2, err)
+	}
+	if entries, _ := os.ReadDir(filepath.Join(dir, "_index")); len(entries) > 3 {
+		for _, e := range entries {
+			t.Logf("_index 下: %s", e.Name())
+		}
+		t.Fatal("_index 下出现了预期外的文件，DSN 可能指向了别的库")
+	}
+	t.Logf("旧库 %d 字节，新版本接管后仍是同一文件", stat.Size())
+
+	// 2. 历史数据全部可见
+	n, err := tr.Count()
+	if err != nil || n != 3 {
+		t.Fatalf("Count=(%d,%v)，应为 (3,nil) —— 历史流量丢失", n, err)
+	}
+	// 3. 历史全文索引仍可搜
+	if tr.fts {
+		rows, err := tr.query("old.example.com", "", "secret-token", 0, 10)
+		if err != nil {
+			t.Fatalf("历史全文搜索失败: %v", err)
+		}
+		if len(rows) != 2 {
+			t.Fatalf("历史全文搜索命中 %d 条，应为 2", len(rows))
+		}
+	}
+	// 4. 历史正文仍可读
+	if _, resp, err := tr.Get("1700000000-0001"); err != nil {
+		t.Fatalf("读取历史正文失败: %v", err)
+	} else if resp == "" {
+		t.Fatal("历史响应为空")
+	}
+	// 5. 旧库不会被误判为已启用增量回收
+	if tr.incrementalVacuum {
+		t.Fatal("旧库被误判为已启用增量回收")
+	}
+	// 6. 删除仍然正常工作，且回收流程在旧库上能收敛
+	deleted, err := tr.DeleteHostsExact([]string{"old.example.com"})
+	if err != nil || deleted != 2 {
+		t.Fatalf("DeleteHostsExact=(%d,%v)，应为 (2,nil)", deleted, err)
+	}
+	tr.reaping.Wait()
+	if n, err := tr.Count(); err != nil || n != 1 {
+		t.Fatalf("删除后 Count=(%d,%v)，应为 (1,nil)", n, err)
+	}
+	// 7. legacy path<>'' 的行没被牵连
+	var legacyPath string
+	if err := tr.DB().QueryRow(`SELECT path FROM exchanges`).Scan(&legacyPath); err != nil {
+		t.Fatal(err)
+	}
+	if legacyPath == "" {
+		t.Fatal("legacy 行的 path 被清空了")
+	}
+}
+
+// TestDowngradeToOldBinary covers a rollback: a database created with
+// auto_vacuum=incremental must stay readable and writable by a build that knows
+// nothing about it. auto_vacuum only changes where SQLite tracks free pages, so
+// the old binary simply goes back to never returning them.
+func TestDowngradeToOldBinary(t *testing.T) {
+	dir := t.TempDir()
+	tr, err := Open(dir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !tr.incrementalVacuum {
+		t.Fatal("新库应启用增量回收")
+	}
+	bulkRecord(tr, "keep.example.com", 5, 100*1024)
+	if err := tr.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	old := openLegacyIndex(t, dir) // 旧版本二进制接管
+	defer old.Close()
+	var n int
+	if err := old.QueryRow(`SELECT COUNT(*) FROM exchanges`).Scan(&n); err != nil || n != 5 {
+		t.Fatalf("旧版本读到 (%d,%v)，应为 (5,nil)", n, err)
+	}
+	if _, err := old.Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
+VALUES('x',1,'new.example.com','GET','/x','http://x/x',200,'',0,0,'')`); err != nil {
+		t.Fatalf("旧版本写入失败: %v", err)
+	}
+	if _, err := old.Exec(`DELETE FROM exchanges WHERE host='keep.example.com'`); err != nil {
+		t.Fatalf("旧版本删除失败: %v", err)
+	}
+}