First Commit

agent/compaction.go

@@ -0,0 +1,548 @@
+package agent
+
+// cold-digest §4/§5/§7: the Compactor ties the pure algorithms (coldgraph.go)
+// to the store (db/digest.go) and the LLM. It runs in two modes:
+//
+//	maintain — cheap, synchronous, once per planner round: bump round_no,
+//	           recompute hot/cold, stamp/clear cold_since_round (§2.3). This is
+//	           the bookkeeping the planner does anyway; it never calls the LLM.
+//	minor/major — background, off the planner hot path (§7): group cold nodes
+//	           and compress each ≥2 block into a digest via the LLM. minor folds
+//	           only the not-yet-covered cold set (tiered append); major re-derives
+//	           the whole grouping from source and merges fragments (§5.1/§5.2),
+//	           reusing bodies whose signature is unchanged (§5.3).
+//
+// Concurrency: one compaction per task at a time (mutex), ≥cooldown between runs,
+// and a commit-time liveness recheck drops any member that revived while the body
+// was being generated so a digest never covers a hot node.
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/Autumn-27/artex/db"
+	"github.com/Autumn-27/norma/llm"
+	"github.com/Autumn-27/norma/transcript"
+)
+
+// Compactor performs background cold-node compaction for many explorations.
+type Compactor struct {
+	prov     llm.Provider
+	model    string
+	params   coldParams
+	n, m     int           // minor / major thresholds (§7 N=20, M=8)
+	cooldown time.Duration // min gap between compactions per task (§7 60s)
+	maxDur   time.Duration // hard cap on one background compaction
+
+	mu      sync.Mutex
+	running map[int64]bool
+	lastRun map[int64]time.Time
+}
+
+// NewCompactor builds a compactor. prov/model are used for the §4 body LLM call
+// (same model the agent runs on, per §4). A nil Compactor is a safe no-op.
+func NewCompactor(prov llm.Provider, model string) *Compactor {
+	return &Compactor{
+		prov:     prov,
+		model:    model,
+		params:   defaultColdParams(),
+		n:        20,
+		m:        8,
+		cooldown: 60 * time.Second,
+		maxDur:   5 * time.Minute,
+		running:  map[int64]bool{},
+		lastRun:  map[int64]time.Time{},
+	}
+}
+
+// OnPlannerRound is the single entry the planner calls each wake-up. It bumps the
+// round, maintains the cold stamps synchronously, then (if a threshold is hit and
+// no compaction is running / cooling down) launches a background compaction that
+// outlives this planner round.
+func (c *Compactor) OnPlannerRound(ctx context.Context, ts *db.ExplorationStore) {
+	if c == nil || c.prov == nil || ts == nil {
+		return
+	}
+	round, uncompressed, activeDigests, err := c.maintain(ts)
+	if err != nil {
+		log.Printf("[compaction] maintain exp=%d: %v", ts.ID(), err)
+		return
+	}
+	needMinor := uncompressed >= c.n
+	needMajor := activeDigests >= c.m
+	if !needMinor && !needMajor {
+		return
+	}
+	if !c.tryStart(ts.ID()) {
+		return // already running, or within cooldown —派生态最终一致，下轮再压
+	}
+	go func() {
+		defer c.finish(ts.ID())
+		bg, cancel := context.WithTimeout(context.WithoutCancel(ctx), c.maxDur)
+		defer cancel()
+		// 压缩是裸 provider 调用（compress 里直接 prov.Complete），不经过 agentcore
+		// 的会话循环，所以 ctx 上没有 session id；按 session-id 头做提示缓存/粘性
+		// 路由的网关（opencode zen 缺 x-opencode-session 直接 400）就收不到该头。
+		// 这里补一个按探索稳定的 id：同一探索的所有压缩请求共享它，既能带上头，
+		// 也让 llmrec 能把这次调用的 token 归因回该探索（此前记不到）。
+		bg = transcript.WithSessionID(bg, fmt.Sprintf("exp%d-compactor", ts.ID()))
+		if needMajor {
+			c.major(bg, ts)
+		} else {
+			c.minor(bg, ts)
+		}
+	}()
+	_ = round
+}
+
+// maintain bumps round_no, recomputes hot/cold over the whole graph, and applies
+// the cold_since_round stamp/clear ops (§2.3). Returns the new round plus the
+// counts that drive the trigger: how many eligible-cold nodes are not yet covered
+// (minor) and how many active digests exist (major).
+func (c *Compactor) maintain(ts *db.ExplorationStore) (round int64, uncompressed, activeDigests int, err error) {
+	round, err = ts.BumpRound()
+	if err != nil {
+		return
+	}
+	g, _, err := loadColdGraph(ts)
+	if err != nil {
+		return
+	}
+	stamps, err := ts.ColdStamps()
+	if err != nil {
+		return
+	}
+	hot := g.hotSet()
+	structCold := g.structuralCold(hot)
+	ops := computeStampOps(structCold, stamps, round)
+	if err = ts.ApplyStampOps(toDBStampOps(ops)); err != nil {
+		return
+	}
+	applyStampsInPlace(stamps, ops)
+	elig := g.eligibleCold(structCold, stamps, round, c.params)
+	covered, err := ts.CoveredMembers()
+	if err != nil {
+		return
+	}
+	for id := range elig {
+		if _, ok := covered[id]; !ok {
+			uncompressed++
+		}
+	}
+	ad, err := ts.ActiveDigests()
+	if err != nil {
+		return
+	}
+	activeDigests = len(ad)
+	return
+}
+
+// minor folds the not-yet-covered eligible-cold set into new digest segments
+// (tiered append, §5). Existing digests are untouched.
+func (c *Compactor) minor(ctx context.Context, ts *db.ExplorationStore) {
+	round, err := ts.RoundNo()
+	if err != nil {
+		return
+	}
+	g, nodeByID, err := loadColdGraph(ts)
+	if err != nil {
+		return
+	}
+	stamps, err := ts.ColdStamps()
+	if err != nil {
+		return
+	}
+	cvers, err := ts.ContentVersions()
+	if err != nil {
+		return
+	}
+	covered, err := ts.CoveredMembers()
+	if err != nil {
+		return
+	}
+	hot := g.hotSet()
+	elig := g.eligibleCold(g.structuralCold(hot), stamps, round, c.params)
+	uncompressed := map[int64]bool{}
+	for id := range elig {
+		if _, ok := covered[id]; !ok {
+			uncompressed[id] = true
+		}
+	}
+	blocks := g.group(uncompressed, c.params)
+	if len(blocks) == 0 {
+		return // this batch has no ≥2 connected/shared-parent block — nothing to fold (§7)
+	}
+	for _, b := range blocks {
+		c.foldBlock(ctx, ts, g, b, nodeByID, cvers, c.generationFor(b, nil))
+	}
+	// A minor may have pushed the segment count over M → merge in the same run.
+	if ad, e := ts.ActiveDigests(); e == nil && len(ad) >= c.m {
+		c.major(ctx, ts)
+	}
+}
+
+// major re-derives the whole grouping from source over ALL eligible-cold nodes
+// (§5.1 回源重压), then reconciles against the active digests by signature:
+// unchanged blocks keep their digest (no LLM), stale digests are superseded, and
+// new/changed blocks are compressed afresh. This is where tiered fragments of one
+// direction merge and where "later became connected" blocks unify (§5.2).
+func (c *Compactor) major(ctx context.Context, ts *db.ExplorationStore) {
+	round, err := ts.RoundNo()
+	if err != nil {
+		return
+	}
+	g, nodeByID, err := loadColdGraph(ts)
+	if err != nil {
+		return
+	}
+	stamps, err := ts.ColdStamps()
+	if err != nil {
+		return
+	}
+	cvers, err := ts.ContentVersions()
+	if err != nil {
+		return
+	}
+	active, err := ts.ActiveDigests()
+	if err != nil {
+		return
+	}
+	hot := g.hotSet()
+	elig := g.eligibleCold(g.structuralCold(hot), stamps, round, c.params)
+	blocks := g.group(elig, c.params)
+
+	bySig := map[string]*db.Node{}
+	for _, d := range active {
+		sig, _ := digestSigGen(d)
+		bySig[sig] = d
+	}
+	desired := map[string]bool{}
+	var toCreate []block
+	for _, b := range blocks {
+		sig := blockSignature(b, cvers)
+		desired[sig] = true
+		if _, ok := bySig[sig]; ok {
+			continue // unchanged → reuse the existing digest, skip LLM (§5.3)
+		}
+		toCreate = append(toCreate, b)
+	}
+	// Supersede stale digests FIRST (atomic drop of their covers edges) so a member
+	// is never covered by both an old and a new digest (§5.1 one-member-one-digest).
+	var stale []int64
+	for _, d := range active {
+		sig, _ := digestSigGen(d)
+		if !desired[sig] {
+			stale = append(stale, d.ID)
+		}
+	}
+	if err := ts.SupersedeDigests(stale); err != nil {
+		log.Printf("[compaction] supersede exp=%d: %v", ts.ID(), err)
+	}
+	for _, b := range toCreate {
+		c.foldBlock(ctx, ts, g, b, nodeByID, cvers, c.generationFor(b, active))
+	}
+}
+
+// foldBlock compresses one block and writes its digest — with a commit-time
+// liveness recheck (§ concurrency): between grouping and write the graph may have
+// changed, so any member that has since gone hot (revived) is dropped from the
+// covers set. If the block dissolves below K it is skipped.
+func (c *Compactor) foldBlock(ctx context.Context, ts *db.ExplorationStore, g *coldGraph, b block, nodeByID map[int64]*db.Node, cvers map[int64]int, generation int) {
+	body, err := c.compress(ctx, g, b, nodeByID)
+	if err != nil {
+		log.Printf("[compaction] compress exp=%d block=%v: %v", ts.ID(), b.Members, err)
+		return
+	}
+	// Re-read fresh state and drop any member that revived while we compressed.
+	fresh, _, err := loadColdGraph(ts)
+	if err != nil {
+		return
+	}
+	freshHot := fresh.hotSet()
+	members := make([]int64, 0, len(b.Members))
+	for _, mID := range b.Members {
+		if !freshHot[mID] {
+			members = append(members, mID)
+		}
+	}
+	if len(members) < c.params.K {
+		return // block revived out from under us — leave those nodes hot, don't fold
+	}
+	final := block{Members: members, Anchors: b.Anchors}
+	payload := digestPayload(body, final, nodeByID, generation, blockSignature(final, cvers))
+	if _, err := ts.AddDigest(payload, members); err != nil {
+		log.Printf("[compaction] add digest exp=%d: %v", ts.ID(), err)
+	}
+}
+
+// generationFor computes a digest's重摘代次 (§1): 1 for a fresh fold; for a major
+// merge, max(generation) over the active digests that overlap this block's
+// members, +1.
+func (c *Compactor) generationFor(b block, active []*db.Node) int {
+	if len(active) == 0 {
+		return 1
+	}
+	memberSet := make(map[int64]bool, len(b.Members))
+	for _, m := range b.Members {
+		memberSet[m] = true
+	}
+	best := 0
+	for _, d := range active {
+		_, gen := digestSigGen(d)
+		for _, m := range digestMemberIDs(d) {
+			if memberSet[m] {
+				if gen > best {
+					best = gen
+				}
+				break
+			}
+		}
+	}
+	return best + 1
+}
+
+// tryStart acquires the per-task compaction lock, honoring the cooldown.
+func (c *Compactor) tryStart(expID int64) bool {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.running[expID] {
+		return false
+	}
+	if t, ok := c.lastRun[expID]; ok && time.Since(t) < c.cooldown {
+		return false
+	}
+	c.running[expID] = true
+	return true
+}
+
+func (c *Compactor) finish(expID int64) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.running[expID] = false
+	c.lastRun[expID] = time.Now()
+}
+
+// --- helpers: db ↔ coldgraph ---
+
+// loadColdGraph reads the exploration's nodes + edges and builds the cold-graph
+// view plus an id→node index (for summaries/payload during compression).
+func loadColdGraph(ts *db.ExplorationStore) (*coldGraph, map[int64]*db.Node, error) {
+	// Compaction must see the WHOLE graph, not the default row caps — pass a very
+	// high limit so the LIMIT clause is effectively unbounded for real task sizes.
+	const allRows = 1 << 30
+	nodes, err := ts.Nodes(allRows)
+	if err != nil {
+		return nil, nil, err
+	}
+	edges, err := ts.Edges(allRows)
+	if err != nil {
+		return nil, nil, err
+	}
+	cgNodes := make([]cgNode, 0, len(nodes))
+	byID := make(map[int64]*db.Node, len(nodes))
+	for _, n := range nodes {
+		cgNodes = append(cgNodes, cgNode{ID: n.ID, Kind: n.Kind, State: n.State})
+		byID[n.ID] = n
+	}
+	cgEdges := make([]cgEdge, 0, len(edges))
+	for _, e := range edges {
+		cgEdges = append(cgEdges, cgEdge{From: e.From, Rel: e.Rel, To: e.To})
+	}
+	return newColdGraph(cgNodes, cgEdges), byID, nil
+}
+
+func toDBStampOps(ops []stampOp) []db.StampOp {
+	out := make([]db.StampOp, len(ops))
+	for i, o := range ops {
+		out[i] = db.StampOp{ID: o.ID, Set: o.Set, Round: o.Round}
+	}
+	return out
+}
+
+// applyStampsInPlace folds the just-applied ops into the in-memory stamp map so
+// eligibility can be computed immediately without a re-read.
+func applyStampsInPlace(stamps map[int64]*int64, ops []stampOp) {
+	for _, o := range ops {
+		if o.Set {
+			r := o.Round
+			stamps[o.ID] = &r
+		} else {
+			stamps[o.ID] = nil
+		}
+	}
+}
+
+// --- helpers: digest payload ---
+
+// digestPayload builds the digest node payload (cold-digest §1): the body, the
+// member ids split by kind (restore cache; source of truth is the covers edges),
+// the anchor ids, the generation, and the change-detection signature.
+func digestPayload(body string, b block, nodeByID map[int64]*db.Node, generation int, signature string) map[string]any {
+	var facts, intents []int64
+	for _, m := range b.Members {
+		if n := nodeByID[m]; n != nil && n.Kind == db.KindIntent {
+			intents = append(intents, m)
+		} else {
+			facts = append(facts, m)
+		}
+	}
+	return map[string]any{
+		"body":       body,
+		"member_ids": map[string]any{"facts": facts, "intents": intents},
+		"anchor_ids": b.Anchors,
+		"generation": generation,
+		"signature":  signature,
+	}
+}
+
+func digestSigGen(n *db.Node) (string, int) {
+	var p struct {
+		Signature  string `json:"signature"`
+		Generation int    `json:"generation"`
+	}
+	_ = json.Unmarshal(n.Payload, &p)
+	return p.Signature, p.Generation
+}
+
+func digestMemberIDs(n *db.Node) []int64 {
+	var p struct {
+		MemberIDs struct {
+			Facts   []int64 `json:"facts"`
+			Intents []int64 `json:"intents"`
+		} `json:"member_ids"`
+	}
+	_ = json.Unmarshal(n.Payload, &p)
+	return append(append([]int64{}, p.MemberIDs.Facts...), p.MemberIDs.Intents...)
+}
+
+// --- helpers: compression input + LLM (§4) ---
+
+func nodeSummary(n *db.Node) string {
+	if n == nil {
+		return ""
+	}
+	var p map[string]any
+	if json.Unmarshal(n.Payload, &p) == nil {
+		if s, ok := p["summary"].(string); ok {
+			return s
+		}
+		if t, ok := p["text"].(string); ok {
+			return t
+		}
+	}
+	return ""
+}
+
+func nodeConfidence(n *db.Node) string {
+	if n == nil {
+		return ""
+	}
+	var p map[string]any
+	if json.Unmarshal(n.Payload, &p) == nil {
+		if c, ok := p["confidence"].(string); ok {
+			return c
+		}
+	}
+	return ""
+}
+
+// buildCompressionInput renders the connected sub-graph for the §4 prompt:
+// member nodes (summary + id + kind + state + confidence), the internal blood
+// edges among members, and — for a §3.1 shared-parent group — the anchor parents
+// as context ("共同父 #p"), which are NOT members.
+func buildCompressionInput(g *coldGraph, b block, nodeByID map[int64]*db.Node) string {
+	memberSet := make(map[int64]bool, len(b.Members))
+	for _, m := range b.Members {
+		memberSet[m] = true
+	}
+	var sb strings.Builder
+	sb.WriteString("【成员节点（要压缩的）】：\n")
+	for _, m := range b.Members {
+		n := nodeByID[m]
+		kind := "fact"
+		if n != nil && n.Kind == db.KindIntent {
+			kind = "intent"
+		}
+		state := ""
+		if n != nil {
+			state = n.State
+		}
+		line := fmt.Sprintf("- #%d [%s/%s] %s", m, kind, state, nodeSummary(n))
+		if conf := nodeConfidence(n); conf != "" {
+			line += fmt.Sprintf(" (confidence=%s)", conf)
+		}
+		sb.WriteString(line)
+		sb.WriteByte('\n')
+	}
+	// internal edges among members
+	var edgeLines []string
+	for _, m := range b.Members {
+		for _, to := range g.children[m] {
+			if memberSet[to] {
+				edgeLines = append(edgeLines, fmt.Sprintf("- #%d 产出/派生→ #%d", m, to))
+			}
+		}
+	}
+	if len(edgeLines) > 0 {
+		sb.WriteString("\n【成员之间的血缘边（父→子）】：\n")
+		sort.Strings(edgeLines)
+		sb.WriteString(strings.Join(edgeLines, "\n"))
+		sb.WriteByte('\n')
+	}
+	if len(b.Anchors) > 0 {
+		sb.WriteString("\n【共同父 / 上下文锚（不是成员，只用于理解这些结果从哪个意图探出）】：\n")
+		for _, a := range b.Anchors {
+			n := nodeByID[a]
+			state := ""
+			if n != nil {
+				state = n.State
+			}
+			fmt.Fprintf(&sb, "- #%d [%s] %s\n", a, state, nodeSummary(n))
+		}
+	}
+	return sb.String()
+}
+
+// compress runs the §4 body LLM call on one block. Uses the same model the agent
+// runs on; thinking disabled (a pure summarization step).
+func (c *Compactor) compress(ctx context.Context, g *coldGraph, b block, nodeByID map[int64]*db.Node) (string, error) {
+	req := llm.CompletionRequest{
+		System:    []string{compressionSystemPrompt},
+		Messages:  []llm.Message{llm.UserText(buildCompressionInput(g, b, nodeByID))},
+		MaxTokens: 1500,
+		Thinking:  "disabled",
+	}
+	msg, _, _, err := c.prov.Complete(ctx, req)
+	if err != nil {
+		return "", err
+	}
+	body := strings.TrimSpace(msg.Text())
+	if body == "" {
+		return "", fmt.Errorf("empty body from model")
+	}
+	return body, nil
+}
+
+// compressionSystemPrompt is the §4 body prompt.
+const compressionSystemPrompt = `你在压缩一组【彼此关联】的探索节点，产出一段综合结论(body)，供规划者快速掌握"这一片已经探明了什么"。
+
+输入是一个连通子图：
+- 节点：每条是一个意图或事实的 summary（一句话），带 id、类型(intent/fact)、state、confidence(若有)。
+- 关系：节点之间的血缘边（A 派生自 B / A 产出 B），说明它们如何串联。
+- 若节点间没有直接血缘边、但同属一个上游意图（会另给出该上游意图作为"共同父 #p"），则按"这个意图（#p）探到了什么"来综合它们——共同父只是上下文锚、不是要压缩的成员。
+
+据此写一段 body：
+1. 综合、不罗列：顺着关系把因果串起来（哪个事实催生哪个意图、哪条意图产出了哪个结论），讲成"这一片探索得出了什么"，不要把每条 summary 抄一遍。
+2. 保留区分度：彼此不同的结论分别说清，别揉成一句笼统的话。
+3. 保留证据强度：带 confidence 的结论标出 observed / inferred；inferred 的否定/存疑结论要点明它只是推断、可复核，别写成定论。
+4. 带上 id：每条结论后标注来源节点 id（如"…（#12,#28）"），让规划者能按 id 还原原节点。
+5. 正向陈述、只写输入里有的：不脑补、不引入输入中没有的判断。
+6. 长度随内容自适应：结论少就短，多且互不相同就写够——但整体显著短于所有输入 summary 的总和。
+
+只输出 body 正文本身。`