First Commit

llmpool/health.go

@@ -0,0 +1,184 @@
+// Package llmpool implements LLM failover ("轮询"): a Provider decorator that
+// walks an ordered chain of LLM profiles and moves to the next one when the
+// current one can't serve the request — out of credit, revoked key, rate-limited
+// past the SDK's own retries, or down.
+//
+// The chain is built by the server from llm_profiles (active profile first, then
+// by priority), so the whole process shares ONE chain and ONE circuit-breaker
+// registry: when a task discovers a profile is out of credit, every other task
+// skips it immediately.
+package llmpool
+
+import (
+	"sync"
+	"time"
+)
+
+// backoff is the cooling-off ladder, indexed by trip count: the 1st trip cools
+// for a minute, the 2nd for five, everything after that for half an hour. A
+// profile that is merely rate-limited recovers quickly; one that keeps failing
+// stops being probed every round.
+var backoff = []time.Duration{time.Minute, 5 * time.Minute, 30 * time.Minute}
+
+// State is one profile's circuit-breaker state.
+type State struct {
+	Fails     int       // consecutive failures; reset by a success
+	Trips     int       // total trips, indexes the backoff ladder
+	OpenUntil time.Time // zero / past = closed (usable)
+	LastError string
+	LastAt    time.Time
+}
+
+// Open reports whether the breaker is currently open (profile should be skipped).
+func (s State) Open() bool { return time.Now().Before(s.OpenUntil) }
+
+// softTripAfter is the DEFAULT number of consecutive TRANSIENT failures (429 /
+// 5xx / network) that trip the breaker. Deterministic failures (no credit, bad
+// key) trip on the first regardless. Overridable via Registry.SetPolicy.
+const softTripAfter = 3
+
+// Registry holds the circuit-breaker state of every profile, keyed by profile id.
+// It is process-wide and outlives any single Pool instance, so rebuilding the pool
+// (saving an unrelated profile, toggling a setting) never clears what we've learned
+// about which backends are broken.
+type Registry struct {
+	mu sync.Mutex
+	m  map[int64]*State
+
+	// persist / forget mirror state to the DB so a cooling-off window survives a
+	// restart. Both may be nil (no DB); both are called off the hot path.
+	persist func(id int64, st State)
+	forget  func(id int64)
+
+	// softTrip / cooldown are the operator-set overrides (0 = use the built-in
+	// default / ladder). They live here rather than being read per failure because
+	// Trip runs on the failure path of every request.
+	softTrip int
+	cooldown time.Duration
+}
+
+// SetPolicy overrides the breaker's two knobs. softTrip: how many consecutive
+// transient failures trip it (0 = default softTripAfter; negative = transient
+// failures never trip it, leaving only the deterministic ones). cooldown: a fixed
+// cooling-off window (0 = the 1min/5min/30min ladder). Safe to call at any time.
+func (r *Registry) SetPolicy(softTrip int, cooldown time.Duration) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.softTrip, r.cooldown = softTrip, cooldown
+}
+
+// tripAfter is the effective consecutive-transient-failure threshold. Callers
+// hold r.mu. A negative override yields 0, which Trip reads as "never soft-trip".
+func (r *Registry) tripAfter() int {
+	if r.softTrip == 0 {
+		return softTripAfter
+	}
+	return max(r.softTrip, 0)
+}
+
+// coolFor is the cooling-off window for the trips-th trip (1-based): the fixed
+// override when set, else the ladder (last rung repeats). Callers hold r.mu.
+func (r *Registry) coolFor(trips int) time.Duration {
+	if r.cooldown > 0 {
+		return r.cooldown
+	}
+	if trips-1 < len(backoff) {
+		return backoff[trips-1]
+	}
+	return backoff[len(backoff)-1]
+}
+
+// NewRegistry builds an empty registry. persist/forget may be nil.
+func NewRegistry(persist func(id int64, st State), forget func(id int64)) *Registry {
+	return &Registry{m: map[int64]*State{}, persist: persist, forget: forget}
+}
+
+// Restore seeds state loaded from the DB at startup (bypasses persistence).
+func (r *Registry) Restore(id int64, st State) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	cp := st
+	r.m[id] = &cp
+}
+
+// Get returns a copy of one profile's state.
+func (r *Registry) Get(id int64) State {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if st := r.m[id]; st != nil {
+		return *st
+	}
+	return State{}
+}
+
+// IsOpen reports whether a profile is in its cooling-off window.
+func (r *Registry) IsOpen(id int64) bool { return r.Get(id).Open() }
+
+// Pass records a successful call: clears the failure counters so a profile that
+// recovers is fully trusted again (and drops the persisted row).
+func (r *Registry) Pass(id int64) {
+	r.mu.Lock()
+	st := r.m[id]
+	if st == nil || (st.Fails == 0 && st.Trips == 0 && st.OpenUntil.IsZero()) {
+		r.mu.Unlock()
+		return // already clean — nothing to write
+	}
+	delete(r.m, id)
+	forget := r.forget
+	r.mu.Unlock()
+	if forget != nil {
+		forget(id)
+	}
+}
+
+// Trip records a failed call. hard=true marks a deterministic failure (no credit,
+// invalid key, missing model) which opens the breaker immediately; hard=false is a
+// transient one (429 / 5xx / network) that needs softTripAfter in a row. Returns
+// true when this call is what opened the breaker, so the caller can log it once.
+func (r *Registry) Trip(id int64, errMsg string, hard bool) (tripped bool) {
+	r.mu.Lock()
+	st := r.m[id]
+	if st == nil {
+		st = &State{}
+		r.m[id] = st
+	}
+	st.Fails++
+	st.LastError = errMsg
+	st.LastAt = time.Now()
+	softTrip := r.tripAfter()
+	if hard || (softTrip > 0 && st.Fails >= softTrip) {
+		st.Trips++
+		st.OpenUntil = time.Now().Add(r.coolFor(st.Trips))
+		st.Fails = 0 // counted into this trip; start fresh for the half-open probe
+		tripped = true
+	}
+	snap := *st
+	persist := r.persist
+	r.mu.Unlock()
+	if persist != nil {
+		persist(id, snap)
+	}
+	return tripped
+}
+
+// Reset clears one profile's state — the UI's "立即恢复" action.
+func (r *Registry) Reset(id int64) {
+	r.mu.Lock()
+	delete(r.m, id)
+	forget := r.forget
+	r.mu.Unlock()
+	if forget != nil {
+		forget(id)
+	}
+}
+
+// Snapshot returns a copy of all tracked state, for the status API.
+func (r *Registry) Snapshot() map[int64]State {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	out := make(map[int64]State, len(r.m))
+	for id, st := range r.m {
+		out[id] = *st
+	}
+	return out
+}