9mark9's picture
Add lethe live demo
8c7d6b7 verified
Raw History Blame Contribute Delete
26 kB
// Package memory is the store: it keeps memories per namespace, finds
// candidates by keyword and by embedding, lets the policy choose which to
// return, and feeds reward back into the policy.
package memory
import (
"container/list"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"math"
// Exploration sampling has to be seedable so experiments can be repeated;
// it is not used for anything secret. IDs come from crypto/rand.
mrand "math/rand/v2" // nosemgrep: go.lang.security.audit.crypto.math_random.math-random-used
"os"
"slices"
"sort"
"sync"
"sync/atomic"
"time"
"github.com/shaikn6/lethe/internal/index"
"github.com/shaikn6/lethe/internal/policy"
)
// Limits on what a caller may store.
const (
MaxTextBytes = 16 << 10
MaxNamespaceBytes = 128
MaxEmbeddingDim = 4096
MaxMetadata = 32
MaxMetadataKeyLen = 128
MaxMetadataValLen = 1024
MaxK = 100
// Neighbours are memories stored next to each other within this gap, such
// as consecutive turns of one conversation.
defaultNeighborGap = time.Hour
neighborReach = 2 // how many positions away still counts
neighborSeeds = 25 // best hits per search method whose neighbours join the pool
compactSuffix = ".compact"
defaultCompactSlack = 1 << 20
defaultMaxMemories = 1_000_000
defaultCompact = 1000
)
var (
// ErrInvalid marks a request the caller has to fix.
ErrInvalid = errors.New("invalid request")
// ErrNotFound marks an unknown memory or recall.
ErrNotFound = errors.New("not found")
// ErrClosed means the store was closed; nothing is accepted any more.
ErrClosed = errors.New("store is closed")
// ErrFull means the store holds as many memories as it is allowed to.
ErrFull = errors.New("store is full")
)
// Memory is one stored item.
type Memory struct {
ID string `json:"id"`
Namespace string `json:"namespace"`
Text string `json:"text"`
Embedding []float32 `json:"embedding,omitempty"`
Importance float64 `json:"importance"`
CreatedAt time.Time `json:"created_at"`
Metadata map[string]string `json:"metadata,omitempty"`
}
// Recalled is a memory returned for a query.
type Recalled struct {
ID string `json:"id"`
Text string `json:"text"`
Importance float64 `json:"importance"`
CreatedAt time.Time `json:"created_at"`
Metadata map[string]string `json:"metadata,omitempty"`
Score float64 `json:"score"`
}
// Recall is the answer to one query. Its ID is what feedback refers to.
type Recall struct {
ID string `json:"recall_id"`
Memories []Recalled `json:"memories"`
}
// Query is one recall request.
type Query struct {
Namespace string
Text string
Embedding []float32
K int
AsOf time.Time // zero means now
Explore bool // sample from the policy so feedback can teach it
}
// Event describes one thing the store did. It carries ids and outcomes
// only, never the text of a memory or a query.
type Event struct {
Type string `json:"type"` // remember, forget, recall or feedback
At time.Time `json:"at"`
Namespace string `json:"namespace"`
RecallID string `json:"recall_id,omitempty"`
MemoryIDs []string `json:"memory_ids,omitempty"`
Explore bool `json:"explore,omitempty"`
Reward *float64 `json:"reward,omitempty"`
Useful []string `json:"useful,omitempty"`
}
// Options configures a Store. The zero value is an in-memory store.
type Options struct {
JournalPath string // empty keeps everything in memory only
Fsync bool // fsync every journal write
Candidates int // per search method, default 50
MaxPending int // recalls kept waiting for feedback, default 4096
MaxMemories int // live memories across all namespaces, default 1,000,000
CompactAfter int // forgets before the journal is rewritten, default 1000
CompactSlack int64 // journal growth in bytes that Compact ignores, default 1 MiB
Seed uint64 // exploration seed, 0 picks a random one
Now func() time.Time
// OnEvent, when set, is told about every successful operation. It is
// called while the store holds its locks, so it must return at once.
OnEvent func(Event)
NeighborGap time.Duration // how close in time adjacent memories must be to count as neighbours, default 1h
LearningRate float64 // 0 keeps the policy default
Temperature float64 // 0 keeps the policy default
}
// Stats is a snapshot for health checks and monitoring.
type Stats struct {
Memories int `json:"memories"`
Namespaces int `json:"namespaces"`
Pending int `json:"pending_recalls"`
Updates int `json:"policy_updates"`
Baseline float64 `json:"reward_baseline"`
Weights policy.Features `json:"weights"`
JournalErrors uint64 `json:"journal_errors"`
// JournalHealthy is false once the journal file can no longer be
// written safely; the store then refuses writes instead of losing them.
JournalHealthy bool `json:"journal_healthy"`
// ForgottenOnDisk counts forgotten memories whose text is still in the
// journal file, waiting for the next compaction. It should return to 0.
ForgottenOnDisk int `json:"forgotten_on_disk"`
}
type entry struct {
Memory
norm float64
recalls int
rewardSum float64
alive bool
}
func (e *entry) utility() float64 { return (e.rewardSum + 1) / (float64(e.recalls) + 2) }
type space struct {
ix *index.Index
entries []*entry // by index document number
byID map[string]uint32
dim int
live int
}
func newSpace() *space { return &space{ix: index.New(), byID: make(map[string]uint32)} }
func (sp *space) add(e *entry) {
sp.byID[e.ID] = sp.ix.Add(e.Text)
sp.entries = append(sp.entries, e)
sp.live++
}
type pending struct {
id string
namespace string
ids []string
features []policy.Features
slate []int
explore bool
}
// Store is safe for concurrent use. When both locks are needed, dataMu is
// taken before learnMu. Every journal write happens under dataMu, which is
// what lets a compaction take a consistent snapshot.
type Store struct {
opts Options
dataMu sync.RWMutex
spaces map[string]*space
closed bool
live int
tombstones int // forgets since the journal was last rewritten
compacted int64 // journal size right after the last rewrite
learnMu sync.Mutex
policy *policy.Policy
rng *mrand.Rand
waiting map[string]*list.Element
order *list.List // oldest pending recall at the front
journal *journal
journalErrors atomic.Uint64
}
// Open creates a store and, when a journal path is set, replays it.
func Open(opts Options) (*Store, error) {
if opts.Candidates <= 0 {
opts.Candidates = 50
}
if opts.MaxPending <= 0 {
opts.MaxPending = 4096
}
if opts.NeighborGap <= 0 {
opts.NeighborGap = defaultNeighborGap
}
if opts.MaxMemories <= 0 {
opts.MaxMemories = defaultMaxMemories
}
if opts.CompactAfter <= 0 {
opts.CompactAfter = defaultCompact
}
if opts.CompactSlack <= 0 {
opts.CompactSlack = defaultCompactSlack
}
if opts.Now == nil {
opts.Now = time.Now
}
seed := opts.Seed
if seed == 0 {
seed = mrand.Uint64()
}
pol := policy.New()
if opts.LearningRate > 0 {
pol.LearningRate = opts.LearningRate
}
if opts.Temperature > 0 {
pol.Temperature = opts.Temperature
}
s := &Store{
opts: opts,
spaces: make(map[string]*space),
policy: pol,
rng: mrand.New(mrand.NewPCG(seed, seed^0x9e3779b97f4a7c15)),
waiting: make(map[string]*list.Element),
order: list.New(),
}
if opts.JournalPath != "" {
// A compaction interrupted by a crash leaves its unfinished copy
// behind; it may hold memories that were forgotten since.
if err := os.Remove(opts.JournalPath + compactSuffix); err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, err
}
j, err := openJournal(opts.JournalPath, opts.Fsync, s.replay)
if err != nil {
return nil, err
}
s.journal = j
s.compacted = j.bytes()
// Forgotten memories are still in the file that was just replayed.
// Rewrite it now so they do not outlive this start, and refuse to
// start if that is impossible rather than keep them silently.
if s.tombstones > 0 {
if err := s.compactLocked(); err != nil {
j.close()
return nil, err
}
}
}
return s, nil
}
// Close erases forgotten memories from the journal and releases it. An
// error means some forgotten text may still be on disk; the next Open will
// try again.
func (s *Store) Close() error {
s.dataMu.Lock()
defer s.dataMu.Unlock()
if s.closed {
return nil
}
s.closed = true
if s.journal == nil {
return nil
}
var erase error
if s.tombstones > 0 {
erase = s.compactLocked()
}
return errors.Join(erase, s.journal.close())
}
func (s *Store) replay(rec record) error {
switch {
case rec.Remember != nil:
e, err := s.insert(*rec.Remember)
if err != nil {
return err
}
e.recalls, e.rewardSum = rec.Recalls, rec.RewardSum
case rec.Forget != nil:
s.remove(rec.Forget.Namespace, rec.Forget.ID)
case rec.Learn != nil:
s.policy.Weights = rec.Learn.Weights
s.policy.Baseline = rec.Learn.Baseline
s.policy.ItemBaseline = rec.Learn.ItemBaseline
s.policy.Updates = rec.Learn.Updates
s.credit(rec.Learn.Namespace, rec.Learn.IDs, rec.Learn.Reward, rec.Learn.ItemRewards)
}
return nil
}
// Remember stores a memory and returns it with its ID and timestamp filled in.
func (s *Store) Remember(m Memory) (Memory, error) {
if err := validate(m.Namespace, m.Text, m.Embedding); err != nil {
return Memory{}, err
}
if len(m.Metadata) > MaxMetadata {
return Memory{}, fmt.Errorf("%w: at most %d metadata entries", ErrInvalid, MaxMetadata)
}
for key, value := range m.Metadata {
if len(key) > MaxMetadataKeyLen || len(value) > MaxMetadataValLen {
return Memory{}, fmt.Errorf("%w: metadata keys are limited to %d bytes and values to %d", ErrInvalid, MaxMetadataKeyLen, MaxMetadataValLen)
}
}
if math.IsNaN(m.Importance) || m.Importance < 0 || m.Importance > 1 {
return Memory{}, fmt.Errorf("%w: importance must be between 0 and 1", ErrInvalid)
}
if m.Text == "" {
return Memory{}, fmt.Errorf("%w: text is empty", ErrInvalid)
}
m.ID = newID()
if m.CreatedAt.IsZero() {
m.CreatedAt = s.opts.Now().UTC()
}
s.dataMu.Lock()
defer s.dataMu.Unlock()
if s.closed {
return Memory{}, ErrClosed
}
if s.live >= s.opts.MaxMemories {
return Memory{}, ErrFull
}
if sp := s.spaces[m.Namespace]; sp != nil && sp.dim != 0 && len(m.Embedding) != 0 && len(m.Embedding) != sp.dim {
return Memory{}, fmt.Errorf("%w: embedding has %d dimensions, namespace uses %d", ErrInvalid, len(m.Embedding), sp.dim)
}
// Journal first: a memory is only acknowledged once it is on disk.
if s.journal != nil {
if err := s.journal.append(record{Remember: &m}); err != nil {
s.journalErrors.Add(1)
return Memory{}, fmt.Errorf("journal write failed: %w", err)
}
}
if _, err := s.insert(m); err != nil {
return Memory{}, err
}
s.emit(Event{Type: "remember", Namespace: m.Namespace, MemoryIDs: []string{m.ID}})
return m, nil
}
// insert adds m to its namespace. The caller holds dataMu or is replaying.
func (s *Store) insert(m Memory) (*entry, error) {
sp := s.spaces[m.Namespace]
if sp == nil {
sp = newSpace()
s.spaces[m.Namespace] = sp
}
if len(m.Embedding) != 0 {
if sp.dim == 0 {
sp.dim = len(m.Embedding)
} else if sp.dim != len(m.Embedding) {
return nil, fmt.Errorf("%w: embedding has %d dimensions, namespace uses %d", ErrInvalid, len(m.Embedding), sp.dim)
}
}
e := &entry{Memory: m, norm: norm(m.Embedding), alive: true}
sp.add(e)
s.live++
return e, nil
}
// Forget deletes a memory. Its content is dropped from RAM at once and from
// the journal file at the next compaction, which runs after CompactAfter
// forgets, on Close, on the next Open, or when Compact is called.
func (s *Store) Forget(namespace, id string) error {
s.dataMu.Lock()
defer s.dataMu.Unlock()
if s.closed {
return ErrClosed
}
sp := s.spaces[namespace]
if sp == nil {
return ErrNotFound
}
if _, ok := sp.byID[id]; !ok {
return ErrNotFound
}
if s.journal != nil {
if err := s.journal.append(record{Forget: &forgetEntry{Namespace: namespace, ID: id}}); err != nil {
s.journalErrors.Add(1)
return fmt.Errorf("journal write failed: %w", err)
}
}
s.remove(namespace, id)
s.emit(Event{Type: "forget", Namespace: namespace, MemoryIDs: []string{id}})
if s.tombstones >= s.opts.CompactAfter {
// The forget is already durable. If the rewrite fails it is counted
// in JournalErrors, ForgottenOnDisk stays above zero, and the next
// forget tries again.
_ = s.compactLocked()
}
return nil
}
func (s *Store) remove(namespace, id string) {
sp := s.spaces[namespace]
if sp == nil {
return
}
doc, ok := sp.byID[id]
if !ok {
return
}
sp.ix.Remove(doc)
*sp.entries[doc] = entry{} // drop the text, embedding and metadata
delete(sp.byID, id)
sp.live--
s.live--
s.tombstones++
}
// Compact rewrites the journal so it holds only live memories and the
// current learning state, and frees what forgotten memories still occupy.
// It does nothing when there is nothing to erase and the journal has not
// grown much, so calling it often is cheap.
func (s *Store) Compact() error {
s.dataMu.Lock()
defer s.dataMu.Unlock()
if s.closed {
return ErrClosed
}
if s.tombstones == 0 && (s.journal == nil || s.journal.bytes() < 2*s.compacted+s.opts.CompactSlack) {
return nil
}
return s.compactLocked()
}
// compactLocked requires dataMu. If the rewrite fails the old journal is
// still complete, so the failure is counted and the store keeps running.
func (s *Store) compactLocked() error {
if s.journal != nil {
err := s.journal.replace(func(emit func(record) error) error {
names := make([]string, 0, len(s.spaces))
for name := range s.spaces {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
for _, e := range s.spaces[name].entries {
if !e.alive {
continue
}
if err := emit(record{Remember: &e.Memory, Recalls: e.recalls, RewardSum: e.rewardSum}); err != nil {
return err
}
}
}
s.learnMu.Lock()
defer s.learnMu.Unlock()
return emit(record{Learn: &learnEntry{
Weights: s.policy.Weights, Baseline: s.policy.Baseline,
ItemBaseline: s.policy.ItemBaseline, Updates: s.policy.Updates,
}})
})
if err != nil {
s.journalErrors.Add(1)
return fmt.Errorf("journal compaction failed: %w", err)
}
s.compacted = s.journal.bytes()
}
for name, sp := range s.spaces {
if sp.live == len(sp.entries) {
continue
}
if sp.live == 0 {
delete(s.spaces, name)
continue
}
fresh := newSpace()
fresh.dim = sp.dim
for _, e := range sp.entries {
if e.alive {
fresh.add(e)
}
}
s.spaces[name] = fresh
}
s.tombstones = 0
return nil
}
// Recall returns up to q.K memories for the query.
func (s *Store) Recall(q Query) (Recall, error) {
if err := validate(q.Namespace, q.Text, q.Embedding); err != nil {
return Recall{}, err
}
if q.K <= 0 || q.K > MaxK {
return Recall{}, fmt.Errorf("%w: k must be between 1 and %d", ErrInvalid, MaxK)
}
asOf := q.AsOf
if asOf.IsZero() {
asOf = s.opts.Now()
}
found, features, err := s.candidates(q, asOf)
if err != nil {
return Recall{}, err
}
p := &pending{id: newID(), namespace: q.Namespace, features: features, explore: q.Explore}
p.ids = make([]string, len(found))
for i, m := range found {
p.ids[i] = m.ID
}
s.learnMu.Lock()
defer s.learnMu.Unlock()
if q.Explore {
p.slate = s.policy.Sample(features, q.K, s.rng)
} else {
p.slate = s.policy.Rank(features, q.K)
}
out := Recall{ID: p.id, Memories: make([]Recalled, len(p.slate))}
for i, at := range p.slate {
m := found[at]
out.Memories[i] = Recalled{
ID: m.ID, Text: m.Text, Importance: m.Importance, CreatedAt: m.CreatedAt,
Metadata: m.Metadata, Score: s.policy.Score(features[at]),
}
}
for s.order.Len() >= s.opts.MaxPending {
oldest := s.order.Remove(s.order.Front()).(*pending)
delete(s.waiting, oldest.id)
}
s.waiting[p.id] = s.order.PushBack(p)
returned := make([]string, len(out.Memories))
for i, m := range out.Memories {
returned[i] = m.ID
}
s.emit(Event{Type: "recall", Namespace: q.Namespace, RecallID: p.id, MemoryIDs: returned, Explore: q.Explore})
return out, nil
}
// candidates gathers the best keyword and semantic matches, adds the
// memories stored right next to the strongest ones, and describes each for
// the policy. Either search method alone is enough to answer.
func (s *Store) candidates(q Query, asOf time.Time) ([]Memory, []policy.Features, error) {
s.dataMu.RLock()
defer s.dataMu.RUnlock()
if s.closed {
return nil, nil, ErrClosed
}
sp := s.spaces[q.Namespace]
if sp == nil {
return nil, nil, nil
}
pool := make(map[uint32]struct{})
keyword := make(map[uint32]float64)
var best float64
for i, hit := range sp.ix.Search(q.Text, s.opts.Candidates) {
keyword[hit.Doc] = hit.Score
best = math.Max(best, hit.Score)
pool[hit.Doc] = struct{}{}
if i < neighborSeeds {
s.eachNeighbor(sp, hit.Doc, func(n uint32, _ int) { pool[n] = struct{}{} })
}
}
semantic := make(map[uint32]bool)
qNorm := norm(q.Embedding)
useVectors := qNorm > 0 && len(q.Embedding) == sp.dim
similarity := func(e *entry) float64 {
if !useVectors || e.norm == 0 {
return 0
}
return math.Max(0, cosine(q.Embedding, e.Embedding, qNorm, e.norm))
}
if useVectors {
type scored struct {
doc uint32
cos float64
}
all := make([]scored, 0, sp.live)
for doc, e := range sp.entries {
if e.alive && e.norm > 0 {
all = append(all, scored{uint32(doc), cosine(q.Embedding, e.Embedding, qNorm, e.norm)})
}
}
sort.Slice(all, func(i, j int) bool {
if all[i].cos != all[j].cos {
return all[i].cos > all[j].cos
}
return all[i].doc < all[j].doc
})
for i, sc := range all[:min(s.opts.Candidates, len(all))] {
semantic[sc.doc] = true
pool[sc.doc] = struct{}{}
if i < neighborSeeds {
s.eachNeighbor(sp, sc.doc, func(n uint32, _ int) { pool[n] = struct{}{} })
}
}
}
docs := make([]uint32, 0, len(pool))
for doc := range pool {
docs = append(docs, doc)
}
sort.Slice(docs, func(i, j int) bool { return docs[i] < docs[j] })
temporal := 0.0
if asksAboutTime(q.Text) {
temporal = 1
}
found := make([]Memory, len(docs))
features := make([]policy.Features, len(docs))
for i, doc := range docs {
e := sp.entries[doc]
found[i] = e.Memory
found[i].Embedding = nil
f := &features[i]
if best > 0 {
f[policy.Keyword] = keyword[doc] / best
}
f[policy.Semantic] = similarity(e)
ageDays := math.Max(0, asOf.Sub(e.CreatedAt).Hours()/24)
f[policy.Recency] = 1 / (1 + ageDays)
f[policy.Utility] = e.utility()
f[policy.Importance] = e.Importance
f[policy.TemporalRecency] = temporal * f[policy.Recency]
if _, inKeyword := keyword[doc]; inKeyword && semantic[doc] {
f[policy.Agreement] = 1
}
s.eachNeighbor(sp, doc, func(n uint32, distance int) {
weight := 1 / float64(distance)
if best > 0 {
f[policy.NeighborKeyword] = math.Max(f[policy.NeighborKeyword], weight*keyword[n]/best)
}
f[policy.NeighborSemantic] = math.Max(f[policy.NeighborSemantic], weight*similarity(sp.entries[n]))
})
}
return found, features, nil
}
// eachNeighbor calls visit for the live memories stored just before and
// after doc, as long as each step between them is within the neighbour gap.
func (s *Store) eachNeighbor(sp *space, doc uint32, visit func(neighbor uint32, distance int)) {
for _, step := range []int{-1, 1} {
at := int(doc)
for distance := 1; distance <= neighborReach; distance++ {
next := at + step
if next < 0 || next >= len(sp.entries) || !sp.entries[next].alive {
break
}
gap := sp.entries[next].CreatedAt.Sub(sp.entries[at].CreatedAt)
if gap < 0 {
gap = -gap
}
if gap > s.opts.NeighborGap {
break
}
visit(uint32(next), distance)
at = next
}
}
}
// Feedback reports how useful a recall was, from 0 (useless) to 1 (it
// answered the question). Each recall accepts feedback once.
//
// useful may name the returned memories that actually helped, for example
// the ones an agent cited. The policy then learns from each memory's own
// outcome instead of one number for the whole slate, which is far less noisy.
//
// The learning step and its journal record succeed or fail together: if the
// record cannot be written, the policy is put back as it was and the recall
// stays open, so the caller can retry and RAM never runs ahead of disk.
func (s *Store) Feedback(recallID string, reward float64, useful []string) error {
if math.IsNaN(reward) || reward < 0 || reward > 1 {
return fmt.Errorf("%w: reward must be between 0 and 1", ErrInvalid)
}
s.dataMu.Lock()
defer s.dataMu.Unlock()
if s.closed {
return ErrClosed
}
s.learnMu.Lock()
defer s.learnMu.Unlock()
el, ok := s.waiting[recallID]
if !ok {
return ErrNotFound
}
p := el.Value.(*pending)
ids := make([]string, len(p.slate))
for i, at := range p.slate {
ids[i] = p.ids[at]
}
var itemRewards []float64
if useful != nil {
itemRewards = make([]float64, len(ids))
for _, id := range useful {
at := slices.Index(ids, id)
if at < 0 {
return fmt.Errorf("%w: memory %q was not returned by this recall", ErrInvalid, id)
}
itemRewards[at] = 1
}
}
before := *s.policy
if p.explore {
s.policy.Update(p.features, p.slate, reward, itemRewards)
}
if s.journal != nil {
err := s.journal.append(record{Learn: &learnEntry{
Weights: s.policy.Weights, Baseline: s.policy.Baseline, ItemBaseline: s.policy.ItemBaseline,
Updates: s.policy.Updates, Namespace: p.namespace, IDs: ids, Reward: reward, ItemRewards: itemRewards,
}})
if err != nil {
*s.policy = before
s.journalErrors.Add(1)
return fmt.Errorf("journal write failed: %w", err)
}
}
s.order.Remove(el)
delete(s.waiting, recallID)
s.credit(p.namespace, ids, reward, itemRewards)
s.emit(Event{Type: "feedback", Namespace: p.namespace, RecallID: recallID, MemoryIDs: ids, Reward: &reward, Useful: useful})
return nil
}
func (s *Store) emit(e Event) {
if s.opts.OnEvent != nil {
e.At = s.opts.Now().UTC()
s.opts.OnEvent(e)
}
}
// credit records the outcome of a slate against each memory in it: its own
// reward when the caller said which memories helped, else the slate's.
func (s *Store) credit(namespace string, ids []string, reward float64, itemRewards []float64) {
sp := s.spaces[namespace]
if sp == nil {
return
}
for i, id := range ids {
if doc, ok := sp.byID[id]; ok {
earned := reward
if len(itemRewards) == len(ids) {
earned = itemRewards[i]
}
sp.entries[doc].recalls++
sp.entries[doc].rewardSum += earned
}
}
}
// SetWeights replaces the learned weights, for example to roll back a policy
// that learned from bad feedback or to pin a fixed ranking.
func (s *Store) SetWeights(weights policy.Features) error {
for _, w := range weights {
if math.IsNaN(w) || math.Abs(w) > policy.MaxWeight {
return fmt.Errorf("%w: weights must be finite and within ±%v", ErrInvalid, policy.MaxWeight)
}
}
s.dataMu.Lock()
defer s.dataMu.Unlock()
if s.closed {
return ErrClosed
}
s.learnMu.Lock()
defer s.learnMu.Unlock()
if s.journal != nil {
err := s.journal.append(record{Learn: &learnEntry{Weights: weights, Baseline: 0.5, Updates: s.policy.Updates}})
if err != nil {
s.journalErrors.Add(1)
return fmt.Errorf("journal write failed: %w", err)
}
}
s.policy.Weights = weights
s.policy.Baseline = 0.5
return nil
}
// Stats returns a snapshot of the store.
func (s *Store) Stats() Stats {
st := Stats{JournalHealthy: true}
s.dataMu.RLock()
st.Namespaces = len(s.spaces)
st.Memories = s.live
s.dataMu.RUnlock()
s.learnMu.Lock()
st.Pending = s.order.Len()
st.Updates = s.policy.Updates
st.Baseline = s.policy.Baseline
st.Weights = s.policy.Weights
s.learnMu.Unlock()
st.JournalErrors = s.journalErrors.Load()
if s.journal != nil {
st.JournalHealthy = s.journal.healthy()
s.dataMu.RLock()
st.ForgottenOnDisk = s.tombstones
s.dataMu.RUnlock()
}
return st
}
func validate(namespace, text string, embedding []float32) error {
if namespace == "" || len(namespace) > MaxNamespaceBytes {
return fmt.Errorf("%w: namespace must be 1 to %d bytes", ErrInvalid, MaxNamespaceBytes)
}
if len(text) > MaxTextBytes {
return fmt.Errorf("%w: text is longer than %d bytes", ErrInvalid, MaxTextBytes)
}
if len(embedding) > MaxEmbeddingDim {
return fmt.Errorf("%w: embedding has more than %d dimensions", ErrInvalid, MaxEmbeddingDim)
}
for _, v := range embedding {
if math.IsNaN(float64(v)) || math.IsInf(float64(v), 0) {
return fmt.Errorf("%w: embedding contains a non-finite value", ErrInvalid)
}
}
return nil
}
var timeWords = map[string]struct{}{}
func init() {
for _, w := range []string{
"when", "last", "latest", "recent", "recently", "newest", "current", "currently", "now",
"today", "yesterday", "tomorrow", "ago", "date", "first", "earliest", "before", "after", "previous",
} {
timeWords[w] = struct{}{}
}
}
func asksAboutTime(query string) bool {
for _, token := range index.Tokenize(query) {
if _, ok := timeWords[token]; ok {
return true
}
}
return false
}
func norm(v []float32) float64 {
var sum float64
for _, x := range v {
sum += float64(x) * float64(x)
}
return math.Sqrt(sum)
}
func cosine(a, b []float32, aNorm, bNorm float64) float64 {
var dot float64
for i := range a {
dot += float64(a[i]) * float64(b[i])
}
return dot / (aNorm * bNorm)
}
func newID() string {
var raw [12]byte
if _, err := rand.Read(raw[:]); err != nil {
panic(err) // the system random source is gone; nothing sensible to do
}
return hex.EncodeToString(raw[:])
}