Spaces:
Running
Running
Download src/internal/memory/journal.go from 9mark9/lethe-agent-memory: direct link, hf CLI and curl.
- Browser
- Download file 5.95 kB
-
https://huggingface.co/spaces/9mark9/lethe-agent-memory/resolve/main/src/internal/memory/journal.go
- Command line
-
hf download hf://spaces/9mark9/lethe-agent-memory/src/internal/memory/journal.go
-
curl -L -o journal.go https://huggingface.co/spaces/9mark9/lethe-agent-memory/resolve/main/src/internal/memory/journal.go
5.95 kB
| package memory | |
| import ( | |
| "bufio" | |
| "bytes" | |
| "encoding/json" | |
| "errors" | |
| "fmt" | |
| "io" | |
| "os" | |
| "path/filepath" | |
| "sync" | |
| "syscall" | |
| "github.com/shaikn6/lethe/internal/policy" | |
| ) | |
| // record is one line of the journal. Exactly one of the pointer fields is set. | |
| type record struct { | |
| Remember *Memory `json:"remember,omitempty"` | |
| Forget *forgetEntry `json:"forget,omitempty"` | |
| Learn *learnEntry `json:"learn,omitempty"` | |
| // Set next to Remember in a compacted journal: what the memory had | |
| // earned from feedback before the compaction. | |
| Recalls int `json:"recalls,omitempty"` | |
| RewardSum float64 `json:"reward_sum,omitempty"` | |
| } | |
| type forgetEntry struct { | |
| Namespace string `json:"namespace"` | |
| ID string `json:"id"` | |
| } | |
| type learnEntry struct { | |
| Weights policy.Features `json:"weights"` | |
| Baseline float64 `json:"baseline"` | |
| ItemBaseline float64 `json:"item_baseline,omitempty"` | |
| Updates int `json:"updates"` | |
| Namespace string `json:"namespace,omitempty"` | |
| IDs []string `json:"ids,omitempty"` | |
| Reward float64 `json:"reward,omitempty"` | |
| ItemRewards []float64 `json:"item_rewards,omitempty"` | |
| } | |
| // journalFile is the part of *os.File the journal uses, so tests can make | |
| // writes fail. | |
| type journalFile interface { | |
| io.Writer | |
| Sync() error | |
| Truncate(size int64) error | |
| Seek(offset int64, whence int) (int64, error) | |
| Close() error | |
| } | |
| // journal is an append-only log of every change. Replaying it rebuilds the | |
| // store, so a process can be killed at any point and lose at most the write | |
| // that was in flight. | |
| type journal struct { | |
| mu sync.Mutex | |
| file journalFile | |
| path string | |
| sync bool | |
| size int64 // bytes of complete records | |
| broken error // set once the file can no longer be trusted | |
| } | |
| // openJournal replays the journal at path through apply and leaves it open | |
| // for appending. A final line cut short by a crash is dropped; damage | |
| // anywhere else is reported rather than skipped, because skipping it would | |
| // silently lose memories. | |
| func openJournal(path string, fsync bool, apply func(record) error) (*journal, error) { | |
| file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600) | |
| if err != nil { | |
| return nil, err | |
| } | |
| reader := bufio.NewReaderSize(file, 1<<20) | |
| var good int64 | |
| for line := 1; ; line++ { | |
| data, err := reader.ReadBytes('\n') | |
| if errors.Is(err, io.EOF) { | |
| break // anything left in data is a torn final write | |
| } | |
| if err != nil { | |
| file.Close() | |
| return nil, err | |
| } | |
| var rec record | |
| if len(bytes.TrimSpace(data)) > 0 { | |
| if err := json.Unmarshal(data, &rec); err != nil { | |
| file.Close() | |
| return nil, fmt.Errorf("journal %s is damaged at line %d: %w", path, line, err) | |
| } | |
| if err := apply(rec); err != nil { | |
| file.Close() | |
| return nil, fmt.Errorf("journal %s line %d: %w", path, line, err) | |
| } | |
| } | |
| good += int64(len(data)) | |
| } | |
| if err := file.Truncate(good); err != nil { | |
| file.Close() | |
| return nil, err | |
| } | |
| if _, err := file.Seek(good, io.SeekStart); err != nil { | |
| file.Close() | |
| return nil, err | |
| } | |
| return &journal{file: file, path: path, sync: fsync, size: good}, nil | |
| } | |
| // append writes one record. If the write fails part-way the file is rolled | |
| // back to the last complete record, so a half-written line can never end up | |
| // in the middle of the journal. If even the rollback fails, the journal is | |
| // marked broken and refuses further writes rather than risk corrupting it. | |
| func (j *journal) append(rec record) error { | |
| data, err := json.Marshal(rec) | |
| if err != nil { | |
| return err | |
| } | |
| data = append(data, '\n') | |
| j.mu.Lock() | |
| defer j.mu.Unlock() | |
| if j.broken != nil { | |
| return j.broken | |
| } | |
| n, err := j.file.Write(data) | |
| if err == nil && j.sync { | |
| err = j.file.Sync() | |
| } | |
| if err != nil { | |
| if rollback := j.rewind(); rollback != nil { | |
| j.broken = fmt.Errorf("journal is unusable: write failed (%v) and rollback failed (%v)", err, rollback) | |
| } | |
| return err | |
| } | |
| j.size += int64(n) | |
| return nil | |
| } | |
| func (j *journal) rewind() error { | |
| if err := j.file.Truncate(j.size); err != nil { | |
| return err | |
| } | |
| _, err := j.file.Seek(j.size, io.SeekStart) | |
| return err | |
| } | |
| // replace swaps the journal for a new one holding exactly the records that | |
| // write emits. The new file is complete and on disk before it takes the old | |
| // one's place, so a crash leaves either the old journal or the new one. | |
| func (j *journal) replace(write func(emit func(record) error) error) error { | |
| j.mu.Lock() | |
| defer j.mu.Unlock() | |
| if j.broken != nil { | |
| return j.broken | |
| } | |
| // The copy has a predictable name, so it is created exclusively and | |
| // without following links: whatever sits at that path is removed, never | |
| // written through. | |
| tmpPath := j.path + compactSuffix | |
| if err := os.Remove(tmpPath); err != nil && !errors.Is(err, os.ErrNotExist) { | |
| return err | |
| } | |
| tmp, err := os.OpenFile(tmpPath, os.O_RDWR|os.O_CREATE|os.O_EXCL|syscall.O_NOFOLLOW, 0o600) | |
| if err != nil { | |
| return err | |
| } | |
| discard := func(err error) error { | |
| tmp.Close() | |
| os.Remove(tmpPath) | |
| return err | |
| } | |
| out := bufio.NewWriterSize(tmp, 1<<20) | |
| var size int64 | |
| emit := func(rec record) error { | |
| data, err := json.Marshal(rec) | |
| if err != nil { | |
| return err | |
| } | |
| n, err := out.Write(append(data, '\n')) | |
| size += int64(n) | |
| return err | |
| } | |
| if err := write(emit); err != nil { | |
| return discard(err) | |
| } | |
| if err := out.Flush(); err != nil { | |
| return discard(err) | |
| } | |
| if err := tmp.Sync(); err != nil { | |
| return discard(err) | |
| } | |
| if err := os.Rename(tmpPath, j.path); err != nil { | |
| return discard(err) | |
| } | |
| if dir, err := os.Open(filepath.Dir(j.path)); err == nil { | |
| _ = dir.Sync() | |
| dir.Close() | |
| } | |
| j.file.Close() | |
| j.file, j.size = tmp, size | |
| return nil | |
| } | |
| func (j *journal) bytes() int64 { | |
| j.mu.Lock() | |
| defer j.mu.Unlock() | |
| return j.size | |
| } | |
| func (j *journal) healthy() bool { | |
| j.mu.Lock() | |
| defer j.mu.Unlock() | |
| return j.broken == nil | |
| } | |
| func (j *journal) close() error { | |
| j.mu.Lock() | |
| defer j.mu.Unlock() | |
| return j.file.Close() | |
| } | |