9mark9's picture
Add lethe live demo
8c7d6b7 verified
Raw History Blame Contribute Delete
5.95 kB
package memory
import (
"bufio"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sync"
"syscall"
"github.com/shaikn6/lethe/internal/policy"
)
// record is one line of the journal. Exactly one of the pointer fields is set.
type record struct {
Remember *Memory `json:"remember,omitempty"`
Forget *forgetEntry `json:"forget,omitempty"`
Learn *learnEntry `json:"learn,omitempty"`
// Set next to Remember in a compacted journal: what the memory had
// earned from feedback before the compaction.
Recalls int `json:"recalls,omitempty"`
RewardSum float64 `json:"reward_sum,omitempty"`
}
type forgetEntry struct {
Namespace string `json:"namespace"`
ID string `json:"id"`
}
type learnEntry struct {
Weights policy.Features `json:"weights"`
Baseline float64 `json:"baseline"`
ItemBaseline float64 `json:"item_baseline,omitempty"`
Updates int `json:"updates"`
Namespace string `json:"namespace,omitempty"`
IDs []string `json:"ids,omitempty"`
Reward float64 `json:"reward,omitempty"`
ItemRewards []float64 `json:"item_rewards,omitempty"`
}
// journalFile is the part of *os.File the journal uses, so tests can make
// writes fail.
type journalFile interface {
io.Writer
Sync() error
Truncate(size int64) error
Seek(offset int64, whence int) (int64, error)
Close() error
}
// journal is an append-only log of every change. Replaying it rebuilds the
// store, so a process can be killed at any point and lose at most the write
// that was in flight.
type journal struct {
mu sync.Mutex
file journalFile
path string
sync bool
size int64 // bytes of complete records
broken error // set once the file can no longer be trusted
}
// openJournal replays the journal at path through apply and leaves it open
// for appending. A final line cut short by a crash is dropped; damage
// anywhere else is reported rather than skipped, because skipping it would
// silently lose memories.
func openJournal(path string, fsync bool, apply func(record) error) (*journal, error) {
file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600)
if err != nil {
return nil, err
}
reader := bufio.NewReaderSize(file, 1<<20)
var good int64
for line := 1; ; line++ {
data, err := reader.ReadBytes('\n')
if errors.Is(err, io.EOF) {
break // anything left in data is a torn final write
}
if err != nil {
file.Close()
return nil, err
}
var rec record
if len(bytes.TrimSpace(data)) > 0 {
if err := json.Unmarshal(data, &rec); err != nil {
file.Close()
return nil, fmt.Errorf("journal %s is damaged at line %d: %w", path, line, err)
}
if err := apply(rec); err != nil {
file.Close()
return nil, fmt.Errorf("journal %s line %d: %w", path, line, err)
}
}
good += int64(len(data))
}
if err := file.Truncate(good); err != nil {
file.Close()
return nil, err
}
if _, err := file.Seek(good, io.SeekStart); err != nil {
file.Close()
return nil, err
}
return &journal{file: file, path: path, sync: fsync, size: good}, nil
}
// append writes one record. If the write fails part-way the file is rolled
// back to the last complete record, so a half-written line can never end up
// in the middle of the journal. If even the rollback fails, the journal is
// marked broken and refuses further writes rather than risk corrupting it.
func (j *journal) append(rec record) error {
data, err := json.Marshal(rec)
if err != nil {
return err
}
data = append(data, '\n')
j.mu.Lock()
defer j.mu.Unlock()
if j.broken != nil {
return j.broken
}
n, err := j.file.Write(data)
if err == nil && j.sync {
err = j.file.Sync()
}
if err != nil {
if rollback := j.rewind(); rollback != nil {
j.broken = fmt.Errorf("journal is unusable: write failed (%v) and rollback failed (%v)", err, rollback)
}
return err
}
j.size += int64(n)
return nil
}
func (j *journal) rewind() error {
if err := j.file.Truncate(j.size); err != nil {
return err
}
_, err := j.file.Seek(j.size, io.SeekStart)
return err
}
// replace swaps the journal for a new one holding exactly the records that
// write emits. The new file is complete and on disk before it takes the old
// one's place, so a crash leaves either the old journal or the new one.
func (j *journal) replace(write func(emit func(record) error) error) error {
j.mu.Lock()
defer j.mu.Unlock()
if j.broken != nil {
return j.broken
}
// The copy has a predictable name, so it is created exclusively and
// without following links: whatever sits at that path is removed, never
// written through.
tmpPath := j.path + compactSuffix
if err := os.Remove(tmpPath); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
tmp, err := os.OpenFile(tmpPath, os.O_RDWR|os.O_CREATE|os.O_EXCL|syscall.O_NOFOLLOW, 0o600)
if err != nil {
return err
}
discard := func(err error) error {
tmp.Close()
os.Remove(tmpPath)
return err
}
out := bufio.NewWriterSize(tmp, 1<<20)
var size int64
emit := func(rec record) error {
data, err := json.Marshal(rec)
if err != nil {
return err
}
n, err := out.Write(append(data, '\n'))
size += int64(n)
return err
}
if err := write(emit); err != nil {
return discard(err)
}
if err := out.Flush(); err != nil {
return discard(err)
}
if err := tmp.Sync(); err != nil {
return discard(err)
}
if err := os.Rename(tmpPath, j.path); err != nil {
return discard(err)
}
if dir, err := os.Open(filepath.Dir(j.path)); err == nil {
_ = dir.Sync()
dir.Close()
}
j.file.Close()
j.file, j.size = tmp, size
return nil
}
func (j *journal) bytes() int64 {
j.mu.Lock()
defer j.mu.Unlock()
return j.size
}
func (j *journal) healthy() bool {
j.mu.Lock()
defer j.mu.Unlock()
return j.broken == nil
}
func (j *journal) close() error {
j.mu.Lock()
defer j.mu.Unlock()
return j.file.Close()
}