run_all.sh

#!/usr/bin/env bash
# CLERK: Consolidated Ledger with Eviction and Rewrite Keys
# End-to-end reproducible pipeline. Every step is seeded; running this
# script twice produces byte-identical data and the same training run.
#
# Prereqs: pip install -r requirements.txt; huggingface-cli login
# Hardware: one 16GB+ GPU (L4/A10G class) for training; CPU suffices for
#           steps 1-3.
set -euo pipefail

TRAIN_PERSONAS=800
VAL_PERSONAS=100
TEST_PERSONAS=100
SESSIONS=8
BUDGET=12
SEED=137
ADAPTER_REPO="Falln87/clerk-qwen2.5-0.5b-lora"

mkdir -p data results

echo "== 1. smoke tests (CPU) =="
python -m clerk.tests_smoke --personas 20 --sessions 8 --budget $BUDGET --seed $SEED

echo "== 2. generate synthetic evolving sessions =="
python -m clerk.generator --personas $TRAIN_PERSONAS --sessions $SESSIONS \
    --budget $BUDGET --seed $SEED --split train --out data/timelines_train.jsonl
python -m clerk.generator --personas $VAL_PERSONAS --sessions $SESSIONS \
    --budget $BUDGET --seed 991 --split train --out data/timelines_val.jsonl
python -m clerk.generator --personas $TEST_PERSONAS --sessions $SESSIONS \
    --budget $BUDGET --seed 4242 --split test --out data/timelines_test.jsonl

echo "== 3. build SFT dataset =="
python -m clerk.build_sft --train data/timelines_train.jsonl \
    --val data/timelines_val.jsonl \
    --out-train data/sft_train.jsonl --out-val data/sft_val.jsonl

echo "== 4. LoRA SFT (GPU) =="
python -m clerk.train_sft --train data/sft_train.jsonl \
    --val data/sft_val.jsonl --hub-model-id $ADAPTER_REPO

echo "== 5. synthetic held-out evaluation (GPU) =="
python -m clerk.eval_synthetic --timelines data/timelines_test.jsonl \
    --adapter $ADAPTER_REPO \
    --systems clerk,prompted_ledger,full,window,rag \
    --out results/results_synthetic.json

echo "== 6. LoCoMo-MC10 zero-shot evaluation (GPU) =="
python -m clerk.eval_locomo --adapter $ADAPTER_REPO \
    --systems clerk,prompted_ledger,full,rag \
    --out results/results_locomo.json

echo "== done. Results in results/ =="