File size: 2,025 Bytes
1c428b7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | #!/usr/bin/env bash
# CLERK: Consolidated Ledger with Eviction and Rewrite Keys
# End-to-end reproducible pipeline. Every step is seeded; running this
# script twice produces byte-identical data and the same training run.
#
# Prereqs: pip install -r requirements.txt; huggingface-cli login
# Hardware: one 16GB+ GPU (L4/A10G class) for training; CPU suffices for
# steps 1-3.
set -euo pipefail
TRAIN_PERSONAS=800
VAL_PERSONAS=100
TEST_PERSONAS=100
SESSIONS=8
BUDGET=12
SEED=137
ADAPTER_REPO="Falln87/clerk-qwen2.5-0.5b-lora"
mkdir -p data results
echo "== 1. smoke tests (CPU) =="
python -m clerk.tests_smoke --personas 20 --sessions 8 --budget $BUDGET --seed $SEED
echo "== 2. generate synthetic evolving sessions =="
python -m clerk.generator --personas $TRAIN_PERSONAS --sessions $SESSIONS \
--budget $BUDGET --seed $SEED --split train --out data/timelines_train.jsonl
python -m clerk.generator --personas $VAL_PERSONAS --sessions $SESSIONS \
--budget $BUDGET --seed 991 --split train --out data/timelines_val.jsonl
python -m clerk.generator --personas $TEST_PERSONAS --sessions $SESSIONS \
--budget $BUDGET --seed 4242 --split test --out data/timelines_test.jsonl
echo "== 3. build SFT dataset =="
python -m clerk.build_sft --train data/timelines_train.jsonl \
--val data/timelines_val.jsonl \
--out-train data/sft_train.jsonl --out-val data/sft_val.jsonl
echo "== 4. LoRA SFT (GPU) =="
python -m clerk.train_sft --train data/sft_train.jsonl \
--val data/sft_val.jsonl --hub-model-id $ADAPTER_REPO
echo "== 5. synthetic held-out evaluation (GPU) =="
python -m clerk.eval_synthetic --timelines data/timelines_test.jsonl \
--adapter $ADAPTER_REPO \
--systems clerk,prompted_ledger,full,window,rag \
--out results/results_synthetic.json
echo "== 6. LoCoMo-MC10 zero-shot evaluation (GPU) =="
python -m clerk.eval_locomo --adapter $ADAPTER_REPO \
--systems clerk,prompted_ledger,full,rag \
--out results/results_locomo.json
echo "== done. Results in results/ ==" |