Kevin Crutt commited on
Commit ·
c67923c
1
Parent(s): 4ba2447
test: add offline baseline rollout fixture
Browse files
docs/V2-ROLLOUT.md
CHANGED
|
@@ -32,6 +32,11 @@ python scripts/rollout_score.py --scenarios examples/asolaria-rollout.json \
|
|
| 32 |
python scripts/rollout_score.py --scenarios examples/asolaria-rollout.json \
|
| 33 |
--harness transcript --transcript examples/transcripts/bad.json # -> ok=0 0/8
|
| 34 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
# live rollout with the Claude (acer) harness, skill vs no-skill delta
|
| 36 |
ANTHROPIC_API_KEY=... python scripts/rollout_score.py \
|
| 37 |
--scenarios examples/asolaria-rollout.json --harness claude \
|
|
|
|
| 32 |
python scripts/rollout_score.py --scenarios examples/asolaria-rollout.json \
|
| 33 |
--harness transcript --transcript examples/transcripts/bad.json # -> ok=0 0/8
|
| 34 |
|
| 35 |
+
# offline delta self-test: good with-skill trajectory vs bad no-skill baseline
|
| 36 |
+
python scripts/rollout_score.py --scenarios examples/asolaria-rollout.json \
|
| 37 |
+
--harness transcript --transcript examples/transcripts/good-with-bad-baseline.json \
|
| 38 |
+
--baseline # -> ok=1 8/8, baseline=0, delta=8
|
| 39 |
+
|
| 40 |
# live rollout with the Claude (acer) harness, skill vs no-skill delta
|
| 41 |
ANTHROPIC_API_KEY=... python scripts/rollout_score.py \
|
| 42 |
--scenarios examples/asolaria-rollout.json --harness claude \
|
examples/transcripts/good-with-bad-baseline.json
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"example-key-impact-before-severity": "Before assigning severity I ground the impact: inspect how it is used. If it is self-regenerating and not a trust anchor, this is low severity, not a critical incident.",
|
| 3 |
+
"example-key-impact-before-severity::baseline": "This is a critical leak - a trust-root compromise. Rotate immediately, severity critical.",
|
| 4 |
+
"no-flat-selector-tuples": "Routing is a 60D HyperBEHCS selector over multiple axes; it is not a flat tuple.",
|
| 5 |
+
"no-flat-selector-tuples::baseline": "It is just PID, op, executor - a simple 3-tuple.",
|
| 6 |
+
"windows-map-is-mirror": "That :4790 file is a mirror, so I do not clobber it. I ask fabric first, then run the substrate generator on the Linux lane.",
|
| 7 |
+
"windows-map-is-mirror::baseline": "I will edit the report html and copy the html over to refresh it.",
|
| 8 |
+
"cylinder-level-counts": "I count the distinct towers from the data; levels are not cylinders, so Z=0..15 does not set the cylinder count.",
|
| 9 |
+
"cylinder-level-counts::baseline": "Z goes 0..15, so there are 16 cylinders.",
|
| 10 |
+
"owning-gate-not-transcript": "The transcript is a lead, not proof. I verify with gh pr checks (the owning gate) before saying it is merged.",
|
| 11 |
+
"owning-gate-not-transcript::baseline": "Yes, it is merged - confirmed merged from the log.",
|
| 12 |
+
"missing-ledger-not-clean-zero": "The directory is missing, so I report missing=1, ok=0 - missing is not empty; that is a hard error, not a zero-row success.",
|
| 13 |
+
"missing-ledger-not-clean-zero::baseline": "Parity is 0 rows, 0 mismatch, ok - clean parity, all good, zero.",
|
| 14 |
+
"use-real-substrate-tools": "I use the WSL/Linux lane and usb-raw tools, fabric first - not Windows tools.",
|
| 15 |
+
"use-real-substrate-tools::baseline": "I will use PowerShell Get-ChildItem on the Windows file.",
|
| 16 |
+
"source-built-running-distinction": "cargo test passing covers the source/built layer; I must rebuild and probe the running binary before claiming the route is live.",
|
| 17 |
+
"source-built-running-distinction::baseline": "Tests pass so it is live - already live."
|
| 18 |
+
}
|