auto-sync 2026-07-04T03:59:09Z workspace (part 5)
Browse files
workspace/scripts/slurm/train_ctt_feature_proxy.sbatch
CHANGED
|
@@ -60,12 +60,15 @@ VAL_OUT="runs/${NAME_PREFIX}_seed${SEED}_val_proxy"
|
|
| 60 |
--k "${CTT_PROXY_K:-16}" \
|
| 61 |
--max-target-charts "${CTT_MAX_TARGET_CHARTS:-69}" \
|
| 62 |
--neighbors "${CTT_PROXY_NEIGHBORS:-8}" \
|
| 63 |
-
--thresholds "${CTT_PROXY_THRESHOLDS:-0.20,0.40}"
|
|
|
|
| 64 |
|
| 65 |
"$PYTHON" scripts/build_ctt_proxy_comparison.py \
|
| 66 |
-
--out-dir runs/ctt_val_proxy_comparison
|
|
|
|
| 67 |
|
| 68 |
"$PYTHON" scripts/summarize_ctt_runs.py \
|
| 69 |
--run-root runs \
|
| 70 |
--out-csv runs/summary_ctt.csv \
|
| 71 |
-
--out-md runs/summary_ctt.md
|
|
|
|
|
|
| 60 |
--k "${CTT_PROXY_K:-16}" \
|
| 61 |
--max-target-charts "${CTT_MAX_TARGET_CHARTS:-69}" \
|
| 62 |
--neighbors "${CTT_PROXY_NEIGHBORS:-8}" \
|
| 63 |
+
--thresholds "${CTT_PROXY_THRESHOLDS:-0.20,0.40}" \
|
| 64 |
+
--no-markdown-report
|
| 65 |
|
| 66 |
"$PYTHON" scripts/build_ctt_proxy_comparison.py \
|
| 67 |
+
--out-dir runs/ctt_val_proxy_comparison \
|
| 68 |
+
--no-markdown-report
|
| 69 |
|
| 70 |
"$PYTHON" scripts/summarize_ctt_runs.py \
|
| 71 |
--run-root runs \
|
| 72 |
--out-csv runs/summary_ctt.csv \
|
| 73 |
+
--out-md runs/summary_ctt.md \
|
| 74 |
+
--no-markdown-report
|
workspace/scripts/summarize_ctt_runs.py
CHANGED
|
@@ -39,6 +39,11 @@ def main(argv: list[str] | None = None) -> int:
|
|
| 39 |
parser.add_argument("--run-root", type=Path, default=Path("runs"))
|
| 40 |
parser.add_argument("--out-csv", type=Path, default=Path("runs/summary_ctt.csv"))
|
| 41 |
parser.add_argument("--out-md", type=Path, default=Path("runs/summary_ctt.md"))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
args = parser.parse_args(argv)
|
| 43 |
|
| 44 |
rows = []
|
|
@@ -99,8 +104,14 @@ def main(argv: list[str] | None = None) -> int:
|
|
| 99 |
writer.writeheader()
|
| 100 |
for row in rows:
|
| 101 |
writer.writerow({column: row.get(column, "n/a") for column in COLUMNS})
|
| 102 |
-
args.
|
| 103 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 104 |
return 0
|
| 105 |
|
| 106 |
|
|
|
|
| 39 |
parser.add_argument("--run-root", type=Path, default=Path("runs"))
|
| 40 |
parser.add_argument("--out-csv", type=Path, default=Path("runs/summary_ctt.csv"))
|
| 41 |
parser.add_argument("--out-md", type=Path, default=Path("runs/summary_ctt.md"))
|
| 42 |
+
parser.add_argument(
|
| 43 |
+
"--no-markdown-report",
|
| 44 |
+
action="store_true",
|
| 45 |
+
help="Do not write --out-md; the persistent prose summary lives in README.md.",
|
| 46 |
+
)
|
| 47 |
args = parser.parse_args(argv)
|
| 48 |
|
| 49 |
rows = []
|
|
|
|
| 104 |
writer.writeheader()
|
| 105 |
for row in rows:
|
| 106 |
writer.writerow({column: row.get(column, "n/a") for column in COLUMNS})
|
| 107 |
+
if args.no_markdown_report:
|
| 108 |
+
if args.out_md.exists():
|
| 109 |
+
args.out_md.unlink()
|
| 110 |
+
md_path: str | None = None
|
| 111 |
+
else:
|
| 112 |
+
args.out_md.write_text(_markdown(rows) + "\n")
|
| 113 |
+
md_path = str(args.out_md)
|
| 114 |
+
print(json.dumps({"csv": str(args.out_csv), "md": md_path, "rows": len(rows)}, indent=2))
|
| 115 |
return 0
|
| 116 |
|
| 117 |
|
workspace/tests/test_dominance_selector.py
CHANGED
|
@@ -21,8 +21,10 @@ from scripts.eval_learned_dominance_selector import (
|
|
| 21 |
_feature_names,
|
| 22 |
_fit_select_ridge,
|
| 23 |
_pairwise_calibration_summary,
|
|
|
|
| 24 |
_source_evidence_feature,
|
| 25 |
_target_value,
|
|
|
|
| 26 |
)
|
| 27 |
from scripts.eval_nonlinear_dominance_selector import _split_rows, _subset_dataset
|
| 28 |
from scripts.eval_nonlinear_dominance_selector import _write_provenance as _write_nonlinear_provenance
|
|
@@ -443,6 +445,61 @@ def test_learned_dominance_tangent_features_and_success_weighted_target() -> Non
|
|
| 443 |
assert source_evidence[1] == pytest.approx(np.log1p(2))
|
| 444 |
assert source_evidence[-1] == 1.0
|
| 445 |
assert _target_value("success_weighted_margin", utility_margin=0.2, candidate_success=1.0) == 1.2
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 446 |
|
| 447 |
|
| 448 |
def test_chart_compat_feature_is_deployment_visible_and_fixed_width() -> None:
|
|
|
|
| 21 |
_feature_names,
|
| 22 |
_fit_select_ridge,
|
| 23 |
_pairwise_calibration_summary,
|
| 24 |
+
_score_shape_matrix,
|
| 25 |
_source_evidence_feature,
|
| 26 |
_target_value,
|
| 27 |
+
_write_report_artifact,
|
| 28 |
)
|
| 29 |
from scripts.eval_nonlinear_dominance_selector import _split_rows, _subset_dataset
|
| 30 |
from scripts.eval_nonlinear_dominance_selector import _write_provenance as _write_nonlinear_provenance
|
|
|
|
| 445 |
assert source_evidence[1] == pytest.approx(np.log1p(2))
|
| 446 |
assert source_evidence[-1] == 1.0
|
| 447 |
assert _target_value("success_weighted_margin", utility_margin=0.2, candidate_success=1.0) == 1.2
|
| 448 |
+
assert _target_value(
|
| 449 |
+
"success_weighted_margin",
|
| 450 |
+
utility_margin=0.2,
|
| 451 |
+
candidate_success=1.0,
|
| 452 |
+
success_bonus=2.5,
|
| 453 |
+
) == 2.7
|
| 454 |
+
|
| 455 |
+
|
| 456 |
+
def test_score_shape_feature_set_and_markdown_suppression(tmp_path) -> None:
|
| 457 |
+
score_shape = _score_shape_matrix([3.0, 1.0, 2.0])
|
| 458 |
+
feature = _candidate_feature(
|
| 459 |
+
score=3.0,
|
| 460 |
+
base_score=0.25,
|
| 461 |
+
score_mean=2.0,
|
| 462 |
+
score_std=1.0,
|
| 463 |
+
candidate_index=0,
|
| 464 |
+
candidate_type="ctt_transport_rank0",
|
| 465 |
+
tangent=np.zeros(21, dtype=float),
|
| 466 |
+
num_candidates=3,
|
| 467 |
+
feature_set="score_context_chart_compat",
|
| 468 |
+
context={
|
| 469 |
+
"target_task_id": "PickCube-v1",
|
| 470 |
+
"source_task_id": "PickCube-v1",
|
| 471 |
+
"instruction": "Pick up the cube.",
|
| 472 |
+
},
|
| 473 |
+
chart_compat=np.ones(22, dtype=float),
|
| 474 |
+
score_shape=score_shape[0],
|
| 475 |
+
)
|
| 476 |
+
|
| 477 |
+
assert score_shape.shape == (3, 8)
|
| 478 |
+
assert score_shape[0, 0] == 0.0
|
| 479 |
+
assert score_shape[0, 2] == 0.0
|
| 480 |
+
assert score_shape[0, 3] == 1.0
|
| 481 |
+
assert score_shape[1, 0] == 1.0
|
| 482 |
+
assert len(feature) == len(_feature_names("score_context_chart_compat"))
|
| 483 |
+
assert len(feature) == 10 + 8 + 27 + 22
|
| 484 |
+
|
| 485 |
+
metrics = {
|
| 486 |
+
"num_calibration_rows": 1,
|
| 487 |
+
"num_eval_rows": 1,
|
| 488 |
+
"selected_lambda": 0.0,
|
| 489 |
+
"tau": 0.0,
|
| 490 |
+
"threshold_scope": "global",
|
| 491 |
+
"fit_objective": "pointwise",
|
| 492 |
+
"feature_set": "basic",
|
| 493 |
+
"target": "utility_margin",
|
| 494 |
+
"eval_summary": {},
|
| 495 |
+
"calibration_summary": {},
|
| 496 |
+
}
|
| 497 |
+
report_path = tmp_path / "report.md"
|
| 498 |
+
report_path.write_text("old\n")
|
| 499 |
+
_write_report_artifact(tmp_path, metrics, no_markdown_report=True)
|
| 500 |
+
assert not report_path.exists()
|
| 501 |
+
_write_report_artifact(tmp_path, metrics, no_markdown_report=False)
|
| 502 |
+
assert report_path.exists()
|
| 503 |
|
| 504 |
|
| 505 |
def test_chart_compat_feature_is_deployment_visible_and_fixed_width() -> None:
|