anhtld commited on
Commit
12a67b5
·
verified ·
1 Parent(s): bf0048e

auto-sync 2026-07-04T03:59:09Z workspace (part 5)

Browse files
workspace/scripts/slurm/train_ctt_feature_proxy.sbatch CHANGED
@@ -60,12 +60,15 @@ VAL_OUT="runs/${NAME_PREFIX}_seed${SEED}_val_proxy"
60
  --k "${CTT_PROXY_K:-16}" \
61
  --max-target-charts "${CTT_MAX_TARGET_CHARTS:-69}" \
62
  --neighbors "${CTT_PROXY_NEIGHBORS:-8}" \
63
- --thresholds "${CTT_PROXY_THRESHOLDS:-0.20,0.40}"
 
64
 
65
  "$PYTHON" scripts/build_ctt_proxy_comparison.py \
66
- --out-dir runs/ctt_val_proxy_comparison
 
67
 
68
  "$PYTHON" scripts/summarize_ctt_runs.py \
69
  --run-root runs \
70
  --out-csv runs/summary_ctt.csv \
71
- --out-md runs/summary_ctt.md
 
 
60
  --k "${CTT_PROXY_K:-16}" \
61
  --max-target-charts "${CTT_MAX_TARGET_CHARTS:-69}" \
62
  --neighbors "${CTT_PROXY_NEIGHBORS:-8}" \
63
+ --thresholds "${CTT_PROXY_THRESHOLDS:-0.20,0.40}" \
64
+ --no-markdown-report
65
 
66
  "$PYTHON" scripts/build_ctt_proxy_comparison.py \
67
+ --out-dir runs/ctt_val_proxy_comparison \
68
+ --no-markdown-report
69
 
70
  "$PYTHON" scripts/summarize_ctt_runs.py \
71
  --run-root runs \
72
  --out-csv runs/summary_ctt.csv \
73
+ --out-md runs/summary_ctt.md \
74
+ --no-markdown-report
workspace/scripts/summarize_ctt_runs.py CHANGED
@@ -39,6 +39,11 @@ def main(argv: list[str] | None = None) -> int:
39
  parser.add_argument("--run-root", type=Path, default=Path("runs"))
40
  parser.add_argument("--out-csv", type=Path, default=Path("runs/summary_ctt.csv"))
41
  parser.add_argument("--out-md", type=Path, default=Path("runs/summary_ctt.md"))
 
 
 
 
 
42
  args = parser.parse_args(argv)
43
 
44
  rows = []
@@ -99,8 +104,14 @@ def main(argv: list[str] | None = None) -> int:
99
  writer.writeheader()
100
  for row in rows:
101
  writer.writerow({column: row.get(column, "n/a") for column in COLUMNS})
102
- args.out_md.write_text(_markdown(rows) + "\n")
103
- print(json.dumps({"csv": str(args.out_csv), "md": str(args.out_md), "rows": len(rows)}, indent=2))
 
 
 
 
 
 
104
  return 0
105
 
106
 
 
39
  parser.add_argument("--run-root", type=Path, default=Path("runs"))
40
  parser.add_argument("--out-csv", type=Path, default=Path("runs/summary_ctt.csv"))
41
  parser.add_argument("--out-md", type=Path, default=Path("runs/summary_ctt.md"))
42
+ parser.add_argument(
43
+ "--no-markdown-report",
44
+ action="store_true",
45
+ help="Do not write --out-md; the persistent prose summary lives in README.md.",
46
+ )
47
  args = parser.parse_args(argv)
48
 
49
  rows = []
 
104
  writer.writeheader()
105
  for row in rows:
106
  writer.writerow({column: row.get(column, "n/a") for column in COLUMNS})
107
+ if args.no_markdown_report:
108
+ if args.out_md.exists():
109
+ args.out_md.unlink()
110
+ md_path: str | None = None
111
+ else:
112
+ args.out_md.write_text(_markdown(rows) + "\n")
113
+ md_path = str(args.out_md)
114
+ print(json.dumps({"csv": str(args.out_csv), "md": md_path, "rows": len(rows)}, indent=2))
115
  return 0
116
 
117
 
workspace/tests/test_dominance_selector.py CHANGED
@@ -21,8 +21,10 @@ from scripts.eval_learned_dominance_selector import (
21
  _feature_names,
22
  _fit_select_ridge,
23
  _pairwise_calibration_summary,
 
24
  _source_evidence_feature,
25
  _target_value,
 
26
  )
27
  from scripts.eval_nonlinear_dominance_selector import _split_rows, _subset_dataset
28
  from scripts.eval_nonlinear_dominance_selector import _write_provenance as _write_nonlinear_provenance
@@ -443,6 +445,61 @@ def test_learned_dominance_tangent_features_and_success_weighted_target() -> Non
443
  assert source_evidence[1] == pytest.approx(np.log1p(2))
444
  assert source_evidence[-1] == 1.0
445
  assert _target_value("success_weighted_margin", utility_margin=0.2, candidate_success=1.0) == 1.2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
446
 
447
 
448
  def test_chart_compat_feature_is_deployment_visible_and_fixed_width() -> None:
 
21
  _feature_names,
22
  _fit_select_ridge,
23
  _pairwise_calibration_summary,
24
+ _score_shape_matrix,
25
  _source_evidence_feature,
26
  _target_value,
27
+ _write_report_artifact,
28
  )
29
  from scripts.eval_nonlinear_dominance_selector import _split_rows, _subset_dataset
30
  from scripts.eval_nonlinear_dominance_selector import _write_provenance as _write_nonlinear_provenance
 
445
  assert source_evidence[1] == pytest.approx(np.log1p(2))
446
  assert source_evidence[-1] == 1.0
447
  assert _target_value("success_weighted_margin", utility_margin=0.2, candidate_success=1.0) == 1.2
448
+ assert _target_value(
449
+ "success_weighted_margin",
450
+ utility_margin=0.2,
451
+ candidate_success=1.0,
452
+ success_bonus=2.5,
453
+ ) == 2.7
454
+
455
+
456
+ def test_score_shape_feature_set_and_markdown_suppression(tmp_path) -> None:
457
+ score_shape = _score_shape_matrix([3.0, 1.0, 2.0])
458
+ feature = _candidate_feature(
459
+ score=3.0,
460
+ base_score=0.25,
461
+ score_mean=2.0,
462
+ score_std=1.0,
463
+ candidate_index=0,
464
+ candidate_type="ctt_transport_rank0",
465
+ tangent=np.zeros(21, dtype=float),
466
+ num_candidates=3,
467
+ feature_set="score_context_chart_compat",
468
+ context={
469
+ "target_task_id": "PickCube-v1",
470
+ "source_task_id": "PickCube-v1",
471
+ "instruction": "Pick up the cube.",
472
+ },
473
+ chart_compat=np.ones(22, dtype=float),
474
+ score_shape=score_shape[0],
475
+ )
476
+
477
+ assert score_shape.shape == (3, 8)
478
+ assert score_shape[0, 0] == 0.0
479
+ assert score_shape[0, 2] == 0.0
480
+ assert score_shape[0, 3] == 1.0
481
+ assert score_shape[1, 0] == 1.0
482
+ assert len(feature) == len(_feature_names("score_context_chart_compat"))
483
+ assert len(feature) == 10 + 8 + 27 + 22
484
+
485
+ metrics = {
486
+ "num_calibration_rows": 1,
487
+ "num_eval_rows": 1,
488
+ "selected_lambda": 0.0,
489
+ "tau": 0.0,
490
+ "threshold_scope": "global",
491
+ "fit_objective": "pointwise",
492
+ "feature_set": "basic",
493
+ "target": "utility_margin",
494
+ "eval_summary": {},
495
+ "calibration_summary": {},
496
+ }
497
+ report_path = tmp_path / "report.md"
498
+ report_path.write_text("old\n")
499
+ _write_report_artifact(tmp_path, metrics, no_markdown_report=True)
500
+ assert not report_path.exists()
501
+ _write_report_artifact(tmp_path, metrics, no_markdown_report=False)
502
+ assert report_path.exists()
503
 
504
 
505
  def test_chart_compat_feature_is_deployment_visible_and_fixed_width() -> None: