diff --git a/AGENTS.md b/AGENTS.md index 46c9344..a2b6b02 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,3 +13,17 @@ Training, eval, and vLLM-launch scripts write reproducibility artifacts so runs **Always pass `--provenance-dir` when running `scripts/launch_vllm.py`** — point it at the associated run's output/checkpoint directory (e.g. `--provenance-dir `) so the vLLM provenance is co-located with the training/eval it belongs to. When publishing a model or eval results (HuggingFace, GitHub comments, etc.), always include the provenance artifacts (`train_command.txt`, `eval_command.txt`, `vllm_command.txt`, `checkpoint_sha256.txt`, `drafter_checkpoint_sha256.txt`, patches) so the run can be reproduced. + +## Agent skills + +### Issue tracker + +Issues and specs are tracked in GitHub Issues in `rahul-tuli/speculators`; use `gh --repo rahul-tuli/speculators`. See `docs/agents/issue-tracker.md`. + +### Triage labels + +Use the default triage labels: `needs-triage`, `needs-info`, `ready-for-agent`, `ready-for-human`, and `wontfix`. See `docs/agents/triage-labels.md`. + +### Domain docs + +This is a single-context repo using root `CONTEXT.md` and `docs/adr/`. See `docs/agents/domain.md`. diff --git a/scripts/evaluate/evaluate.py b/scripts/evaluate/evaluate.py index f121402..14a6dea 100644 --- a/scripts/evaluate/evaluate.py +++ b/scripts/evaluate/evaluate.py @@ -263,14 +263,11 @@ def _run_subset( logger.warning("[%s] No speculative decoding metrics found", subset) if is_sweep: - rows = parse_sweep_results( - run_output, - spec if has_spec else None, - ) + # Counters bracket the entire sweep, not individual load points. + rows = parse_sweep_results(run_output) if rows: if perf_csv is None: - acc_cols = acceptance_csv_columns(spec) if has_spec else [] - cols = BASE_CSV_COLUMNS + acc_cols + cols = BASE_CSV_COLUMNS perf_csv = CsvWriter( output_dir / "perf_results.csv", cols, @@ -357,7 +354,7 @@ def run_benchmark(args: argparse.Namespace) -> None: if mt is not None: all_max_tokens[label] = mt - if acceptance_csv is None: + if acceptance_csv is None and not args.target_only: logger.error("No acceptance metrics collected from any subset") sys.exit(1) @@ -389,12 +386,19 @@ def main() -> None: ) parser.add_argument( "mode", - choices=["throughput", "sweep"], + choices=["throughput", "sweep", "point"], help=( "throughput: max-rate run for acceptance rates; " "sweep: full benchmarking pipeline" ), ) + parser.add_argument("--target-only", action="store_true") + parser.add_argument( + "--identity", type=Path, help="Immutable point run identity JSON" + ) + parser.add_argument("--warmup-requests", type=int, default=2) + parser.add_argument("--max-tokens", type=int, default=4096) + parser.add_argument("--eval-seed", type=int, default=0) parser.add_argument( "--target", required=True, @@ -470,7 +474,12 @@ def main() -> None: else: args.output_dir = f"perf_results_{timestamp}" - run_benchmark(args) + if args.mode == "point": + from point import run_point # noqa: PLC0415 + + run_point(args) + else: + run_benchmark(args) if __name__ == "__main__": diff --git a/scripts/evaluate/perf_utils.py b/scripts/evaluate/perf_utils.py index cc2001a..f6048d6 100644 --- a/scripts/evaluate/perf_utils.py +++ b/scripts/evaluate/perf_utils.py @@ -567,7 +567,7 @@ def run_guidellm( data = f"kind=huggingface,source={dataset}" data += f",load_kwargs.data_files={subset}.jsonl,load_kwargs.split=train" else: - data = f"kind=json_file,path={dataset}" + data = f"kind=json_file,path={dataset},load_kwargs.split=train" cmd.extend(["--data", data]) cmd.extend(["--data-column-mapper", data_column_mapper]) diff --git a/scripts/launch_vllm.py b/scripts/launch_vllm.py index 47eb77b..2bafbbd 100644 --- a/scripts/launch_vllm.py +++ b/scripts/launch_vllm.py @@ -244,8 +244,8 @@ def parse_args(): eval_parser.add_argument( "--spec-model", type=str, - required=True, - help="Drafter model name or path", + default=None, + help="Drafter model name or path (omit for target-only evaluation)", ) eval_parser.add_argument( "--spec-tokens", @@ -503,9 +503,11 @@ def _build_eval_cmd(args, vllm_args): "vllm.entrypoints.cli.main", "serve", args.model, - "--spec-model", - args.spec_model, ] + if args.spec_model is not None: + cmd.extend(["--spec-model", args.spec_model]) + elif args.spec_tokens is not None or args.spec_method is not None: + raise ValueError("--spec-tokens/--spec-method require --spec-model") if args.spec_tokens is not None: cmd.extend(["--spec-tokens", str(args.spec_tokens)]) if args.spec_method is not None: diff --git a/tests/e2e/smoke/test_finetuning_sanity.py b/tests/e2e/smoke/test_finetuning_sanity.py index 4a13150..fcc93a1 100644 --- a/tests/e2e/smoke/test_finetuning_sanity.py +++ b/tests/e2e/smoke/test_finetuning_sanity.py @@ -42,6 +42,7 @@ def test_finetuning_weight_sanity(tmp_path: Path): # at least MIN_CHANGED tensors must have rel_l1 > REL_L1_MIN # to ensure weights actually changed. LR = "1e-5" + MUON_LR = "1e-4" REL_L1_MAX = 0.05 REL_L1_MIN = 1e-4 MIN_CHANGED = 3 @@ -93,6 +94,8 @@ def test_finetuning_weight_sanity(tmp_path: Path): "2", "--lr", LR, + "--muon-lr", + MUON_LR, "--total-seq-len", "2048", "--num-workers", diff --git a/tests/unit/cli/test_cli.py b/tests/unit/cli/test_cli.py index fbc2559..ebecb73 100644 --- a/tests/unit/cli/test_cli.py +++ b/tests/unit/cli/test_cli.py @@ -5,7 +5,9 @@ from typer.testing import CliRunner from speculators.cli import app -runner = CliRunner() +# Rich uses COLUMNS when rendering help, so keep assertions independent of the +# terminal width provided by the environment running the tests. +runner = CliRunner(env={"COLUMNS": "200"}) def unstyled_output(result): diff --git a/tests/unit/scripts/test_launch_vllm.py b/tests/unit/scripts/test_launch_vllm.py index de34b5c..b751480 100644 --- a/tests/unit/scripts/test_launch_vllm.py +++ b/tests/unit/scripts/test_launch_vllm.py @@ -1,7 +1,9 @@ import os +from argparse import Namespace from scripts.launch_vllm import ( DEFAULT_RENDERER_NUM_WORKERS, + _build_eval_cmd, _enable_scale_out_endpoints, _preprocessing_workers, _set_render_thread_defaults, @@ -87,3 +89,12 @@ def test_sizing_uses_one_combined_budget(): assert preprocessing_workers == 30 assert api_servers == 7 assert preprocessing_workers * 3 + api_servers * 4 <= int(160 * 0.75) + + +def test_target_only_eval_omits_drafter_flags(): + args = Namespace( + model="target", spec_model=None, spec_tokens=None, spec_method=None + ) + cmd = _build_eval_cmd(args, ["--dtype", "bfloat16"]) + assert cmd[-3:] == ["target", "--dtype", "bfloat16"] + assert "--spec-model" not in cmd