| |
| |
| |
| |
| @@ -13,3 +13,17 @@ Training, eval, and vLLM-launch scripts write reproducibility artifacts so runs |
| **Always pass `--provenance-dir` when running `scripts/launch_vllm.py`** — point it at the associated run's output/checkpoint directory (e.g. `--provenance-dir <output_dir>`) so the vLLM provenance is co-located with the training/eval it belongs to. |
| |
| When publishing a model or eval results (HuggingFace, GitHub comments, etc.), always include the provenance artifacts (`train_command.txt`, `eval_command.txt`, `vllm_command.txt`, `checkpoint_sha256.txt`, `drafter_checkpoint_sha256.txt`, patches) so the run can be reproduced. |
| + |
| +## Agent skills |
| + |
| +### Issue tracker |
| + |
| +Issues and specs are tracked in GitHub Issues in `rahul-tuli/speculators`; use `gh --repo rahul-tuli/speculators`. See `docs/agents/issue-tracker.md`. |
| + |
| +### Triage labels |
| + |
| +Use the default triage labels: `needs-triage`, `needs-info`, `ready-for-agent`, `ready-for-human`, and `wontfix`. See `docs/agents/triage-labels.md`. |
| + |
| +### Domain docs |
| + |
| +This is a single-context repo using root `CONTEXT.md` and `docs/adr/`. See `docs/agents/domain.md`. |
| |
| |
| |
| |
| @@ -263,14 +263,11 @@ def _run_subset( |
| logger.warning("[%s] No speculative decoding metrics found", subset) |
| |
| if is_sweep: |
| - rows = parse_sweep_results( |
| - run_output, |
| - spec if has_spec else None, |
| - ) |
| + # Counters bracket the entire sweep, not individual load points. |
| + rows = parse_sweep_results(run_output) |
| if rows: |
| if perf_csv is None: |
| - acc_cols = acceptance_csv_columns(spec) if has_spec else [] |
| - cols = BASE_CSV_COLUMNS + acc_cols |
| + cols = BASE_CSV_COLUMNS |
| perf_csv = CsvWriter( |
| output_dir / "perf_results.csv", |
| cols, |
| @@ -357,7 +354,7 @@ def run_benchmark(args: argparse.Namespace) -> None: |
| if mt is not None: |
| all_max_tokens[label] = mt |
| |
| - if acceptance_csv is None: |
| + if acceptance_csv is None and not args.target_only: |
| logger.error("No acceptance metrics collected from any subset") |
| sys.exit(1) |
| |
| @@ -389,12 +386,19 @@ def main() -> None: |
| ) |
| parser.add_argument( |
| "mode", |
| - choices=["throughput", "sweep"], |
| + choices=["throughput", "sweep", "point"], |
| help=( |
| "throughput: max-rate run for acceptance rates; " |
| "sweep: full benchmarking pipeline" |
| ), |
| ) |
| + parser.add_argument("--target-only", action="store_true") |
| + parser.add_argument( |
| + "--identity", type=Path, help="Immutable point run identity JSON" |
| + ) |
| + parser.add_argument("--warmup-requests", type=int, default=2) |
| + parser.add_argument("--max-tokens", type=int, default=4096) |
| + parser.add_argument("--eval-seed", type=int, default=0) |
| parser.add_argument( |
| "--target", |
| required=True, |
| @@ -470,7 +474,12 @@ def main() -> None: |
| else: |
| args.output_dir = f"perf_results_{timestamp}" |
| |
| - run_benchmark(args) |
| + if args.mode == "point": |
| + from point import run_point # noqa: PLC0415 |
| + |
| + run_point(args) |
| + else: |
| + run_benchmark(args) |
| |
| |
| if __name__ == "__main__": |
| |
| |
| |
| |
| @@ -567,7 +567,7 @@ def run_guidellm( |
| data = f"kind=huggingface,source={dataset}" |
| data += f",load_kwargs.data_files={subset}.jsonl,load_kwargs.split=train" |
| else: |
| - data = f"kind=json_file,path={dataset}" |
| + data = f"kind=json_file,path={dataset},load_kwargs.split=train" |
| cmd.extend(["--data", data]) |
| |
| cmd.extend(["--data-column-mapper", data_column_mapper]) |
| |
| |
| |
| |
| @@ -244,8 +244,8 @@ def parse_args(): |
| eval_parser.add_argument( |
| "--spec-model", |
| type=str, |
| - required=True, |
| - help="Drafter model name or path", |
| + default=None, |
| + help="Drafter model name or path (omit for target-only evaluation)", |
| ) |
| eval_parser.add_argument( |
| "--spec-tokens", |
| @@ -503,9 +503,11 @@ def _build_eval_cmd(args, vllm_args): |
| "vllm.entrypoints.cli.main", |
| "serve", |
| args.model, |
| - "--spec-model", |
| - args.spec_model, |
| ] |
| + if args.spec_model is not None: |
| + cmd.extend(["--spec-model", args.spec_model]) |
| + elif args.spec_tokens is not None or args.spec_method is not None: |
| + raise ValueError("--spec-tokens/--spec-method require --spec-model") |
| if args.spec_tokens is not None: |
| cmd.extend(["--spec-tokens", str(args.spec_tokens)]) |
| if args.spec_method is not None: |
| |
| |
| |
| |
| @@ -42,6 +42,7 @@ def test_finetuning_weight_sanity(tmp_path: Path): |
| # at least MIN_CHANGED tensors must have rel_l1 > REL_L1_MIN |
| # to ensure weights actually changed. |
| LR = "1e-5" |
| + MUON_LR = "1e-4" |
| REL_L1_MAX = 0.05 |
| REL_L1_MIN = 1e-4 |
| MIN_CHANGED = 3 |
| @@ -93,6 +94,8 @@ def test_finetuning_weight_sanity(tmp_path: Path): |
| "2", |
| "--lr", |
| LR, |
| + "--muon-lr", |
| + MUON_LR, |
| "--total-seq-len", |
| "2048", |
| "--num-workers", |
| |
| |
| |
| |
| @@ -5,7 +5,9 @@ from typer.testing import CliRunner |
| |
| from speculators.cli import app |
| |
| -runner = CliRunner() |
| +# Rich uses COLUMNS when rendering help, so keep assertions independent of the |
| +# terminal width provided by the environment running the tests. |
| +runner = CliRunner(env={"COLUMNS": "200"}) |
| |
| |
| def unstyled_output(result): |
| |
| |
| |
| |
| @@ -1,7 +1,9 @@ |
| import os |
| +from argparse import Namespace |
| |
| from scripts.launch_vllm import ( |
| DEFAULT_RENDERER_NUM_WORKERS, |
| + _build_eval_cmd, |
| _enable_scale_out_endpoints, |
| _preprocessing_workers, |
| _set_render_thread_defaults, |
| @@ -87,3 +89,12 @@ def test_sizing_uses_one_combined_budget(): |
| assert preprocessing_workers == 30 |
| assert api_servers == 7 |
| assert preprocessing_workers * 3 + api_servers * 4 <= int(160 * 0.75) |
| + |
| + |
| +def test_target_only_eval_omits_drafter_flags(): |
| + args = Namespace( |
| + model="target", spec_model=None, spec_tokens=None, spec_method=None |
| + ) |
| + cmd = _build_eval_cmd(args, ["--dtype", "bfloat16"]) |
| + assert cmd[-3:] == ["target", "--dtype", "bfloat16"] |
| + assert "--spec-model" not in cmd |
|
|
