File size: 7,431 Bytes
9e05253 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 | diff --git a/AGENTS.md b/AGENTS.md
index 46c9344..a2b6b02 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -13,3 +13,17 @@ Training, eval, and vLLM-launch scripts write reproducibility artifacts so runs
**Always pass `--provenance-dir` when running `scripts/launch_vllm.py`** — point it at the associated run's output/checkpoint directory (e.g. `--provenance-dir <output_dir>`) so the vLLM provenance is co-located with the training/eval it belongs to.
When publishing a model or eval results (HuggingFace, GitHub comments, etc.), always include the provenance artifacts (`train_command.txt`, `eval_command.txt`, `vllm_command.txt`, `checkpoint_sha256.txt`, `drafter_checkpoint_sha256.txt`, patches) so the run can be reproduced.
+
+## Agent skills
+
+### Issue tracker
+
+Issues and specs are tracked in GitHub Issues in `rahul-tuli/speculators`; use `gh --repo rahul-tuli/speculators`. See `docs/agents/issue-tracker.md`.
+
+### Triage labels
+
+Use the default triage labels: `needs-triage`, `needs-info`, `ready-for-agent`, `ready-for-human`, and `wontfix`. See `docs/agents/triage-labels.md`.
+
+### Domain docs
+
+This is a single-context repo using root `CONTEXT.md` and `docs/adr/`. See `docs/agents/domain.md`.
diff --git a/scripts/evaluate/evaluate.py b/scripts/evaluate/evaluate.py
index f121402..14a6dea 100644
--- a/scripts/evaluate/evaluate.py
+++ b/scripts/evaluate/evaluate.py
@@ -263,14 +263,11 @@ def _run_subset(
logger.warning("[%s] No speculative decoding metrics found", subset)
if is_sweep:
- rows = parse_sweep_results(
- run_output,
- spec if has_spec else None,
- )
+ # Counters bracket the entire sweep, not individual load points.
+ rows = parse_sweep_results(run_output)
if rows:
if perf_csv is None:
- acc_cols = acceptance_csv_columns(spec) if has_spec else []
- cols = BASE_CSV_COLUMNS + acc_cols
+ cols = BASE_CSV_COLUMNS
perf_csv = CsvWriter(
output_dir / "perf_results.csv",
cols,
@@ -357,7 +354,7 @@ def run_benchmark(args: argparse.Namespace) -> None:
if mt is not None:
all_max_tokens[label] = mt
- if acceptance_csv is None:
+ if acceptance_csv is None and not args.target_only:
logger.error("No acceptance metrics collected from any subset")
sys.exit(1)
@@ -389,12 +386,19 @@ def main() -> None:
)
parser.add_argument(
"mode",
- choices=["throughput", "sweep"],
+ choices=["throughput", "sweep", "point"],
help=(
"throughput: max-rate run for acceptance rates; "
"sweep: full benchmarking pipeline"
),
)
+ parser.add_argument("--target-only", action="store_true")
+ parser.add_argument(
+ "--identity", type=Path, help="Immutable point run identity JSON"
+ )
+ parser.add_argument("--warmup-requests", type=int, default=2)
+ parser.add_argument("--max-tokens", type=int, default=4096)
+ parser.add_argument("--eval-seed", type=int, default=0)
parser.add_argument(
"--target",
required=True,
@@ -470,7 +474,12 @@ def main() -> None:
else:
args.output_dir = f"perf_results_{timestamp}"
- run_benchmark(args)
+ if args.mode == "point":
+ from point import run_point # noqa: PLC0415
+
+ run_point(args)
+ else:
+ run_benchmark(args)
if __name__ == "__main__":
diff --git a/scripts/evaluate/perf_utils.py b/scripts/evaluate/perf_utils.py
index cc2001a..f6048d6 100644
--- a/scripts/evaluate/perf_utils.py
+++ b/scripts/evaluate/perf_utils.py
@@ -567,7 +567,7 @@ def run_guidellm(
data = f"kind=huggingface,source={dataset}"
data += f",load_kwargs.data_files={subset}.jsonl,load_kwargs.split=train"
else:
- data = f"kind=json_file,path={dataset}"
+ data = f"kind=json_file,path={dataset},load_kwargs.split=train"
cmd.extend(["--data", data])
cmd.extend(["--data-column-mapper", data_column_mapper])
diff --git a/scripts/launch_vllm.py b/scripts/launch_vllm.py
index 47eb77b..2bafbbd 100644
--- a/scripts/launch_vllm.py
+++ b/scripts/launch_vllm.py
@@ -244,8 +244,8 @@ def parse_args():
eval_parser.add_argument(
"--spec-model",
type=str,
- required=True,
- help="Drafter model name or path",
+ default=None,
+ help="Drafter model name or path (omit for target-only evaluation)",
)
eval_parser.add_argument(
"--spec-tokens",
@@ -503,9 +503,11 @@ def _build_eval_cmd(args, vllm_args):
"vllm.entrypoints.cli.main",
"serve",
args.model,
- "--spec-model",
- args.spec_model,
]
+ if args.spec_model is not None:
+ cmd.extend(["--spec-model", args.spec_model])
+ elif args.spec_tokens is not None or args.spec_method is not None:
+ raise ValueError("--spec-tokens/--spec-method require --spec-model")
if args.spec_tokens is not None:
cmd.extend(["--spec-tokens", str(args.spec_tokens)])
if args.spec_method is not None:
diff --git a/tests/e2e/smoke/test_finetuning_sanity.py b/tests/e2e/smoke/test_finetuning_sanity.py
index 4a13150..fcc93a1 100644
--- a/tests/e2e/smoke/test_finetuning_sanity.py
+++ b/tests/e2e/smoke/test_finetuning_sanity.py
@@ -42,6 +42,7 @@ def test_finetuning_weight_sanity(tmp_path: Path):
# at least MIN_CHANGED tensors must have rel_l1 > REL_L1_MIN
# to ensure weights actually changed.
LR = "1e-5"
+ MUON_LR = "1e-4"
REL_L1_MAX = 0.05
REL_L1_MIN = 1e-4
MIN_CHANGED = 3
@@ -93,6 +94,8 @@ def test_finetuning_weight_sanity(tmp_path: Path):
"2",
"--lr",
LR,
+ "--muon-lr",
+ MUON_LR,
"--total-seq-len",
"2048",
"--num-workers",
diff --git a/tests/unit/cli/test_cli.py b/tests/unit/cli/test_cli.py
index fbc2559..ebecb73 100644
--- a/tests/unit/cli/test_cli.py
+++ b/tests/unit/cli/test_cli.py
@@ -5,7 +5,9 @@ from typer.testing import CliRunner
from speculators.cli import app
-runner = CliRunner()
+# Rich uses COLUMNS when rendering help, so keep assertions independent of the
+# terminal width provided by the environment running the tests.
+runner = CliRunner(env={"COLUMNS": "200"})
def unstyled_output(result):
diff --git a/tests/unit/scripts/test_launch_vllm.py b/tests/unit/scripts/test_launch_vllm.py
index de34b5c..b751480 100644
--- a/tests/unit/scripts/test_launch_vllm.py
+++ b/tests/unit/scripts/test_launch_vllm.py
@@ -1,7 +1,9 @@
import os
+from argparse import Namespace
from scripts.launch_vllm import (
DEFAULT_RENDERER_NUM_WORKERS,
+ _build_eval_cmd,
_enable_scale_out_endpoints,
_preprocessing_workers,
_set_render_thread_defaults,
@@ -87,3 +89,12 @@ def test_sizing_uses_one_combined_budget():
assert preprocessing_workers == 30
assert api_servers == 7
assert preprocessing_workers * 3 + api_servers * 4 <= int(160 * 0.75)
+
+
+def test_target_only_eval_omits_drafter_flags():
+ args = Namespace(
+ model="target", spec_model=None, spec_tokens=None, spec_method=None
+ )
+ cmd = _build_eval_cmd(args, ["--dtype", "bfloat16"])
+ assert cmd[-3:] == ["target", "--dtype", "bfloat16"]
+ assert "--spec-model" not in cmd
|