File size: 8,391 Bytes
5cdf702
 
 
 
 
 
 
 
 
 
 
7ab5064
 
 
5cdf702
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7ab5064
5cdf702
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
"""Public portability adaptation of the recorded coding benchmark.
Prompts and throughput formulas retained; server, credentials and output directory configurable.
Original SHA256: dca2d3c234730fafadd6acc56a66ceab28a2b04b2c7b90d90edd520f5239a474
"""
import argparse
import hashlib
import json
import os
from pathlib import Path
import time
import urllib.request
BASE = os.environ.get("QWEN_API_BASE", "http://localhost:8000").rstrip("/")
MODEL = os.environ.get("QWEN_MODEL", "qwen3.8-27b")
KEY = os.environ.get("QWEN_API_KEY", "")

SUITE_ID = "m23-holdout-v2-9f04c2b7"

TASKS = [
    (
        "python_index",
        "Write a complete Python 3.12 module implementing an incremental suffix-occurrence "
        "index over integer token chunks. It must answer the longest suffix match in a bounded "
        "6..12-token window without rescanning all history, support eviction, use typed "
        "dataclasses, and include pytest tests. Put the literal suite marker " + SUITE_ID +
        " in a module constant. Return code only.",
    ),
    (
        "rust_queue",
        "Write a complete stable-Rust module for a bounded MPMC queue using per-slot atomic "
        "sequence counters. Include close semantics, try_push, try_pop, timeout wrappers, safe "
        "Drop, and multithreaded tests. No external crates. Include the literal marker " +
        SUITE_ID + ". Return code only.",
    ),
    (
        "typescript_patch",
        "Write a complete strict TypeScript module that applies transactional text edits. "
        "Edits use UTF-16 offsets, may arrive unsorted, must reject overlap, preserve an inverse "
        "patch, and expose compose and rollback operations. Include node:test cases and the "
        "literal marker " + SUITE_ID + ". Return code only.",
    ),
    (
        "go_reload",
        "Write a complete Go package implementing an atomic configuration reloader. Parse JSON, "
        "validate cross-field invariants, publish immutable snapshots, debounce file events, "
        "retain the last good snapshot, and expose observable error state. Include table-driven "
        "tests and literal marker " + SUITE_ID + ". Return code only.",
    ),
    (
        "cpp_parser",
        "Write a complete C++20 incremental WebSocket frame parser with bounded buffering, "
        "fragment reassembly, masking, control-frame validation, protocol errors, and unit tests "
        "using only the standard library. Include literal marker " + SUITE_ID +
        ". Return code only.",
    ),
    (
        "zig_pool",
        "Write a complete Zig 0.14 module for a generation-counted object pool. It must detect "
        "stale handles, support O(1) allocate/free, iterate live objects, and include tests for "
        "wraparound and double free. Include literal marker " + SUITE_ID + ". Return code only.",
    ),
    (
        "sql_ledger",
        "Write a PostgreSQL 17 migration and verification script for an append-only double-entry "
        "ledger. Enforce balanced transactions with deferred constraints, idempotency keys, "
        "tenant isolation, reversal entries, useful indexes, and pgTAP-style checks. Include "
        "literal marker " + SUITE_ID + ". Return SQL only.",
    ),
    (
        "cuda_scan",
        "Write a complete CUDA C++ program implementing an exclusive prefix sum for arbitrary "
        "length float arrays using hierarchical block scans. Avoid bank conflicts, handle tails, "
        "check CUDA errors, compare against a CPU reference, and benchmark. Include literal "
        "marker " + SUITE_ID + ". Return code only.",
    ),
]

def metrics() -> tuple[float, float]:
    request = urllib.request.Request(
        BASE + "/metrics", headers={"Authorization": "Bearer " + KEY}
    )
    values: dict[str, float] = {}
    for line in urllib.request.urlopen(request, timeout=10).read().decode().splitlines():
        for name in (
            "vllm:spec_decode_num_drafts_total",
            "vllm:spec_decode_num_accepted_tokens_total",
        ):
            if line.startswith(name + " ") or line.startswith(name + "{"):
                values[name] = float(line.split()[-1])
    return (
        values.get("vllm:spec_decode_num_drafts_total", 0.0),
        values.get("vllm:spec_decode_num_accepted_tokens_total", 0.0),
    )

def run_request(prompt: str, max_tokens: int) -> dict[str, object]:
    payload = {
        "model": MODEL,
        "messages": [{"role": "user", "content": prompt}],
        "max_tokens": max_tokens,
        "temperature": 0,
        "stream": True,
        "stream_options": {"include_usage": True},
        "chat_template_kwargs": {"enable_thinking": False},
    }
    request = urllib.request.Request(
        BASE + "/v1/chat/completions",
        data=json.dumps(payload).encode(),
        headers={
            "Content-Type": "application/json",
            "Authorization": "Bearer " + KEY,
        },
    )
    drafts0, accepted0 = metrics()
    started = time.perf_counter()
    first: float | None = None
    usage: dict[str, int] = {}
    chunks: list[str] = []
    with urllib.request.urlopen(request, timeout=1800) as response:
        for raw in response:
            line = raw.decode().strip()
            if not line.startswith("data: "):
                continue
            body = line[6:]
            if body == "[DONE]":
                break
            event = json.loads(body)
            if event.get("usage"):
                usage = event["usage"]
            for choice in event.get("choices", []):
                content = choice.get("delta", {}).get("content")
                if content:
                    first = first or time.perf_counter()
                    chunks.append(content)
    ended = time.perf_counter()
    drafts1, accepted1 = metrics()
    output = "".join(chunks)
    output_tokens = int(usage.get("completion_tokens", len(chunks)))
    steps = drafts1 - drafts0
    accepted = accepted1 - accepted0
    decode_seconds = max(ended - (first or ended), 1e-6)
    return {
        "prompt_tokens": int(usage.get("prompt_tokens", 0)),
        "output_tokens": output_tokens,
        "steps": steps,
        "accepted": accepted,
        "tokens_per_step": 1.0 + accepted / max(steps, 1.0),
        "decode_tps": max(output_tokens - 1, 0) / decode_seconds,
        "ttft": (first or ended) - started,
        "sha256": hashlib.sha256(output.encode()).hexdigest()[:16],
    }

def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("tag")
    parser.add_argument("--max-tokens", type=int, default=384)
    args = parser.parse_args()

    # Warm the exact graph length without reusing any benchmark prompt.
    run_request(
        "Return a Python function that adds two integers. Marker warmup-" + SUITE_ID,
        96,
    )
    rows: list[dict[str, object]] = []
    for name, prompt in TASKS:
        row = {"task": name, **run_request(prompt, args.max_tokens)}
        rows.append(row)
        print(
            f"HOLDOUT {args.tag} {name:16s} out={row['output_tokens']:4d} "
            f"steps={row['steps']:5.0f} tok/step={row['tokens_per_step']:5.2f} "
            f"decode={row['decode_tps']:6.1f} ttft={row['ttft']:5.2f}s "
            f"sha={row['sha256']}",
            flush=True,
        )

    total_out = sum(int(row["output_tokens"]) for row in rows)
    total_steps = sum(float(row["steps"]) for row in rows)
    total_accepted = sum(float(row["accepted"]) for row in rows)
    total_time = sum(
        max(int(row["output_tokens"]) - 1, 0) / max(float(row["decode_tps"]), 1e-9)
        for row in rows
    )
    summary = {
        "suite_id": SUITE_ID,
        "tag": args.tag,
        "rows": rows,
        "total_tokens_per_step": 1.0 + total_accepted / max(total_steps, 1.0),
        "total_decode_tps": max(total_out - len(rows), 0) / max(total_time, 1e-9),
    }
    print(
        f"HOLDOUT {args.tag} TOTAL out={total_out} steps={total_steps:.0f} "
        f"tok/step={summary['total_tokens_per_step']:.2f} "
        f"decode={summary['total_decode_tps']:.1f} tok/s",
        flush=True,
    )
    output_dir = Path(os.environ.get("QWEN_RESULTS_DIR", "./results"))
    output_dir.mkdir(parents=True, exist_ok=True)
    (output_dir / f"coding_holdout_v2_{args.tag}.json").write_text(
        json.dumps(summary, indent=2) + "\n"
    )

if __name__ == "__main__":
    main()