"""Native Qwen thought experiment; no paid launch or power operations in this file. Run against an already registered vLLM server. Preparation and audit are CPU-only. All intermediate distributions are untempered; gate 0.7 precedes calibration. """ import argparse import concurrent.futures import datetime as dt import gzip import hashlib import json import math from pathlib import Path import threading import time THOUGHT_SYSTEM = ('You are a calibration engine inside a decision system. You are given a state, a question and lettered options. ' 'Think it through briefly first (check every condition, date and number that matters), then reply with one capital letter, no words, no punctuation.') VERSION = 'native-chat-gate/1' CONTROL_TOKENS = ('', '', '<|im_start|>', '<|im_end|>', '<|endoftext|>') def sha(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() def rows(path): with (gzip.open(path, 'rt') if str(path).endswith('.gz') else open(path)) as f: return [json.loads(line) for line in f if line.strip()] def dump(path, obj): Path(path).write_text(json.dumps(obj, indent=2, ensure_ascii=False) + '\n') def remap(p, order): """Displayed position -> canonical option position, with a complete permutation.""" if len(p) != len(order) or sorted(order) != list(range(len(order))): raise ValueError('option map must be a complete permutation') out = [0.0] * len(p) for value, canonical in zip(p, order): out[canonical] = value return out def average_orders(reads): if not reads: raise ValueError('no order reads') mapped = [remap(p, order) for p, order in reads] n = len(mapped[0]) if any(len(p) != n for p in mapped): raise ValueError('order reads differ in option count') return [sum(p[i] for p in mapped) / len(mapped) for i in range(n)] def gate(p, threshold=0.7): if not p or any(not math.isfinite(x) or x < 0 for x in p) or not math.isclose(sum(p), 1.0, abs_tol=1e-6): raise ValueError('invalid option distribution') return len(p) > 1 and max(p) < threshold def prepare(args): """Bind exact historical suites; keep private payloads outside the repository.""" di = rows(args.di_suite) xl = [r for r in rows(args.xl_source) if r['set'] == 'xl'] if len(di) != 6948 or len(xl) != 2691: raise ValueError('expected original DI6948 and XL2691 suites') ids = [r['_evaluation']['run_id'] for r in di] if len(set(ids)) != len(ids) or len({r['id'] for r in xl}) != len(xl): raise ValueError('duplicate suite identities') out = Path(args.out); out.mkdir(parents=True, exist_ok=True) tasks = [] for r in di: # Metadata/expected are retained for CPU kit scoring, never sent to the model. tasks.append({'id': r['_evaluation']['run_id'], 'suite': 'di', 'request': {'state': r['state'], 'questions': r['questions']}}) for r in xl: tasks.append({'id': r['id'], 'suite': 'xl', 'request': r['request'], 'gold_key': r['keys'][r['gold']]}) tasks.sort(key=lambda r: hashlib.sha256(('native-c16-1005:' + r['suite'] + ':' + r['id']).encode()).hexdigest()) path = out / 'requests.jsonl' path.write_text(''.join(json.dumps(r, ensure_ascii=False) + '\n' for r in tasks)) receipt = {'status': 'prepared_not_run', 'version': VERSION, 'di_requests': len(di), 'xl_items': len(xl), 'requests_sha256': sha(path), 'di_suite_sha256': sha(args.di_suite), 'xl_source_sha256': sha(args.xl_source), 'temperature_sha256': sha(args.temperature), 'native_thinking_measured': False, 'gate': 0.7, 'budget': 512, 'think_k': 1, 'order': 'original_only', 'wide': 'historical_knockout_onepass', 'di_prompt_format': 'repeat_state_plain', 'xl_prompt_format': 'plain', 'calibration': 'fixed historical A table for every policy; native B has no fitted table; additionally report T=1', 'selection': 'no new selection; original registered historical DI/XL diagnostic replay'} dump(out / 'prepared.json', receipt) if args.receipt: dump(args.receipt, receipt) print(json.dumps(receipt)) def native_read(cl, sov, state, q, threshold, budget, seed_text, usage, raw): n = len(q['options']) if n > 26: p, mode, pa = sov.read_question(cl, state, q, 0, budget, seed_text, usage, raw=raw) raw['native'] = {'status': 'wide_onepass', 'generated_tokens': 0} return p, mode, pa ida = sov.prompt_ids(cl, state, q) p, mass = cl.readout(ida, n) usage['input_tokens'] += len(ida) raw.update(A=p, A_mass=mass) if not gate(p, threshold): raw['native'] = {'status': 'not_gated', 'generated_tokens': 0} return p, 'A', None from eval.vllm_letter import chat_messages pre = cl.post('/tokenize', {'model': cl.served, 'messages': chat_messages(sov.chat_user(q['chat'], cl.prompt_format), system=THOUGHT_SYSTEM), 'add_generation_prompt': True, 'add_special_tokens': False, 'chat_template_kwargs': {'enable_thinking': True}})['tokens'] # The native Qwen template must actually open thought, not merely accept a flag. suffix = cl.ids('\n') if not suffix or pre[-len(suffix):] != suffix: raise ValueError('native thinking prefix does not end in \\n') close = cl.ids('') + cl.ids('\n\n') if len(pre) + budget + len(close) + 1 > cl.max_len: raw['native'] = {'status': 'capacity_onepass', 'generated_tokens': 0} return p, 'A', None t0 = time.perf_counter() response = cl.post('/v1/completions', {'model': cl.served, 'prompt': pre, 'max_tokens': budget, 'temperature': 0.6, 'top_p': 0.95, 'top_k': 20, 'seed': sov.hash_seed(seed_text), 'stop': [''], 'include_stop_str_in_output': False}) c = response['choices'][0] text = (c.get('text') or '').strip() ntok = response.get('usage', {}).get('completion_tokens') if not isinstance(ntok, int) or not 0 <= ntok <= budget: raise ValueError('missing or out-of-budget completion usage') usage['output_tokens'] += ntok # Final typed answers are scored by the benchmark adapter. Generated control # text alone is not an official answer-error condition; retain it privately. ids = pre + cl.ids(text + '\n') + close pb, bmass = cl.readout(ids, n) usage['input_tokens'] += len(pre) + len(ids) raw.update(B=[pb], B_mass=bmass, native={'status': 'thought_read', 'text': text, 'generated_tokens': ntok, 'naturally_closed': c.get('finish_reason') == 'stop' and c.get('stop_reason') == '', 'finish_reason': c.get('finish_reason'), 'stop_reason': c.get('stop_reason'), 'truncated': c.get('finish_reason') == 'length', 'generation_seconds': time.perf_counter() - t0, 'prefix_sha256': hashlib.sha256(json.dumps(pre).encode()).hexdigest()}) return pb, 'B', p def make_client(endpoint, served, max_len, deadline=None): from eval import systemone_vllm as sov class StrictClient(sov.Client): def __init__(self, *args, **kwargs): self.deadline = deadline super().__init__(*args, **kwargs) def post(self, path, body, retries=1): if self.deadline is not None: remaining = self.deadline - time.monotonic() - 120 if remaining <= 0: raise TimeoutError('registered evaluation window ended') self.timeout = min(180, remaining) return super().post(path, body, retries=1) def readout(self, ids, n): if len(ids) + 1 > self.max_len: raise sov.Capacity('native prompt exceeds context') want = [t for letters in self.letter_ids[:n] for t in letters] r = self.post('/v1/completions', {'model': self.served, 'prompt': ids, 'max_tokens': 1, 'temperature': 0.0, 'logprobs': 20, 'logprob_token_ids': want, 'return_tokens_as_token_ids': True}) from eval.vllm_letter import letter_readout p, _, mass = letter_readout(r['choices'][0]['logprobs']['top_logprobs'][0], dict(zip('ABCDEFGHIJKLMNOPQRSTUVWXYZ', self.letter_ids)), n) return p, mass cl = StrictClient(endpoint, served, max_len, timeout=180) cl.template = 'chat'; cl.wide = 'knockout'; cl.topk = 0; cl.wide_residual = 0.001 return cl, sov def answer_task(cl, sov, task, table, threshold, budget): from kev.api import SystemOneRequest, to_record req = task['request']; rec, meta = to_record(SystemOneRequest.model_validate(req)) chat = sov.chat_parts(req, rec, meta) # Client is per worker/task so DI/XL prompt formatting cannot race. cl.prompt_format = 'repeat_state_plain' if task['suite'] == 'di' else 'plain' answers = {}; usage = {'input_tokens': 0, 'output_tokens': 0} for rq, m in zip(rec['questions'], meta): q = {'id': m['id'], 'type': rq['qtype'], 'instructions': rq['instr'], 'options': rq['options'], 'option_texts': rq['options'], 'chat': chat[m['id']]} raw = {}; p, mode, pa = native_read(cl, sov, rec['state'], q, threshold, budget, json.dumps(req, sort_keys=True) + m['id'], usage, raw) # Fixed historical A table, no pretending a paren B table calibrates native B. calibrated = sov.temper(p, sov.temperature(table, m['type'], len(p))) keys = m['keys']; j = max(range(len(p)), key=p.__getitem__) value = {'type': m['type'], 'mode': mode, 'raw': {**raw, 'keys': keys}, 'probabilities': dict(zip(keys, calibrated)), 'probabilities_t1': dict(zip(keys, p))} if m['type'] == 'noul': value['noul'] = calibrated[keys.index('true')] elif m['type'] == 'score': value['score'] = sum(i * v for i, v in enumerate(calibrated)) else: value['choice'] = keys[j] value['confidence'] = max(calibrated) answers[m['id']] = value return {'model': cl.served, 'answers': answers, 'usage': usage} def run(args): inputs = Path(args.inputs); prepared = json.loads((inputs / 'prepared.json').read_text()) if sha(inputs / 'requests.jsonl') != prepared['requests_sha256'] or sha(args.temperature) != prepared['temperature_sha256']: raise ValueError('frozen input/calibration SHA mismatch') if args.wall_seconds > 14400 or args.wall_seconds <= 0: raise ValueError('evaluation window must be within 1..14400 seconds under registered USD2 outer guard') tasks = rows(inputs / 'requests.jsonl'); out = Path(args.out); out.mkdir(parents=True, exist_ok=True) if (out / 'reads.jsonl').exists(): raise ValueError('refuse overwrite or ambiguous resume') # Read-only inference endpoint has no per-request API billing. Provider box # power is registered/guarded by the operator before this process starts. table = json.loads(Path(args.temperature).read_text()) if 'A' in table: table = table['A'] # The GPU paid window began before checkpoint restoration, not at the first # inference. Honour its absolute deadline and reserve four minutes for evidence # transfer/shutdown before the independent provider guard. budget_path = Path(__file__).with_name('budget.json') if budget_path.exists(): paid = json.loads(budget_path.read_text()) args.wall_seconds = min(args.wall_seconds, int(paid['gpu_deadline_utc_ts'] - time.time() - 240)) if args.wall_seconds <= 180: raise TimeoutError('restoration consumed evaluation window') tls = threading.local(); start = time.monotonic(); deadline = start + args.wall_seconds completed = 0; errors = 0; generated = 0; stopped = threading.Event() dump(out / 'run.json', {'status': 'evaluating', 'version': VERSION, 'inputs': prepared, 'endpoint': args.endpoint, 'served': args.served, 'started_sgt': dt.datetime.now(dt.timezone(dt.timedelta(hours=8))).isoformat(), 'wall_seconds': args.wall_seconds, 'cap_usd': 2.0, 'no_per_request_api_charge': True, 'operator_power_guard_required': True, 'source_sha256': sha(__file__)}) def one(task): if stopped.is_set() or time.monotonic() > deadline - 180: stopped.set(); return None if not hasattr(tls, 'client'): tls.client, tls.sov = make_client(args.endpoint, args.served, args.max_model_len, deadline) t0 = time.perf_counter() try: response = answer_task(tls.client, tls.sov, task, table, 0.7, 512) return {'id': task['id'], 'suite': task['suite'], 'status': 'ok', 'response': response, 'gold_key': task.get('gold_key'), 'wall_s': time.perf_counter() - t0} except Exception as e: return {'id': task['id'], 'suite': task['suite'], 'status': 'error', 'error': str(e)[:400]} # A bounded queue prevents thousands of queued tasks continuing after a stop. with (out / 'reads.jsonl').open('w') as f, concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as pool: iterator = iter(tasks); pending = {} for _ in range(args.workers): task = next(iterator, None) if task: pending[pool.submit(one, task)] = task while pending: done, _ = concurrent.futures.wait(pending, timeout=1, return_when=concurrent.futures.FIRST_COMPLETED) if time.monotonic() > deadline - 180: stopped.set() for future in done: pending.pop(future); r = future.result() if r is not None: f.write(json.dumps(r, ensure_ascii=False) + '\n'); f.flush() completed += 1; errors += r['status'] != 'ok' generated += r.get('response', {}).get('usage', {}).get('output_tokens', 0) elapsed = time.monotonic() - start # Projection starts only after 256 completed tasks; task order is # frozen and early rates are diagnostic, never a success guarantee. projected = elapsed * len(tasks) / max(completed, 1) if completed >= 256 and projected > args.wall_seconds - 180: stopped.set() dump(out / 'progress.json', {'completed': completed, 'total': len(tasks), 'errors': errors, 'generated_tokens': generated, 'elapsed_s': elapsed, 'projected_s': projected, 'stop_requested': stopped.is_set()}) if not stopped.is_set(): task = next(iterator, None) if task: pending[pool.submit(one, task)] = task dump(out / 'DONE.json', {'status': 'completed' if completed == len(tasks) and not errors else 'stopped_or_failed', 'completed': completed, 'total': len(tasks), 'errors': errors, 'generated_tokens': generated, 'wall_s': time.monotonic() - start, 'reads_sha256': sha(out / 'reads.jsonl'), 'gpu_shutdown_required_now': True}) def aggregate(args): """CPU-only native diagnostics and DI kit envelope preparation, never DI rescoring.""" read = rows(args.reads); suite = rows(args.di_suite) if sha(args.di_suite) != 'be10d45f2cfbc4872aa32a5f1d9b0c5f75a53cceee204463d68198108c689846': raise ValueError('historical DI suite SHA mismatch') byid = {r['_evaluation']['run_id']: r for r in suite} seen = set(); di = []; xl = []; native = []; errors = [] for r in read: key = (r['suite'], r['id']) if key in seen: raise ValueError('duplicate read identity') seen.add(key) if r['status'] != 'ok': errors.append(key); continue for answer in r['response']['answers'].values(): native.append(answer['raw']['native']) if r['suite'] == 'di': if r['id'] not in byid: raise ValueError('DI read outside frozen suite') meta = byid[r['id']]['_evaluation'] # Preserve the historical frozen kit envelope rather than inventing a # custom index; omitted/failed identities require explicit partial label. di.append({**meta, 'run_id': r['id'], 'status': 'ok', 'engine': 'http', 'total_wall_ms': 1000*r['wall_s'], 'response': r['response']}) elif r['suite'] == 'xl': if len(r['response']['answers']) != 1: raise ValueError('XL item must have one question') answer = next(iter(r['response']['answers'].values())); raw = answer['raw']; keys = raw['keys'] pa = raw['A']; pb = list(answer['probabilities_t1'].values()); gold = keys.index(r['gold_key']) ca = max(range(len(pa)), key=pa.__getitem__) == gold cb = max(range(len(pb)), key=pb.__getitem__) == gold xl.append((ca, cb)) out = Path(args.out); out.mkdir(parents=True, exist_ok=True) (out / 'di-results.jsonl').write_text(''.join(json.dumps(r, ensure_ascii=False) + '\n' for r in di)) thought = [r for r in native if r['status'] == 'thought_read'] result = {'status': 'completed' if len(di) == 6948 and len(xl) == 2691 and not errors else 'partial_or_failed', 'reads_sha256': sha(args.reads), 'di_requests': len(di), 'xl_items': len(xl), 'errors': len(errors), 'native_questions': len(thought), 'nonempty_thoughts': sum(bool(r['text'].strip()) for r in thought), 'naturally_closed': sum(r['naturally_closed'] for r in thought), 'truncated': sum(r['truncated'] for r in thought), 'generated_tokens': sum(r['generated_tokens'] for r in thought), 'wide_onepass_questions': sum(r['status'] == 'wide_onepass' for r in native), 'xl_A_correct': sum(a for a,b in xl), 'xl_gate_correct': sum(b for a,b in xl), 'xl_wrong_to_right': sum(not a and b for a,b in xl), 'xl_correct_to_wrong': sum(a and not b for a,b in xl), 'order_average': 'not_run; remapping CPU-validated only', 'interpretation': 'A versus native B also changes system instruction and template branch; useful-thought causal effect needs matched empty-thought control', 'di_index': 'requires original frozen decision_index kit score plus boot edition0.2.1 B300, after GPU shutdown', 'native_calibration': 'unfitted; T1 and historical A-table distributions preserved; no TEST fitting'} dump(out / 'aggregate.json', result); print(json.dumps(result)) def main(): ap = argparse.ArgumentParser(description=__doc__); sub = ap.add_subparsers(dest='command', required=True) p = sub.add_parser('prepare'); p.add_argument('--di-suite', required=True); p.add_argument('--xl-source', required=True) p.add_argument('--temperature', required=True); p.add_argument('--out', required=True); p.add_argument('--receipt') r = sub.add_parser('run'); r.add_argument('--inputs', required=True); r.add_argument('--temperature', required=True) r.add_argument('--endpoint', required=True); r.add_argument('--served', required=True); r.add_argument('--out', required=True) r.add_argument('--wall-seconds', type=int, default=3600); r.add_argument('--max-model-len', type=int, default=131072) r.add_argument('--workers', type=int, default=16) g = sub.add_parser('aggregate'); g.add_argument('--reads', required=True); g.add_argument('--di-suite', required=True); g.add_argument('--out', required=True) args = ap.parse_args(); {'prepare': prepare, 'run': run, 'aggregate': aggregate}[args.command](args) if __name__ == '__main__': main()