"""Closed-loop corpus evaluation against a persistent, real Host session. Only the first authored memory snapshot seeds a trajectory. Later model inputs come from actual commits. No answer repair, forced kinds, retries or gold fallback. """ import argparse from collections import OrderedDict import json from pathlib import Path import subprocess import uuid import hashlib ROOT = Path(__file__).resolve().parents[1] class Host: def __init__(self, directory): self.process = subprocess.Popen( ["dotnet", str(ROOT / "src/Y.Host/bin/Debug/net10.0/Y.Host.dll"), "d3-corpus-session", str(directory)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, encoding="utf-8") def call(self, op, **values): self.process.stdin.write(json.dumps(dict(op=op, **values), ensure_ascii=False) + "\n") self.process.stdin.flush() line = self.process.stdout.readline().lstrip("\ufeff") if not line: raise RuntimeError("Host exited") return json.loads(line) def close(self): self.process.stdin.close() if self.process.wait(timeout=10): raise RuntimeError("Host failed") def replay(rows, directory, generate=None, progress_path=None): host = Host(directory) trajectories = OrderedDict() for row in rows: name, number, _ = row["sample_id"].split("/") trajectories.setdefault(name, OrderedDict()).setdefault(int(number), []).append(row) results = [] try: for name, steps in trajectories.items(): first = next(iter(steps.values()))[0] start = host.call("start", persona=name, memories=first["memories"]) assert start["status"] == "started", start # An unrelated seed or an optional model write cannot stand in for a # required authored encoding that the model skipped. Bind only the # actual trace created at that declared encoding slot. refs = {} active_ref = None previous = start["state"] roots = {} for number, batch in steps.items(): if number > 1: interrupted = host.call("event", kind="interrupt") assert interrupted["status"] == "Applied", interrupted restored = host.call("restart") assert restored["status"] == "restored", restored assert restored["state"]["memories"] == previous["memories"] resumed = host.call("event", kind="resume") assert resumed["status"] == "Applied", resumed transition = batch[0].get("harness_transition", {}) loss = transition.get("loss") loss_ref = loss.get("memory_ref", active_ref) if loss else None if loss and loss_ref in refs: terms, source = loss["erase_terms"], loss["lose_source"] erased = host.call("erase", memory_id=refs[loss_ref], terms=terms, lose_source=source) assert erased["status"] == "Applied" activity = host.call("event", kind="activity", activity=dict(id="current", progress=batch[0]["activity"])) assert activity["status"] == "Applied", activity previous = activity["state"] scene = batch[0]["scene"] if scene: source_number = transition.get("source_step", number) root = roots.setdefault(source_number, str(uuid.uuid4())) observation = host.call("event", kind="observation", text=scene, root=root) assert observation["status"] == "Applied", observation # External consumes the current perception before memory commit clears it. for row in sorted(batch, key=lambda x: x["channel"] != "external"): values = dict(channel=row["channel"]) reference = row.get("memory_ref", "main") # Older single-reference corpus exports. if row["channel"] in ("memory_conflict", "memory_reconstruction"): if reference not in refs: results.append(dict(sample_id=row["sample_id"], status="MissingActualMemory", expected=row["proposal"])) continue values["memory_id"] = refs[reference] request = host.call("begin", **values) if request["status"] != "ready": results.append(dict(sample_id=row["sample_id"], status=request["status"], error=request.get("error"))) continue raw = generate(row["channel"], request["model_visible"]) if generate else row["proposal"] receipt = host.call("commit", proposal=raw) after = receipt.get("state", request["state"]) old_ids = {m["id"] for m in request["state"]["memories"]} new_ids = {m["id"] for m in after["memories"]} - old_ids if new_ids: assert len(new_ids) == 1 if row["proposal"]["kind"] == "encode_memory" and receipt["status"] == "Accepted": refs[reference] = next(iter(new_ids)) active_ref = reference results.append(dict(sample_id=row["sample_id"], group=row["group"], status=receipt["status"], model_visible=request["model_visible"], expected=row["proposal"], proposal=raw, before=request["state"]["memories"], after=after["memories"], played=receipt.get("played"))) previous = after print(f"sequence {name}/{number}", flush=True) if progress_path: pending = progress_path.with_suffix(".tmp") pending.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8") pending.replace(progress_path) finally: host.close() return results def generator(adapter, prompt_function=None, external_adapter_scale=1.0, memory_adapter=None): import torch from transformers import StoppingCriteria, StoppingCriteriaList from .models import load_core from .common import seed from .d3_corpus import prompt seed() weights = torch.load(adapter, map_location="cpu", weights_only=True) weights = weights.get("trainable", weights) scope = "language" if any(".in_proj_qkv." in n for n in weights) else "qv" core, tokenizer, _ = load_core(scope) params = {n: p for n, p in core.named_parameters() if p.requires_grad} assert weights.keys() == params.keys() with torch.no_grad(): for n, p in params.items(): p.copy_(weights[n]) core.eval() core.gradient_checkpointing_disable() generate=apply_external_scale(core,generate_with(core, tokenizer, prompt_function),external_adapter_scale) if memory_adapter is None:return generate memory_weights=torch.load(memory_adapter,map_location='cpu',weights_only=True) memory_weights=memory_weights.get('trainable',memory_weights) if memory_weights.keys()!=params.keys():raise ValueError('Memory adapter architecture mismatch') active='external' def routed(channel,visible): nonlocal active mode='external' if channel=='external' else 'memory' if mode!=active: with torch.no_grad(): for name,p in params.items(): value=(weights if mode=='external' else memory_weights)[name] if value.shape!=p.shape or value.dtype!=p.dtype:raise ValueError('Adapter parameter mismatch') p.copy_(value) active=mode return generate(channel,visible) return routed def apply_external_scale(core, generate, external_adapter_scale): if external_adapter_scale == 1.0:return generate if not 0 <= external_adapter_scale <= 1:raise ValueError('Invalid adapter scale') # Controlled inference ablation. Other channels retain their trained scale. layers=[m for m in core.modules() if hasattr(m,'scaling') and isinstance(m.scaling,dict)] def scaled(channel,visible): if channel!='external':return generate(channel,visible) original=[dict(m.scaling) for m in layers] try: for module,values in zip(layers,original): for name,value in values.items():module.scaling[name]=value*external_adapter_scale return generate(channel,visible) finally: for module,values in zip(layers,original):module.scaling.update(values) return scaled def generate_with(core, tokenizer, prompt_function=None): """Use the same free decoder with a loaded model during checkpoint evaluation.""" import torch from transformers import StoppingCriteria, StoppingCriteriaList from .d3_corpus import prompt def generate(channel, visible): p = (prompt_function or prompt)(channel, visible) ids = torch.tensor([tokenizer.encode(p, add_special_tokens=False)], device="cuda") width = ids.shape[1] class Done(StoppingCriteria): def __call__(self, input_ids, scores, **kwargs): try: return isinstance(json.loads(tokenizer.decode(input_ids[0, width:], skip_special_tokens=True)), dict) except ValueError: return False with torch.inference_mode(): output = core.generate(input_ids=ids, attention_mask=torch.ones_like(ids), do_sample=False, use_cache=True, max_new_tokens=320, pad_token_id=tokenizer.eos_token_id, stopping_criteria=StoppingCriteriaList([Done()])) return tokenizer.decode(output[0, width:], skip_special_tokens=True).strip() return generate if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("directory", type=Path) parser.add_argument("--adapter", type=Path) parser.add_argument("--all", action="store_true") parser.add_argument("--label", default="model") parser.add_argument("--external-profile", choices=['standard','conversation_v1','focused_v1'], default='standard') parser.add_argument("--external-adapter-scale",type=float,default=1.0) parser.add_argument("--memory-adapter",type=Path) args = parser.parse_args() rows = json.loads((args.directory / "gold.json").read_text(encoding="utf-8")) split = json.loads((args.directory / "split.json").read_text(encoding="utf-8")) if not args.all: rows = [r for r in rows if r["group"] in split["holdout"]] label = args.label if args.adapter or args.label != "model" else "gold" run = args.directory / ("sequence-" + label) run.mkdir(parents=True, exist_ok=False) manifest = dict(gold_sha256=hashlib.sha256((args.directory / "gold.json").read_bytes()).hexdigest(), split_sha256=hashlib.sha256((args.directory / "split.json").read_bytes()).hexdigest(), adapter_sha256=hashlib.sha256(args.adapter.read_bytes()).hexdigest() if args.adapter else None, expected_ids=[r["sample_id"] for r in rows], closed_loop=True) manifest.update(external_profile=args.external_profile,external_adapter_scale=args.external_adapter_scale) if args.memory_adapter: manifest['memory_adapter_sha256']=hashlib.sha256(args.memory_adapter.read_bytes()).hexdigest() (run / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") from contextlib import nullcontext from .gpu_lease import gpu_lease from .conversation_prompt import conversation_prompt,focused_prompt prompt_function={'standard':None,'conversation_v1':conversation_prompt,'focused_v1':focused_prompt}[args.external_profile] with gpu_lease() if args.adapter else nullcontext(): result = replay(rows, run / "state", generator(args.adapter,prompt_function,args.external_adapter_scale,args.memory_adapter) if args.adapter else None, run / "results.partial.json") (run / "results.json").write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") print(f"Accepted {sum(r['status'] == 'Accepted' for r in result)}/{len(result)}")