| |
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| import os |
| import sys |
| import tempfile |
| from pathlib import Path |
|
|
| HERE = Path(__file__).resolve().parent |
| sys.path.insert(0, str(HERE)) |
|
|
| from backend_factory import build_backend |
| from endpoint_provenance import ( |
| validate_binding, |
| write_binding_manifest, |
| verify_binding_manifest, |
| ) |
| from runtime_lifecycle import choose_free_port |
| from runtime_registry import RUNTIME_REGISTRY |
|
|
|
|
| DUMMY_MODEL_ID = "dummy-model" |
| DUMMY_MODEL_SHA = ( |
| "7de8c857d7b2379e7807323cc718397efa2f994607477d417a47725476d144b7" |
| ) |
| DUMMY_RUNTIME_ID = "dummy-runtime-v2" |
|
|
|
|
| def make_configs(run_dir: Path): |
| port = choose_free_port("127.0.0.1") |
| nonce = "binding-" + os.urandom(8).hex() |
| runtime_dir = run_dir / "runtime" |
|
|
| runtime_cfg = { |
| "runtime_adapter_id": |
| "process_http.runtime.v1", |
| "run_dir": str(runtime_dir), |
| "host": "127.0.0.1", |
| "port": port, |
| "readiness_url": |
| f"http://127.0.0.1:{port}/health", |
| "readiness_nonce": nonce, |
| "startup_timeout": 10, |
| "stop_timeout": 5, |
| "launch_plan": { |
| "plan_id": "dummy-binding-runtime", |
| "executable": sys.executable, |
| "argv": [ |
| str( |
| (HERE / "dummy_runtime_server.py") |
| .resolve() |
| ), |
| "--host", "127.0.0.1", |
| "--port", str(port), |
| "--nonce", nonce, |
| "--model-id", DUMMY_MODEL_ID, |
| "--model-sha256", DUMMY_MODEL_SHA, |
| "--runtime-id", DUMMY_RUNTIME_ID, |
| ], |
| "env": {}, |
| "cwd": str(HERE), |
| "ownership_role": |
| "v47_binding_dummy_runtime", |
| "ownership_nonce": nonce, |
| }, |
| } |
|
|
| backend_spec = { |
| "adapter_id": "openai_http.generator.v1", |
| "config": { |
| "backend_id": |
| "v47-dummy-generator-backend", |
| "base_url": |
| f"http://127.0.0.1:{port}/v1", |
| "model": DUMMY_MODEL_ID, |
| "model_path": "dummy://model", |
| "model_sha256": DUMMY_MODEL_SHA, |
| "runtime_id": DUMMY_RUNTIME_ID, |
| "timeout_seconds": 3, |
| "temperature": 0.0, |
| "top_p": 1.0, |
| "max_tokens": 16, |
| }, |
| } |
| return runtime_cfg, backend_spec |
|
|
|
|
| def build_runtime(cfg): |
| return RUNTIME_REGISTRY.instantiate( |
| cfg["runtime_adapter_id"], |
| cfg, |
| ) |
|
|
|
|
| def self_test(): |
| with tempfile.TemporaryDirectory() as td: |
| root = Path(td) |
| runtime_cfg, backend_spec = make_configs(root) |
|
|
| runtime = build_runtime(runtime_cfg) |
| backend = build_backend( |
| backend_spec, |
| "generator", |
| ) |
|
|
| runtime.start() |
| try: |
| rp = runtime.probe() |
| assert rp["status"] == "PASS" |
|
|
| bp = backend.preflight() |
| assert bp["status"] == "PASS" |
|
|
| binding = validate_binding( |
| runtime_probe=rp, |
| runtime_identity= |
| runtime.identity().to_dict(), |
| backend_identity= |
| backend.identity().to_dict(), |
| backend_config=backend_spec["config"], |
| runtime_config=runtime_cfg, |
| ) |
| assert binding["status"] == "PASS" |
|
|
| manifest = root / "BINDING_MANIFEST.json" |
| frozen = write_binding_manifest( |
| manifest, |
| binding, |
| ) |
| assert frozen["status"] == "PASS" |
| assert ( |
| verify_binding_manifest(manifest)["status"] |
| == "PASS" |
| ) |
|
|
| |
| bad_spec = json.loads( |
| json.dumps(backend_spec) |
| ) |
| bad_spec["config"]["model"] = "wrong-model" |
| bad_backend = build_backend( |
| bad_spec, |
| "generator", |
| ) |
| bad = validate_binding( |
| runtime_probe=rp, |
| runtime_identity= |
| runtime.identity().to_dict(), |
| backend_identity= |
| bad_backend.identity().to_dict(), |
| backend_config=bad_spec["config"], |
| runtime_config=runtime_cfg, |
| ) |
| assert bad["status"] == "BLOCK" |
| assert any( |
| "MODEL" in x |
| for x in bad["failures"] |
| ) |
|
|
| |
| bad_sha_spec = json.loads( |
| json.dumps(backend_spec) |
| ) |
| bad_sha_spec["config"]["model_sha256"] = ( |
| "0" * 64 |
| ) |
| bad_sha_backend = build_backend( |
| bad_sha_spec, |
| "generator", |
| ) |
| bad_sha = validate_binding( |
| runtime_probe=rp, |
| runtime_identity= |
| runtime.identity().to_dict(), |
| backend_identity= |
| bad_sha_backend.identity().to_dict(), |
| backend_config=bad_sha_spec["config"], |
| runtime_config=runtime_cfg, |
| ) |
| assert bad_sha["status"] == "BLOCK" |
| assert ( |
| "PROVENANCE_MODEL_SHA_MISMATCH" |
| in bad_sha["failures"] |
| ) |
|
|
| |
| bad_rt_spec = json.loads( |
| json.dumps(backend_spec) |
| ) |
| bad_rt_spec["config"]["runtime_id"] = ( |
| "wrong-runtime" |
| ) |
| bad_rt_backend = build_backend( |
| bad_rt_spec, |
| "generator", |
| ) |
| bad_rt = validate_binding( |
| runtime_probe=rp, |
| runtime_identity= |
| runtime.identity().to_dict(), |
| backend_identity= |
| bad_rt_backend.identity().to_dict(), |
| backend_config=bad_rt_spec["config"], |
| runtime_config=runtime_cfg, |
| ) |
| assert bad_rt["status"] == "BLOCK" |
| assert ( |
| "PROVENANCE_RUNTIME_ID_MISMATCH" |
| in bad_rt["failures"] |
| ) |
|
|
| |
| bad_ep_spec = json.loads( |
| json.dumps(backend_spec) |
| ) |
| bad_ep_spec["config"]["base_url"] = ( |
| "http://127.0.0.1:1/v1" |
| ) |
| bad_ep_backend = build_backend( |
| bad_ep_spec, |
| "generator", |
| ) |
| bad_ep = validate_binding( |
| runtime_probe=rp, |
| runtime_identity= |
| runtime.identity().to_dict(), |
| backend_identity= |
| bad_ep_backend.identity().to_dict(), |
| backend_config=bad_ep_spec["config"], |
| runtime_config=runtime_cfg, |
| ) |
| assert bad_ep["status"] == "BLOCK" |
| assert ( |
| "BACKEND_RUNTIME_HOST_PORT_MISMATCH" |
| in bad_ep["failures"] |
| ) |
|
|
| finally: |
| stop = runtime.stop() |
| assert stop["status"] == "PASS" |
|
|
| print("BACKEND_RUNTIME_BINDING=PASS") |
| print("ENDPOINT_HOST_PORT_BINDING=PASS") |
| print("RUNTIME_OWNERSHIP_BINDING=PASS") |
| print("PROVENANCE_NONCE_BINDING=PASS") |
| print("PROVENANCE_PID_BINDING=PASS") |
| print("MODEL_ID_BINDING=PASS") |
| print("MODEL_SHA_BINDING=PASS") |
| print("RUNTIME_ID_BINDING=PASS") |
| print("OPENAI_MODELS_PREFLIGHT=PASS") |
| print("BINDING_MANIFEST_FREEZE=PASS") |
| print("BINDING_MANIFEST_SHA_VERIFY=PASS") |
| print("MODEL_ID_MISMATCH_BLOCK=PASS") |
| print("MODEL_SHA_MISMATCH_BLOCK=PASS") |
| print("RUNTIME_ID_MISMATCH_BLOCK=PASS") |
| print("ENDPOINT_MISMATCH_BLOCK=PASS") |
| print("MODEL_LOAD=NO") |
| print("VLLM_START=NO") |
| print("CANARY64_GENERATION=NO") |
| print("TRAINING=NO") |
| print("STABLE_MUTATION=NO") |
| print("SELF_TEST=PASS") |
|
|
|
|
| def live_smoke(report: Path): |
| report.mkdir(parents=True, exist_ok=True) |
| runtime_cfg, backend_spec = make_configs(report) |
|
|
| (report / "RUNTIME_CONFIG.json").write_text( |
| json.dumps( |
| runtime_cfg, |
| ensure_ascii=False, |
| indent=2, |
| ) + "\n", |
| encoding="utf-8", |
| ) |
| (report / "BACKEND_CONFIG.json").write_text( |
| json.dumps( |
| backend_spec, |
| ensure_ascii=False, |
| indent=2, |
| ) + "\n", |
| encoding="utf-8", |
| ) |
|
|
| runtime = build_runtime(runtime_cfg) |
| backend = build_backend( |
| backend_spec, |
| "generator", |
| ) |
|
|
| try: |
| rs = runtime.start() |
| if rs["status"] != "PASS": |
| node-7.example.invalid RuntimeError("RUNTIME_START_BLOCK") |
|
|
| rp = runtime.probe() |
| if rp["status"] != "PASS": |
| node-7.example.invalid RuntimeError("RUNTIME_PROBE_BLOCK") |
|
|
| bp = backend.preflight() |
| if bp["status"] != "PASS": |
| node-7.example.invalid RuntimeError("BACKEND_PREFLIGHT_BLOCK") |
|
|
| binding = validate_binding( |
| runtime_probe=rp, |
| runtime_identity= |
| runtime.identity().to_dict(), |
| backend_identity= |
| backend.identity().to_dict(), |
| backend_config=backend_spec["config"], |
| runtime_config=runtime_cfg, |
| ) |
| if binding["status"] != "PASS": |
| node-7.example.invalid RuntimeError( |
| "BINDING_BLOCK:" |
| + json.dumps( |
| binding, |
| ensure_ascii=False, |
| ) |
| ) |
|
|
| frozen = write_binding_manifest( |
| report / "BINDING_MANIFEST.json", |
| binding, |
| ) |
| verify_binding_manifest( |
| report / "BINDING_MANIFEST.json" |
| ) |
|
|
| |
| item = { |
| "index": 0, |
| "category": "binding_contract_probe", |
| "prompt": "binding contract probe", |
| "prompt_sha256": "not-a-dataset-item", |
| } |
| candidate = backend.generate(item) |
| if ( |
| candidate["answer"] |
| != "DUMMY_BOUND_RESPONSE" |
| ): |
| node-7.example.invalid RuntimeError( |
| "BOUND_ENDPOINT_RESPONSE_MISMATCH" |
| ) |
|
|
| (report / "BOUND_ENDPOINT_PROBE.json").write_text( |
| json.dumps( |
| candidate, |
| ensure_ascii=False, |
| indent=2, |
| ) + "\n", |
| encoding="utf-8", |
| ) |
|
|
| finally: |
| stop = runtime.stop() |
| if stop["status"] != "PASS": |
| node-7.example.invalid RuntimeError("RUNTIME_STOP_BLOCK") |
|
|
| print("RUNTIME_START=PASS") |
| print("RUNTIME_OWNERSHIP_PROBE=PASS") |
| print("BACKEND_OPENAI_PREFLIGHT=PASS") |
| print("ENDPOINT_PROVENANCE=PASS") |
| print("BACKEND_RUNTIME_BINDING=PASS") |
| print("BINDING_MANIFEST=PASS") |
| print("BOUND_ENDPOINT_CONTRACT_PROBE=PASS") |
| print("RUNTIME_EXACT_STOP=PASS") |
| print("CANARY64_GENERATION=NO") |
| print("DATASET_BUILD=NO") |
| print("TRAINING=NO") |
| print("STABLE_MUTATION=NO") |
| print("LIVE_BINDING_SMOKE=PASS") |
| print(f"REPORT={report}") |
|
|
|
|
| def main(): |
| ap = argparse.ArgumentParser() |
| sp = ap.add_subparsers( |
| dest="cmd", |
| required=True, |
| ) |
|
|
| s = sp.add_parser("self-test") |
| s.set_defaults( |
| fn=lambda a: self_test() |
| ) |
|
|
| l = sp.add_parser("live-smoke") |
| l.add_argument("--report", required=True) |
| l.set_defaults( |
| fn=lambda a: live_smoke(Path(a.report)) |
| ) |
|
|
| args = ap.parse_args() |
| args.fn(args) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
