vela3-fast-CoreAI / smoke.py
devilyouwei's picture
Replace fixed graph with dynamic sequence and option dimensions
6aaa9cc verified
Raw History Blame Contribute Delete
4.51 kB
"""Run package-only inference and compare with pinned upstream reference probabilities."""
import asyncio
import json
import sys
from pathlib import Path
import numpy as np
from vela_coreai import VelaCoreAI # type: ignore[import-not-found]
async def main() -> None:
root = Path(__file__).parent
cases = json.loads((root / "validation_cases.json").read_text())
rows = []
async with VelaCoreAI.open(root) as model:
for case in cases:
arrays, labels = model.pack(case["payload"]["state"], case["payload"]["question"])
for name, expected in case["inputs"].items():
np.testing.assert_array_equal(arrays[name].ravel(), expected)
probabilities = await model.probabilities(
case["payload"]["state"], case["payload"]["question"]
)
values = np.array([probabilities[label] for label in labels])
reference = np.array(case["reference_probabilities"])
np.testing.assert_allclose(values, reference, atol=0.02, rtol=0.02)
decision = max(probabilities, key=probabilities.__getitem__)
assert decision == case["reference_decision"]
if case.get("expected") is not None:
assert decision == case["expected"]
rows.append(
{
"id": case["id"],
"sequence_length": int(arrays["attention_mask"].sum()),
"option_count": len(labels),
"decision": decision,
"max_probability_error": float(np.max(np.abs(values - reference))),
"probabilities": probabilities,
}
)
first = cases[0]
arrays, labels = model.pack(first["payload"]["state"], first["payload"]["question"])
expected = await model.probabilities(
first["payload"]["state"], first["payload"]["question"]
)
fixture = {name: value.ravel().tolist() for name, value in arrays.items()}
fixture.update(
expected=[expected[label] for label in labels], labels=labels, output="logits"
)
(root / "swift_fixture.json").write_text(json.dumps(fixture, indent=2) + "\n")
rejected = 0
for state, question in [
("text", {"type": "span"}),
("text", {"type": "choice", "criteria": {}}),
("word " * (model.manifest["sequence_length"] + 1), first["payload"]["question"]),
]:
try:
model.pack(state, question)
except ValueError:
rejected += 1
assert rejected == 3
# Host adapter must preserve the public choice/noul response shape.
answer = await model.predict(first["payload"]["state"], first["payload"]["question"])
assert answer["type"] == "choice" and answer["choice"] == first["expected"]
noul = cases[2]
answer = await model.predict(noul["payload"]["state"], noul["payload"]["question"])
assert answer["type"] == "noul" and 0 <= answer["noul"] <= 1
questions = {
"topic": first["payload"]["question"],
"software": noul["payload"]["question"],
}
answers, usage = await model.predict_questions(first["payload"]["state"], questions)
assert list(answers) == list(questions) and usage > 0
for key, question in questions.items():
assert answers[key] == await model.predict(first["payload"]["state"], question)
list_question = {"type": "choice", "criteria": ["coding", "cooking", "sports"]}
map_question = {**list_question, "criteria": dict.fromkeys(list_question["criteria"])}
left, left_labels = model.pack(None, list_question)
right, right_labels = model.pack(None, map_question)
assert left_labels == right_labels
for name in left:
np.testing.assert_array_equal(left[name], right[name])
assert "torch" not in sys.modules
report = {
"status": "passed",
"scope": "dynamic decision graph",
"device": "cpu",
"cases": rows,
"invalid_requests_rejected": rejected,
"pytorch_imported": False,
"requires_original_weights": False,
"multiple_questions": "passed",
"list_criteria": "passed",
}
(root / "package_smoke.json").write_text(json.dumps(report, indent=2) + "\n")
print(json.dumps(report, indent=2))
if __name__ == "__main__":
asyncio.run(main())