Instructions to use XXXpilar/Fastfill-test with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use XXXpilar/Fastfill-test with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
File size: 2,925 Bytes
e6f80f3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | """Self-check of stratify's selection-rule attribution on new-data rows (CPU only).
python selftest_shift.py --rows runs/analysis/ablation-sample-500/new.jsonl --work runs/analysis/test/shift
Writes the labels as predictions with +0.5 m along x on every target whose selection_rule is not frozen_prep, scores
them with ``evaluate --predictions`` (full condition) and checks via stratify: report reproduced exactly; position error
0 on frozen_prep and 0.5 on each added rule; size and yaw error 0 everywhere.
"""
import argparse
import json
from pathlib import Path
import subprocess
import sys
import stratify as S
SHIFT = .5
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--rows", required=True, type=Path)
p.add_argument("--work", required=True, type=Path)
a = p.parse_args()
a.work.mkdir(parents=True, exist_ok=False)
with open(a.rows) as src, open(a.work / "predictions.jsonl", "w") as out:
for line in src:
row = json.loads(line)
rules = [(e or {}).get("selection_rule") or "frozen_prep" for e in row["provenance"]["field_evidence"]]
objects = []
for t, rule in zip(row["target"]["objects"], rules):
position = [v if v is not None else 0. for v in t["bottom_center_m"]]
size = [v if v else 1. for v in t["target_size_local_m"]]
objects.append({"id": t["id"], "bottom_center_m": [position[0] + (rule != "frozen_prep") * SHIFT, *position[1:]],
"target_size_local_m": size, "yaw_rad": t["yaw_rad"] if t["yaw_rad"] is not None else 0.})
out.write(json.dumps({"layout": {"schema_version": "fastfill.v2", "objects": objects}}) + "\n")
subprocess.run([sys.executable, "-m", "fastfill.v2.evaluate", "--data", str(a.rows), "--predictions",
str(a.work / "predictions.jsonl"), "--output", str(a.work / "eval")], check=True, stdout=subprocess.DEVNULL)
meta, rooms, checks = S.collect(a.work / "eval", "outcomes.jsonl", a.rows, 6)
strata = S.aggregate(rooms, boot=200)
assert checks["reproduces_report_exactly"], checks
seen = {}
for rule, entry in strata["rule"].items():
m = entry["metrics"]
seen[rule] = (m["bottom_center_error_m"]["mean"], m["bottom_center_error_m"]["n"])
assert abs(m["bottom_center_error_m"]["mean"] - (0. if rule == "frozen_prep" else SHIFT)) < 1e-9, (rule, m)
assert m["yaw_error_rad"]["mean"] < 1e-9 and m.get("log_size_error", {"mean": 0.})["mean"] < 1e-9, (rule, m)
assert set(seen) == set(S.RULES), seen
print(json.dumps({"ok": True, "rooms": len(rooms), "layouts": checks["requests_with_layout"],
"position_error_mean_and_objects_by_rule": seen,
"places": {k: v["objects"] for k, v in strata["place"].items()}}, indent=1))
if __name__ == "__main__":
main()
|