Fastfill-test / code /fastfill /v2 /ops /remask_ablation.py
XXXpilar's picture
FastFill v4 base (step 6552): LoRA adapter, geometry heads, tokenizer, training code, predict.py
e6f80f3 verified
Raw History Blame Contribute Delete
10.2 kB
"""One checkpoint, several decodes: one-shot argmax / spread against remasking decoding (``remask``).
Every mode decodes the same rows (full condition, as deployed) from the same checkpoint and is scored by
``evaluate --predictions``; the table compares the raw layouts:
argmax one forward pass, argmax cell (the raw head)
spread one forward pass, ``spread_grid_xy`` post-processing (the deployed decode)
remaskR R rounds of remasking decoding, argmax cells (needs pose_conditioning when R > 1)
remaskRL R rounds, cells restricted to the least-cost cells (outside / door zone / fixed objects /
committed footprints)
remaskRLX ... and SceneNAT retraction: low-confidence committed objects are masked again (``remask``)
``--strip-support`` removes every support declaration (``support_parent`` and ``on`` constraints) from the
conditions before decoding. ``support_accuracy`` scores a support-head model's predicted ``support_parent`` only
against declarations the decoded condition does not show (a shown one is copied, not predicted): without
``--strip-support`` there are none and it is None; ``declared_in_input`` counts the shown ones. Overall and for
floor / wall / object parents; None without a support head.
python -m fastfill.v2.ops.remask_ablation --checkpoint RUN/model --data rows.jsonl --output DIR \\
--modes argmax spread remask1L remask4 remask4L remask4LX [--max-samples N] [--strip-support] [--device cuda]
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import re
import subprocess
import sys
import time
import torch
from fastfill.v2.batch import collate_samples, load_tokenizer
from fastfill.v2.evaluate import serialize_predictions
from fastfill.v2.io import load_checkpoint_config, to_device
from fastfill.v2.model import load_model, model_inputs
from fastfill.v2.remask import remask_predictions
from fastfill.v2.schema import migrate_legacy_row
MODE = re.compile(r"^(argmax|spread|remask(\d+)(L?)(X?))$")
def decode(model, batch, mode):
m = MODE.match(mode)
if not m:
raise ValueError(f"unknown mode {mode}")
if m.group(1) in ("argmax", "spread"):
with torch.no_grad():
return serialize_predictions(model(**model_inputs(batch)), batch, grid_decode=m.group(1))[0]
out = remask_predictions(model, batch, rounds=int(m.group(2)), legal=bool(m.group(3)), retract=bool(m.group(4)))
return serialize_predictions(out, batch, grid_decode="argmax")[0]
def strip_support(row: dict) -> dict:
"""The row with no support declaration left in its condition."""
condition = {**row["condition"], "objects": [{k: v for k, v in obj.items() if k != "support_parent"}
for obj in row["condition"]["objects"]],
"constraints": [c for c in row["condition"].get("constraints", []) if c.get("type") != "on"]}
return {**row, "condition": condition}
def support_labels(row: dict) -> dict:
"""id -> declared parent (support_parent or hard on), for the requests that declare one."""
from fastfill.v2.validation import effective_support_requests
return {r["id"]: r["support_parent"] for r in effective_support_requests(row["condition"])
if r.get("support_parent") is not None}
def support_accuracy(predictions: list, labels: list, shown: list | None = None) -> dict | None:
"""Predicted support_parent against the declarations hidden from the decoded condition (``shown``: the
declarations it still carries, per row): overall and by kind of declared parent."""
hits = {"all": [], "floor": [], "wall": [], "object": []}
visible = 0
for k, (prediction, label) in enumerate(zip(predictions, labels)):
objects = {o["id"]: o for o in (prediction["layout"] or {}).get("objects", [])}
for ident, parent in label.items():
if ident not in objects:
continue
if "support_parent" not in objects[ident]:
return None
if shown is not None and ident in shown[k]:
visible += 1
continue
hit = objects[ident]["support_parent"] == parent
hits["all"].append(hit)
hits[parent if parent in ("floor", "wall") else "object"].append(hit)
return {k: (sum(v) / len(v) if v else None) for k, v in hits.items()} | {"objects": len(hits["all"]),
"declared_in_input": visible}
def summarize(report: dict) -> dict:
collapse = report["collapse"]["predicted_matched"]
checks = report["target_validation_checks"]
reference = report["model"]["reference"]
rate = lambda key: (checks[key]["pass"] / (checks[key]["pass"] + checks[key]["violation"])
if key in checks and checks[key]["pass"] + checks[key]["violation"] else None)
return {"position_m": reference["bottom_center_error_m"]["mean"], "log_size": reference["log_size_error"]["mean"],
"yaw_rad": reference["yaw_error_rad"]["mean"], "bev_iou": reference["bev_iou"]["mean"],
"footprint_outside": collapse["footprint_outside_fraction"],
"door_blocking": collapse["door_blocking_fraction_of_floor_objects"],
"overlap_iou_0.3": collapse["bev_overlap_rate_iou_gt_0.3"], "central": collapse["central_quarter_fraction"],
"collision_pairs_per_room": (report["validation"]["model"] or {}).get("collision_pairs", 0) / max(report["requests"], 1),
"constraint_pass": rate("constraint"), "clean_room_rate": (report["validation"]["model"] or {}).get("clean_room_rate")}
def main(argv=None):
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--checkpoint", type=Path, required=True, help="RUN/model (RUN/tokenizer beside it)")
p.add_argument("--data", type=Path, required=True)
p.add_argument("--output", type=Path, required=True)
p.add_argument("--modes", nargs="+", default=["argmax", "spread", "remask1L", "remask4", "remask4L", "remask4LX"])
p.add_argument("--max-samples", type=int)
p.add_argument("--strip-support", action="store_true", help="decode without any support declaration")
p.add_argument("--device", default="cpu")
a = p.parse_args(argv)
a.output.mkdir(parents=True, exist_ok=True)
model = load_model(a.checkpoint, device=a.device).eval()
tokenizer = load_tokenizer("tiny" if model.config.backbone == "tiny" else str(a.checkpoint.parent / "tokenizer"),
local_files_only=True)
max_length = load_checkpoint_config(a.checkpoint)["max_length"] or 4096
lines = [l for l in a.data.read_text().splitlines() if l.strip()][:a.max_samples]
labels = [support_labels(migrate_legacy_row(json.loads(l))) for l in lines]
if a.strip_support:
lines = [json.dumps(strip_support(migrate_legacy_row(json.loads(l)))) for l in lines]
shown = [support_labels(migrate_legacy_row(json.loads(l))) for l in lines] # what the decoded condition declares
rows = a.output / "rows.jsonl"
rows.write_text("\n".join(lines) + "\n")
batches = []
for l in lines: # a room over the object or context budget is a failed request, as in evaluate, not a crash
try:
batches.append(to_device(collate_samples([migrate_legacy_row(json.loads(l))], tokenizer, max_length=max_length,
max_objects=model.config.max_objects), a.device))
except ValueError as e:
batches.append(str(e))
print(f"{sum(isinstance(b, str) for b in batches)} of {len(lines)} rows over the model's budget (max_length "
f"{max_length}, max_objects {model.config.max_objects}): failed requests", flush=True)
table = {}
for mode in a.modes:
predictions, seconds = [], []
for batch in batches:
start = time.perf_counter()
if isinstance(batch, str):
predictions.append({"layout": None, "error": batch, "latency_s": None})
continue
try:
layout, error = decode(model, batch, mode), None
except ValueError as e:
layout, error = None, str(e)
seconds.append(time.perf_counter() - start)
predictions.append({"layout": layout, "error": error, "latency_s": seconds[-1]})
path = a.output / f"predictions_{mode}.jsonl"
path.write_text("".join(json.dumps(r) + "\n" for r in predictions))
scored = a.output / f"eval_{mode}"
subprocess.run([sys.executable, "-m", "fastfill.v2.evaluate", "--data", str(rows), "--predictions", str(path),
"--output", str(scored)], check=True, stdout=subprocess.DEVNULL)
report = json.loads((scored / "report.json").read_text())
table[mode] = {**summarize(report),
"decode_ms": 1000 * sum(seconds) / max(len(seconds), 1), "failed": sum(r["layout"] is None for r in predictions),
"support_accuracy": support_accuracy(predictions, labels, shown)}
print(mode, json.dumps({k: round(v, 4) if isinstance(v, float) else v for k, v in table[mode].items()}), flush=True)
table[mode]["by_constraints"] = {name: {"requests": cohort["requests"], **summarize(cohort)}
for name, cohort in report.get("by_constraints", {}).items()}
ground_truth = json.loads((a.output / f"eval_{a.modes[0]}" / "report.json").read_text())["collapse"]["ground_truth"]
table["ground_truth"] = {"footprint_outside": ground_truth["footprint_outside_fraction"],
"door_blocking": ground_truth["door_blocking_fraction_of_floor_objects"],
"overlap_iou_0.3": ground_truth["bev_overlap_rate_iou_gt_0.3"],
"central": ground_truth["central_quarter_fraction"]}
(a.output / "ablation.json").write_text(json.dumps(table, indent=1))
if __name__ == "__main__":
main()