Publish evaluated Forma3D-Tiny v0.2

e80b8d8 verified 3 days ago

11.8 kB

"""Evaluate a completed run, stage a self-contained release, optionally publish it."""
import argparse
import hashlib
import json
import shutil
from pathlib import Path

import torch
from torch.utils.data import DataLoader

from image3d_student.data import CachedDataset, load_manifest
from image3d_student.engine import device_for, evaluate, generate
from image3d_student.model import Student


def sha(path):
    return hashlib.sha256(Path(path).read_bytes()).hexdigest()


def evaluate_release(args):
    run, dataset = Path(args.run), Path(args.data)
    meta = json.loads((run / "run.json").read_text())
    checkpoint = torch.load(run / "best.pt", map_location="cpu", weights_only=True)
    last = torch.load(run / "last.pt", map_location="cpu", weights_only=True)
    if last["step"] != meta["steps_requested"]:
        raise ValueError("Training did not complete the requested steps")
    if checkpoint.get("step", 0) < 1 or not meta.get("initialization"):
        raise ValueError("Expected completed post-training from a base checkpoint")
    if sha(args.base) != meta["initialization"]["sha256"]:
        raise ValueError("Base checkpoint differs from the training initialization")
    if sha(dataset / "manifest.jsonl") != meta["manifest_sha256"]:
        raise ValueError("Training manifest changed")
    development = load_manifest(dataset / "manifest.jsonl")
    test = load_manifest(dataset / "test-manifest.jsonl")
    selected_ids = set(meta["train_ids"]) | set(meta["val_ids"])
    if selected_ids != {r["id"] for r in development}:
        raise ValueError("Run split does not match development manifest")
    if {r["metadata"]["object_id"] for r in development} & {r["metadata"]["object_id"] for r in test}:
        raise ValueError("Test object leakage")
    for key in ["image_sha256", "target_sha256"]:
        if {r[key] for r in development} & {r[key] for r in test}:
            raise ValueError("Duplicate data across development and test")
    device = device_for(args.device)
    model = Student(**checkpoint["config"]).to(device)
    model.load_state_dict(checkpoint["model"])
    loader = DataLoader(CachedDataset(dataset / "test-manifest.jsonl"), batch_size=1)
    final = evaluate(model, loader, device)
    base = torch.load(args.base, map_location="cpu", weights_only=True)
    initial = Student(**checkpoint["config"]).to(device)
    result = initial.load_state_dict({k: v for k, v in base["model"].items() if k not in ("coords", "centers")}, strict=False)
    if set(result.missing_keys) != {"coords", "centers"} or result.unexpected_keys:
        raise ValueError("Base architecture mismatch")
    baseline = evaluate(initial, loader, device)
    best_validation = min(json.loads(line)["validation"]["loss"] for line in
                          (run / "metrics.jsonl").read_text().splitlines())
    checks = {"validation_loss_improved": best_validation < meta["baseline_validation"]["loss"],
              "test_loss_improved": final["loss"] < baseline["loss"],
              "test_occupancy_iou_improved": final["occupancy_iou"] > baseline["occupancy_iou"],
              "test_occupancy_iou_at_least_005": final["occupancy_iou"] >= .05}
    report = {"checkpoint_sha256": sha(run / "best.pt"), "base_sha256": sha(args.base),
              "test_manifest_sha256": sha(dataset / "test-manifest.jsonl"),
              "test_objects": len({r["metadata"]["object_id"] for r in test}),
              "test_samples": len(test), "baseline_test": baseline, "posttrained_test": final,
              "checks": checks, "passed": all(checks.values()),
              "scope": "Rendered mesh-proxy evaluation only; does not establish real-photo or mesh reconstruction quality"}
    (run / "evaluation.json").write_text(json.dumps(report, indent=2))
    print(json.dumps(report, indent=2), flush=True)
    # Keep qualitative inspection files local; all held-out objects, first view.
    example_dir = run / "inspection"
    example_dir.mkdir(exist_ok=True)
    seen = set()
    from image3d_student.splats import preview, read_ply
    from PIL import Image, ImageDraw
    for row in test:
        group = row["metadata"]["object_id"]
        if group in seen:
            continue
        seen.add(group)
        out = example_dir / (row["id"] + ".ply")
        if not out.exists():
            try:
                generate(run / "best.pt", dataset / row["image"], out, args.device)
            except ValueError as exc:
                if "No occupied cells" not in str(exc):
                    raise
                report["checks"]["nonempty_examples"] = False
                report["passed"] = False
                continue
        sheet = Image.new("RGB", (512, 152), "black")
        sheet.paste(Image.open(dataset / row["image"]).convert("RGB").resize((128, 128)), (0, 24))
        splats = read_ply(out)
        for i, angle in enumerate([0, 1.5707963, 3.1415927]):
            sheet.paste(preview(splats, size=128, yaw=angle), (128 * (i + 1), 24))
        ImageDraw.Draw(sheet).text((5, 5), "Input proxy | prediction front | side | back", fill="white")
        sheet.save(out.with_suffix(".png"))
        if len(seen) >= 8:
            break
    (run / "evaluation.json").write_text(json.dumps(report, indent=2))
    if not report["passed"]:
        raise ValueError("Evaluation gate failed. Nothing was published. Inspect evaluation.json and improve data/training.")
    return checkpoint, meta, report


def stage(args, checkpoint, meta, report):
    source = Path(__file__).resolve().parents[1]
    release = Path(args.release)
    if release.exists():
        raise ValueError("Release directory already exists; choose a new --release path")
    release.mkdir(parents=True)
    shutil.copytree(source / "src", release / "src", ignore=shutil.ignore_patterns("__pycache__", "*.egg-info"))
    shutil.copytree(source / "scripts", release / "scripts", ignore=shutil.ignore_patterns("__pycache__"))
    shutil.copytree(source / "licenses", release / "licenses")
    for filename in ["pyproject.toml", "LICENSE", "NOTICE", "THIRD_PARTY.md", "DATASETS.md", "COMMANDS.md", "setup-mac.sh", "train-mac.sh", "requirements-tested-macos.txt"]:
        shutil.copyfile(source / filename, release / filename)
    # Publish inference weights only; optimizer and local absolute paths stay local.
    public_meta = {k: v for k, v in meta.items() if k != "dataset"}
    torch.save({"format": 1, "config": checkpoint["config"], "model": checkpoint["model"],
                "step": checkpoint["step"], "metadata": public_meta}, release / "model.pt")
    (release / "config.json").write_text(json.dumps(checkpoint["config"], indent=2))
    (release / "training.json").write_text(json.dumps(public_meta, indent=2))
    (release / "evaluation.json").write_text(json.dumps({**report,
        "published_model_sha256": sha(release / "model.pt")}, indent=2))
    shutil.copyfile(Path(args.data) / "sources.json", release / "ATTRIBUTIONS.json")
    used = sorted({r["repo"] for r in json.loads((release / "ATTRIBUTIONS.json").read_text())})
    datasets_yaml = "\n".join("- " + repo for repo in used)
    text = f'''---
license: apache-2.0
pipeline_tag: image-to-3d
base_model: Lucasgitguy-co/image3d-student
model_name: Forma3D-Tiny v0.2
datasets:
{datasets_yaml}
tags:
- pytorch
- gaussian-splatting
- research-prototype
- mesh-proxy-supervision
---
# Forma3D-Tiny v0.2
An experimental compact image-to-Gaussian model post-trained from Image3D Student's
synthetic demonstration checkpoint. This is a coarse research baseline, not a
general-purpose photo-to-3D model. Its grid is {checkpoint["config"]["grid"]} cubed and
it has {meta["parameters"]:,} parameters. Selected checkpoint: step {checkpoint["step"]};
completed run: {meta["steps_requested"]} steps.
## Actual training data and preprocessing
Only the datasets listed above were used, with per-asset provenance in ATTRIBUTIONS.json.
Scenario assets are CC-BY-4.0; the Objaverse subset is CC0. Original objects were centered
and uniformly scaled into a 0.9-wide cube, surface sampled, and converted to isotropic
Gaussian proxy targets. Orthographic images were rendered from these same proxy clouds.
This is not teacher distillation from Idol, Hunyuan, InstantMesh, or Hi3D, and it did not
train on natural photographs, MultiScan, 3D-Front, Show3D, or Voxel51 scene splats.
Related source inputs and all views stay in the same object group. A separate 15% test
partition is excluded from training and validation-based checkpoint selection.
## Measured results
Independent test objects: {report["test_objects"]}; test views: {report["test_samples"]}.
Base test loss: {report["baseline_test"]["loss"]:.6f}; post-trained test loss:
{report["posttrained_test"]["loss"]:.6f}. Base occupancy IoU:
{report["baseline_test"]["occupancy_iou"]:.6f}; post-trained occupancy IoU:
{report["posttrained_test"]["occupancy_iou"]:.6f}.
See evaluation.json for the full results and checks. These scores measure coarse proxy
reconstruction on the rendered data distribution; they do not prove photo realism,
watertight meshes, production quality, or texture fidelity. Near duplicates can remain
despite source grouping and exact-byte deduplication.
## Run on a Mac
Download this repository, then run `bash setup-mac.sh` and
`source "$HOME/Library/Application Support/Forma3D/venv/bin/activate"`.
```sh
image3d generate --checkpoint model.pt --image your-centered-cutout.png \\
  --out results/object.ply --device mps
image3d preview --ply results/object.ply --out results/preview.png
```
Inputs are centered objects on black, matching the training framing. Natural photos
are outside the verified distribution. Output is Gaussian PLY, not a triangle mesh.
CPU works with `--device cpu`; CUDA support is implemented but was not tested on the
Mac used to prepare these scripts.
## License and attribution
Source code and these weights are offered under Apache-2.0. Dataset and individual
asset licenses remain separate; retain ATTRIBUTIONS.json and consult DATASETS.md.
This license statement does not grant rights to third-party trademarks or likenesses.
See COMMANDS.md for reproduction commands.
'''
    (release / "README.md").write_text(text)
    sums = [sha(p) + "  " + str(p.relative_to(release)) for p in sorted(release.rglob("*")) if p.is_file()]
    (release / "SHA256SUMS").write_text("\n".join(sums) + "\n")
    return release


def main():
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument("--run", required=True)
    p.add_argument("--data", required=True)
    p.add_argument("--base", required=True)
    p.add_argument("--release", default="release/forma3d-tiny")
    p.add_argument("--repo", default="Lucasgitguy-co/forma3d-tiny")
    p.add_argument("--device", choices=["mps", "cpu", "cuda", "auto"], default="mps")
    p.add_argument("--publish", action="store_true")
    args = p.parse_args()
    checkpoint, meta, report = evaluate_release(args)
    release = stage(args, checkpoint, meta, report)
    if args.publish:
        from huggingface_hub import HfApi
        api = HfApi()
        # Refuse accidental replacement of a previously published model.
        if api.repo_exists(args.repo, repo_type="model"):
            raise ValueError("Repository already exists. Choose a new --repo; no overwrite was performed.")
        api.create_repo(args.repo, repo_type="model", private=True)
        commit = api.upload_folder(repo_id=args.repo, folder_path=release,
                                   commit_message="Publish evaluated Forma3D-Tiny v0.2")
        api.update_repo_settings(args.repo, private=False)
        print(f"Published https://huggingface.co/{args.repo} at {commit.oid}")
    else:
        print(f"Evaluated release staged at {release}; nothing published.")


if __name__ == "__main__":
    main()

	"""Evaluate a completed run, stage a self-contained release, optionally publish it."""
	import argparse
	import hashlib
	import json
	import shutil
	from pathlib import Path

	import torch
	from torch.utils.data import DataLoader

	from image3d_student.data import CachedDataset, load_manifest
	from image3d_student.engine import device_for, evaluate, generate
	from image3d_student.model import Student


	def sha(path):
	return hashlib.sha256(Path(path).read_bytes()).hexdigest()


	def evaluate_release(args):
	run, dataset = Path(args.run), Path(args.data)
	meta = json.loads((run / "run.json").read_text())
	checkpoint = torch.load(run / "best.pt", map_location="cpu", weights_only=True)
	last = torch.load(run / "last.pt", map_location="cpu", weights_only=True)
	if last["step"] != meta["steps_requested"]:
	raise ValueError("Training did not complete the requested steps")
	if checkpoint.get("step", 0) < 1 or not meta.get("initialization"):
	raise ValueError("Expected completed post-training from a base checkpoint")
	if sha(args.base) != meta["initialization"]["sha256"]:
	raise ValueError("Base checkpoint differs from the training initialization")
	if sha(dataset / "manifest.jsonl") != meta["manifest_sha256"]:
	raise ValueError("Training manifest changed")
	development = load_manifest(dataset / "manifest.jsonl")
	test = load_manifest(dataset / "test-manifest.jsonl")
	selected_ids = set(meta["train_ids"]) \| set(meta["val_ids"])
	if selected_ids != {r["id"] for r in development}:
	raise ValueError("Run split does not match development manifest")
	if {r["metadata"]["object_id"] for r in development} & {r["metadata"]["object_id"] for r in test}:
	raise ValueError("Test object leakage")
	for key in ["image_sha256", "target_sha256"]:
	if {r[key] for r in development} & {r[key] for r in test}:
	raise ValueError("Duplicate data across development and test")
	device = device_for(args.device)
	model = Student(**checkpoint["config"]).to(device)
	model.load_state_dict(checkpoint["model"])
	loader = DataLoader(CachedDataset(dataset / "test-manifest.jsonl"), batch_size=1)
	final = evaluate(model, loader, device)
	base = torch.load(args.base, map_location="cpu", weights_only=True)
	initial = Student(**checkpoint["config"]).to(device)
	result = initial.load_state_dict({k: v for k, v in base["model"].items() if k not in ("coords", "centers")}, strict=False)
	if set(result.missing_keys) != {"coords", "centers"} or result.unexpected_keys:
	raise ValueError("Base architecture mismatch")
	baseline = evaluate(initial, loader, device)
	best_validation = min(json.loads(line)["validation"]["loss"] for line in
	(run / "metrics.jsonl").read_text().splitlines())
	checks = {"validation_loss_improved": best_validation < meta["baseline_validation"]["loss"],
	"test_loss_improved": final["loss"] < baseline["loss"],
	"test_occupancy_iou_improved": final["occupancy_iou"] > baseline["occupancy_iou"],
	"test_occupancy_iou_at_least_005": final["occupancy_iou"] >= .05}
	report = {"checkpoint_sha256": sha(run / "best.pt"), "base_sha256": sha(args.base),
	"test_manifest_sha256": sha(dataset / "test-manifest.jsonl"),
	"test_objects": len({r["metadata"]["object_id"] for r in test}),
	"test_samples": len(test), "baseline_test": baseline, "posttrained_test": final,
	"checks": checks, "passed": all(checks.values()),
	"scope": "Rendered mesh-proxy evaluation only; does not establish real-photo or mesh reconstruction quality"}
	(run / "evaluation.json").write_text(json.dumps(report, indent=2))
	print(json.dumps(report, indent=2), flush=True)
	# Keep qualitative inspection files local; all held-out objects, first view.
	example_dir = run / "inspection"
	example_dir.mkdir(exist_ok=True)
	seen = set()
	from image3d_student.splats import preview, read_ply
	from PIL import Image, ImageDraw
	for row in test:
	group = row["metadata"]["object_id"]
	if group in seen:
	continue
	seen.add(group)
	out = example_dir / (row["id"] + ".ply")
	if not out.exists():
	try:
	generate(run / "best.pt", dataset / row["image"], out, args.device)
	except ValueError as exc:
	if "No occupied cells" not in str(exc):
	raise
	report["checks"]["nonempty_examples"] = False
	report["passed"] = False
	continue
	sheet = Image.new("RGB", (512, 152), "black")
	sheet.paste(Image.open(dataset / row["image"]).convert("RGB").resize((128, 128)), (0, 24))
	splats = read_ply(out)
	for i, angle in enumerate([0, 1.5707963, 3.1415927]):
	sheet.paste(preview(splats, size=128, yaw=angle), (128 * (i + 1), 24))
	ImageDraw.Draw(sheet).text((5, 5), "Input proxy \| prediction front \| side \| back", fill="white")
	sheet.save(out.with_suffix(".png"))
	if len(seen) >= 8:
	break
	(run / "evaluation.json").write_text(json.dumps(report, indent=2))
	if not report["passed"]:
	raise ValueError("Evaluation gate failed. Nothing was published. Inspect evaluation.json and improve data/training.")
	return checkpoint, meta, report


	def stage(args, checkpoint, meta, report):
	source = Path(__file__).resolve().parents[1]
	release = Path(args.release)
	if release.exists():
	raise ValueError("Release directory already exists; choose a new --release path")
	release.mkdir(parents=True)
	shutil.copytree(source / "src", release / "src", ignore=shutil.ignore_patterns("__pycache__", "*.egg-info"))
	shutil.copytree(source / "scripts", release / "scripts", ignore=shutil.ignore_patterns("__pycache__"))
	shutil.copytree(source / "licenses", release / "licenses")
	for filename in ["pyproject.toml", "LICENSE", "NOTICE", "THIRD_PARTY.md", "DATASETS.md", "COMMANDS.md", "setup-mac.sh", "train-mac.sh", "requirements-tested-macos.txt"]:
	shutil.copyfile(source / filename, release / filename)
	# Publish inference weights only; optimizer and local absolute paths stay local.
	public_meta = {k: v for k, v in meta.items() if k != "dataset"}
	torch.save({"format": 1, "config": checkpoint["config"], "model": checkpoint["model"],
	"step": checkpoint["step"], "metadata": public_meta}, release / "model.pt")
	(release / "config.json").write_text(json.dumps(checkpoint["config"], indent=2))
	(release / "training.json").write_text(json.dumps(public_meta, indent=2))
	(release / "evaluation.json").write_text(json.dumps({**report,
	"published_model_sha256": sha(release / "model.pt")}, indent=2))
	shutil.copyfile(Path(args.data) / "sources.json", release / "ATTRIBUTIONS.json")
	used = sorted({r["repo"] for r in json.loads((release / "ATTRIBUTIONS.json").read_text())})
	datasets_yaml = "\n".join("- " + repo for repo in used)
	text = f'''---
	license: apache-2.0
	pipeline_tag: image-to-3d
	base_model: Lucasgitguy-co/image3d-student
	model_name: Forma3D-Tiny v0.2
	datasets:
	{datasets_yaml}
	tags:
	- pytorch
	- gaussian-splatting
	- research-prototype
	- mesh-proxy-supervision
	---
	# Forma3D-Tiny v0.2

	An experimental compact image-to-Gaussian model post-trained from Image3D Student's
	synthetic demonstration checkpoint. This is a coarse research baseline, not a
	general-purpose photo-to-3D model. Its grid is {checkpoint["config"]["grid"]} cubed and
	it has {meta["parameters"]:,} parameters. Selected checkpoint: step {checkpoint["step"]};
	completed run: {meta["steps_requested"]} steps.

	## Actual training data and preprocessing

	Only the datasets listed above were used, with per-asset provenance in ATTRIBUTIONS.json.
	Scenario assets are CC-BY-4.0; the Objaverse subset is CC0. Original objects were centered
	and uniformly scaled into a 0.9-wide cube, surface sampled, and converted to isotropic
	Gaussian proxy targets. Orthographic images were rendered from these same proxy clouds.
	This is not teacher distillation from Idol, Hunyuan, InstantMesh, or Hi3D, and it did not
	train on natural photographs, MultiScan, 3D-Front, Show3D, or Voxel51 scene splats.

	Related source inputs and all views stay in the same object group. A separate 15% test
	partition is excluded from training and validation-based checkpoint selection.

	## Measured results

	Independent test objects: {report["test_objects"]}; test views: {report["test_samples"]}.
	Base test loss: {report["baseline_test"]["loss"]:.6f}; post-trained test loss:
	{report["posttrained_test"]["loss"]:.6f}. Base occupancy IoU:
	{report["baseline_test"]["occupancy_iou"]:.6f}; post-trained occupancy IoU:
	{report["posttrained_test"]["occupancy_iou"]:.6f}.
	See evaluation.json for the full results and checks. These scores measure coarse proxy
	reconstruction on the rendered data distribution; they do not prove photo realism,
	watertight meshes, production quality, or texture fidelity. Near duplicates can remain
	despite source grouping and exact-byte deduplication.

	## Run on a Mac

	Download this repository, then run `bash setup-mac.sh` and
	`source "$HOME/Library/Application Support/Forma3D/venv/bin/activate"`.

	```sh
	image3d generate --checkpoint model.pt --image your-centered-cutout.png \\
	--out results/object.ply --device mps
	image3d preview --ply results/object.ply --out results/preview.png
	```

	Inputs are centered objects on black, matching the training framing. Natural photos
	are outside the verified distribution. Output is Gaussian PLY, not a triangle mesh.
	CPU works with `--device cpu`; CUDA support is implemented but was not tested on the
	Mac used to prepare these scripts.

	## License and attribution

	Source code and these weights are offered under Apache-2.0. Dataset and individual
	asset licenses remain separate; retain ATTRIBUTIONS.json and consult DATASETS.md.
	This license statement does not grant rights to third-party trademarks or likenesses.
	See COMMANDS.md for reproduction commands.
	'''
	(release / "README.md").write_text(text)
	sums = [sha(p) + " " + str(p.relative_to(release)) for p in sorted(release.rglob("*")) if p.is_file()]
	(release / "SHA256SUMS").write_text("\n".join(sums) + "\n")
	return release


	def main():
	p = argparse.ArgumentParser(description=__doc__)
	p.add_argument("--run", required=True)
	p.add_argument("--data", required=True)
	p.add_argument("--base", required=True)
	p.add_argument("--release", default="release/forma3d-tiny")
	p.add_argument("--repo", default="Lucasgitguy-co/forma3d-tiny")
	p.add_argument("--device", choices=["mps", "cpu", "cuda", "auto"], default="mps")
	p.add_argument("--publish", action="store_true")
	args = p.parse_args()
	checkpoint, meta, report = evaluate_release(args)
	release = stage(args, checkpoint, meta, report)
	if args.publish:
	from huggingface_hub import HfApi
	api = HfApi()
	# Refuse accidental replacement of a previously published model.
	if api.repo_exists(args.repo, repo_type="model"):
	raise ValueError("Repository already exists. Choose a new --repo; no overwrite was performed.")
	api.create_repo(args.repo, repo_type="model", private=True)
	commit = api.upload_folder(repo_id=args.repo, folder_path=release,
	commit_message="Publish evaluated Forma3D-Tiny v0.2")
	api.update_repo_settings(args.repo, private=False)
	print(f"Published https://huggingface.co/{args.repo} at {commit.oid}")
	else:
	print(f"Evaluated release staged at {release}; nothing published.")


	if __name__ == "__main__":
	main()