Instructions to use XXXpilar/Fastfill-test with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use XXXpilar/Fastfill-test with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
File size: 21,299 Bytes
e6f80f3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 | """Read-only, bounded inventory and documented geometry semantics of local sources.
Inventory is not a full numerical/mesh audit of every large dataset. CSV counts
are measured; supplied manifests and README totals retain their evidence label.
Only MultiScan's admitted annotations undergo full per-row geometry validation.
"""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
from pathlib import Path
TOP_DIRECTORIES = ("BillLin66__3D_Room_Collections", "imChuling__3D_Room_Collections",
"liantian__3D_Room_Scene_Collections", "XXXpilar__hssd_clean",
"XXXpilar__multiscan-clean")
BILL, IM, SAGE = TOP_DIRECTORIES[:3]
CANONICAL_SOURCE_ROOT = Path("/Volumes/harddisk/3D_Room_Collections")
def _source(name, directory, sample_file, **fields):
return {"name": name, "directory": directory, "sample_file": sample_file,
"category_description": "source category; no trustworthy scene-language assumed",
"object_asset_id": "instance ID retained in provenance only; asset IDs never enter main condition",
"support": "unknown unless explicit source metadata", "relations": "not used as condition by default",
"semantic_front": "not independently established for all assets",
"eligibility": "audit_only", **fields}
def source_catalog():
"""Versioned semantics, established from local raw records and export code."""
return [
_source("3D-FRONT (BillLin66)", f"{BILL}/3DFront_exported", "3dfront.jsonl",
room_geometry="floor mesh convex hull; planar floor/ceiling height; openings",
room_type="source room.type", size_semantics="asset-definition local size/bbox times instance scale",
extent_kind="full; source bbox variants require independent verification", units="meters",
position_reference="asset pivot/origin, not proven bottom/center", rotation="raw quaternion retained in houses; export yaw degrees",
up_handedness="source Y-up -> (x,-z,y), right-handed Z-up", scale="export already applies abs(instance scale)",
meshes="room structure vertices retained; furniture mesh not included",
blockers="pivot-to-bbox offset and semantic front missing; many category/size nulls",
declared_counts={"rooms": 56129, "objects": 533837},
code_evidence=["kits/convert_3dfront.py"]),
_source("SpatialGen", f"{BILL}/SpatialGen_exported", "spatialgen.jsonl",
room_geometry="floor line loop; known self-intersection scene_00019", room_type="caption-derived mapping",
size_semantics="instance-scaled local XYZ full extents", extent_kind="full", units="bbox meters; floor/ceiling millimeters",
position_reference="box geometric center", rotation="full transform, mirrored/tilted rows retained",
up_handedness="right-handed Z-up; reflection separately recorded", scale="norm of transform axis columns, once",
meshes="not present", blockers="official test-only; semantic front must be established; reject tilt/mirrors and invalid rooms",
declared_counts={"scenes": 48, "objects": 977}, code_evidence=["kits/convert_spatialgen.py"]),
_source("3RScan", f"{IM}/3RScan_exported", "3rscan.json",
room_geometry="no true room polygon/height; scan container", room_type="missing",
size_semantics="obb.axesLengths along raw normalizedAxes", extent_kind="full", units="source metric OBBs, no export normalization",
position_reference="obb.centroid", rotation="raw normalizedAxes; exported yaw null",
up_handedness="scan frame, source axes need per-record verification", scale="OBB already observed metric geometry",
support="3DSSG attributes/affordances", relations="3DSSG relationships preserved",
meshes="not present", blockers="room reconstruction and semantic-front evidence required",
declared_counts={"scans": 1482, "objects": 43450}, code_evidence=["kits/extract_3rscan.py"]),
_source("ARKitScenes", f"{IM}/ARKitScenes_exported", "arkitscenes.json",
room_geometry="no true room boundary/height", room_type="missing",
size_semantics="segments.obbAligned.axesLengths local OBB", extent_kind="full", units="meters for obbAligned; raw obb is different",
position_reference="export world AABB bottom; raw obbAligned.centroid is OBB center",
rotation="normalizedAxes.T; raw orientation includes tilt", up_handedness="right-handed Z-up aligned frame",
scale="aligned OBB metric extents; no asset scaling", meshes="not present; RGB keyframes archive only",
object_asset_id="modelId custom.abandon is a placeholder, never a real CAD identity",
blockers="front/room missing; official visit_id has one cross-split location group",
declared_counts={"scans": 5047, "objects": 56605}, code_evidence=["extract_arkitscenes_json.py"]),
_source("IL3D (3D-FRONT/HSSD/synthetic)", f"{IM}/IL3D_exported", "3dfront.json",
room_geometry="source floor mesh; partial 3D-FRONT height/opening supplement", room_type="source category",
size_semantics="layout bbox XYZ mapped to local X/Z/Y; verify asset metadata/model orientation",
extent_kind="full", units="meters; USDA output separately multiplies by 100",
position_reference="claimed bottom center; requires per-asset pivot verification",
rotation="USD rotateXYZ matrix -> Z-up; complete matrix kept", up_handedness="right-handed Y-up -> (x,-z,y)",
scale="source bbox vs scale requires asset audit; do not reapply automatically",
support="placement_flags available; not mesh support surfaces", meshes="CAD USDZ asset archives not downloaded",
blockers="semantic-front/pivot/asset axis audit; source NaNs and extreme outlier; opening world-span defect in v1",
declared_counts={"scenes": 27816, "objects": 226978}, code_evidence=["aggregate_scenes.py"]),
_source("InteriorGS", f"{IM}/InteriorGS_exported", "interiorgs.json",
room_geometry="structure room profiles and holes", room_type="missing",
size_semantics="long/short horizontal dimensions recovered from source 8 bbox corners",
extent_kind="full", units="meters", position_reference="bbox bottom center",
rotation="geometric long-axis angle modulo pi, no semantic front", up_handedness="right-handed Z-up; X right/Y back",
scale="observed corners already metric", meshes="not present",
blockers="geometry-only yaw cannot become semantic-front label; zero extents and ambiguous room assignments",
declared_counts={"scenes": 1000, "rooms": 6607, "objects": 506061}, code_evidence=["aggregate_scenes.py"]),
_source("InternScenes (Real2Sim/Gen)", f"{IM}/InternScenes_exported", "internscenes_gen_exported.json",
room_geometry="floor convex hull; Real2Sim scan/region containers, Gen region labels", room_type="partial",
size_semantics="source bbox[3:6] local extents before full ZXY rotation", extent_kind="full", units="meters",
position_reference="bbox geometric center", rotation="bbox angle_z/x/y radians; ZXY order",
up_handedness="right-handed Z-up layout; floor mesh Y-up converted", scale="box sizes already instance geometry",
meshes="floor geometry only; full asset library absent", blockers="semantic front unverified; v1 geometric fallback must not become a front label",
declared_counts={"scenes": 23672, "objects": 798954}, code_evidence=["kits/extract_internscenes_json.py"]),
_source("MansionWorld", f"{IM}/MansionWorld_exported", "mansionworld.jsonl",
room_geometry="room floor polygon, height, doors/windows from THOR source", room_type="source roomType",
size_semantics="Objathor annotation bounding box; floor/wall/surface paths differ",
extent_kind="full", units="meters", position_reference="floor/wall bottom center; surface path must inspect raw parent geometry",
rotation="source THOR yaw mapped to output degrees", up_handedness="THOR Y-up -> (x,-z,y), right-handed Z-up",
scale="metadata bbox; any asset scale must be checked against raw annotation",
support="surface_groups with source parent associations", meshes="metadata/annotations only, no glTF/SDF",
blockers="per-asset canonical front/pivot and support surfaces require audit; room/asset descriptions may leak target geometry",
declared_counts={"buildings": 1000, "rooms": 27884, "objects": 405253}, code_evidence=["kits/convert_mansionworld.py"]),
_source("OptiScene / 3D-SynthPlace", f"{IM}/OptiScene_exported", "holodeck.json",
room_geometry="source rectangular proxy, no original floor polygon", room_type="source labels",
size_semantics="bbox [height,width,depth] claimed local geometry; mesh verification missing",
extent_kind="full", units="bbox centimeters; positions meters", position_reference="source_position_anchor, pivot semantics not established",
rotation="source rotation.y is horizontal angle; full Euler order unspecified",
up_handedness="source Y-up; export (x,z,y) reverses handedness; needs explicit yaw correction",
scale="bbox source geometry; no unique future mesh implied", meshes="JSON-only export; scenes.tar.gz and prompts retained",
blockers="handedness/yaw/pivot and known invalid geometry; do not reuse prompt bbox as v2 condition",
declared_counts={"scenes": 17528, "objects": 147222}, code_evidence=["extract_optiscene_json.py", "OptiScene/gprompt.py"]),
_source("SceneCAD / Scan2CAD", f"{IM}/SceneCAD_Scan2CAD_exported", "scenecad_scan2cad_combined.json",
room_geometry="SceneCAD shell; scan may have multiple rooms; no true room partition", room_type="ScanNet sceneType",
size_semantics="ScanNet objects world AABB; separate CAD local bbox half-extents × TRS scale",
extent_kind="AABB full; raw CAD bbox half", units="meters", position_reference="ScanNet AABB bottom; CAD geometric OBB center",
rotation="ScanNet yaw unavailable; CAD full transform incl axisAlignment", up_handedness="ShapeNet Y-up -> right-handed Z-up",
scale="CAD 2*bbox*scale; no rotated AABB used as local size", meshes="not present",
blockers="v1 tilted-CAD defect; semantic front and room grouping need raw-source adapter",
declared_counts={"scenes": 1509, "cad_scenes": 1506, "cad_objects": 14225}, code_evidence=["aggregate_scenecad_scan2cad.py"]),
_source("SceneSmith", f"{IM}/SceneSmith_exported", "json_single.json",
room_geometry="floor-plan mesh AABB; Room/House frames", room_type="frame-name derived",
size_semantics="visual glTF/SDF mesh local extent; origin bounds must be retained",
extent_kind="full", units="meters", position_reference="DMD asset origin; mixed bottom and centered wall assets",
rotation="AngleAxis -> Rz Ry Rx Euler degrees", up_handedness="DMD Z-up; glTF Y-up -> (x,-z,y)",
scale="SDF visual scale/node transforms applied during extent extraction", meshes="geometry metadata/DMD; full tar meshes not present",
blockers="v1 46 wall assets have wrong center assumption; per-asset bounds/front missing",
declared_counts={"scenes": 210, "rooms": 364, "objects": 19360}, code_evidence=["kits/scenesmith_aggregate.py", "kits/scenesmith_extract_geometry_metadata.py"]),
_source("SpatialLM", f"{IM}/SpatialLM_exported", "spatiallm.json",
room_geometry="wall-loop polygon, known invalid rings; source floor plan", room_type="source room category",
size_semantics="Bbox scale_x/y/z local extents", extent_kind="full", units="meters",
position_reference="box geometric center", rotation="angle_z radians, yaw-only format",
up_handedness="Z-up source frame", scale="box extents already final geometry", meshes="not present; layout archive retained",
blockers="semantic front calibrated in v1 but not per-asset proven; cross-export source-house duplicates require common split",
declared_counts={"scenes": 12328, "rooms": 54775}, code_evidence=["aggregate_spatiallm.py"]),
_source("Structured3D", f"{IM}/Structured3D_exported", "structured3d.jsonl",
room_geometry="annotation junction/line floor polygons, opening planes, house-room split", room_type="source semantic room labels",
size_semantics="raw_coeffs along raw_basis, not heuristic exported axis reorder", extent_kind="raw half extents",
units="raw bbox millimeters; exported meters", position_reference="raw bbox centroid; exported room-local center",
rotation="raw basis rows; geometric vs semantic front not guaranteed", up_handedness="right-handed Z-up",
scale="raw half extents × 2/1000 once", meshes="not present; annotation and rendered labels retained",
blockers="heuristic yaw axes/front, 68 zero-sized boxes, source errata, unknown category/tilt",
declared_counts={"houses": 3488, "rooms": 22894, "assigned_objects": 398485}, code_evidence=["kits/convert_structured3d.py"]),
_source("SAGE10k", f"{SAGE}/SAGE10k", "json_single.json",
room_geometry="authored rectangle/wall loop and ceiling height", room_type="source room_type",
size_semantics="local dimensions width/length/height; raw mesh scale/frame must verify",
extent_kind="full", units="positions/dimensions meters; placement_constraints XY centimeters",
position_reference="asset bottom-center origin; raw source physics pose rotates that origin",
rotation="Rz Ry Rx degrees incl real pitch/roll; README nonsemantic-tilt claim is not sufficient",
up_handedness="Z-up; right-handed from source matrix/render code", scale="object-generation scale/mesh size requires raw asset check",
support="place_id floor/wall/instance retained in per-scene source", relations="placement_constraints include exact target pose: exclude from main condition",
meshes="local scene metadata; mesh catalog/semantic-front completeness unverified",
blockers="semantic front currently empirical +Y; significant physics tilt must explicitly reject; support surfaces not bbox tops",
declared_counts={"scenes": 10000, "rooms": 10027, "objects": 853428}, code_evidence=["aggregate_scenes.py", "sage10k/kits/tex_utils_local.py"]),
_source("HSSD", "XXXpilar__hssd_clean", "objects.csv",
room_geometry="regions.csv polygon/floor/ceiling; 2351 regions", room_type="human region label",
size_semantics="rigid mesh-axis OBB half extents; articulated rows lack box", extent_kind="half", units="meters",
position_reference="obb_center; translation is asset_local pivot; openings metadata center may be wrong",
rotation="obb_rotation_wxyz; full rigid transform", up_handedness="right-handed Y-up -> (x,-z,y)",
scale="obb_half_extents already scaled; NEVER multiply non_uniform_scale again",
support="asset support/affordance metadata, not verified mesh surfaces", relations="unavailable",
meshes="GLB stages/objects/openings plus URDFs present", blockers="per-asset front/up not retained in catalog; empirical v1 fronts differ; opening/pivot and articulated twins need reliable repair",
declared_counts={"scenes": 168, "regions": 2351, "objects": 56647}, code_evidence=[]),
_source("MultiScan", "XXXpilar__multiscan-clean", "objects.csv",
room_geometry="partial floor-slab convex hull; slab top not a verified contact plane", room_type="source scan room_type",
size_semantics="annotation OBB axes local extents plus annotated front/up", extent_kind="half", units="meters",
position_reference="obb_center -> each own full-height bottom center", rotation="obb_axes plus annotated front/up",
up_handedness="right-handed aligned Z-up", scale="annotated box already metric; no extra scale",
semantic_front="explicit source front vector; admitted only horizontal OBB-aligned fronts",
support="geometry-derived rests_on_floor/supported_by; not trusted physical labels",
relations="geometry-derived; omitted to prevent reference-layout leakage", meshes="273 PLY+textured OBJ scans, extracted per-object assets not built",
blockers="strict tilt/frame reject; incomplete boundary/floor prevent full room-validity claims",
eligibility="full_geometry_supervision_after_strict_filter", declared_counts={"scans": 273, "scenes": 117, "objects": 10957},
code_evidence=[]),
]
def _file_evidence(path):
text = path.read_bytes()
return {"path": path.name, "bytes": len(text), "sha256": hashlib.sha256(text).hexdigest()}
def _sample(path):
"""Read at most 4 MiB of a large array; never load whole corpus into RAM."""
if path.suffix == ".csv":
with path.open(newline="") as fh:
reader = csv.DictReader(fh)
first = next(reader, {})
count = bool(first) + sum(1 for _ in reader)
return {"measured_records": count, "first_record_keys": sorted(first)}
with path.open(encoding="utf-8") as fh:
prefix = fh.read(4 * 1024 * 1024).lstrip()
prefix = prefix[1:].lstrip() if prefix.startswith("[") else prefix
first, _ = json.JSONDecoder().raw_decode(prefix)
rooms = first.get("rooms") or ([first["room"]] if "room" in first else [])
room = rooms[0] if rooms else first.get("room_context", {})
objects = room.get("furniture") or first.get("layout", {}).get("floor_layout", {}).get("objects") or []
return {"first_record_keys": sorted(first), "first_room_keys": sorted(room),
"first_object_keys": sorted(objects[0]) if objects else [], "sampling": "first record only; max 4 MiB"}
def audit_sources(source_root):
root = Path(source_root).resolve()
entries = []
for definition in source_catalog():
base = root / definition["directory"]
evidence = []
for rel in ["README.md", "manifest.json", "dataset_manifest.json", *definition["code_evidence"]]:
path = base / rel
if path.is_file():
evidence.append({**_file_evidence(path), "path": rel})
sample = {}
path = base / definition["sample_file"]
if path.is_file():
try:
sample = _sample(path)
except (ValueError, TypeError, json.JSONDecodeError) as exc:
sample = {"sample_error": str(exc)}
entries.append({**definition, "present": base.is_dir(), "evidence": evidence, "actual_sample": sample,
"count_scope": "declared_counts are retained source documentation, not a new full-corpus recount"})
return {"schema_version": "fastfill.v2.source_audit", "source_root": str(root),
"auditor_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
"top_directories": [{"name": name, "present": (root / name).is_dir()} for name in TOP_DIRECTORIES],
"sources": entries, "scope": "README+converter+bounded real record inventory; CSV rows counted; no in-place normalization",
"source_data_modified": False}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--source-root", type=Path, default=Path("/Volumes/harddisk/3D_Room_Collections"))
parser.add_argument("--output", type=Path, help="New JSON output file outside source data")
args = parser.parse_args()
report = audit_sources(args.source_root)
if args.output:
path = args.output.resolve()
protected = (args.source_root.resolve(), CANONICAL_SOURCE_ROOT.resolve())
if any(path == root or root in path.parents or path in root.parents for root in protected):
raise ValueError("audit output must be outside read-only source data")
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("x") as fh:
json.dump(report, fh, indent=2, ensure_ascii=False, allow_nan=False)
fh.write("\n")
print(json.dumps({"output": str(path), "sources_present": sum(s["present"] for s in report["sources"])}))
else:
print(json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False))
if __name__ == "__main__":
main()
|