File size: 21,299 Bytes
e6f80f3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
"""Read-only, bounded inventory and documented geometry semantics of local sources.

Inventory is not a full numerical/mesh audit of every large dataset. CSV counts
are measured; supplied manifests and README totals retain their evidence label.
Only MultiScan's admitted annotations undergo full per-row geometry validation.
"""
from __future__ import annotations

import argparse
import csv
import hashlib
import json
from pathlib import Path

TOP_DIRECTORIES = ("BillLin66__3D_Room_Collections", "imChuling__3D_Room_Collections",
                   "liantian__3D_Room_Scene_Collections", "XXXpilar__hssd_clean",
                   "XXXpilar__multiscan-clean")
BILL, IM, SAGE = TOP_DIRECTORIES[:3]
CANONICAL_SOURCE_ROOT = Path("/Volumes/harddisk/3D_Room_Collections")


def _source(name, directory, sample_file, **fields):
    return {"name": name, "directory": directory, "sample_file": sample_file,
            "category_description": "source category; no trustworthy scene-language assumed",
            "object_asset_id": "instance ID retained in provenance only; asset IDs never enter main condition",
            "support": "unknown unless explicit source metadata", "relations": "not used as condition by default",
            "semantic_front": "not independently established for all assets",
            "eligibility": "audit_only", **fields}


def source_catalog():
    """Versioned semantics, established from local raw records and export code."""
    return [
        _source("3D-FRONT (BillLin66)", f"{BILL}/3DFront_exported", "3dfront.jsonl",
                room_geometry="floor mesh convex hull; planar floor/ceiling height; openings",
                room_type="source room.type", size_semantics="asset-definition local size/bbox times instance scale",
                extent_kind="full; source bbox variants require independent verification", units="meters",
                position_reference="asset pivot/origin, not proven bottom/center", rotation="raw quaternion retained in houses; export yaw degrees",
                up_handedness="source Y-up -> (x,-z,y), right-handed Z-up", scale="export already applies abs(instance scale)",
                meshes="room structure vertices retained; furniture mesh not included",
                blockers="pivot-to-bbox offset and semantic front missing; many category/size nulls",
                declared_counts={"rooms": 56129, "objects": 533837},
                code_evidence=["kits/convert_3dfront.py"]),
        _source("SpatialGen", f"{BILL}/SpatialGen_exported", "spatialgen.jsonl",
                room_geometry="floor line loop; known self-intersection scene_00019", room_type="caption-derived mapping",
                size_semantics="instance-scaled local XYZ full extents", extent_kind="full", units="bbox meters; floor/ceiling millimeters",
                position_reference="box geometric center", rotation="full transform, mirrored/tilted rows retained",
                up_handedness="right-handed Z-up; reflection separately recorded", scale="norm of transform axis columns, once",
                meshes="not present", blockers="official test-only; semantic front must be established; reject tilt/mirrors and invalid rooms",
                declared_counts={"scenes": 48, "objects": 977}, code_evidence=["kits/convert_spatialgen.py"]),
        _source("3RScan", f"{IM}/3RScan_exported", "3rscan.json",
                room_geometry="no true room polygon/height; scan container", room_type="missing",
                size_semantics="obb.axesLengths along raw normalizedAxes", extent_kind="full", units="source metric OBBs, no export normalization",
                position_reference="obb.centroid", rotation="raw normalizedAxes; exported yaw null",
                up_handedness="scan frame, source axes need per-record verification", scale="OBB already observed metric geometry",
                support="3DSSG attributes/affordances", relations="3DSSG relationships preserved",
                meshes="not present", blockers="room reconstruction and semantic-front evidence required",
                declared_counts={"scans": 1482, "objects": 43450}, code_evidence=["kits/extract_3rscan.py"]),
        _source("ARKitScenes", f"{IM}/ARKitScenes_exported", "arkitscenes.json",
                room_geometry="no true room boundary/height", room_type="missing",
                size_semantics="segments.obbAligned.axesLengths local OBB", extent_kind="full", units="meters for obbAligned; raw obb is different",
                position_reference="export world AABB bottom; raw obbAligned.centroid is OBB center",
                rotation="normalizedAxes.T; raw orientation includes tilt", up_handedness="right-handed Z-up aligned frame",
                scale="aligned OBB metric extents; no asset scaling", meshes="not present; RGB keyframes archive only",
                object_asset_id="modelId custom.abandon is a placeholder, never a real CAD identity",
                blockers="front/room missing; official visit_id has one cross-split location group",
                declared_counts={"scans": 5047, "objects": 56605}, code_evidence=["extract_arkitscenes_json.py"]),
        _source("IL3D (3D-FRONT/HSSD/synthetic)", f"{IM}/IL3D_exported", "3dfront.json",
                room_geometry="source floor mesh; partial 3D-FRONT height/opening supplement", room_type="source category",
                size_semantics="layout bbox XYZ mapped to local X/Z/Y; verify asset metadata/model orientation",
                extent_kind="full", units="meters; USDA output separately multiplies by 100",
                position_reference="claimed bottom center; requires per-asset pivot verification",
                rotation="USD rotateXYZ matrix -> Z-up; complete matrix kept", up_handedness="right-handed Y-up -> (x,-z,y)",
                scale="source bbox vs scale requires asset audit; do not reapply automatically",
                support="placement_flags available; not mesh support surfaces", meshes="CAD USDZ asset archives not downloaded",
                blockers="semantic-front/pivot/asset axis audit; source NaNs and extreme outlier; opening world-span defect in v1",
                declared_counts={"scenes": 27816, "objects": 226978}, code_evidence=["aggregate_scenes.py"]),
        _source("InteriorGS", f"{IM}/InteriorGS_exported", "interiorgs.json",
                room_geometry="structure room profiles and holes", room_type="missing",
                size_semantics="long/short horizontal dimensions recovered from source 8 bbox corners",
                extent_kind="full", units="meters", position_reference="bbox bottom center",
                rotation="geometric long-axis angle modulo pi, no semantic front", up_handedness="right-handed Z-up; X right/Y back",
                scale="observed corners already metric", meshes="not present",
                blockers="geometry-only yaw cannot become semantic-front label; zero extents and ambiguous room assignments",
                declared_counts={"scenes": 1000, "rooms": 6607, "objects": 506061}, code_evidence=["aggregate_scenes.py"]),
        _source("InternScenes (Real2Sim/Gen)", f"{IM}/InternScenes_exported", "internscenes_gen_exported.json",
                room_geometry="floor convex hull; Real2Sim scan/region containers, Gen region labels", room_type="partial",
                size_semantics="source bbox[3:6] local extents before full ZXY rotation", extent_kind="full", units="meters",
                position_reference="bbox geometric center", rotation="bbox angle_z/x/y radians; ZXY order",
                up_handedness="right-handed Z-up layout; floor mesh Y-up converted", scale="box sizes already instance geometry",
                meshes="floor geometry only; full asset library absent", blockers="semantic front unverified; v1 geometric fallback must not become a front label",
                declared_counts={"scenes": 23672, "objects": 798954}, code_evidence=["kits/extract_internscenes_json.py"]),
        _source("MansionWorld", f"{IM}/MansionWorld_exported", "mansionworld.jsonl",
                room_geometry="room floor polygon, height, doors/windows from THOR source", room_type="source roomType",
                size_semantics="Objathor annotation bounding box; floor/wall/surface paths differ",
                extent_kind="full", units="meters", position_reference="floor/wall bottom center; surface path must inspect raw parent geometry",
                rotation="source THOR yaw mapped to output degrees", up_handedness="THOR Y-up -> (x,-z,y), right-handed Z-up",
                scale="metadata bbox; any asset scale must be checked against raw annotation",
                support="surface_groups with source parent associations", meshes="metadata/annotations only, no glTF/SDF",
                blockers="per-asset canonical front/pivot and support surfaces require audit; room/asset descriptions may leak target geometry",
                declared_counts={"buildings": 1000, "rooms": 27884, "objects": 405253}, code_evidence=["kits/convert_mansionworld.py"]),
        _source("OptiScene / 3D-SynthPlace", f"{IM}/OptiScene_exported", "holodeck.json",
                room_geometry="source rectangular proxy, no original floor polygon", room_type="source labels",
                size_semantics="bbox [height,width,depth] claimed local geometry; mesh verification missing",
                extent_kind="full", units="bbox centimeters; positions meters", position_reference="source_position_anchor, pivot semantics not established",
                rotation="source rotation.y is horizontal angle; full Euler order unspecified",
                up_handedness="source Y-up; export (x,z,y) reverses handedness; needs explicit yaw correction",
                scale="bbox source geometry; no unique future mesh implied", meshes="JSON-only export; scenes.tar.gz and prompts retained",
                blockers="handedness/yaw/pivot and known invalid geometry; do not reuse prompt bbox as v2 condition",
                declared_counts={"scenes": 17528, "objects": 147222}, code_evidence=["extract_optiscene_json.py", "OptiScene/gprompt.py"]),
        _source("SceneCAD / Scan2CAD", f"{IM}/SceneCAD_Scan2CAD_exported", "scenecad_scan2cad_combined.json",
                room_geometry="SceneCAD shell; scan may have multiple rooms; no true room partition", room_type="ScanNet sceneType",
                size_semantics="ScanNet objects world AABB; separate CAD local bbox half-extents × TRS scale",
                extent_kind="AABB full; raw CAD bbox half", units="meters", position_reference="ScanNet AABB bottom; CAD geometric OBB center",
                rotation="ScanNet yaw unavailable; CAD full transform incl axisAlignment", up_handedness="ShapeNet Y-up -> right-handed Z-up",
                scale="CAD 2*bbox*scale; no rotated AABB used as local size", meshes="not present",
                blockers="v1 tilted-CAD defect; semantic front and room grouping need raw-source adapter",
                declared_counts={"scenes": 1509, "cad_scenes": 1506, "cad_objects": 14225}, code_evidence=["aggregate_scenecad_scan2cad.py"]),
        _source("SceneSmith", f"{IM}/SceneSmith_exported", "json_single.json",
                room_geometry="floor-plan mesh AABB; Room/House frames", room_type="frame-name derived",
                size_semantics="visual glTF/SDF mesh local extent; origin bounds must be retained",
                extent_kind="full", units="meters", position_reference="DMD asset origin; mixed bottom and centered wall assets",
                rotation="AngleAxis -> Rz Ry Rx Euler degrees", up_handedness="DMD Z-up; glTF Y-up -> (x,-z,y)",
                scale="SDF visual scale/node transforms applied during extent extraction", meshes="geometry metadata/DMD; full tar meshes not present",
                blockers="v1 46 wall assets have wrong center assumption; per-asset bounds/front missing",
                declared_counts={"scenes": 210, "rooms": 364, "objects": 19360}, code_evidence=["kits/scenesmith_aggregate.py", "kits/scenesmith_extract_geometry_metadata.py"]),
        _source("SpatialLM", f"{IM}/SpatialLM_exported", "spatiallm.json",
                room_geometry="wall-loop polygon, known invalid rings; source floor plan", room_type="source room category",
                size_semantics="Bbox scale_x/y/z local extents", extent_kind="full", units="meters",
                position_reference="box geometric center", rotation="angle_z radians, yaw-only format",
                up_handedness="Z-up source frame", scale="box extents already final geometry", meshes="not present; layout archive retained",
                blockers="semantic front calibrated in v1 but not per-asset proven; cross-export source-house duplicates require common split",
                declared_counts={"scenes": 12328, "rooms": 54775}, code_evidence=["aggregate_spatiallm.py"]),
        _source("Structured3D", f"{IM}/Structured3D_exported", "structured3d.jsonl",
                room_geometry="annotation junction/line floor polygons, opening planes, house-room split", room_type="source semantic room labels",
                size_semantics="raw_coeffs along raw_basis, not heuristic exported axis reorder", extent_kind="raw half extents",
                units="raw bbox millimeters; exported meters", position_reference="raw bbox centroid; exported room-local center",
                rotation="raw basis rows; geometric vs semantic front not guaranteed", up_handedness="right-handed Z-up",
                scale="raw half extents × 2/1000 once", meshes="not present; annotation and rendered labels retained",
                blockers="heuristic yaw axes/front, 68 zero-sized boxes, source errata, unknown category/tilt",
                declared_counts={"houses": 3488, "rooms": 22894, "assigned_objects": 398485}, code_evidence=["kits/convert_structured3d.py"]),
        _source("SAGE10k", f"{SAGE}/SAGE10k", "json_single.json",
                room_geometry="authored rectangle/wall loop and ceiling height", room_type="source room_type",
                size_semantics="local dimensions width/length/height; raw mesh scale/frame must verify",
                extent_kind="full", units="positions/dimensions meters; placement_constraints XY centimeters",
                position_reference="asset bottom-center origin; raw source physics pose rotates that origin",
                rotation="Rz Ry Rx degrees incl real pitch/roll; README nonsemantic-tilt claim is not sufficient",
                up_handedness="Z-up; right-handed from source matrix/render code", scale="object-generation scale/mesh size requires raw asset check",
                support="place_id floor/wall/instance retained in per-scene source", relations="placement_constraints include exact target pose: exclude from main condition",
                meshes="local scene metadata; mesh catalog/semantic-front completeness unverified",
                blockers="semantic front currently empirical +Y; significant physics tilt must explicitly reject; support surfaces not bbox tops",
                declared_counts={"scenes": 10000, "rooms": 10027, "objects": 853428}, code_evidence=["aggregate_scenes.py", "sage10k/kits/tex_utils_local.py"]),
        _source("HSSD", "XXXpilar__hssd_clean", "objects.csv",
                room_geometry="regions.csv polygon/floor/ceiling; 2351 regions", room_type="human region label",
                size_semantics="rigid mesh-axis OBB half extents; articulated rows lack box", extent_kind="half", units="meters",
                position_reference="obb_center; translation is asset_local pivot; openings metadata center may be wrong",
                rotation="obb_rotation_wxyz; full rigid transform", up_handedness="right-handed Y-up -> (x,-z,y)",
                scale="obb_half_extents already scaled; NEVER multiply non_uniform_scale again",
                support="asset support/affordance metadata, not verified mesh surfaces", relations="unavailable",
                meshes="GLB stages/objects/openings plus URDFs present", blockers="per-asset front/up not retained in catalog; empirical v1 fronts differ; opening/pivot and articulated twins need reliable repair",
                declared_counts={"scenes": 168, "regions": 2351, "objects": 56647}, code_evidence=[]),
        _source("MultiScan", "XXXpilar__multiscan-clean", "objects.csv",
                room_geometry="partial floor-slab convex hull; slab top not a verified contact plane", room_type="source scan room_type",
                size_semantics="annotation OBB axes local extents plus annotated front/up", extent_kind="half", units="meters",
                position_reference="obb_center -> each own full-height bottom center", rotation="obb_axes plus annotated front/up",
                up_handedness="right-handed aligned Z-up", scale="annotated box already metric; no extra scale",
                semantic_front="explicit source front vector; admitted only horizontal OBB-aligned fronts",
                support="geometry-derived rests_on_floor/supported_by; not trusted physical labels",
                relations="geometry-derived; omitted to prevent reference-layout leakage", meshes="273 PLY+textured OBJ scans, extracted per-object assets not built",
                blockers="strict tilt/frame reject; incomplete boundary/floor prevent full room-validity claims",
                eligibility="full_geometry_supervision_after_strict_filter", declared_counts={"scans": 273, "scenes": 117, "objects": 10957},
                code_evidence=[]),
    ]


def _file_evidence(path):
    text = path.read_bytes()
    return {"path": path.name, "bytes": len(text), "sha256": hashlib.sha256(text).hexdigest()}


def _sample(path):
    """Read at most 4 MiB of a large array; never load whole corpus into RAM."""
    if path.suffix == ".csv":
        with path.open(newline="") as fh:
            reader = csv.DictReader(fh)
            first = next(reader, {})
            count = bool(first) + sum(1 for _ in reader)
        return {"measured_records": count, "first_record_keys": sorted(first)}
    with path.open(encoding="utf-8") as fh:
        prefix = fh.read(4 * 1024 * 1024).lstrip()
    prefix = prefix[1:].lstrip() if prefix.startswith("[") else prefix
    first, _ = json.JSONDecoder().raw_decode(prefix)
    rooms = first.get("rooms") or ([first["room"]] if "room" in first else [])
    room = rooms[0] if rooms else first.get("room_context", {})
    objects = room.get("furniture") or first.get("layout", {}).get("floor_layout", {}).get("objects") or []
    return {"first_record_keys": sorted(first), "first_room_keys": sorted(room),
            "first_object_keys": sorted(objects[0]) if objects else [], "sampling": "first record only; max 4 MiB"}


def audit_sources(source_root):
    root = Path(source_root).resolve()
    entries = []
    for definition in source_catalog():
        base = root / definition["directory"]
        evidence = []
        for rel in ["README.md", "manifest.json", "dataset_manifest.json", *definition["code_evidence"]]:
            path = base / rel
            if path.is_file():
                evidence.append({**_file_evidence(path), "path": rel})
        sample = {}
        path = base / definition["sample_file"]
        if path.is_file():
            try:
                sample = _sample(path)
            except (ValueError, TypeError, json.JSONDecodeError) as exc:
                sample = {"sample_error": str(exc)}
        entries.append({**definition, "present": base.is_dir(), "evidence": evidence, "actual_sample": sample,
                        "count_scope": "declared_counts are retained source documentation, not a new full-corpus recount"})
    return {"schema_version": "fastfill.v2.source_audit", "source_root": str(root),
            "auditor_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
            "top_directories": [{"name": name, "present": (root / name).is_dir()} for name in TOP_DIRECTORIES],
            "sources": entries, "scope": "README+converter+bounded real record inventory; CSV rows counted; no in-place normalization",
            "source_data_modified": False}


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--source-root", type=Path, default=Path("/Volumes/harddisk/3D_Room_Collections"))
    parser.add_argument("--output", type=Path, help="New JSON output file outside source data")
    args = parser.parse_args()
    report = audit_sources(args.source_root)
    if args.output:
        path = args.output.resolve()
        protected = (args.source_root.resolve(), CANONICAL_SOURCE_ROOT.resolve())
        if any(path == root or root in path.parents or path in root.parents for root in protected):
            raise ValueError("audit output must be outside read-only source data")
        path.parent.mkdir(parents=True, exist_ok=True)
        with path.open("x") as fh:
            json.dump(report, fh, indent=2, ensure_ascii=False, allow_nan=False)
            fh.write("\n")
        print(json.dumps({"output": str(path), "sources_present": sum(s["present"] for s in report["sources"])}))
    else:
        print(json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False))


if __name__ == "__main__":
    main()