File size: 8,324 Bytes
e6f80f3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
"""Dataset safety, immutable outputs and reproducible run metadata."""
from __future__ import annotations

import hashlib
import json
from pathlib import Path
import subprocess

import torch

from fastfill.v2.schema import migrate_legacy_row

SOURCE_ROOT = Path("/Volumes/harddisk/3D_Room_Collections")
# RoomGenBench benchmark rooms (SAGE layouts) never train; qualification writes them to test
# and every training-data reader refuses them. multisource_verify keeps an independent copy.
ROOMGENBENCH_HOLDOUT_GROUPS = ("sage:layout_61ebde9f", "sage:layout_6b049b06", "sage:layout_fef15043",
                               "sage:layout_60ee2ae3", "sage:layout_46cdcce3")
HOLDOUT_REASON = "roomgenbench_benchmark_room"


def safe_output(path, *, create=False):
    target = Path(path).resolve()
    source = SOURCE_ROOT.resolve()
    if target == source or source in target.parents or target in source.parents:
        raise ValueError("generated outputs must stay outside the read-only source root and ancestors")
    if target.exists():
        raise FileExistsError(f"output already exists; choose a new run path: {target}")
    if create:
        target.mkdir(parents=True, exist_ok=False)
    return target


def read_samples(path, *, training=False, max_samples=None):
    """Read audited v2 rows; pre-C1 rows get their exchangeable groups migrated into validity once, here."""
    if max_samples is not None and (isinstance(max_samples, bool) or not isinstance(max_samples, int) or max_samples < 1):
        raise ValueError("max_samples must be a positive integer")
    path = Path(path)
    with path.open() as stream:
        rows = []
        for line in stream:
            if line.strip():
                rows.append(json.loads(line))
                if max_samples is not None and len(rows) >= max_samples:
                    break
    if not rows:
        raise ValueError("dataset is empty")
    for row in rows:
        if row.get("schema_version") != "fastfill.v2" or not isinstance(row.get("provenance"), dict):
            raise ValueError("audited v2 sample with provenance is required")
        if training and row["provenance"].get("split") != "train":
            raise ValueError("training accepts only explicitly marked training-split rows")
        if training and (row["provenance"].get("group") in ROOMGENBENCH_HOLDOUT_GROUPS or row["provenance"].get("holdout_reason")):
            raise ValueError("RoomGenBench benchmark room in training data; build training data through qualification")
    return [migrate_legacy_row(row) for row in rows]


def ensure_disjoint(train, holdout):
    def identity(row):
        p = row["provenance"]
        if not p.get("source") or not p.get("house_id"):
            raise ValueError("split integrity needs source and underlying house identity")
        return p["source"], p["house_id"]
    if {identity(r) for r in train} & {identity(r) for r in holdout}:
        raise ValueError("training and held-out scenes share an underlying source house")


def fingerprint(path):
    digest = hashlib.sha256()
    with Path(path).open("rb") as stream:
        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def fingerprint_tree(directory):
    """One digest over every file of a directory (sorted relative path + content hash); e.g. a saved tokenizer."""
    directory = Path(directory)
    files = sorted(path for path in directory.rglob("*") if path.is_file())
    listing = [(str(path.relative_to(directory)), fingerprint(path)) for path in files]
    return hashlib.sha256(json.dumps(listing).encode()).hexdigest()


def backbone_provenance(backbone, *, local_files_only=True):
    """Backbone identity for run manifests: resolved directory, HF snapshot revision (hub id) and config.json sha256.

    A hub id resolves inside the HF cache to ``.../snapshots/<revision>/config.json``;
    a local directory has no revision, so its config.json digest identifies it
    (that digest does not pin the weights).
    """
    if backbone == "tiny":
        return {"path": "tiny", "revision": None, "config_sha256": None}
    from transformers.utils import cached_file
    config_path = Path(cached_file(backbone, "config.json", local_files_only=local_files_only))
    # Resolve the directory, not config.json: a relative path or a symlink to a snapshot becomes the absolute
    # snapshots/<revision> directory, while its files (symlinks into blobs/) would lose the revision.
    snapshot = config_path.parent.resolve()
    return {"path": str(snapshot), "revision": snapshot.name if snapshot.parent.name == "snapshots" else None,
            "config_sha256": fingerprint(config_path)}


CHECKPOINT_MANIFEST = "checkpoint_manifest.json"
CHECKPOINT_KEYS = ("step", "max_length", "data_sha256", "validation_data_sha256", "config_sha256", "implementation_sha256")


def load_checkpoint_config(model_dir):
    """Settings bound to an exported checkpoint, so evaluation and prediction never guess them.

    Structured exports hold ``model_config.json`` (position head, grid, size
    reference, yaw bins, ...; ``model.load_model`` applies it) and the
    ``checkpoint_manifest.json`` train.py writes beside it (``CHECKPOINT_KEYS``
    plus backbone and tokenizer identity). Exports older than that manifest read
    max_length and hashes from the run's ``run_manifest.json`` one directory up.
    Text SFT exports (``text_config.json``) carry their own max_length and run
    metadata. Values that no file records are None; ``source`` names the file used.
    """
    model_dir = Path(model_dir)
    text = model_dir / "text_config.json"
    if text.exists():
        config = json.loads(text.read_text())
        values = {**config.get("run_metadata", {}), "step": config.get("steps"), "max_length": config.get("max_length")}
        return {"model": None, **{key: values.get(key) for key in CHECKPOINT_KEYS}, "source": str(text)}
    model = json.loads((model_dir / "model_config.json").read_text()) if (model_dir / "model_config.json").exists() else None
    values, source = {}, None
    if (model_dir / CHECKPOINT_MANIFEST).exists():
        source = model_dir / CHECKPOINT_MANIFEST
        values = json.loads(source.read_text())
    elif (model_dir.parent / "run_manifest.json").exists():
        source = model_dir.parent / "run_manifest.json"
        manifest = json.loads(source.read_text())
        # A run directory holds several exports; only the run-wide settings bind here, not the step.
        values = {**manifest, "step": None, "max_length": manifest.get("training", {}).get("max_length")}
    return {"model": model, **{key: values.get(key) for key in CHECKPOINT_KEYS},
            "source": str(source) if source else None}


def run_metadata(data_path):
    import importlib.metadata
    packages = {}
    for name in ("torch", "transformers", "peft", "scipy", "accelerate"):
        try:
            packages[name] = importlib.metadata.version(name)
        except importlib.metadata.PackageNotFoundError:
            packages[name] = None
    repo_root = Path(__file__).resolve().parents[2]
    result = subprocess.run(["git", "rev-parse", "HEAD"], cwd=repo_root, capture_output=True, text=True, check=False)
    status = subprocess.run(["git", "status", "--porcelain"], cwd=repo_root, capture_output=True, text=True, check=False)
    source_files = sorted(Path(__file__).parent.glob("*.py"))
    hashes = {str(path.relative_to(repo_root)): fingerprint(path) for path in source_files}
    tree_hash = hashlib.sha256(json.dumps(hashes, sort_keys=True).encode()).hexdigest()
    return {"data_path": str(Path(data_path).resolve()), "data_sha256": fingerprint(data_path),
            "code_commit": result.stdout.strip() if result.returncode == 0 else None,
            "code_dirty": bool(status.stdout.strip()) if status.returncode == 0 else None,
            "implementation_files_sha256": hashes, "implementation_sha256": tree_hash,
            "packages": packages}


def to_device(value, device):
    if isinstance(value, torch.Tensor):
        return value.to(device)
    if isinstance(value, dict):
        return {k: to_device(v, device) for k, v in value.items()}
    # Condition metadata lists remain Python values; tensor fields live in dicts.
    return value