| """ABO mesh-proxy continuation of Forma3D Tiny 2. No large teacher models run.""" |
| import argparse |
| import fcntl |
| import getpass |
| import hashlib |
| import importlib.util |
| import importlib.metadata |
| import json |
| import math |
| from pathlib import Path |
| import random |
| import re |
| import shutil |
| import subprocess |
| import sys |
| import time |
|
|
| BASE = 'Lucasgitguy-co/forma3d-tiny-2' |
| BASE_REV = '1648582b07a0f7f9d482a8c0f4cf89e6ecd86972' |
| DATA = 'codepawl/abo-glb' |
| DATA_REV = 'c7c8293fed3b7ab122ef5d7896521a187e0962d7' |
| DEST = 'Lucasgitguy-co/forma3d-tiny-3' |
| HERE = Path(__file__).resolve().parent |
|
|
|
|
| def dump(path, value): |
| path = Path(path) |
| tmp = path.with_suffix(path.suffix + '.tmp') |
| tmp.write_text(json.dumps(value, indent=2, allow_nan=False) + '\n') |
| tmp.replace(path) |
|
|
|
|
| def sha(path): |
| h = hashlib.sha256() |
| with Path(path).open('rb') as f: |
| for chunk in iter(lambda: f.read(1024 * 1024), b''): |
| h.update(chunk) |
| return h.hexdigest() |
|
|
|
|
| def read(path): |
| return json.loads(Path(path).read_text()) |
|
|
|
|
| def log(**kw): |
| print(json.dumps(kw, allow_nan=False), flush=True) |
|
|
|
|
| def authenticate(): |
| from huggingface_hub import HfApi, get_token, login |
| token = get_token() |
| if not token: |
| print('Enter a Hugging Face WRITE token for Lucasgitguy-co. Input is hidden.\n' |
| 'Create one at https://huggingface.co/settings/tokens', flush=True) |
| token = getpass.getpass('Hugging Face token: ') |
| login(token=token, add_to_git_credential=False) |
| api = HfApi(token=token) |
| identity = api.whoami() |
| owner = DEST.split('/')[0] |
| valid_owner = identity['name'].lower() == owner.lower() |
| valid_org = any(o['name'].lower() == owner.lower() and o.get('roleInOrg') in ('admin', 'write') |
| for o in identity.get('orgs', [])) |
| if not (valid_owner or valid_org): |
| raise RuntimeError(f'This login cannot publish under {owner}. Use a token for that account.') |
| if identity.get('auth', {}).get('accessToken', {}).get('role') == 'read': |
| raise RuntimeError('This is a read-only token. Use a write token.') |
| print('Hugging Face login verified. No repository is created until evaluation passes.', flush=True) |
| return api |
|
|
|
|
| def upstream(root): |
| from huggingface_hub import snapshot_download |
| source = root / 'upstream' |
| snapshot_download(BASE, revision=BASE_REV, local_dir=source, |
| allow_patterns=['src/*', 'scripts/prepare_hf.py', 'model.pt', 'config.json', |
| 'pyproject.toml', 'LICENSE', 'NOTICE', 'THIRD_PARTY.md', |
| 'licenses/*', 'ATTRIBUTIONS.json', 'README.md'], max_workers=2) |
| sys.path.insert(0, str(source / 'src')) |
| spec = importlib.util.spec_from_file_location('base_prepare', source / 'scripts/prepare_hf.py') |
| mod = importlib.util.module_from_spec(spec) |
| spec.loader.exec_module(mod) |
| return source, mod |
|
|
|
|
| def prepare(root, source, helper, args): |
| import numpy as np |
| from huggingface_hub import hf_hub_download |
| from image3d_student.data import cache_sample, CachedDataset |
| from image3d_student.splats import Splats, preview |
| from image3d_student.engine import split_rows |
| folder = root / 'data' |
| folder.mkdir(exist_ok=True) |
| ready = folder / 'preparation.json' |
| if ready.exists(): |
| for split in ('train', 'validation', 'test'): |
| CachedDataset(folder / f'{split}.jsonl') |
| print('Reusing verified prepared data.', flush=True) |
| return folder |
| metadata = hf_hub_download(DATA, 'metadata.jsonl', repo_type='dataset', revision=DATA_REV, |
| local_dir=root / 'metadata') |
| license_path = hf_hub_download(DATA, 'LICENSE-CC-BY-4.0.txt', repo_type='dataset', |
| revision=DATA_REV, local_dir=root / 'metadata') |
| shutil.copy2(license_path, folder / 'LICENSE-ABO-CC-BY-4.0.txt') |
| records = [json.loads(line) for line in Path(metadata).read_text().splitlines() if line.strip()] |
| candidates = [r for r in records if 0 < r.get('triangles', 0) <= 200_000 |
| and 0 < r.get('pack_bytes', 0) <= 8 * 1024**2 |
| and re.fullmatch(r'[A-Za-z0-9_-]+', r['id'])] |
| candidates.sort(key=lambda r: r['id']) |
| random.Random(args.seed).shuffle(candidates) |
| progress = folder / 'progress.json' |
| state = read(progress) if progress.exists() else {'rows': [], 'sources': [], 'skipped': []} |
| rows, sources = state['rows'], state['sources'] |
| completed = {s['asset_id'] for s in sources} |
| prior = read(source / 'ATTRIBUTIONS.json') |
| blocked_hashes = set() |
| def visit(x): |
| if isinstance(x, dict): |
| for k, v in x.items(): |
| if k == 'sha256' and isinstance(v, str): |
| blocked_hashes.add(v) |
| visit(v) |
| elif isinstance(x, list): |
| for v in x: |
| visit(v) |
| visit(prior) |
| seen_hashes = {s['sha256'] for s in sources} | blocked_hashes |
| fetch = helper.Downloader(root / 'downloads', 8, 3072, 4) |
| for record in candidates: |
| if len(sources) >= args.objects: |
| break |
| uid = record['id'] |
| if uid in completed: |
| continue |
| stage = folder / '_staging' |
| if stage.exists(): |
| shutil.rmtree(stage) |
| stage.mkdir() |
| try: |
| filename = f'objects/{uid}/{uid}.glb' |
| raw = fetch.get(DATA, DATA_REV, filename) |
| digest = sha(raw) |
| if digest in seen_hashes: |
| continue |
| decoded = stage / 'decoded.glb' |
| subprocess.run(['node', str(HERE / 'decode.mjs'), str(raw), str(decoded)], |
| check=True, capture_output=True, text=True, timeout=120) |
| sample_seed = int(digest[:8], 16) ^ args.seed |
| splats, normalization = helper.mesh_splats(decoded, 4096, 16, sample_seed) |
| info = {'repo': DATA, 'revision': DATA_REV, 'path': filename, 'sha256': digest, |
| 'decoded_sha256': sha(decoded), 'asset_id': uid, 'object_id': 'abo-' + uid, |
| 'amazon_item_ids': record.get('amazon_item_ids', []), |
| 'title': record.get('name', uid), 'product_type': record.get('product_type'), |
| 'author': 'Amazon.com; Amazon Berkeley Objects contributors', |
| 'conversion': 'CodePawl GLB conversion; decompressed, normalized and sampled for training', |
| 'license': 'cc-by-4.0', 'license_url': 'https://creativecommons.org/licenses/by/4.0/', |
| 'source_url': f'https://huggingface.co/datasets/{DATA}/blob/{DATA_REV}/{filename}'} |
| new_rows = [] |
| for view in range(4): |
| yaw = 2 * np.pi * view / 4 |
| c, s = np.cos(yaw), np.sin(yaw) |
| rotation = np.array([[c, 0, s], [0, 1, 0], [-s, 0, c]], np.float32) |
| rotated = Splats(splats.xyz @ rotation.T, splats.rgb, splats.scale, |
| splats.quat, splats.opacity) |
| picture = preview(rotated, size=128) |
| if picture.getbbox() is None: |
| raise ValueError('Invisible proxy render') |
| picture.save(stage / 'input.png') |
| new_rows.append(cache_sample(stage, f'abo-{uid}-v{view}', stage / 'input.png', |
| rotated, 16, source='mesh-proxy:' + DATA, |
| extra={**info, 'normalization_matrix': normalization, 'yaw_radians': float(yaw), |
| 'render': 'orthographic Gaussian proxy on black; camera +z', |
| 'target': 'mesh-derived isotropic surface samples; no pretrained teacher'})) |
| for name in ('images', 'targets'): |
| (folder / name).mkdir(exist_ok=True) |
| for p in (stage / name).iterdir(): |
| p.replace(folder / name / p.name) |
| rows.extend(new_rows) |
| sources.append(info) |
| seen_hashes.add(digest) |
| completed.add(uid) |
| dump(progress, state) |
| log(prepared_objects=len(sources), requested=args.objects, asset=uid) |
| except RuntimeError: |
| raise |
| except Exception as exc: |
| error = str(exc)[:300] |
| if isinstance(exc, subprocess.CalledProcessError): |
| error = exc.stderr[-500:] |
| state['skipped'].append({'id': uid, 'error': error}) |
| dump(progress, state) |
| log(skipped=uid, error=error) |
| if len(state['skipped']) >= 20 and not sources: |
| raise RuntimeError('Twenty objects failed. Inspect data/progress.json before continuing.') from exc |
| finally: |
| shutil.rmtree(stage, ignore_errors=True) |
| if len(sources) < args.objects: |
| raise RuntimeError(f'Only {len(sources)} usable objects; need {args.objects}. Progress is saved.') |
| |
| parent = {s['object_id']: s['object_id'] for s in sources} |
| def find(x): |
| while parent[x] != x: |
| parent[x] = parent[parent[x]] |
| x = parent[x] |
| return x |
| def union(a, b): |
| a, b = find(a), find(b) |
| parent[max(a, b)] = min(a, b) |
| identities = {} |
| for info in sources: |
| for key in [('mesh', info['sha256'])] + [('amazon', a) for a in info['amazon_item_ids']]: |
| if key in identities: |
| union(info['object_id'], identities[key]) |
| identities[key] = info['object_id'] |
| for row in rows: |
| for field in ('image_sha256', 'target_sha256'): |
| key = (field, row[field]) |
| if key in identities: |
| union(row['metadata']['object_id'], identities[key]) |
| identities[key] = row['metadata']['object_id'] |
| for row in rows: |
| row['metadata']['object_id'] = find(row['metadata']['object_id']) |
| for info in sources: |
| info['object_id'] = find(info['object_id']) |
| development, test = split_rows(rows, .15, args.seed + 1) |
| training, validation = split_rows(development, .20, args.seed + 2) |
| splits = {'train': training, 'validation': validation, 'test': test} |
| for name, items in splits.items(): |
| (folder / f'{name}.jsonl').write_text(''.join(json.dumps(r) + '\n' for r in items)) |
| CachedDataset(folder / f'{name}.jsonl') |
| dump(folder / 'sources.json', sources) |
| dump(ready, {'dataset': DATA, 'revision': DATA_REV, 'objects': len(sources), 'views': len(rows), |
| 'groups': {k: len({r['metadata']['object_id'] for r in v}) for k, v in splits.items()}, |
| 'split_views': {k: len(v) for k, v in splits.items()}, |
| 'manifest_sha256': {k: sha(folder / f'{k}.jsonl') for k in splits}, |
| 'grouping': 'object IDs, shared Amazon item IDs, exact mesh/image/target hashes', |
| 'limitations': 'Near duplicates can remain. Cross-dataset overlap with ancestor training ' |
| 'is not fully known; only recorded exact source hashes were excluded.'}) |
| return folder |
|
|
|
|
| def cpu_tree(value): |
| import torch |
| if isinstance(value, torch.Tensor): |
| return value.detach().cpu() |
| if isinstance(value, dict): |
| return {k: cpu_tree(v) for k, v in value.items()} |
| if isinstance(value, list): |
| return [cpu_tree(v) for v in value] |
| if isinstance(value, tuple): |
| return tuple(cpu_tree(v) for v in value) |
| return value |
|
|
|
|
| def train(root, source, folder, args): |
| import torch |
| from torch.utils.data import DataLoader, default_collate |
| from image3d_student.data import CachedDataset |
| from image3d_student.model import Student, distillation_loss |
| from image3d_student.engine import device_for, evaluate, on_device |
| device = device_for(args.device) |
| torch.manual_seed(args.seed) |
| run = root / 'training' |
| run.mkdir(exist_ok=True) |
| train_data = CachedDataset(folder / 'train.jsonl') |
| val_data = CachedDataset(folder / 'validation.jsonl') |
| val_loader = DataLoader(val_data, batch_size=1, num_workers=0) |
| base = torch.load(source / 'model.pt', map_location='cpu', weights_only=True) |
| if base['config'] != {'grid': 16, 'width': 64} or base['format'] != 1: |
| raise RuntimeError('Unexpected base model architecture.') |
| model = Student(**base['config']).to(device) |
| model.load_state_dict(base['model'], strict=True) |
| optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=1e-4) |
| rng = random.Random(args.seed) |
| meta = {'base_model': BASE, 'base_revision': BASE_REV, 'base_sha256': sha(source / 'model.pt'), |
| 'parameters': sum(p.numel() for p in model.parameters()), 'device': str(device), |
| 'torch': str(torch.__version__), 'dataset': DATA, 'dataset_revision': DATA_REV, |
| 'steps_requested': args.steps, 'batch_size': args.batch_size, 'learning_rate': args.lr, |
| 'seed': args.seed, 'teacher_models_run': [], 'contains_synthetic': False, |
| 'rendered_mesh_proxies': True, 'optimizer': 'fresh AdamW, resumed on interruption', |
| 'data': read(folder / 'preparation.json')} |
| step, best_step = 0, 0 |
| if (run / 'last.pt').exists(): |
| last = torch.load(run / 'last.pt', map_location='cpu', weights_only=True) |
| model.load_state_dict(last['model']) |
| optimizer.load_state_dict(last['optimizer']) |
| rng.setstate(last['python_rng']) |
| step, best_step, best_loss = last['step'], last['best_step'], last['best_loss'] |
| meta = last['metadata'] |
| log(resuming_from_step=step) |
| else: |
| meta['baseline_validation'] = evaluate(model, val_loader, device) |
| best_loss = meta['baseline_validation']['loss'] |
| if not math.isfinite(best_loss): |
| raise RuntimeError('Nonfinite baseline evaluation.') |
| dump(run / 'training.json', meta) |
| started = time.monotonic() |
| starting_step = step |
| def save(path, include_optimizer): |
| payload = {'format': 1, 'config': model.config, 'model': cpu_tree(model.state_dict()), |
| 'step': step, 'metadata': meta} |
| if include_optimizer: |
| payload.update(optimizer=cpu_tree(optimizer.state_dict()), python_rng=rng.getstate(), |
| best_step=best_step, best_loss=best_loss) |
| tmp = path.with_suffix('.tmp') |
| torch.save(payload, tmp) |
| tmp.replace(path) |
| try: |
| while step < args.steps: |
| images, target = default_collate([train_data[rng.randrange(len(train_data))] |
| for _ in range(args.batch_size)]) |
| model.train() |
| optimizer.zero_grad(set_to_none=True) |
| prediction = model(images.to(device)) |
| loss, _ = distillation_loss(prediction, on_device(target, device), model.grid) |
| if not torch.isfinite(loss): |
| raise RuntimeError('Nonfinite training loss; publication disabled.') |
| loss.backward() |
| torch.nn.utils.clip_grad_norm_(model.parameters(), 1., error_if_nonfinite=True) |
| optimizer.step() |
| step += 1 |
| if step == 1 or step % 50 == 0: |
| log(step=step, total=args.steps, loss=float(loss.detach()), |
| elapsed_seconds=round(time.monotonic() - started), |
| steps_per_second=round((step - starting_step) / max(time.monotonic()-started, .001), 3)) |
| if step % args.eval_every == 0 or step == args.steps: |
| metrics = evaluate(model, val_loader, device) |
| if not all(math.isfinite(v) for v in metrics.values()): |
| raise RuntimeError('Nonfinite validation metrics.') |
| if metrics['loss'] < best_loss: |
| best_loss, best_step = metrics['loss'], step |
| save(run / 'best.pt', False) |
| record = {'step': step, 'validation': metrics, 'best_step': best_step} |
| with (run / 'metrics.jsonl').open('a') as f: |
| f.write(json.dumps(record) + '\n') |
| log(**record) |
| save(run / 'last.pt', True) |
| finally: |
| if step: |
| save(run / 'last.pt', True) |
| if step != args.steps: |
| raise RuntimeError('Training incomplete; no publication.') |
| if not best_step: |
| raise RuntimeError('Validation did not improve over v2. Checkpoints are local; no publication.') |
| dump(run / 'completed.json', {'completed_steps': step, 'selected_step': best_step, |
| 'best_validation_loss': best_loss}) |
| return run |
|
|
|
|
| def assess(root, source, folder, run, args): |
| import torch |
| from torch.utils.data import DataLoader |
| from image3d_student.data import CachedDataset |
| from image3d_student.model import Student |
| from image3d_student.engine import device_for, evaluate |
| device = device_for(args.device) |
| loader = DataLoader(CachedDataset(folder / 'test.jsonl'), batch_size=1, num_workers=0) |
| result = {} |
| for name, checkpoint in [('base_v2', source / 'model.pt'), ('candidate_v3', run / 'best.pt')]: |
| state = torch.load(checkpoint, map_location='cpu', weights_only=True) |
| model = Student(**state['config']).to(device) |
| model.load_state_dict(state['model'], strict=True) |
| metrics = evaluate(model, loader, device) |
| if not all(math.isfinite(v) for v in metrics.values()): |
| raise RuntimeError('Nonfinite test metrics.') |
| result[name] = {'sha256': sha(checkpoint), 'metrics': metrics} |
| del model |
| before, after = result['base_v2']['metrics'], result['candidate_v3']['metrics'] |
| meta = read(run / 'training.json') |
| completed = read(run / 'completed.json') |
| gates = {'training_complete': completed['completed_steps'] == args.steps, |
| 'validation_loss_improved': completed['best_validation_loss'] < meta['baseline_validation']['loss'], |
| 'test_loss_improved_at_least_half_percent': after['loss'] <= before['loss'] * .995, |
| 'test_occupancy_iou_not_worse': after['occupancy_iou'] >= before['occupancy_iou'], |
| 'weights_changed': result['base_v2']['sha256'] != result['candidate_v3']['sha256']} |
| report = {**result, 'gates': gates, 'passed': all(gates.values()), |
| 'test_manifest_sha256': sha(folder / 'test.jsonl'), |
| 'measurement': 'mean per-view proxy loss and occupancy IoU, threshold 0.5, batch size 1', |
| 'limitation': 'Not evidence of natural-photo, mesh, texture or photorealistic rendering quality.'} |
| dump(root / 'evaluation.json', report) |
| log(evaluation=report) |
| return report |
|
|
|
|
| def package(root, source, folder, run, report): |
| release = root / 'release' |
| release.mkdir(exist_ok=True) |
| shutil.copy2(run / 'best.pt', release / 'model.pt') |
| for name in ('LICENSE', 'NOTICE', 'THIRD_PARTY.md', 'config.json'): |
| shutil.copy2(source / name, release / name) |
| for name in ('src', 'licenses'): |
| shutil.copytree(source / name, release / name, dirs_exist_ok=True, |
| ignore=shutil.ignore_patterns('__pycache__', '*.pyc')) |
| project = (source / 'pyproject.toml').read_text().replace('forma3d-tiny-2', 'forma3d-tiny-3') |
| project = project.replace('version = "0.3.0"', 'version = "0.4.0"') |
| (release / 'pyproject.toml').write_text(project) |
| shutil.copy2(folder / 'LICENSE-ABO-CC-BY-4.0.txt', release / 'licenses/LICENSE-ABO-CC-BY-4.0.txt') |
| for path in (run / 'training.json', run / 'completed.json', run / 'metrics.jsonl', |
| folder / 'preparation.json', root / 'evaluation.json', root / 'settings.json'): |
| shutil.copy2(path, release / path.name) |
| dump(release / 'ATTRIBUTIONS.json', {'inherited': read(source / 'ATTRIBUTIONS.json'), |
| 'abo_continuation': read(folder / 'sources.json')}) |
| (release / 'splits').mkdir(exist_ok=True) |
| for name in ('train', 'validation', 'test'): |
| shutil.copy2(folder / f'{name}.jsonl', release / 'splits' / f'{name}.jsonl') |
| recipe = release / 'recipe' |
| recipe.mkdir(exist_ok=True) |
| for name in ('pipeline.py', 'decode.mjs', 'requirements.txt', 'package.json', 'package-lock.json', 'run.command'): |
| shutil.copy2(HERE / name, recipe / name) |
| inherited_notice = (release / 'NOTICE').read_text() |
| (release / 'NOTICE').write_text(inherited_notice + '\nABO continuation data: Amazon.com, Amazon Berkeley ' |
| 'Objects contributors; converted to GLB by CodePawl. CC BY 4.0. Training preprocessing ' |
| 'decompresses geometry, normalizes objects, samples surface colors and renders Gaussian proxies. ' |
| 'See ATTRIBUTIONS.json and licenses/LICENSE-ABO-CC-BY-4.0.txt. No endorsement implied.\n') |
| base = report['base_v2']['metrics'] |
| candidate = report['candidate_v3']['metrics'] |
| prep = read(folder / 'preparation.json') |
| completed = read(run / 'completed.json') |
| (release / 'README.md').write_text(f'''--- |
| license: apache-2.0 |
| pipeline_tag: image-to-3d |
| base_model: {BASE} |
| base_model_relation: finetune |
| model_name: Forma3D Tiny 3 |
| datasets: |
| - codepawl/abo-glb |
| tags: |
| - pytorch |
| - gaussian-splatting |
| - mesh-proxy-supervision |
| - research-prototype |
| --- |
| # Forma3D Tiny 3 |
| |
| Continued from [{BASE}](https://huggingface.co/{BASE}) at `{BASE_REV}`. |
| Same 306,321-parameter architecture, 16 cubed grid, width 64, maximum 4,096 Gaussian slots. |
| Fresh AdamW optimizer at the start of continuation. Completed {completed['completed_steps']} steps; |
| selected step {completed['selected_step']} by validation loss only. |
| |
| ## Data and scope |
| |
| {prep['objects']} ABO objects, {prep['views']} orthographic Gaussian-proxy images on black. |
| Data revision: `{DATA_REV}`. Targets are isotropic mesh surface samples, compressed to |
| one Gaussian per occupied grid cell. Texture surface colors are sampled; full PBR rendering |
| and differentiable rendering are not used. No natural-photo training is claimed. |
| MeshAnythingV2, Idol Diffusers, SHARP, Pixal3D, Idol Turbo and Qwen3.5-4B-3D-Coder |
| were **not run, merged, or distilled** in this Mac-only recipe. |
| |
| Objects, shared Amazon item IDs and exact mesh/image/target hashes are grouped before splitting. |
| Split groups: {json.dumps(prep['groups'])}. Test data is excluded from training and checkpoint selection. |
| Near duplicates can remain; overlap with ancestor training datasets is not fully known. |
| |
| ## Evaluation |
| |
| Both checkpoints were evaluated on the identical held-out ABO proxy views. |
| |
| | Metric | Starting v2 | Selected v3 | |
| |---|---:|---:| |
| | Mean test loss | {base['loss']:.6f} | {candidate['loss']:.6f} | |
| | Mean view occupancy IoU, threshold 0.5 | {base['occupancy_iou']:.6f} | {candidate['occupancy_iou']:.6f} | |
| |
| These measurements do not establish natural-photo reconstruction, photorealism, mesh quality, |
| novel-view rendering quality or improvement on v2's original domains. No confidence intervals |
| or broad generalization claim. See evaluation.json, preparation.json and split manifests. |
| |
| ## Use |
| |
| With Python 3.11, install this repository using `pip install .`, then: |
| |
| ```bash |
| image3d generate --checkpoint model.pt --image centered-object-on-black.png --out object.ply --device mps |
| image3d preview --ply object.ply --out preview.png |
| ``` |
| |
| Output is Gaussian PLY, not a textured GLB mesh. Inputs should match centered objects on black. |
| The model has no text prompt conditioning. Reproduction launcher and locked dependencies are in recipe/. |
| |
| ## Attribution |
| |
| Publisher source and weights: Apache 2.0. Retain inherited NOTICE, THIRD_PARTY.md and ATTRIBUTIONS.json. |
| ABO models, images and listing text: Amazon.com and the Amazon Berkeley Objects contributors, |
| CC BY 4.0; converted to GLB by CodePawl. The recipe further decompresses geometry, |
| normalizes coordinates, samples colored surfaces and renders proxy images. |
| See [ABO dataset](https://huggingface.co/datasets/codepawl/abo-glb), |
| ATTRIBUTIONS.json and licenses/LICENSE-ABO-CC-BY-4.0.txt. Raw assets are not bundled. |
| ''') |
| freeze = ''.join(sorted(f"{d.metadata['Name']}=={d.version}\n" |
| for d in importlib.metadata.distributions())) |
| (release / 'environment.txt').write_text(freeze) |
| sums = ''.join(f'{sha(p)} {p.relative_to(release)}\n' for p in sorted(release.rglob('*')) |
| if p.is_file() and p.name != 'SHA256SUMS') |
| (release / 'SHA256SUMS').write_text(sums) |
| return release |
|
|
|
|
| def publish(api, root, release): |
| from huggingface_hub import hf_hub_download |
| from huggingface_hub.errors import RepositoryNotFoundError, EntryNotFoundError |
| marker = root / 'publication.json' |
| expected = sha(release / 'model.pt') |
| try: |
| info = api.model_info(DEST) |
| except RepositoryNotFoundError: |
| info = None |
| if info is not None: |
| try: |
| remote = hf_hub_download(DEST, 'model.pt', revision=info.sha, token=api.token) |
| except EntryNotFoundError: |
| if not marker.exists(): |
| raise RuntimeError(f'{DEST} already exists. Refusing to overwrite an unrelated repository.') |
| else: |
| if sha(remote) == expected: |
| print(f'Already published and verified: https://huggingface.co/{DEST}', flush=True) |
| return |
| raise RuntimeError(f'{DEST} already has different weights; refusing to overwrite them.') |
| else: |
| api.create_repo(DEST, repo_type='model', private=False, exist_ok=False) |
| dump(marker, {'repo_id': DEST, 'expected_model_sha256': expected}) |
| commit = api.upload_folder(repo_id=DEST, repo_type='model', folder_path=release, |
| commit_message='Publish evaluated Forma3D Tiny 3 ABO continuation') |
| remote = hf_hub_download(DEST, 'model.pt', revision=commit.oid, token=api.token) |
| if sha(remote) != expected: |
| raise RuntimeError('Uploaded checkpoint verification failed.') |
| dump(marker, {'repo_id': DEST, 'model_sha256': expected, 'commit': commit.oid, 'verified': True}) |
| print(f'Published and verified: https://huggingface.co/{DEST}', flush=True) |
|
|
|
|
| def main(): |
| p = argparse.ArgumentParser(description=__doc__) |
| p.add_argument('--root', type=Path, default=HERE / 'run') |
| p.add_argument('--objects', type=int, default=512) |
| p.add_argument('--steps', type=int, default=6000) |
| p.add_argument('--batch-size', type=int, default=4) |
| p.add_argument('--lr', type=float, default=5e-5) |
| p.add_argument('--eval-every', type=int, default=250) |
| p.add_argument('--seed', type=int, default=44) |
| p.add_argument('--device', choices=['mps', 'cpu'], default='mps') |
| p.add_argument('--publish', action='store_true') |
| p.add_argument('--prepare-only', action='store_true') |
| args = p.parse_args() |
| if args.objects < 12 or min(args.steps, args.batch_size, args.eval_every) < 1 or args.lr <= 0: |
| p.error('Need at least 12 objects and positive training settings.') |
| if args.publish and (args.objects < 128 or args.steps < 1000 or args.device != 'mps'): |
| p.error('Publication requires >=128 objects, >=1000 steps and MPS; small runs are tests only.') |
| root = args.root.resolve() |
| root.mkdir(parents=True, exist_ok=True) |
| with (root / '.lock').open('w') as lock: |
| try: |
| fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) |
| except BlockingIOError: |
| raise RuntimeError('Another run is already using this directory.') |
| settings = {k: v for k, v in vars(args).items() if k not in ('root', 'publish', 'prepare_only')} |
| settings.update(base_revision=BASE_REV, data_revision=DATA_REV, recipe_sha256=sha(__file__)) |
| settings_path = root / 'settings.json' |
| if settings_path.exists() and read(settings_path) != settings: |
| raise RuntimeError('Run settings/code changed. Use --root with a new directory.') |
| dump(settings_path, settings) |
| if shutil.disk_usage(root).free < 8 * 1024**3: |
| raise RuntimeError('Need at least 8 GiB free disk space before starting.') |
| api = authenticate() if args.publish and not args.prepare_only else None |
| source, helper = upstream(root) |
| folder = prepare(root, source, helper, args) |
| if args.prepare_only: |
| return |
| run = train(root, source, folder, args) |
| report = assess(root, source, folder, run, args) |
| if not report['passed']: |
| print('Release checks did not pass. Results and checkpoints are saved locally; nothing published.') |
| sys.exit(2) |
| release = package(root, source, folder, run, report) |
| if args.publish: |
| publish(api, root, release) |
| else: |
| print(f'Local test only; no publication. Release package: {release}') |
|
|
|
|
| if __name__ == '__main__': |
| try: |
| main() |
| except KeyboardInterrupt: |
| print('\nInterrupted. Rerun the same command to continue from saved progress.', file=sys.stderr) |
| sys.exit(130) |
|
|
