| { | |
| "model_name": "ShadeNet-5-35M", | |
| "task": "intrinsic-decomposition", | |
| "architecture": "ParallelUNet + frozen DINOv2 ViT-S/14", | |
| "backbone": "facebook/dinov2-small", | |
| "parameters": 34652431, | |
| "trainable_parameters": 12595855, | |
| "image_size": 448, | |
| "supported_image_sizes": [ | |
| 448, | |
| 896 | |
| ], | |
| "input": { | |
| "name": "input_rgb", | |
| "channels": 3, | |
| "range": [ | |
| -1.0, | |
| 1.0 | |
| ], | |
| "note": "RGB photo, resized and padded to a supported size" | |
| }, | |
| "output": { | |
| "name": "maps", | |
| "channels": 7, | |
| "range": [ | |
| -1.0, | |
| 1.0 | |
| ], | |
| "layout": { | |
| "albedo": [ | |
| 0, | |
| 3 | |
| ], | |
| "depth": [ | |
| 3, | |
| 4 | |
| ], | |
| "normal": [ | |
| 4, | |
| 7 | |
| ] | |
| }, | |
| "note": "depth is relative; normals are x right, y up, z toward the viewer" | |
| }, | |
| "checkpoint": "checkpoints/best-l1-pruned10-ep19.ckpt", | |
| "inference_script": "inference/run_onnx.py", | |
| "onnx": { | |
| "onnx/shadenet5_mix_int4_int8.onnx": "ViT fc1/fc2 int8, everything else int4", | |
| "onnx/shadenet5_mix2_int4_int8.onnx": "ViT fc2 int8, everything else int4", | |
| "onnx/shadenet5_refiner.onnx": "the small refiner, fp32, fixed 448x448 (inputs photo and maps, output refined maps)", | |
| "onnx/shadenet5_mix_refined.onnx": "the MIX model + the refiner + clamp + unit-length normals in one file: input `input_rgb` [1,3,448,448] in [-1,1], output `refined` [1,7,448,448]" | |
| }, | |
| "recommended_inference": { | |
| "mode": "refine", | |
| "command": "python inference/run_onnx.py <photos> --refine --out out/", | |
| "six_pass_alternative": { | |
| "flip_avg": true, | |
| "rot_avg_deg": 4.0, | |
| "unit_normals": true, | |
| "forward_passes": 6, | |
| "command": "python inference/run_onnx.py <photos> --enhanced --out out/" | |
| }, | |
| "single_file": "onnx/shadenet5_mix_refined.onnx", | |
| "single_file_command": "python inference/run_onnx.py <photos> --merged --out out/" | |
| }, | |
| "validation_mean_l1": { | |
| "single_pass_unit_normals": 0.1757, | |
| "single_pass_plus_refiner": 0.1673, | |
| "six_passes": 0.1628, | |
| "mix_onnx_single": 0.1755, | |
| "mix_onnx_plus_refiner": 0.1676, | |
| "mix_onnx_six_passes": 0.1645, | |
| "mix_onnx_single_merged_file": 0.1676 | |
| }, | |
| "refiner": { | |
| "pytorch": "refiner/refiner.pt", | |
| "onnx": "onnx/shadenet5_refiner.onnx", | |
| "parameters": 910000, | |
| "training_photos": "Open Images V7 (CC BY 2.0), credited in refiner/attribution.csv", | |
| "note": "applied to the single-pass maps; trained to reproduce the six-pass average" | |
| } | |
| } |