Audited anonymous-review AccessPath release
Browse files- .gitignore +10 -0
- README.md +171 -0
- THIRD_PARTY_NOTICES.md +37 -0
- accessibilityamodal/__init__.py +25 -0
- accessibilityamodal/category_overrides.py +86 -0
- accessibilityamodal/depth.py +598 -0
- accessibilityamodal/geometry_analysis.py +607 -0
- accessibilityamodal/pipeline.py +886 -0
- accessibilityamodal/reconstruct.py +1641 -0
- accessibilityamodal/sam_refinement.py +173 -0
- accessibilityamodal/verification.py +764 -0
- accessibilityamodal/visual_completion.py +493 -0
- accesspath.py +44 -0
- accesspath3r/__init__.py +3 -0
- accesspath3r/cli.py +809 -0
- accesspath3r/privacy.py +145 -0
- checksums.sha256 +39 -0
- configs/accessibility_mask_prompts_e5_v2.json +134 -0
- configs/accessibility_stairs_outdoor_no_occlusion_prompts.json +28 -0
- configs/accessibility_taxonomy.json +84 -0
- docs/DEPENDENCIES_AND_WEIGHTS.md +309 -0
- docs/REPRODUCIBLE_PIPELINE.md +69 -0
- docs/SOURCE_MANIFEST.md +22 -0
- requirements/runtime.txt +6 -0
- slurm/run_accesspath_demo.sbatch +573 -0
- tools/accessibility_2d_completion.py +604 -0
- tools/accessibility_3d_completion.py +1150 -0
- tools/accessibility_3d_variants.py +858 -0
- tools/accessibility_amodal_mask.py +265 -0
- tools/accessibility_fast_2d_baseline.py +581 -0
- tools/accessibility_mask_proposals.py +1332 -0
- tools/audit_accessibility_3d_candidate.py +97 -0
- tools/build_accessibility_review_bundle.py +746 -0
- tools/build_accessibility_solid_mesh_showcase.py +0 -0
- tools/render_accessibility_turntable.py +1316 -0
- tools/sanitize_log_stream.py +42 -0
- tools/sanitize_output_metadata.py +95 -0
- tools/train_accessibility_amodal_adapter.py +462 -0
- tools/update_accessibility_inference_progress.py +93 -0
- tools/validate_accessibility_reviewed_visible_workspace.py +163 -0
.gitignore
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Inputs, weights, and generated outputs are intentionally never versioned.
|
| 2 |
+
data/
|
| 3 |
+
weights/
|
| 4 |
+
outputs/
|
| 5 |
+
logs/
|
| 6 |
+
.env/
|
| 7 |
+
.venv/
|
| 8 |
+
__pycache__/
|
| 9 |
+
*.py[cod]
|
| 10 |
+
.DS_Store
|
README.md
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: other
|
| 3 |
+
tags:
|
| 4 |
+
- accessibility
|
| 5 |
+
- amodal-completion
|
| 6 |
+
- image-inpainting
|
| 7 |
+
- 3d-reconstruction
|
| 8 |
+
---
|
| 9 |
+
|
| 10 |
+
# AccessPath
|
| 11 |
+
|
| 12 |
+
AccessPath is an anonymous-review release for accessibility-scene amodal
|
| 13 |
+
completion. It includes the full, finite GPU execution path from mask proposal
|
| 14 |
+
through quality-gated 2D completion and visual 3D completion. The package also
|
| 15 |
+
keeps two direct, mask-driven entry points:
|
| 16 |
+
|
| 17 |
+
- **2D completion:** GPU inpainting that only changes the reviewed occluded
|
| 18 |
+
region and preserves source pixels elsewhere.
|
| 19 |
+
- **Visual 3D completion:** a wrapper around the separately released
|
| 20 |
+
[Amodal3R](https://huggingface.co/Sm0kyWu/Amodal3R) backend that produces
|
| 21 |
+
Gaussian-splat and dense-mesh visual reconstructions from an RGB image and
|
| 22 |
+
an aligned three-value amodal mask.
|
| 23 |
+
|
| 24 |
+
The five supported accessibility categories are `stairs`, `ramp`, `curb_cut`,
|
| 25 |
+
`tactile_paving`, and `walkway`.
|
| 26 |
+
|
| 27 |
+
## Relationship to prior work
|
| 28 |
+
|
| 29 |
+
AccessPath is **inspired by and adapted from** established research on promptable
|
| 30 |
+
segmentation, amodal completion, diffusion inpainting, monocular geometry, and
|
| 31 |
+
visual 3D reconstruction. It is not presented as a reimplementation or a copy
|
| 32 |
+
of any one prior system. The project-level contribution is the accessibility
|
| 33 |
+
adaptation: category-specific target/obstacle prompts, a visible-hidden-amodal
|
| 34 |
+
mask schema, constrained mask relations, mask-restricted 2D completion,
|
| 35 |
+
geometry/quality gates, a finite Slurm workflow, and review-oriented outputs.
|
| 36 |
+
|
| 37 |
+
When AccessPath actually calls an external model, that model remains an
|
| 38 |
+
independent backend with its own code, weights, and license. The precise
|
| 39 |
+
boundary between inspiration, optional comparison, and runtime use is listed
|
| 40 |
+
in [docs/DEPENDENCIES_AND_WEIGHTS.md](docs/DEPENDENCIES_AND_WEIGHTS.md).
|
| 41 |
+
|
| 42 |
+
## What is included
|
| 43 |
+
|
| 44 |
+
This repository contains project-level pipeline, mask, 2D, depth/geometry,
|
| 45 |
+
visual-3D adapter, verification, rendering, review-bundle, Slurm, and prompt
|
| 46 |
+
configuration code. The complete process is documented in
|
| 47 |
+
[docs/REPRODUCIBLE_PIPELINE.md](docs/REPRODUCIBLE_PIPELINE.md), and the
|
| 48 |
+
stage-to-source mapping is in [docs/SOURCE_MANIFEST.md](docs/SOURCE_MANIFEST.md).
|
| 49 |
+
For a beginner-oriented installation guide, exact dependency status, official
|
| 50 |
+
model links, weight-download commands, and citation information, read
|
| 51 |
+
[docs/DEPENDENCIES_AND_WEIGHTS.md](docs/DEPENDENCIES_AND_WEIGHTS.md) before
|
| 52 |
+
running an experiment.
|
| 53 |
+
|
| 54 |
+
It deliberately contains **no** source images, masks, model checkpoints,
|
| 55 |
+
environments, experiment outputs, logs, author information, or
|
| 56 |
+
machine-specific paths.
|
| 57 |
+
|
| 58 |
+
The 100-image reviewer subset is not distributed yet. Its source-image and
|
| 59 |
+
derived-mask redistribution status requires source-specific license and
|
| 60 |
+
privacy clearance. A data card and a release manifest will be added only after
|
| 61 |
+
that review is complete.
|
| 62 |
+
|
| 63 |
+
## Setup
|
| 64 |
+
|
| 65 |
+
Use Linux, Python 3.10+, a CUDA-capable GPU, and an environment compatible
|
| 66 |
+
with the chosen models. Create a fresh environment, install PyTorch for the
|
| 67 |
+
local CUDA version, then install the lightweight utilities:
|
| 68 |
+
|
| 69 |
+
```bash
|
| 70 |
+
python -m pip install -r requirements/runtime.txt
|
| 71 |
+
python -m pip install diffusers transformers accelerate safetensors
|
| 72 |
+
```
|
| 73 |
+
|
| 74 |
+
The visual-3D backend additionally needs the official Amodal3R environment
|
| 75 |
+
and its CUDA rasterizer dependencies. Follow the upstream installation guide;
|
| 76 |
+
do not copy the upstream source tree or checkpoints into this repository.
|
| 77 |
+
|
| 78 |
+
## Obtain models separately
|
| 79 |
+
|
| 80 |
+
No weights are redistributed here. Download each dependency only after
|
| 81 |
+
accepting its own license and access conditions.
|
| 82 |
+
|
| 83 |
+
| Component | Official source | Used by |
|
| 84 |
+
| --- | --- | --- |
|
| 85 |
+
| Stable Diffusion inpainting | [`sd-legacy/stable-diffusion-inpainting`](https://huggingface.co/sd-legacy/stable-diffusion-inpainting) | 2D completion |
|
| 86 |
+
| Amodal3R | [`Sm0kyWu/Amodal3R`](https://huggingface.co/Sm0kyWu/Amodal3R) | visual 3D completion |
|
| 87 |
+
| SAM 3 (optional mask proposal stage) | [`facebook/sam3`](https://huggingface.co/facebook/sam3) | mask proposals only |
|
| 88 |
+
|
| 89 |
+
Project-trained checkpoints are intentionally withheld during anonymous review.
|
| 90 |
+
They should be released only after verifying the training-data permissions,
|
| 91 |
+
base-model terms, privacy risk, and the paper's release policy.
|
| 92 |
+
|
| 93 |
+
The optional VGGT path and the non-runtime related-work references
|
| 94 |
+
(pix2gestalt, Open-World AMODAL, and Amodal Completion in the Wild) are
|
| 95 |
+
identified explicitly in [docs/DEPENDENCIES_AND_WEIGHTS.md](docs/DEPENDENCIES_AND_WEIGHTS.md).
|
| 96 |
+
They are not silently downloaded or executed by the default pipeline.
|
| 97 |
+
|
| 98 |
+
## Input masks
|
| 99 |
+
|
| 100 |
+
All masks must match the input RGB resolution. For 2D completion, provide the
|
| 101 |
+
visible-target, amodal-target, and obstacle masks. For visual 3D completion,
|
| 102 |
+
provide one aligned PNG with exactly these values:
|
| 103 |
+
|
| 104 |
+
```text
|
| 105 |
+
255 background
|
| 106 |
+
188 visible target
|
| 107 |
+
0 hidden target
|
| 108 |
+
```
|
| 109 |
+
|
| 110 |
+
The intended relations are `hidden = amodal AND NOT visible`, visible and
|
| 111 |
+
obstacle are disjoint, and hidden is a subset of obstacle. Automatic masks are
|
| 112 |
+
proposals and should be reviewed before they drive a completion result.
|
| 113 |
+
|
| 114 |
+
## Run 2D completion
|
| 115 |
+
|
| 116 |
+
```bash
|
| 117 |
+
python accesspath.py 2d -- \
|
| 118 |
+
--image path/to/image.jpg \
|
| 119 |
+
--target-visible-mask path/to/target_visible.png \
|
| 120 |
+
--target-amodal-mask path/to/target_amodal.png \
|
| 121 |
+
--obstacle-mask path/to/obstacle.png \
|
| 122 |
+
--category stairs \
|
| 123 |
+
--model path/to/stable-diffusion-inpainting \
|
| 124 |
+
--output-dir outputs/example_2d \
|
| 125 |
+
--device cuda
|
| 126 |
+
```
|
| 127 |
+
|
| 128 |
+
The selected RGB result and its quality/provenance metadata are written under
|
| 129 |
+
the output directory. Run `python accesspath.py 2d -- --help` for all options.
|
| 130 |
+
|
| 131 |
+
## Run the complete pipeline
|
| 132 |
+
|
| 133 |
+
The complete pipeline uses a bounded Slurm GPU job, so it queues for a GPU,
|
| 134 |
+
persists its result directory, and exits once the requested image is complete:
|
| 135 |
+
|
| 136 |
+
```bash
|
| 137 |
+
python accesspath.py pipeline -- \
|
| 138 |
+
--image path/to/image.jpg \
|
| 139 |
+
--category stairs \
|
| 140 |
+
--output-dir outputs/stairs_demo
|
| 141 |
+
```
|
| 142 |
+
|
| 143 |
+
It runs mask proposals (or validates supplied reviewed masks), constrained
|
| 144 |
+
amodal-mask inference, 2D completion, depth/geometry diagnostics, visual 3D,
|
| 145 |
+
verification, and a compact review bundle. See
|
| 146 |
+
[docs/REPRODUCIBLE_PIPELINE.md](docs/REPRODUCIBLE_PIPELINE.md) for environment
|
| 147 |
+
variables, stage-by-stage behavior, outputs, and optional stage switches.
|
| 148 |
+
|
| 149 |
+
## Run visual 3D completion
|
| 150 |
+
|
| 151 |
+
Install the official Amodal3R runtime first, then run:
|
| 152 |
+
|
| 153 |
+
```bash
|
| 154 |
+
python accesspath.py 3d -- \
|
| 155 |
+
--image path/to/image.jpg \
|
| 156 |
+
--mask path/to/amodal_3value.png \
|
| 157 |
+
--model Sm0kyWu/Amodal3R \
|
| 158 |
+
--output-dir outputs/example_3d
|
| 159 |
+
```
|
| 160 |
+
|
| 161 |
+
This is a learned visual reconstruction, not metric geometry, a navigation
|
| 162 |
+
safety label, or a guarantee of a watertight or scaled mesh. Run
|
| 163 |
+
`python accesspath.py 3d -- --help` for output and rendering options.
|
| 164 |
+
|
| 165 |
+
## Attribution and release boundary
|
| 166 |
+
|
| 167 |
+
The 3D adapter calls Amodal3R and does not claim to reimplement that upstream
|
| 168 |
+
model. Consult [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md) and the model
|
| 169 |
+
cards for all dependency terms. Repository code is provided for anonymous
|
| 170 |
+
academic review; a final license and any project checkpoint release will be
|
| 171 |
+
announced with the archival paper release.
|
THIRD_PARTY_NOTICES.md
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Third-party notices
|
| 2 |
+
|
| 3 |
+
AccessPath contains a project-level 2D inpainting orchestration layer and a
|
| 4 |
+
small adapter for external visual-3D inference. It does not redistribute model
|
| 5 |
+
weights, upstream source trees, datasets, or derived image assets.
|
| 6 |
+
|
| 7 |
+
The project is inspired by prior work in amodal completion and visual 3D
|
| 8 |
+
reconstruction, then adapted to accessibility-scene masks, categories, quality
|
| 9 |
+
gates, and review outputs. Inspiration does not transfer authorship: an
|
| 10 |
+
external model remains an external model whenever the runtime calls it.
|
| 11 |
+
|
| 12 |
+
## External models and libraries
|
| 13 |
+
|
| 14 |
+
- **Amodal3R** is an independently authored visual-3D backend. The adapter in
|
| 15 |
+
`tools/accessibility_3d_completion.py` imports its public Python package and
|
| 16 |
+
defaults to the upstream `Sm0kyWu/Amodal3R` model identifier. Its source,
|
| 17 |
+
checkpoint, and license remain those of the upstream project.
|
| 18 |
+
- **Stable Diffusion inpainting** is accessed through Hugging Face Diffusers.
|
| 19 |
+
The user must obtain the selected checkpoint separately and comply with its
|
| 20 |
+
model card and license.
|
| 21 |
+
- **SAM 3**, when used outside this minimal package to create mask proposals,
|
| 22 |
+
is separately licensed and may require access approval.
|
| 23 |
+
- **Depth Anything V2** is the default depth/geometry diagnostic backend in
|
| 24 |
+
the full launcher. **VGGT** is an optional alternative depth/point-map
|
| 25 |
+
backend. Their code and checkpoints are separate downloads.
|
| 26 |
+
- **pix2gestalt**, **Open-World AMODAL**, and **Amodal Completion in the Wild**
|
| 27 |
+
are not imported or executed by the AccessPath runtime; their URLs occur
|
| 28 |
+
only in related-work/comparison metadata.
|
| 29 |
+
- PyTorch, OpenCV, Pillow, NumPy, ImageIO, Trimesh, Diffusers, Transformers,
|
| 30 |
+
Accelerate, and Safetensors retain their respective licenses.
|
| 31 |
+
|
| 32 |
+
Renaming a wrapper or output does not transfer authorship or change any
|
| 33 |
+
upstream license. No local model cache, environment, credential, dataset, or
|
| 34 |
+
experiment artifact is included in this release.
|
| 35 |
+
|
| 36 |
+
See [docs/DEPENDENCIES_AND_WEIGHTS.md](docs/DEPENDENCIES_AND_WEIGHTS.md) for
|
| 37 |
+
the exact use status, official URLs, download commands, and citation guidance.
|
accessibilityamodal/__init__.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""AccessibilityAmodal: project-owned accessibility completion utilities."""
|
| 2 |
+
|
| 3 |
+
from .visual_completion import (
|
| 4 |
+
NEGATIVE_PROMPT,
|
| 5 |
+
PROMPTS,
|
| 6 |
+
build_visual_removal_mask,
|
| 7 |
+
candidate_quality,
|
| 8 |
+
derive_hidden_mask,
|
| 9 |
+
mask_statistics,
|
| 10 |
+
quality_flags_for_metrics,
|
| 11 |
+
select_candidate,
|
| 12 |
+
)
|
| 13 |
+
|
| 14 |
+
__all__ = [
|
| 15 |
+
"NEGATIVE_PROMPT",
|
| 16 |
+
"PROMPTS",
|
| 17 |
+
"build_visual_removal_mask",
|
| 18 |
+
"candidate_quality",
|
| 19 |
+
"derive_hidden_mask",
|
| 20 |
+
"mask_statistics",
|
| 21 |
+
"quality_flags_for_metrics",
|
| 22 |
+
"select_candidate",
|
| 23 |
+
]
|
| 24 |
+
|
| 25 |
+
__version__ = "0.1.0"
|
accessibilityamodal/category_overrides.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Apply human-reviewed accessibility category overrides by sample id."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from typing import Any, Mapping
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
| 11 |
+
DEFAULT_CATEGORY_OVERRIDES_PATH = PROJECT_ROOT / "configs" / "accessibility_review_overrides.json"
|
| 12 |
+
GEOMETRY_REVIEW_REQUIRED_CATEGORIES = frozenset(
|
| 13 |
+
{"stairs", "ramp", "curb_cut", "raised_curb"}
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def load_reviewed_category_overrides(
|
| 18 |
+
path: str | Path | None = None,
|
| 19 |
+
) -> dict[str, dict[str, Any]]:
|
| 20 |
+
source = DEFAULT_CATEGORY_OVERRIDES_PATH if path is None else Path(path)
|
| 21 |
+
if not source.is_absolute():
|
| 22 |
+
source = PROJECT_ROOT / source
|
| 23 |
+
if not source.is_file():
|
| 24 |
+
return {}
|
| 25 |
+
value = json.loads(source.read_text(encoding="utf-8"))
|
| 26 |
+
if not isinstance(value, dict):
|
| 27 |
+
raise ValueError(f"Category override config must contain an object: {source}")
|
| 28 |
+
return {
|
| 29 |
+
str(sample_id): entry
|
| 30 |
+
for sample_id, entry in value.items()
|
| 31 |
+
if isinstance(entry, dict)
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def resolve_reviewed_category(
|
| 36 |
+
sample_id: str,
|
| 37 |
+
fallback_category: str,
|
| 38 |
+
overrides: Mapping[str, Mapping[str, Any]],
|
| 39 |
+
) -> str:
|
| 40 |
+
fallback_category = str(fallback_category or "").strip().lower()
|
| 41 |
+
entry = overrides.get(sample_id)
|
| 42 |
+
if not entry or entry.get("category_reviewed") is not True:
|
| 43 |
+
return fallback_category
|
| 44 |
+
category = entry.get("category")
|
| 45 |
+
if not isinstance(category, str) or not category.strip():
|
| 46 |
+
raise ValueError(
|
| 47 |
+
f"Reviewed category override for {sample_id!r} must contain a non-empty category"
|
| 48 |
+
)
|
| 49 |
+
return category.strip().lower()
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def geometry_category_is_explicitly_reviewed(
|
| 53 |
+
sample_id: str,
|
| 54 |
+
metadata: Mapping[str, Any] | None,
|
| 55 |
+
overrides: Mapping[str, Mapping[str, Any]],
|
| 56 |
+
) -> bool:
|
| 57 |
+
"""Return whether the geometry-driving category itself was reviewed."""
|
| 58 |
+
if metadata and metadata.get("category_reviewed") is True:
|
| 59 |
+
return True
|
| 60 |
+
override = overrides.get(sample_id)
|
| 61 |
+
return bool(override and override.get("category_reviewed") is True)
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def resolve_geometry_category_for_modeling(
|
| 65 |
+
sample_id: str,
|
| 66 |
+
fallback_category: str,
|
| 67 |
+
metadata: Mapping[str, Any] | None,
|
| 68 |
+
overrides: Mapping[str, Mapping[str, Any]],
|
| 69 |
+
*,
|
| 70 |
+
allow_unreviewed_level_change: bool = False,
|
| 71 |
+
) -> str:
|
| 72 |
+
"""Resolve category and fail closed for unreviewed level-change geometry."""
|
| 73 |
+
category = resolve_reviewed_category(sample_id, fallback_category, overrides)
|
| 74 |
+
if not category:
|
| 75 |
+
raise ValueError(f"Sample {sample_id!r} has no category for 3D geometry routing")
|
| 76 |
+
if (
|
| 77 |
+
not allow_unreviewed_level_change
|
| 78 |
+
and category in GEOMETRY_REVIEW_REQUIRED_CATEGORIES
|
| 79 |
+
and not geometry_category_is_explicitly_reviewed(sample_id, metadata, overrides)
|
| 80 |
+
):
|
| 81 |
+
raise ValueError(
|
| 82 |
+
f"Sample {sample_id!r} category {category!r} requires explicit category review "
|
| 83 |
+
"before selecting a 3D geometry prior. Run "
|
| 84 |
+
"tools/audit_accessibility_geometry_categories.py or add a reviewed override."
|
| 85 |
+
)
|
| 86 |
+
return category
|
accessibilityamodal/depth.py
ADDED
|
@@ -0,0 +1,598 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Generate a depth map for the accessibility geometry pipeline.
|
| 2 |
+
|
| 3 |
+
This script keeps depth estimation separate from reconstruction so the core
|
| 4 |
+
pipeline can run in lightweight environments and can optionally use heavier
|
| 5 |
+
models such as Depth Anything V2 when their dependencies are installed.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import argparse
|
| 11 |
+
import json
|
| 12 |
+
import sys
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
from typing import Any
|
| 15 |
+
|
| 16 |
+
import numpy as np
|
| 17 |
+
from PIL import Image, ImageOps
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
DEFAULT_DEPTH_MODEL = 'depth-anything/Depth-Anything-V2-Small-hf'
|
| 21 |
+
DEFAULT_VGGT_MODEL = 'facebook/VGGT-1B'
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def depth_model_identity(
|
| 25 |
+
*,
|
| 26 |
+
engine: str,
|
| 27 |
+
model: str,
|
| 28 |
+
checkpoint: str | Path | None,
|
| 29 |
+
encoder: str,
|
| 30 |
+
metric_depth: bool,
|
| 31 |
+
vggt_model: str,
|
| 32 |
+
) -> dict[str, Any]:
|
| 33 |
+
"""Return an auditable third-party model identity without guessing revision."""
|
| 34 |
+
|
| 35 |
+
if engine == 'depth_anything_v2':
|
| 36 |
+
size = {
|
| 37 |
+
'vits': 'Small',
|
| 38 |
+
'vitb': 'Base',
|
| 39 |
+
'vitl': 'Large',
|
| 40 |
+
'vitg': 'Giant',
|
| 41 |
+
}.get(encoder, encoder)
|
| 42 |
+
checkpoint_name = Path(checkpoint).name.lower() if checkpoint else ''
|
| 43 |
+
dataset = (
|
| 44 |
+
'Hypersim'
|
| 45 |
+
if 'hypersim' in checkpoint_name
|
| 46 |
+
else 'VKITTI'
|
| 47 |
+
if 'vkitti' in checkpoint_name
|
| 48 |
+
else None
|
| 49 |
+
)
|
| 50 |
+
if metric_depth and dataset:
|
| 51 |
+
model_id = f'depth-anything/Depth-Anything-V2-Metric-{dataset}-{size}'
|
| 52 |
+
else:
|
| 53 |
+
model_id = f'depth-anything/Depth-Anything-V2-{size}'
|
| 54 |
+
return {
|
| 55 |
+
'model_id': model_id,
|
| 56 |
+
'family': 'Depth Anything V2',
|
| 57 |
+
'encoder': encoder,
|
| 58 |
+
'metric_training_dataset': dataset,
|
| 59 |
+
'license': 'Apache-2.0' if encoder == 'vits' else 'CC-BY-NC-4.0',
|
| 60 |
+
'third_party': True,
|
| 61 |
+
'revision': 'local_checkpoint_revision_unrecorded',
|
| 62 |
+
'revision_review_required_before_public_release': True,
|
| 63 |
+
}
|
| 64 |
+
if engine == 'transformers':
|
| 65 |
+
return {
|
| 66 |
+
'model_id': model,
|
| 67 |
+
'family': 'Transformers depth-estimation pipeline',
|
| 68 |
+
'license': 'see_upstream_model_card',
|
| 69 |
+
'third_party': True,
|
| 70 |
+
'revision': 'runtime_default_or_local_cache',
|
| 71 |
+
'revision_review_required_before_public_release': True,
|
| 72 |
+
}
|
| 73 |
+
if engine == 'vggt':
|
| 74 |
+
return {
|
| 75 |
+
'model_id': vggt_model,
|
| 76 |
+
'family': 'VGGT',
|
| 77 |
+
'license': 'see_upstream_model_card',
|
| 78 |
+
'third_party': True,
|
| 79 |
+
'revision': 'local_checkpoint_or_runtime_default',
|
| 80 |
+
'revision_review_required_before_public_release': True,
|
| 81 |
+
}
|
| 82 |
+
return {
|
| 83 |
+
'model_id': 'opencv_depth_fallback',
|
| 84 |
+
'family': 'nonlearned_fallback',
|
| 85 |
+
'third_party': True,
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def read_rgb(path: str | Path, max_size: int | None = None) -> Image.Image:
|
| 90 |
+
image = ImageOps.exif_transpose(Image.open(path)).convert('RGB')
|
| 91 |
+
if max_size and max(image.size) > max_size:
|
| 92 |
+
scale = max_size / max(image.size)
|
| 93 |
+
size = (round(image.size[0] * scale), round(image.size[1] * scale))
|
| 94 |
+
image = image.resize(size, Image.Resampling.LANCZOS)
|
| 95 |
+
return image
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def normalize_depth(depth: np.ndarray, near: float, far: float, invert: bool) -> np.ndarray:
|
| 99 |
+
depth = depth.astype(np.float32)
|
| 100 |
+
valid = np.isfinite(depth)
|
| 101 |
+
if not np.any(valid):
|
| 102 |
+
raise ValueError('Depth prediction contains no finite values')
|
| 103 |
+
|
| 104 |
+
values = depth[valid]
|
| 105 |
+
lo, hi = np.percentile(values, [1, 99])
|
| 106 |
+
if hi <= lo:
|
| 107 |
+
hi = lo + 1.0
|
| 108 |
+
norm = np.clip((depth - lo) / (hi - lo), 0.0, 1.0)
|
| 109 |
+
if invert:
|
| 110 |
+
norm = 1.0 - norm
|
| 111 |
+
return near + norm * (far - near)
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def resize_float_map(values: np.ndarray, size: tuple[int, int]) -> np.ndarray:
|
| 115 |
+
image = Image.fromarray(values.astype(np.float32), mode='F')
|
| 116 |
+
image = image.resize(size, Image.Resampling.BILINEAR)
|
| 117 |
+
return np.asarray(image, dtype=np.float32)
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def colorize_confidence(confidence: np.ndarray) -> Image.Image:
|
| 121 |
+
confidence = confidence.astype(np.float32)
|
| 122 |
+
valid = np.isfinite(confidence)
|
| 123 |
+
if not np.any(valid):
|
| 124 |
+
return Image.fromarray(np.zeros(confidence.shape, dtype=np.uint8), mode='L')
|
| 125 |
+
lo, hi = np.percentile(confidence[valid], [2, 98])
|
| 126 |
+
if hi <= lo:
|
| 127 |
+
hi = lo + 1.0
|
| 128 |
+
vis = np.clip((confidence - lo) / (hi - lo), 0.0, 1.0)
|
| 129 |
+
vis[~valid] = 0
|
| 130 |
+
return Image.fromarray((vis * 255).astype(np.uint8), mode='L')
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def preprocess_square_tensor(image: Image.Image, target_size: int):
|
| 134 |
+
import torch
|
| 135 |
+
|
| 136 |
+
width, height = image.size
|
| 137 |
+
max_dim = max(width, height)
|
| 138 |
+
left = (max_dim - width) // 2
|
| 139 |
+
top = (max_dim - height) // 2
|
| 140 |
+
scale = target_size / max_dim
|
| 141 |
+
crop = (
|
| 142 |
+
left * scale,
|
| 143 |
+
top * scale,
|
| 144 |
+
(left + width) * scale,
|
| 145 |
+
(top + height) * scale,
|
| 146 |
+
)
|
| 147 |
+
square = Image.new('RGB', (max_dim, max_dim), (0, 0, 0))
|
| 148 |
+
square.paste(image, (left, top))
|
| 149 |
+
square = square.resize((target_size, target_size), Image.Resampling.BICUBIC)
|
| 150 |
+
array = np.asarray(square, dtype=np.float32) / 255.0
|
| 151 |
+
tensor = torch.from_numpy(array).permute(2, 0, 1).contiguous()
|
| 152 |
+
return tensor[None], crop
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def crop_and_resize_prediction(values: np.ndarray, crop: tuple[float, float, float, float], size: tuple[int, int]) -> np.ndarray:
|
| 156 |
+
left, top, right, bottom = crop
|
| 157 |
+
h, w = values.shape[:2]
|
| 158 |
+
left_i = max(0, min(w - 1, int(np.floor(left))))
|
| 159 |
+
top_i = max(0, min(h - 1, int(np.floor(top))))
|
| 160 |
+
right_i = max(left_i + 1, min(w, int(np.ceil(right))))
|
| 161 |
+
bottom_i = max(top_i + 1, min(h, int(np.ceil(bottom))))
|
| 162 |
+
cropped = values[top_i:bottom_i, left_i:right_i]
|
| 163 |
+
if values.ndim == 2:
|
| 164 |
+
return resize_float_map(cropped, size)
|
| 165 |
+
channels = [resize_float_map(cropped[..., channel], size) for channel in range(values.shape[-1])]
|
| 166 |
+
return np.stack(channels, axis=-1).astype(np.float32)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def write_point_cloud_ply(
|
| 170 |
+
path: Path,
|
| 171 |
+
points: np.ndarray,
|
| 172 |
+
colors: np.ndarray,
|
| 173 |
+
confidence: np.ndarray | None,
|
| 174 |
+
confidence_threshold: float,
|
| 175 |
+
max_points: int,
|
| 176 |
+
) -> int:
|
| 177 |
+
valid = np.all(np.isfinite(points), axis=-1) & (points[..., 2] > 0)
|
| 178 |
+
if confidence is not None:
|
| 179 |
+
valid &= np.isfinite(confidence) & (confidence >= confidence_threshold)
|
| 180 |
+
ys, xs = np.where(valid)
|
| 181 |
+
if ys.size == 0:
|
| 182 |
+
selected = np.array([], dtype=np.int64)
|
| 183 |
+
elif max_points > 0 and ys.size > max_points:
|
| 184 |
+
selected = np.linspace(0, ys.size - 1, max_points, dtype=np.int64)
|
| 185 |
+
else:
|
| 186 |
+
selected = np.arange(ys.size, dtype=np.int64)
|
| 187 |
+
pts = points[ys[selected], xs[selected]]
|
| 188 |
+
cols = colors[ys[selected], xs[selected]]
|
| 189 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 190 |
+
with path.open('w', encoding='ascii') as handle:
|
| 191 |
+
handle.write('ply\nformat ascii 1.0\n')
|
| 192 |
+
handle.write(f'element vertex {len(pts)}\n')
|
| 193 |
+
handle.write('property float x\nproperty float y\nproperty float z\n')
|
| 194 |
+
handle.write('property uchar red\nproperty uchar green\nproperty uchar blue\n')
|
| 195 |
+
handle.write('end_header\n')
|
| 196 |
+
for point, color in zip(pts, cols):
|
| 197 |
+
handle.write(
|
| 198 |
+
f'{point[0]:.6f} {point[1]:.6f} {point[2]:.6f} '
|
| 199 |
+
f'{int(color[0])} {int(color[1])} {int(color[2])}\n'
|
| 200 |
+
)
|
| 201 |
+
return int(len(pts))
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def heuristic_depth(image: Image.Image, near: float, far: float) -> np.ndarray:
|
| 205 |
+
"""Build a deterministic perspective prior from image position and edges."""
|
| 206 |
+
rgb = np.asarray(image, dtype=np.float32) / 255.0
|
| 207 |
+
h, w = rgb.shape[:2]
|
| 208 |
+
yy = np.linspace(0.0, 1.0, h, dtype=np.float32)[:, None]
|
| 209 |
+
depth = far - yy * (far - near)
|
| 210 |
+
depth = np.repeat(depth, w, axis=1)
|
| 211 |
+
|
| 212 |
+
gray = 0.299 * rgb[..., 0] + 0.587 * rgb[..., 1] + 0.114 * rgb[..., 2]
|
| 213 |
+
grad_y = np.abs(np.gradient(gray, axis=0))
|
| 214 |
+
if np.isfinite(grad_y).any() and float(grad_y.max()) > 0:
|
| 215 |
+
grad_y = grad_y / float(grad_y.max())
|
| 216 |
+
depth -= 0.08 * (far - near) * grad_y.astype(np.float32)
|
| 217 |
+
return np.clip(depth, min(near, far), max(near, far)).astype(np.float32)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def resolve_device(device: str) -> int | str:
|
| 221 |
+
if device == 'cpu':
|
| 222 |
+
return -1
|
| 223 |
+
if device == 'auto':
|
| 224 |
+
import torch
|
| 225 |
+
|
| 226 |
+
return 0 if torch.cuda.is_available() else -1
|
| 227 |
+
if device.startswith('cuda'):
|
| 228 |
+
if ':' in device:
|
| 229 |
+
return int(device.split(':', 1)[1])
|
| 230 |
+
return 0
|
| 231 |
+
return device
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def transformers_depth(image: Image.Image, model: str, device: str) -> np.ndarray:
|
| 235 |
+
try:
|
| 236 |
+
from transformers import pipeline
|
| 237 |
+
except ImportError as exc:
|
| 238 |
+
raise RuntimeError('transformers is not installed; install optional depth dependencies first') from exc
|
| 239 |
+
|
| 240 |
+
estimator = pipeline('depth-estimation', model=model, device=resolve_device(device))
|
| 241 |
+
result: dict[str, Any] = estimator(image)
|
| 242 |
+
if 'predicted_depth' in result:
|
| 243 |
+
predicted = result['predicted_depth']
|
| 244 |
+
if hasattr(predicted, 'detach'):
|
| 245 |
+
predicted = predicted.detach().cpu().numpy()
|
| 246 |
+
depth = np.asarray(predicted, dtype=np.float32)
|
| 247 |
+
if depth.ndim == 3:
|
| 248 |
+
depth = depth.squeeze()
|
| 249 |
+
elif 'depth' in result:
|
| 250 |
+
depth = np.asarray(result['depth'], dtype=np.float32)
|
| 251 |
+
else:
|
| 252 |
+
raise RuntimeError(f'Unexpected depth-estimation result keys: {sorted(result)}')
|
| 253 |
+
|
| 254 |
+
if depth.shape[:2] != (image.height, image.width):
|
| 255 |
+
depth_img = Image.fromarray(depth.astype(np.float32), mode='F')
|
| 256 |
+
depth_img = depth_img.resize(image.size, Image.Resampling.BILINEAR)
|
| 257 |
+
depth = np.asarray(depth_img, dtype=np.float32)
|
| 258 |
+
return depth.astype(np.float32)
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
def resolve_torch_device(device: str) -> str:
|
| 262 |
+
if device != 'auto':
|
| 263 |
+
return device
|
| 264 |
+
import torch
|
| 265 |
+
|
| 266 |
+
return 'cuda' if torch.cuda.is_available() else 'cpu'
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
def depth_anything_v2_depth(
|
| 270 |
+
image: Image.Image,
|
| 271 |
+
repo: str | Path,
|
| 272 |
+
checkpoint: str | Path,
|
| 273 |
+
encoder: str,
|
| 274 |
+
device: str,
|
| 275 |
+
input_size: int,
|
| 276 |
+
metric: bool,
|
| 277 |
+
max_depth: float,
|
| 278 |
+
) -> np.ndarray:
|
| 279 |
+
repo = Path(repo).resolve()
|
| 280 |
+
checkpoint = Path(checkpoint).resolve()
|
| 281 |
+
if not checkpoint.exists():
|
| 282 |
+
raise FileNotFoundError(f'Depth Anything V2 checkpoint not found: {checkpoint}')
|
| 283 |
+
if not repo.exists():
|
| 284 |
+
raise FileNotFoundError(f'Depth Anything V2 repo not found: {repo}')
|
| 285 |
+
source_root = repo / 'metric_depth' if metric else repo
|
| 286 |
+
if not source_root.exists():
|
| 287 |
+
raise FileNotFoundError(f'Depth Anything V2 source root not found: {source_root}')
|
| 288 |
+
|
| 289 |
+
sys.path.insert(0, str(source_root))
|
| 290 |
+
import torch
|
| 291 |
+
from depth_anything_v2.dpt import DepthAnythingV2
|
| 292 |
+
import depth_anything_v2.dinov2_layers.attention as da_attention
|
| 293 |
+
import depth_anything_v2.dinov2_layers.block as da_block
|
| 294 |
+
|
| 295 |
+
model_configs = {
|
| 296 |
+
'vits': {'encoder': 'vits', 'features': 64, 'out_channels': [48, 96, 192, 384]},
|
| 297 |
+
'vitb': {'encoder': 'vitb', 'features': 128, 'out_channels': [96, 192, 384, 768]},
|
| 298 |
+
'vitl': {'encoder': 'vitl', 'features': 256, 'out_channels': [256, 512, 1024, 1024]},
|
| 299 |
+
'vitg': {'encoder': 'vitg', 'features': 384, 'out_channels': [1536, 1536, 1536, 1536]},
|
| 300 |
+
}
|
| 301 |
+
if encoder not in model_configs:
|
| 302 |
+
raise ValueError(f'Unsupported Depth Anything V2 encoder: {encoder}')
|
| 303 |
+
config = dict(model_configs[encoder])
|
| 304 |
+
if metric:
|
| 305 |
+
config['max_depth'] = max_depth
|
| 306 |
+
|
| 307 |
+
torch_device = resolve_torch_device(device)
|
| 308 |
+
# The vendored Depth Anything V2 DINOv2 blocks prefer xFormers when it is
|
| 309 |
+
# importable, but the available xFormers kernels do not support every
|
| 310 |
+
# server GPU/dtype combination, including RTX 5090 sm_120 with float32.
|
| 311 |
+
# Use plain PyTorch attention for reproducible CPU and Slurm inference.
|
| 312 |
+
da_attention.XFORMERS_AVAILABLE = False
|
| 313 |
+
da_block.XFORMERS_AVAILABLE = False
|
| 314 |
+
|
| 315 |
+
model = DepthAnythingV2(**config)
|
| 316 |
+
model.load_state_dict(torch.load(str(checkpoint), map_location='cpu'))
|
| 317 |
+
model = model.to(torch_device).eval()
|
| 318 |
+
|
| 319 |
+
# ``image`` was EXIF-normalized and optionally downscaled by ``read_rgb``.
|
| 320 |
+
# Depth Anything's infer_image receives BGR when loaded via OpenCV, so
|
| 321 |
+
# preserve that channel convention without reopening the raw JPEG.
|
| 322 |
+
raw = np.asarray(image.convert('RGB'))[:, :, ::-1].copy()
|
| 323 |
+
|
| 324 |
+
with torch.inference_mode():
|
| 325 |
+
depth = model.infer_image(raw, input_size)
|
| 326 |
+
return depth.astype(np.float32)
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
def load_vggt_model(
|
| 330 |
+
repo: str | Path,
|
| 331 |
+
model_id: str,
|
| 332 |
+
checkpoint: str | Path | None,
|
| 333 |
+
device: str,
|
| 334 |
+
):
|
| 335 |
+
repo = Path(repo).resolve()
|
| 336 |
+
if not repo.exists():
|
| 337 |
+
raise FileNotFoundError(f'VGGT repo not found: {repo}')
|
| 338 |
+
if str(repo) not in sys.path:
|
| 339 |
+
sys.path.insert(0, str(repo))
|
| 340 |
+
|
| 341 |
+
import torch
|
| 342 |
+
from vggt.models.vggt import VGGT
|
| 343 |
+
|
| 344 |
+
torch_device = resolve_torch_device(device)
|
| 345 |
+
# We only need camera/depth/point for accessibility geometry. Disabling
|
| 346 |
+
# track avoids loading the point-tracking branch and lowers memory use.
|
| 347 |
+
model = VGGT(enable_track=False)
|
| 348 |
+
if checkpoint:
|
| 349 |
+
checkpoint = Path(checkpoint).resolve()
|
| 350 |
+
if not checkpoint.exists():
|
| 351 |
+
raise FileNotFoundError(f'VGGT checkpoint not found: {checkpoint}')
|
| 352 |
+
state = torch.load(str(checkpoint), map_location='cpu')
|
| 353 |
+
if isinstance(state, dict) and 'model' in state:
|
| 354 |
+
state = state['model']
|
| 355 |
+
else:
|
| 356 |
+
local_model = Path(model_id)
|
| 357 |
+
if local_model.exists():
|
| 358 |
+
model_path = local_model / 'model.pt' if local_model.is_dir() else local_model
|
| 359 |
+
if not model_path.exists():
|
| 360 |
+
raise FileNotFoundError(f'VGGT local model checkpoint not found: {model_path}')
|
| 361 |
+
else:
|
| 362 |
+
from huggingface_hub import hf_hub_download
|
| 363 |
+
|
| 364 |
+
model_path = Path(hf_hub_download(repo_id=model_id, filename='model.pt'))
|
| 365 |
+
state = torch.load(str(model_path), map_location='cpu')
|
| 366 |
+
if isinstance(state, dict) and 'model' in state:
|
| 367 |
+
state = state['model']
|
| 368 |
+
missing, unexpected = model.load_state_dict(state, strict=False)
|
| 369 |
+
if missing:
|
| 370 |
+
print(f'VGGT checkpoint missing keys: {len(missing)}', file=sys.stderr)
|
| 371 |
+
if unexpected:
|
| 372 |
+
print(f'VGGT checkpoint unexpected keys: {len(unexpected)}', file=sys.stderr)
|
| 373 |
+
model = model.to(torch_device).eval()
|
| 374 |
+
return model, torch_device
|
| 375 |
+
|
| 376 |
+
|
| 377 |
+
def vggt_predict_depth(
|
| 378 |
+
image: Image.Image,
|
| 379 |
+
model,
|
| 380 |
+
device: str,
|
| 381 |
+
input_size: int,
|
| 382 |
+
near: float,
|
| 383 |
+
far: float,
|
| 384 |
+
invert: bool,
|
| 385 |
+
keep_raw_depth: bool,
|
| 386 |
+
) -> dict[str, np.ndarray]:
|
| 387 |
+
import torch
|
| 388 |
+
|
| 389 |
+
tensor, crop = preprocess_square_tensor(image, input_size)
|
| 390 |
+
tensor = tensor.to(device)
|
| 391 |
+
if device.startswith('cuda'):
|
| 392 |
+
major = torch.cuda.get_device_capability()[0]
|
| 393 |
+
dtype = torch.bfloat16 if major >= 8 else torch.float16
|
| 394 |
+
autocast = torch.cuda.amp.autocast(dtype=dtype)
|
| 395 |
+
else:
|
| 396 |
+
autocast = torch.autocast(device_type='cpu', enabled=False)
|
| 397 |
+
|
| 398 |
+
with torch.inference_mode():
|
| 399 |
+
with autocast:
|
| 400 |
+
predictions = model(tensor)
|
| 401 |
+
|
| 402 |
+
raw_depth = predictions['depth'][0, 0, ..., 0].detach().float().cpu().numpy()
|
| 403 |
+
depth_conf = predictions['depth_conf'][0, 0].detach().float().cpu().numpy()
|
| 404 |
+
point_map = predictions['world_points'][0, 0].detach().float().cpu().numpy()
|
| 405 |
+
point_conf = predictions['world_points_conf'][0, 0].detach().float().cpu().numpy()
|
| 406 |
+
|
| 407 |
+
raw_depth = crop_and_resize_prediction(raw_depth, crop, image.size)
|
| 408 |
+
depth_conf = crop_and_resize_prediction(depth_conf, crop, image.size)
|
| 409 |
+
point_map = crop_and_resize_prediction(point_map, crop, image.size)
|
| 410 |
+
point_conf = crop_and_resize_prediction(point_conf, crop, image.size)
|
| 411 |
+
depth = raw_depth if keep_raw_depth else normalize_depth(raw_depth, near, far, invert)
|
| 412 |
+
return {
|
| 413 |
+
'depth': depth.astype(np.float32),
|
| 414 |
+
'raw_depth': raw_depth.astype(np.float32),
|
| 415 |
+
'depth_conf': depth_conf.astype(np.float32),
|
| 416 |
+
'world_points': point_map.astype(np.float32),
|
| 417 |
+
'world_points_conf': point_conf.astype(np.float32),
|
| 418 |
+
}
|
| 419 |
+
|
| 420 |
+
|
| 421 |
+
def save_depth_vis(path: str | Path, depth: np.ndarray) -> None:
|
| 422 |
+
valid = np.isfinite(depth)
|
| 423 |
+
values = depth[valid]
|
| 424 |
+
if values.size == 0:
|
| 425 |
+
vis = np.zeros(depth.shape, dtype=np.uint8)
|
| 426 |
+
else:
|
| 427 |
+
lo, hi = np.percentile(values, [2, 98])
|
| 428 |
+
if hi <= lo:
|
| 429 |
+
hi = lo + 1.0
|
| 430 |
+
vis = np.clip((depth - lo) / (hi - lo), 0.0, 1.0)
|
| 431 |
+
vis = (vis * 255).astype(np.uint8)
|
| 432 |
+
Image.fromarray(vis, mode='L').save(path)
|
| 433 |
+
|
| 434 |
+
|
| 435 |
+
def build_parser() -> argparse.ArgumentParser:
|
| 436 |
+
parser = argparse.ArgumentParser(description='Generate relative depth for accessibility geometry completion.')
|
| 437 |
+
parser.add_argument('--image', required=True, help='Input RGB image.')
|
| 438 |
+
parser.add_argument('--output-depth', required=True, help='Output .npy depth path.')
|
| 439 |
+
parser.add_argument('--output-vis', default=None, help='Optional grayscale depth visualization path.')
|
| 440 |
+
parser.add_argument('--manifest', default=None, help='Optional JSON manifest path.')
|
| 441 |
+
parser.add_argument('--engine', choices=['heuristic', 'transformers', 'depth_anything_v2', 'vggt'], default='heuristic')
|
| 442 |
+
parser.add_argument('--model', default=DEFAULT_DEPTH_MODEL, help='Transformers depth-estimation model id or local path.')
|
| 443 |
+
parser.add_argument('--device', default='auto', help='auto, cpu, cuda, cuda:0, or a transformers device string.')
|
| 444 |
+
parser.add_argument('--depth-anything-repo', default='../diffusion-vas/models/Depth_Anything_V2', help='Local Depth Anything V2 source repo.')
|
| 445 |
+
parser.add_argument('--checkpoint', default=None, help='Depth Anything V2 .pth checkpoint for --engine depth_anything_v2.')
|
| 446 |
+
parser.add_argument('--encoder', choices=['vits', 'vitb', 'vitl', 'vitg'], default='vitl')
|
| 447 |
+
parser.add_argument('--metric-depth', action='store_true', help='Treat the Depth Anything V2 checkpoint as a metric-depth model and do not near/far normalize.')
|
| 448 |
+
parser.add_argument('--max-depth', type=float, default=20.0, help='Metric Depth Anything max depth in meters.')
|
| 449 |
+
parser.add_argument('--input-size', type=int, default=518, help='Depth Anything V2 inference input size.')
|
| 450 |
+
parser.add_argument('--near', type=float, default=1.0, help='Depth value assigned to the near end after normalization.')
|
| 451 |
+
parser.add_argument('--far', type=float, default=6.0, help='Depth value assigned to the far end after normalization.')
|
| 452 |
+
parser.add_argument('--invert-depth', action='store_true', help='Invert predicted relative depth before near/far normalization.')
|
| 453 |
+
parser.add_argument('--max-size', type=int, default=1280, help='Resize longest side before inference. Use 0 to keep original size.')
|
| 454 |
+
parser.add_argument('--vggt-repo', default='vggt', help='Local facebookresearch/VGGT checkout.')
|
| 455 |
+
parser.add_argument('--vggt-model', default=DEFAULT_VGGT_MODEL, help='Hugging Face model id or local VGGT model directory.')
|
| 456 |
+
parser.add_argument('--vggt-checkpoint', default=None, help='Optional local VGGT model.pt checkpoint.')
|
| 457 |
+
parser.add_argument('--vggt-input-size', type=int, default=518, help='Square VGGT inference resolution.')
|
| 458 |
+
parser.add_argument('--vggt-keep-raw-depth', action='store_true', help='Do not near/far normalize VGGT depth before saving --output-depth.')
|
| 459 |
+
parser.add_argument('--output-conf', default=None, help='Optional output .npy confidence map path.')
|
| 460 |
+
parser.add_argument('--output-conf-vis', default=None, help='Optional output confidence visualization path.')
|
| 461 |
+
parser.add_argument('--output-raw-depth', default=None, help='Optional output raw VGGT depth .npy path.')
|
| 462 |
+
parser.add_argument('--output-world-points', default=None, help='Optional output VGGT world point map .npy path.')
|
| 463 |
+
parser.add_argument('--output-point-conf', default=None, help='Optional output VGGT world point confidence .npy path.')
|
| 464 |
+
parser.add_argument('--output-point-cloud', default=None, help='Optional output VGGT point cloud .ply path.')
|
| 465 |
+
parser.add_argument('--point-conf-threshold', type=float, default=1.0, help='VGGT point confidence threshold for --output-point-cloud.')
|
| 466 |
+
parser.add_argument('--max-point-cloud-points', type=int, default=120000, help='Maximum VGGT point-cloud vertices to export.')
|
| 467 |
+
return parser
|
| 468 |
+
|
| 469 |
+
|
| 470 |
+
def main() -> None:
|
| 471 |
+
args = build_parser().parse_args()
|
| 472 |
+
max_size = None if args.max_size == 0 else args.max_size
|
| 473 |
+
image = read_rgb(args.image, max_size=max_size)
|
| 474 |
+
|
| 475 |
+
if args.engine == 'heuristic':
|
| 476 |
+
raw_depth = heuristic_depth(image, args.near, args.far)
|
| 477 |
+
depth = raw_depth
|
| 478 |
+
elif args.engine == 'transformers':
|
| 479 |
+
raw_depth = transformers_depth(image, args.model, args.device)
|
| 480 |
+
depth = normalize_depth(raw_depth, args.near, args.far, args.invert_depth)
|
| 481 |
+
elif args.engine == 'depth_anything_v2':
|
| 482 |
+
raw_depth = depth_anything_v2_depth(
|
| 483 |
+
image,
|
| 484 |
+
args.depth_anything_repo,
|
| 485 |
+
args.checkpoint,
|
| 486 |
+
args.encoder,
|
| 487 |
+
args.device,
|
| 488 |
+
args.input_size,
|
| 489 |
+
args.metric_depth,
|
| 490 |
+
args.max_depth,
|
| 491 |
+
)
|
| 492 |
+
depth = raw_depth if args.metric_depth else normalize_depth(raw_depth, args.near, args.far, args.invert_depth)
|
| 493 |
+
else:
|
| 494 |
+
model, torch_device = load_vggt_model(
|
| 495 |
+
args.vggt_repo,
|
| 496 |
+
args.vggt_model,
|
| 497 |
+
args.vggt_checkpoint,
|
| 498 |
+
args.device,
|
| 499 |
+
)
|
| 500 |
+
prediction = vggt_predict_depth(
|
| 501 |
+
image,
|
| 502 |
+
model,
|
| 503 |
+
torch_device,
|
| 504 |
+
args.vggt_input_size,
|
| 505 |
+
args.near,
|
| 506 |
+
args.far,
|
| 507 |
+
args.invert_depth,
|
| 508 |
+
args.vggt_keep_raw_depth,
|
| 509 |
+
)
|
| 510 |
+
depth = prediction['depth']
|
| 511 |
+
raw_depth = prediction['raw_depth']
|
| 512 |
+
if args.output_conf:
|
| 513 |
+
Path(args.output_conf).parent.mkdir(parents=True, exist_ok=True)
|
| 514 |
+
np.save(Path(args.output_conf), prediction['depth_conf'].astype(np.float32))
|
| 515 |
+
if args.output_conf_vis:
|
| 516 |
+
Path(args.output_conf_vis).parent.mkdir(parents=True, exist_ok=True)
|
| 517 |
+
colorize_confidence(prediction['depth_conf']).save(args.output_conf_vis)
|
| 518 |
+
if args.output_raw_depth:
|
| 519 |
+
Path(args.output_raw_depth).parent.mkdir(parents=True, exist_ok=True)
|
| 520 |
+
np.save(Path(args.output_raw_depth), prediction['raw_depth'].astype(np.float32))
|
| 521 |
+
if args.output_world_points:
|
| 522 |
+
Path(args.output_world_points).parent.mkdir(parents=True, exist_ok=True)
|
| 523 |
+
np.save(Path(args.output_world_points), prediction['world_points'].astype(np.float32))
|
| 524 |
+
if args.output_point_conf:
|
| 525 |
+
Path(args.output_point_conf).parent.mkdir(parents=True, exist_ok=True)
|
| 526 |
+
np.save(Path(args.output_point_conf), prediction['world_points_conf'].astype(np.float32))
|
| 527 |
+
if args.output_point_cloud:
|
| 528 |
+
rgb = np.asarray(image.convert('RGB'), dtype=np.uint8)
|
| 529 |
+
point_count = write_point_cloud_ply(
|
| 530 |
+
Path(args.output_point_cloud),
|
| 531 |
+
prediction['world_points'],
|
| 532 |
+
rgb,
|
| 533 |
+
prediction['world_points_conf'],
|
| 534 |
+
args.point_conf_threshold,
|
| 535 |
+
args.max_point_cloud_points,
|
| 536 |
+
)
|
| 537 |
+
print(f'Wrote VGGT point cloud to {args.output_point_cloud} ({point_count} points)')
|
| 538 |
+
|
| 539 |
+
output_depth = Path(args.output_depth)
|
| 540 |
+
output_depth.parent.mkdir(parents=True, exist_ok=True)
|
| 541 |
+
np.save(output_depth, depth.astype(np.float32))
|
| 542 |
+
|
| 543 |
+
output_vis = Path(args.output_vis) if args.output_vis else output_depth.with_suffix('.png')
|
| 544 |
+
output_vis.parent.mkdir(parents=True, exist_ok=True)
|
| 545 |
+
save_depth_vis(output_vis, depth)
|
| 546 |
+
|
| 547 |
+
note = 'Metric monocular depth estimate; calibrate intrinsics/scale before path-planning use.'
|
| 548 |
+
if args.engine == 'vggt':
|
| 549 |
+
note = 'VGGT single-view/few-view geometry estimate; scale is not calibrated metric ground truth.'
|
| 550 |
+
elif args.engine != 'depth_anything_v2' or not args.metric_depth:
|
| 551 |
+
note = 'Relative monocular depth; use calibrated metric depth for path-planning truth.'
|
| 552 |
+
|
| 553 |
+
model_identity = depth_model_identity(
|
| 554 |
+
engine=args.engine,
|
| 555 |
+
model=args.model,
|
| 556 |
+
checkpoint=args.checkpoint,
|
| 557 |
+
encoder=args.encoder,
|
| 558 |
+
metric_depth=args.metric_depth,
|
| 559 |
+
vggt_model=args.vggt_model,
|
| 560 |
+
)
|
| 561 |
+
manifest = {
|
| 562 |
+
'image': args.image,
|
| 563 |
+
'raster_orientation_policy': 'RGB is decoded with PIL ImageOps.exif_transpose before depth inference.',
|
| 564 |
+
'display_raster_size': {'width': image.width, 'height': image.height},
|
| 565 |
+
'engine': args.engine,
|
| 566 |
+
'model': model_identity['model_id'],
|
| 567 |
+
'model_identity': model_identity,
|
| 568 |
+
'depth_anything_repo': args.depth_anything_repo if args.engine == 'depth_anything_v2' else None,
|
| 569 |
+
'checkpoint': args.checkpoint if args.engine == 'depth_anything_v2' else None,
|
| 570 |
+
'encoder': args.encoder if args.engine == 'depth_anything_v2' else None,
|
| 571 |
+
'vggt_repo': args.vggt_repo if args.engine == 'vggt' else None,
|
| 572 |
+
'vggt_model': args.vggt_model if args.engine == 'vggt' else None,
|
| 573 |
+
'vggt_checkpoint': args.vggt_checkpoint if args.engine == 'vggt' else None,
|
| 574 |
+
'vggt_input_size': args.vggt_input_size if args.engine == 'vggt' else None,
|
| 575 |
+
'vggt_keep_raw_depth': args.vggt_keep_raw_depth if args.engine == 'vggt' else None,
|
| 576 |
+
'metric_depth': args.metric_depth if args.engine == 'depth_anything_v2' else None,
|
| 577 |
+
'max_depth': args.max_depth if args.metric_depth else None,
|
| 578 |
+
'device': args.device if args.engine in {'transformers', 'depth_anything_v2', 'vggt'} else None,
|
| 579 |
+
'output_depth': str(output_depth),
|
| 580 |
+
'output_vis': str(output_vis),
|
| 581 |
+
'output_conf': args.output_conf if args.engine == 'vggt' else None,
|
| 582 |
+
'output_raw_depth': args.output_raw_depth if args.engine == 'vggt' else None,
|
| 583 |
+
'output_world_points': args.output_world_points if args.engine == 'vggt' else None,
|
| 584 |
+
'output_point_cloud': args.output_point_cloud if args.engine == 'vggt' else None,
|
| 585 |
+
'near': args.near,
|
| 586 |
+
'far': args.far,
|
| 587 |
+
'invert_depth': args.invert_depth,
|
| 588 |
+
'shape': list(depth.shape),
|
| 589 |
+
'note': note,
|
| 590 |
+
}
|
| 591 |
+
manifest_path = Path(args.manifest) if args.manifest else output_depth.with_name('depth_manifest.json')
|
| 592 |
+
manifest_path.write_text(json.dumps(manifest, indent=2), encoding='utf-8')
|
| 593 |
+
print(f'Wrote depth to {output_depth}')
|
| 594 |
+
print(f'Wrote depth visualization to {output_vis}')
|
| 595 |
+
|
| 596 |
+
|
| 597 |
+
if __name__ == '__main__':
|
| 598 |
+
main()
|
accessibilityamodal/geometry_analysis.py
ADDED
|
@@ -0,0 +1,607 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Structured accessibility geometry constraints for 3D amodal completion.
|
| 2 |
+
|
| 3 |
+
The functions in this module convert the deterministic outputs of
|
| 4 |
+
``accessibilityamodal.reconstruct`` into a JSON-ready report. The report is intended for
|
| 5 |
+
navigation-risk review and downstream reconstruction code, not for visual
|
| 6 |
+
plausibility scoring.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
from typing import Any
|
| 12 |
+
|
| 13 |
+
import cv2
|
| 14 |
+
import numpy as np
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
VALID_CATEGORIES = {
|
| 18 |
+
"stairs",
|
| 19 |
+
"ramp",
|
| 20 |
+
"walkway",
|
| 21 |
+
"curb_cut",
|
| 22 |
+
"raised_curb",
|
| 23 |
+
"tactile_paving",
|
| 24 |
+
"unknown",
|
| 25 |
+
}
|
| 26 |
+
CONTINUOUS_CATEGORIES = {"ramp", "walkway", "curb_cut", "raised_curb", "tactile_paving"}
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _round(value: float | int | None, digits: int = 6) -> float | None:
|
| 30 |
+
if value is None:
|
| 31 |
+
return None
|
| 32 |
+
return round(float(value), digits)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def _ratio(numerator: int | float, denominator: int | float) -> float:
|
| 36 |
+
if denominator <= 0:
|
| 37 |
+
return 0.0
|
| 38 |
+
return float(numerator) / float(denominator)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def normalize_category(category: str | None, geometry_mode: str) -> str:
|
| 42 |
+
value = (category or "").strip().lower()
|
| 43 |
+
if value == "walkable":
|
| 44 |
+
value = "walkway"
|
| 45 |
+
if value in VALID_CATEGORIES:
|
| 46 |
+
return value
|
| 47 |
+
if geometry_mode == "stairs":
|
| 48 |
+
return "stairs"
|
| 49 |
+
if geometry_mode == "ramp":
|
| 50 |
+
return "ramp"
|
| 51 |
+
if geometry_mode == "walkable":
|
| 52 |
+
return "walkway"
|
| 53 |
+
return "unknown"
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _step_interval_consistency(edges_y: list[int]) -> str:
|
| 57 |
+
if len(edges_y) < 3:
|
| 58 |
+
return "unknown"
|
| 59 |
+
gaps = np.diff(np.array(sorted(edges_y), dtype=np.float32))
|
| 60 |
+
mean_gap = float(np.mean(gaps))
|
| 61 |
+
if mean_gap <= 1e-6:
|
| 62 |
+
return "unknown"
|
| 63 |
+
variation = float(np.std(gaps) / mean_gap)
|
| 64 |
+
if variation < 0.18:
|
| 65 |
+
return "high"
|
| 66 |
+
if variation < 0.35:
|
| 67 |
+
return "medium"
|
| 68 |
+
return "low"
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _slope_direction_from_depth(depth: np.ndarray, target: np.ndarray) -> str:
|
| 72 |
+
if not np.any(target):
|
| 73 |
+
return "unknown"
|
| 74 |
+
valid = target & np.isfinite(depth) & (depth > 0)
|
| 75 |
+
if int(valid.sum()) < 32:
|
| 76 |
+
return "unknown"
|
| 77 |
+
ys = np.where(valid)[0]
|
| 78 |
+
middle = float(np.median(ys))
|
| 79 |
+
upper = valid & (np.indices(valid.shape)[0] <= middle)
|
| 80 |
+
lower = valid & (np.indices(valid.shape)[0] > middle)
|
| 81 |
+
if int(upper.sum()) < 16 or int(lower.sum()) < 16:
|
| 82 |
+
return "unknown"
|
| 83 |
+
upper_depth = float(np.median(depth[upper]))
|
| 84 |
+
lower_depth = float(np.median(depth[lower]))
|
| 85 |
+
scale = max(abs(upper_depth), abs(lower_depth), 1e-6)
|
| 86 |
+
if abs(upper_depth - lower_depth) / scale < 0.02:
|
| 87 |
+
return "unknown"
|
| 88 |
+
return "away_from_camera" if upper_depth > lower_depth else "toward_camera"
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def _visible_evidence(
|
| 92 |
+
category: str,
|
| 93 |
+
visible_mask: np.ndarray,
|
| 94 |
+
amodal_mask: np.ndarray,
|
| 95 |
+
hidden_mask: np.ndarray,
|
| 96 |
+
obstacle_mask: np.ndarray,
|
| 97 |
+
depth: np.ndarray,
|
| 98 |
+
edges_y: list[int],
|
| 99 |
+
stair_edge_confidence: float,
|
| 100 |
+
stair_edge_coverage: float,
|
| 101 |
+
completion_method: str,
|
| 102 |
+
) -> list[dict[str, Any]]:
|
| 103 |
+
evidence: list[dict[str, Any]] = []
|
| 104 |
+
amodal_area = int(amodal_mask.sum())
|
| 105 |
+
visible_area = int(visible_mask.sum())
|
| 106 |
+
hidden_area = int(hidden_mask.sum())
|
| 107 |
+
obstacle_overlap = int((obstacle_mask & amodal_mask).sum())
|
| 108 |
+
|
| 109 |
+
if visible_area > 0:
|
| 110 |
+
evidence.append(
|
| 111 |
+
{
|
| 112 |
+
"type": "boundary",
|
| 113 |
+
"description": "visible target mask defines the observed support boundary",
|
| 114 |
+
"confidence": _round(min(0.95, 0.35 + 0.60 * _ratio(visible_area, max(amodal_area, 1)))),
|
| 115 |
+
}
|
| 116 |
+
)
|
| 117 |
+
|
| 118 |
+
if hidden_area > 0 or obstacle_overlap > 0:
|
| 119 |
+
evidence.append(
|
| 120 |
+
{
|
| 121 |
+
"type": "occlusion",
|
| 122 |
+
"description": "hidden target support is constrained by amodal-minus-visible and obstacle overlap",
|
| 123 |
+
"confidence": _round(min(0.95, 0.30 + 0.60 * _ratio(obstacle_overlap, max(hidden_area, 1)))),
|
| 124 |
+
}
|
| 125 |
+
)
|
| 126 |
+
|
| 127 |
+
if category == "stairs" and edges_y:
|
| 128 |
+
evidence.append(
|
| 129 |
+
{
|
| 130 |
+
"type": "repeated_step",
|
| 131 |
+
"description": f"{len(edges_y)} candidate stair tread/riser boundary rows detected",
|
| 132 |
+
"confidence": _round(max(stair_edge_confidence, min(stair_edge_coverage, 1.0) * 0.75)),
|
| 133 |
+
}
|
| 134 |
+
)
|
| 135 |
+
|
| 136 |
+
if category in CONTINUOUS_CATEGORIES and "plane" in completion_method:
|
| 137 |
+
evidence.append(
|
| 138 |
+
{
|
| 139 |
+
"type": (
|
| 140 |
+
"slope"
|
| 141 |
+
if category == "ramp"
|
| 142 |
+
else "curb_boundary"
|
| 143 |
+
if category == "raised_curb"
|
| 144 |
+
else "boundary"
|
| 145 |
+
),
|
| 146 |
+
"description": (
|
| 147 |
+
"visible curb support is completed as one continuous surface without repeated steps"
|
| 148 |
+
if category == "raised_curb"
|
| 149 |
+
else "visible support is completed with a robust continuous-surface prior"
|
| 150 |
+
),
|
| 151 |
+
"confidence": 0.70,
|
| 152 |
+
}
|
| 153 |
+
)
|
| 154 |
+
|
| 155 |
+
valid_depth = amodal_mask & np.isfinite(depth) & (depth > 0)
|
| 156 |
+
if int(valid_depth.sum()) >= 128:
|
| 157 |
+
rows = np.where(valid_depth)[0]
|
| 158 |
+
row_span = max(int(rows.max() - rows.min()), 1)
|
| 159 |
+
depth_values = depth[valid_depth]
|
| 160 |
+
depth_span = float(np.percentile(depth_values, 90) - np.percentile(depth_values, 10))
|
| 161 |
+
scale = max(float(np.median(depth_values)), 1e-6)
|
| 162 |
+
if depth_span / scale > 0.02 and row_span > 8:
|
| 163 |
+
evidence.append(
|
| 164 |
+
{
|
| 165 |
+
"type": "depth_gradient",
|
| 166 |
+
"description": "target depth varies coherently across the support region",
|
| 167 |
+
"confidence": _round(min(0.80, depth_span / scale)),
|
| 168 |
+
}
|
| 169 |
+
)
|
| 170 |
+
return evidence
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def _connected_hidden_regions(
|
| 174 |
+
category: str,
|
| 175 |
+
hidden_mask: np.ndarray,
|
| 176 |
+
completion_method: str,
|
| 177 |
+
mean_hidden_confidence: float,
|
| 178 |
+
edges_y: list[int],
|
| 179 |
+
) -> list[dict[str, Any]]:
|
| 180 |
+
if not np.any(hidden_mask):
|
| 181 |
+
return []
|
| 182 |
+
labels_count, labels, stats, _ = cv2.connectedComponentsWithStats(
|
| 183 |
+
hidden_mask.astype(np.uint8), connectivity=8
|
| 184 |
+
)
|
| 185 |
+
regions: list[dict[str, Any]] = []
|
| 186 |
+
areas = [
|
| 187 |
+
(idx, int(stats[idx, cv2.CC_STAT_AREA]))
|
| 188 |
+
for idx in range(1, labels_count)
|
| 189 |
+
if int(stats[idx, cv2.CC_STAT_AREA]) > 0
|
| 190 |
+
]
|
| 191 |
+
areas.sort(key=lambda item: item[1], reverse=True)
|
| 192 |
+
total = max(int(hidden_mask.sum()), 1)
|
| 193 |
+
for serial, (idx, area) in enumerate(areas[:12], start=1):
|
| 194 |
+
if category == "stairs" and len(edges_y) >= 2:
|
| 195 |
+
basis = "repeated_steps"
|
| 196 |
+
description = "complete the hidden support by repeating the visible tread/riser pattern"
|
| 197 |
+
elif category in {"ramp", "walkway", "curb_cut", "raised_curb"} and "plane" in completion_method:
|
| 198 |
+
basis = "plane_continuity"
|
| 199 |
+
description = (
|
| 200 |
+
"continue one curb top/face support through the hidden region without stair bands"
|
| 201 |
+
if category == "raised_curb"
|
| 202 |
+
else "project hidden pixels onto the fitted visible support plane"
|
| 203 |
+
)
|
| 204 |
+
elif category == "tactile_paving":
|
| 205 |
+
basis = "boundary_continuity"
|
| 206 |
+
description = "preserve the narrow tactile strip footprint through the hidden region"
|
| 207 |
+
elif "inpaint" in completion_method:
|
| 208 |
+
basis = "depth_interpolation"
|
| 209 |
+
description = "use local depth interpolation because stronger structural evidence is unavailable"
|
| 210 |
+
else:
|
| 211 |
+
basis = "uncertain"
|
| 212 |
+
description = "insufficient structural evidence for confident hidden completion"
|
| 213 |
+
regions.append(
|
| 214 |
+
{
|
| 215 |
+
"region_id": f"hidden_{serial}",
|
| 216 |
+
"completion_basis": basis,
|
| 217 |
+
"description": description,
|
| 218 |
+
"confidence": _round(min(0.95, mean_hidden_confidence * (0.50 + 0.50 * area / total))),
|
| 219 |
+
}
|
| 220 |
+
)
|
| 221 |
+
return regions
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def _amodal_quality(
|
| 225 |
+
target_present: bool,
|
| 226 |
+
visible_mask: np.ndarray,
|
| 227 |
+
amodal_mask: np.ndarray,
|
| 228 |
+
hidden_mask: np.ndarray,
|
| 229 |
+
mean_hidden_confidence: float,
|
| 230 |
+
used_regular_fallback_edges: bool,
|
| 231 |
+
) -> str:
|
| 232 |
+
if not target_present:
|
| 233 |
+
return "bad"
|
| 234 |
+
amodal_area = int(amodal_mask.sum())
|
| 235 |
+
visible_ratio = _ratio(int(visible_mask.sum()), amodal_area)
|
| 236 |
+
hidden_ratio = _ratio(int(hidden_mask.sum()), amodal_area)
|
| 237 |
+
if visible_ratio < 0.02:
|
| 238 |
+
return "bad"
|
| 239 |
+
if used_regular_fallback_edges or mean_hidden_confidence < 0.30:
|
| 240 |
+
return "uncertain"
|
| 241 |
+
if visible_ratio >= 0.20 and (hidden_ratio == 0.0 or mean_hidden_confidence >= 0.50):
|
| 242 |
+
return "good"
|
| 243 |
+
if visible_ratio >= 0.05:
|
| 244 |
+
return "partial"
|
| 245 |
+
return "uncertain"
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
def _geometry_model_type(category: str, geometry_mode: str) -> str:
|
| 249 |
+
if category == "raised_curb":
|
| 250 |
+
return "raised_curb_prism"
|
| 251 |
+
if category == "stairs" or geometry_mode == "stairs":
|
| 252 |
+
return "stair_steps"
|
| 253 |
+
if category == "curb_cut":
|
| 254 |
+
return "curb_cut_planes"
|
| 255 |
+
if category == "tactile_paving":
|
| 256 |
+
return "tactile_strip"
|
| 257 |
+
if category in {"ramp", "walkway"}:
|
| 258 |
+
return "single_plane"
|
| 259 |
+
return "uncertain"
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
def _hidden_depth_rule(category: str, completion_method: str) -> str:
|
| 263 |
+
if category == "stairs":
|
| 264 |
+
return "repeat_stair_geometry"
|
| 265 |
+
if category in {"ramp", "walkway", "curb_cut", "raised_curb", "tactile_paving"} and "plane" in completion_method:
|
| 266 |
+
return "fit_to_plane"
|
| 267 |
+
if "inpaint" in completion_method:
|
| 268 |
+
return "interpolate_between_boundaries"
|
| 269 |
+
return "uncertain"
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
def _recommended_action(category: str, geometry_confidence: float) -> tuple[str, str]:
|
| 273 |
+
if geometry_confidence < 0.30:
|
| 274 |
+
return "manual_review", "Evidence is weak; do not generate a navigation mesh without review."
|
| 275 |
+
if category == "stairs":
|
| 276 |
+
return (
|
| 277 |
+
"stair_regularization",
|
| 278 |
+
"Fit repeated tread/riser bands and complete hidden depth by stair periodicity.",
|
| 279 |
+
)
|
| 280 |
+
if category == "curb_cut":
|
| 281 |
+
return (
|
| 282 |
+
"curb_cut_plane_decomposition",
|
| 283 |
+
"Decompose sidewalk, road, and sloped transition planes before meshing.",
|
| 284 |
+
)
|
| 285 |
+
if category == "raised_curb":
|
| 286 |
+
return (
|
| 287 |
+
"raised_curb_prism_fit",
|
| 288 |
+
"Fit one continuous curb top and vertical face, then extrude a solid prism without repeated stair bands.",
|
| 289 |
+
)
|
| 290 |
+
if category == "tactile_paving":
|
| 291 |
+
return (
|
| 292 |
+
"tactile_centerline_completion",
|
| 293 |
+
"Keep a narrow tactile strip, continue its centerline, and avoid expanding to the full sidewalk.",
|
| 294 |
+
)
|
| 295 |
+
if category in {"ramp", "walkway"}:
|
| 296 |
+
return (
|
| 297 |
+
"plane_fit",
|
| 298 |
+
"Fit visible support only, reject obstacle/depth outliers, and project hidden pixels to the plane.",
|
| 299 |
+
)
|
| 300 |
+
return "manual_review", "Unknown category; keep the sample out of automatic 3D completion."
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
def build_accessibility_geometry_analysis(
|
| 304 |
+
*,
|
| 305 |
+
sample_id: str | None,
|
| 306 |
+
category: str | None,
|
| 307 |
+
geometry_mode: str,
|
| 308 |
+
visible_mask: np.ndarray,
|
| 309 |
+
amodal_mask: np.ndarray,
|
| 310 |
+
hidden_mask: np.ndarray,
|
| 311 |
+
obstacle_mask: np.ndarray,
|
| 312 |
+
depth: np.ndarray,
|
| 313 |
+
completed_depth: np.ndarray,
|
| 314 |
+
confidence: np.ndarray,
|
| 315 |
+
completion_method: str,
|
| 316 |
+
edges_y: list[int],
|
| 317 |
+
stair_edge_slope: float,
|
| 318 |
+
edge_source: str,
|
| 319 |
+
stair_edge_confidence: float,
|
| 320 |
+
stair_edge_coverage: float,
|
| 321 |
+
used_regular_fallback_edges: bool,
|
| 322 |
+
visible_depth_unchanged: bool,
|
| 323 |
+
completed_target_depth_finite: bool,
|
| 324 |
+
point_cloud_vertices: int,
|
| 325 |
+
mesh_vertices: int,
|
| 326 |
+
mesh_faces: int,
|
| 327 |
+
) -> dict[str, Any]:
|
| 328 |
+
"""Build the JSON-ready geometry constraint report."""
|
| 329 |
+
|
| 330 |
+
normalized_category = normalize_category(category, geometry_mode)
|
| 331 |
+
target_present = bool(np.any(amodal_mask) or np.any(visible_mask))
|
| 332 |
+
hidden_nonempty = bool(np.any(hidden_mask))
|
| 333 |
+
hidden_confidence_values = confidence[hidden_mask] if hidden_nonempty else np.array([1.0], dtype=np.float32)
|
| 334 |
+
mean_hidden_confidence = float(np.mean(hidden_confidence_values)) if hidden_confidence_values.size else 0.0
|
| 335 |
+
visible_area = int(visible_mask.sum())
|
| 336 |
+
amodal_area = int(amodal_mask.sum())
|
| 337 |
+
hidden_area = int(hidden_mask.sum())
|
| 338 |
+
obstacle_area = int(obstacle_mask.sum())
|
| 339 |
+
obstacle_target_overlap = int((obstacle_mask & amodal_mask).sum())
|
| 340 |
+
obstacle_visible_overlap_ratio = _ratio(int((obstacle_mask & visible_mask).sum()), max(visible_area, 1))
|
| 341 |
+
|
| 342 |
+
if normalized_category == "stairs":
|
| 343 |
+
structure_confidence = float(stair_edge_confidence)
|
| 344 |
+
if used_regular_fallback_edges:
|
| 345 |
+
structure_confidence = min(structure_confidence, 0.30)
|
| 346 |
+
elif normalized_category in CONTINUOUS_CATEGORIES:
|
| 347 |
+
structure_confidence = 0.72 if "plane" in completion_method else 0.38
|
| 348 |
+
else:
|
| 349 |
+
structure_confidence = 0.20
|
| 350 |
+
|
| 351 |
+
target_evidence_score = min(1.0, 0.25 + 0.75 * _ratio(visible_area, max(amodal_area, 1)))
|
| 352 |
+
mesh_score = 1.0 if point_cloud_vertices > 0 and mesh_vertices > 0 and mesh_faces > 0 else 0.25
|
| 353 |
+
geometry_confidence = min(
|
| 354 |
+
0.99,
|
| 355 |
+
max(
|
| 356 |
+
0.0,
|
| 357 |
+
0.35 * structure_confidence
|
| 358 |
+
+ 0.25 * mean_hidden_confidence
|
| 359 |
+
+ 0.20 * target_evidence_score
|
| 360 |
+
+ 0.20 * mesh_score,
|
| 361 |
+
),
|
| 362 |
+
)
|
| 363 |
+
if not target_present:
|
| 364 |
+
geometry_confidence = 0.0
|
| 365 |
+
if not completed_target_depth_finite:
|
| 366 |
+
geometry_confidence *= 0.50
|
| 367 |
+
|
| 368 |
+
amodal_quality = _amodal_quality(
|
| 369 |
+
target_present,
|
| 370 |
+
visible_mask,
|
| 371 |
+
amodal_mask,
|
| 372 |
+
hidden_mask,
|
| 373 |
+
mean_hidden_confidence,
|
| 374 |
+
used_regular_fallback_edges,
|
| 375 |
+
)
|
| 376 |
+
if amodal_quality in {"uncertain", "bad"}:
|
| 377 |
+
geometry_confidence = min(geometry_confidence, 0.55 if amodal_quality == "uncertain" else 0.20)
|
| 378 |
+
|
| 379 |
+
visible_evidence = _visible_evidence(
|
| 380 |
+
normalized_category,
|
| 381 |
+
visible_mask,
|
| 382 |
+
amodal_mask,
|
| 383 |
+
hidden_mask,
|
| 384 |
+
obstacle_mask,
|
| 385 |
+
depth,
|
| 386 |
+
edges_y,
|
| 387 |
+
stair_edge_confidence,
|
| 388 |
+
stair_edge_coverage,
|
| 389 |
+
completion_method,
|
| 390 |
+
)
|
| 391 |
+
|
| 392 |
+
occluders = []
|
| 393 |
+
if obstacle_area > 0:
|
| 394 |
+
occluders.append(
|
| 395 |
+
{
|
| 396 |
+
"class": "other",
|
| 397 |
+
"overlaps_target": bool(obstacle_target_overlap > 0),
|
| 398 |
+
"should_exclude_from_mesh": True,
|
| 399 |
+
"description": (
|
| 400 |
+
"aggregate obstacle mask overlaps the target support"
|
| 401 |
+
if obstacle_target_overlap > 0
|
| 402 |
+
else "aggregate obstacle mask is outside the target support"
|
| 403 |
+
),
|
| 404 |
+
}
|
| 405 |
+
)
|
| 406 |
+
|
| 407 |
+
hidden_regions = _connected_hidden_regions(
|
| 408 |
+
normalized_category,
|
| 409 |
+
hidden_mask,
|
| 410 |
+
completion_method,
|
| 411 |
+
mean_hidden_confidence,
|
| 412 |
+
edges_y,
|
| 413 |
+
)
|
| 414 |
+
|
| 415 |
+
plane_applies = normalized_category in CONTINUOUS_CATEGORIES
|
| 416 |
+
stair_applies = normalized_category == "stairs"
|
| 417 |
+
model_type = _geometry_model_type(normalized_category, geometry_mode)
|
| 418 |
+
slope_direction = _slope_direction_from_depth(completed_depth, amodal_mask)
|
| 419 |
+
expected_surface = (
|
| 420 |
+
"segmented"
|
| 421 |
+
if normalized_category in {"stairs", "curb_cut"}
|
| 422 |
+
else "continuous_top_with_vertical_face"
|
| 423 |
+
if normalized_category == "raised_curb"
|
| 424 |
+
else "sloped"
|
| 425 |
+
if normalized_category == "ramp"
|
| 426 |
+
else "flat"
|
| 427 |
+
if normalized_category in {"walkway", "tactile_paving"}
|
| 428 |
+
else "segmented"
|
| 429 |
+
)
|
| 430 |
+
hidden_depth_rule = _hidden_depth_rule(normalized_category, completion_method)
|
| 431 |
+
step_lines = [
|
| 432 |
+
{
|
| 433 |
+
"y": int(y),
|
| 434 |
+
"slope": _round(stair_edge_slope),
|
| 435 |
+
"source": edge_source,
|
| 436 |
+
}
|
| 437 |
+
for y in edges_y
|
| 438 |
+
]
|
| 439 |
+
|
| 440 |
+
if normalized_category == "stairs":
|
| 441 |
+
passable = False
|
| 442 |
+
risk_level = "high" if geometry_confidence >= 0.30 else "unknown"
|
| 443 |
+
risks = ["stairs present", "wheeled passability is blocked or requires alternate route"]
|
| 444 |
+
if used_regular_fallback_edges:
|
| 445 |
+
risks.append("stair geometry relies on fallback edge positions")
|
| 446 |
+
reason_short = "Stair geometry is detected; treat as high risk for wheeled accessibility."
|
| 447 |
+
elif normalized_category == "raised_curb":
|
| 448 |
+
passable = False
|
| 449 |
+
risk_level = "high" if geometry_confidence >= 0.30 else "unknown"
|
| 450 |
+
risks = [
|
| 451 |
+
"raised curb is a non-walkable level-change barrier",
|
| 452 |
+
"wheeled passability is blocked or requires a curb cut or alternate route",
|
| 453 |
+
]
|
| 454 |
+
reason_short = "A raised curb is present; model it as a continuous high obstacle, not as stairs."
|
| 455 |
+
elif geometry_confidence < 0.30 or amodal_quality in {"uncertain", "bad"}:
|
| 456 |
+
passable = False
|
| 457 |
+
risk_level = "unknown"
|
| 458 |
+
risks = ["hidden support geometry is uncertain"]
|
| 459 |
+
reason_short = "Evidence is insufficient for an automatic passability decision."
|
| 460 |
+
else:
|
| 461 |
+
obstacle_intrusion = _ratio(obstacle_target_overlap, max(amodal_area, 1))
|
| 462 |
+
passable = obstacle_intrusion < 0.25 and visible_depth_unchanged
|
| 463 |
+
risk_level = "medium" if hidden_nonempty or obstacle_intrusion > 0.10 else "low"
|
| 464 |
+
risks = []
|
| 465 |
+
if hidden_nonempty:
|
| 466 |
+
risks.append("hidden region may contain unresolved hazards")
|
| 467 |
+
if obstacle_intrusion > 0.10:
|
| 468 |
+
risks.append("obstacle intrudes into the target support")
|
| 469 |
+
if not risks:
|
| 470 |
+
risks.append("no major geometry risk from available masks")
|
| 471 |
+
reason_short = "Continuous support appears geometrically consistent, but monocular depth is not metric truth."
|
| 472 |
+
|
| 473 |
+
triangle_fan_passed = True
|
| 474 |
+
if normalized_category in CONTINUOUS_CATEGORIES:
|
| 475 |
+
triangle_fan_passed = bool(mesh_faces > 0 and ("plane" in completion_method or "continuous" in completion_method))
|
| 476 |
+
stair_not_smoothed_passed = True
|
| 477 |
+
if normalized_category == "stairs":
|
| 478 |
+
stair_not_smoothed_passed = bool(
|
| 479 |
+
"stair" in completion_method and len(edges_y) >= 2 and stair_edge_confidence > 0.0
|
| 480 |
+
)
|
| 481 |
+
raised_curb_not_stepped_passed = True
|
| 482 |
+
if normalized_category == "raised_curb":
|
| 483 |
+
raised_curb_not_stepped_passed = bool(
|
| 484 |
+
geometry_mode != "stairs" and "stair" not in completion_method and not edges_y
|
| 485 |
+
)
|
| 486 |
+
obstacle_exclusion_passed = obstacle_visible_overlap_ratio <= 0.02
|
| 487 |
+
hidden_follows_amodal = bool(np.array_equal(hidden_mask, amodal_mask & ~visible_mask))
|
| 488 |
+
|
| 489 |
+
next_step, instruction = _recommended_action(normalized_category, geometry_confidence)
|
| 490 |
+
|
| 491 |
+
return {
|
| 492 |
+
"sample_id": sample_id or "",
|
| 493 |
+
"category": normalized_category,
|
| 494 |
+
"target_present": target_present,
|
| 495 |
+
"geometry_confidence": _round(geometry_confidence),
|
| 496 |
+
"visible_evidence": visible_evidence,
|
| 497 |
+
"occluders": occluders,
|
| 498 |
+
"amodal_completion": {
|
| 499 |
+
"target_amodal_region_quality": amodal_quality,
|
| 500 |
+
"hidden_regions": hidden_regions,
|
| 501 |
+
"do_not_complete_regions": [
|
| 502 |
+
{
|
| 503 |
+
"description": "obstacle mask outside the hidden target support",
|
| 504 |
+
"reason": "foreground occluder geometry is not part of the accessible support surface",
|
| 505 |
+
},
|
| 506 |
+
{
|
| 507 |
+
"description": "outside target_amodal mask",
|
| 508 |
+
"reason": "completion must be clipped to reviewed or inferred target support",
|
| 509 |
+
},
|
| 510 |
+
{
|
| 511 |
+
"description": "thin mask boundary band, shadows, reflections, and wall/railing regions",
|
| 512 |
+
"reason": "these pixels are unstable depth or non-walkable geometry",
|
| 513 |
+
},
|
| 514 |
+
],
|
| 515 |
+
},
|
| 516 |
+
"geometry_prior": {
|
| 517 |
+
"model_type": model_type,
|
| 518 |
+
"plane_prior": {
|
| 519 |
+
"applies": plane_applies,
|
| 520 |
+
"slope_direction_image": slope_direction,
|
| 521 |
+
"expected_surface": expected_surface,
|
| 522 |
+
"fit_visible_only_then_extend_to_hidden": True,
|
| 523 |
+
"reject_depth_outliers": True,
|
| 524 |
+
},
|
| 525 |
+
"stairs_prior": {
|
| 526 |
+
"applies": stair_applies,
|
| 527 |
+
"step_direction_image": slope_direction,
|
| 528 |
+
"visible_step_lines": step_lines,
|
| 529 |
+
"estimated_step_count": len(edges_y) + 1 if edges_y else None,
|
| 530 |
+
"step_interval_consistency": _step_interval_consistency(edges_y),
|
| 531 |
+
"hidden_completion_rule": "repeat_visible_tread_riser_pattern",
|
| 532 |
+
},
|
| 533 |
+
"curb_cut_prior": {
|
| 534 |
+
"applies": normalized_category == "curb_cut",
|
| 535 |
+
"has_sidewalk_plane": None if normalized_category != "curb_cut" else "plane" in completion_method,
|
| 536 |
+
"has_road_plane": None,
|
| 537 |
+
"has_sloped_transition": None if normalized_category != "curb_cut" else "plane" in completion_method,
|
| 538 |
+
"boundary_or_hinge_lines": [],
|
| 539 |
+
},
|
| 540 |
+
"raised_curb_prior": {
|
| 541 |
+
"applies": normalized_category == "raised_curb",
|
| 542 |
+
"is_walkable_surface": False if normalized_category == "raised_curb" else None,
|
| 543 |
+
"hazard_class": "high_obstacle" if normalized_category == "raised_curb" else None,
|
| 544 |
+
"has_continuous_top_surface": (
|
| 545 |
+
None if normalized_category != "raised_curb" else "plane" in completion_method
|
| 546 |
+
),
|
| 547 |
+
"vertical_face_required": True if normalized_category == "raised_curb" else None,
|
| 548 |
+
"repeated_step_profile_allowed": False if normalized_category == "raised_curb" else None,
|
| 549 |
+
"boundary_or_hinge_lines": [],
|
| 550 |
+
},
|
| 551 |
+
},
|
| 552 |
+
"depth_completion_constraints": {
|
| 553 |
+
"trusted_depth_regions": [
|
| 554 |
+
"visible target surface excluding obstacle and mask boundary noise",
|
| 555 |
+
],
|
| 556 |
+
"untrusted_depth_regions": [
|
| 557 |
+
"obstacle mask",
|
| 558 |
+
"hidden mask raw depth",
|
| 559 |
+
"thin railings",
|
| 560 |
+
"mask boundary band",
|
| 561 |
+
"specular/shadow regions",
|
| 562 |
+
],
|
| 563 |
+
"hidden_depth_rule": hidden_depth_rule,
|
| 564 |
+
"mesh_generation_rule": (
|
| 565 |
+
"fit one continuous curb top, add a vertical face, and forbid repeated stair bands"
|
| 566 |
+
if normalized_category == "raised_curb"
|
| 567 |
+
else "clip mesh by target_amodal mask, remove obstacle geometry, regularize hidden region"
|
| 568 |
+
),
|
| 569 |
+
},
|
| 570 |
+
"passability_assessment": {
|
| 571 |
+
"is_likely_passable": passable,
|
| 572 |
+
"risk_level": risk_level,
|
| 573 |
+
"risks": risks,
|
| 574 |
+
"reason_short": reason_short,
|
| 575 |
+
},
|
| 576 |
+
"failure_checks": [
|
| 577 |
+
{
|
| 578 |
+
"check": "ramp_or_walkway_should_not_collapse_into_triangle_fan",
|
| 579 |
+
"passed": triangle_fan_passed,
|
| 580 |
+
"fix_if_failed": "use RANSAC plane fitting on visible target and project hidden mask to fitted plane",
|
| 581 |
+
},
|
| 582 |
+
{
|
| 583 |
+
"check": "stairs_should_not_be_smoothed_into_single_ramp",
|
| 584 |
+
"passed": stair_not_smoothed_passed,
|
| 585 |
+
"fix_if_failed": "fit repeated tread/riser geometry",
|
| 586 |
+
},
|
| 587 |
+
{
|
| 588 |
+
"check": "raised_curb_should_not_use_repeated_stair_bands",
|
| 589 |
+
"passed": raised_curb_not_stepped_passed,
|
| 590 |
+
"fix_if_failed": "switch to continuous-surface completion and fit one raised curb prism",
|
| 591 |
+
},
|
| 592 |
+
{
|
| 593 |
+
"check": "obstacles_should_not_be_included_in_accessible_mesh",
|
| 594 |
+
"passed": obstacle_exclusion_passed,
|
| 595 |
+
"fix_if_failed": "subtract obstacle mask before point cloud and mesh creation",
|
| 596 |
+
},
|
| 597 |
+
{
|
| 598 |
+
"check": "hidden_region_should_follow_amodal_mask_not_visible_mask_only",
|
| 599 |
+
"passed": hidden_follows_amodal,
|
| 600 |
+
"fix_if_failed": "use target_amodal and hidden masks as reconstruction constraints",
|
| 601 |
+
},
|
| 602 |
+
],
|
| 603 |
+
"recommended_3d_action": {
|
| 604 |
+
"next_step": next_step,
|
| 605 |
+
"short_instruction_for_reconstruction_code": instruction,
|
| 606 |
+
},
|
| 607 |
+
}
|
accessibilityamodal/pipeline.py
ADDED
|
@@ -0,0 +1,886 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""AccessibilityAmodal mask construction and geometry completion.
|
| 2 |
+
|
| 3 |
+
The script is an orchestration layer. It does not vendor the external projects;
|
| 4 |
+
instead it consumes their outputs or calls a running LISA Gradio server when
|
| 5 |
+
available. This keeps the project-owned AccessibilityAmodal workflow separate
|
| 6 |
+
from the incompatible dependency trees of LISA, Grounded-SAM, Amodal-Wild,
|
| 7 |
+
Diffusion-VAS, pix2gestalt, and the optional licensed visual 3D backend.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import argparse
|
| 13 |
+
import json
|
| 14 |
+
import shutil
|
| 15 |
+
import subprocess
|
| 16 |
+
import sys
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
from typing import Any
|
| 19 |
+
from urllib.request import urlopen
|
| 20 |
+
|
| 21 |
+
import cv2
|
| 22 |
+
import numpy as np
|
| 23 |
+
from PIL import Image, ImageOps
|
| 24 |
+
|
| 25 |
+
from accessibilityamodal.category_overrides import (
|
| 26 |
+
DEFAULT_CATEGORY_OVERRIDES_PATH,
|
| 27 |
+
load_reviewed_category_overrides,
|
| 28 |
+
resolve_reviewed_category,
|
| 29 |
+
)
|
| 30 |
+
from accessibilityamodal.visual_completion import (
|
| 31 |
+
build_completion_envelope,
|
| 32 |
+
build_visual_removal_mask,
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
ACCESSIBILITY_PRESETS = {
|
| 40 |
+
'stairs': {
|
| 41 |
+
'target_prompt': 'Can you segment all visible stair steps and stair walking surfaces in this image? Please output segmentation mask.',
|
| 42 |
+
'obstacle_prompt': 'Can you segment the suitcase, luggage, person, vehicle, or any object occluding the stairs in this image? Please output segmentation mask.',
|
| 43 |
+
},
|
| 44 |
+
'tactile_paving': {
|
| 45 |
+
'target_prompt': 'Can you segment the visible tactile paving or blind sidewalk guiding tiles in this image? Please output segmentation mask.',
|
| 46 |
+
'obstacle_prompt': 'Can you segment the object blocking the tactile paving path in this image? Please output segmentation mask.',
|
| 47 |
+
},
|
| 48 |
+
'ramp': {
|
| 49 |
+
'target_prompt': 'Can you segment the visible wheelchair ramp or accessible sloped walking surface in this image? Please output segmentation mask.',
|
| 50 |
+
'obstacle_prompt': 'Can you segment the object blocking the wheelchair ramp or accessible route in this image? Please output segmentation mask.',
|
| 51 |
+
},
|
| 52 |
+
'curb_cut': {
|
| 53 |
+
'target_prompt': 'Can you segment the visible curb cut ramp, sidewalk transition, and accessible sloped crossing surface in this image? Please output segmentation mask.',
|
| 54 |
+
'obstacle_prompt': 'Can you segment objects blocking the curb cut, sidewalk transition, or accessible crossing in this image? Please output segmentation mask.',
|
| 55 |
+
},
|
| 56 |
+
'raised_curb': {
|
| 57 |
+
'target_prompt': 'Can you segment the visible raised road curb, including its continuous top edge and vertical face, in this image? Exclude stairs and curb-cut ramps. Please output segmentation mask.',
|
| 58 |
+
'obstacle_prompt': 'Can you segment people, vehicles, vegetation, or other objects occluding the raised curb in this image? Please output segmentation mask.',
|
| 59 |
+
},
|
| 60 |
+
'walkway': {
|
| 61 |
+
'target_prompt': 'Can you segment the visible accessible pedestrian walkway or sidewalk support surface in this image? Please output segmentation mask.',
|
| 62 |
+
'obstacle_prompt': 'Can you segment obstacles blocking the accessible walkway or sidewalk path in this image? Please output segmentation mask.',
|
| 63 |
+
},
|
| 64 |
+
'walkable': {
|
| 65 |
+
'target_prompt': 'Can you segment the visible accessible walkable path for a wheelchair or delivery robot in this image? Please output segmentation mask.',
|
| 66 |
+
'obstacle_prompt': 'Can you segment obstacles blocking the accessible walking path in this image? Please output segmentation mask.',
|
| 67 |
+
},
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def read_rgb(path: str | Path) -> np.ndarray:
|
| 72 |
+
"""Read RGB in the same display orientation used by the mask proposal stage."""
|
| 73 |
+
return np.array(ImageOps.exif_transpose(Image.open(path)).convert('RGB'))
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def save_mask(path: str | Path, mask: np.ndarray) -> None:
|
| 77 |
+
Image.fromarray((mask.astype(np.uint8) * 255)).save(path)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def read_mask(path: str | Path, shape: tuple[int, int]) -> np.ndarray:
|
| 81 |
+
mask = np.array(ImageOps.exif_transpose(Image.open(path)).convert('L')) > 127
|
| 82 |
+
h, w = shape
|
| 83 |
+
if mask.shape != (h, w):
|
| 84 |
+
raise ValueError(
|
| 85 |
+
f'Mask/RGB raster mismatch for {path}: mask={mask.shape}, rgb={(h, w)}. '
|
| 86 |
+
'Refusing to resize because it can hide EXIF-orientation misalignment.'
|
| 87 |
+
)
|
| 88 |
+
return mask
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def parse_box(text: str) -> tuple[float, float, float, float]:
|
| 92 |
+
values = tuple(float(v) for v in text.split(','))
|
| 93 |
+
if len(values) != 4:
|
| 94 |
+
raise argparse.ArgumentTypeError('box must be x1,y1,x2,y2')
|
| 95 |
+
x1, y1, x2, y2 = values
|
| 96 |
+
if x2 <= x1 or y2 <= y1:
|
| 97 |
+
raise argparse.ArgumentTypeError('box must satisfy x2>x1 and y2>y1')
|
| 98 |
+
return values
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def boxes_to_mask(boxes: list[tuple[float, float, float, float]], shape: tuple[int, int]) -> np.ndarray:
|
| 102 |
+
h, w = shape
|
| 103 |
+
mask = np.zeros(shape, dtype=bool)
|
| 104 |
+
for x1, y1, x2, y2 in boxes:
|
| 105 |
+
if max(x1, y1, x2, y2) <= 1.0:
|
| 106 |
+
x1, x2 = x1 * w, x2 * w
|
| 107 |
+
y1, y2 = y1 * h, y2 * h
|
| 108 |
+
left = int(np.clip(round(x1), 0, w - 1))
|
| 109 |
+
right = int(np.clip(round(x2), left + 1, w))
|
| 110 |
+
top = int(np.clip(round(y1), 0, h - 1))
|
| 111 |
+
bottom = int(np.clip(round(y2), top + 1, h))
|
| 112 |
+
mask[top:bottom, left:right] = True
|
| 113 |
+
return mask
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def mask_from_lisa_json(path: str | Path, shape: tuple[int, int]) -> np.ndarray:
|
| 117 |
+
with open(path, 'r', encoding='utf-8') as f:
|
| 118 |
+
data: Any = json.load(f)
|
| 119 |
+
if isinstance(data, dict) and 'data' in data:
|
| 120 |
+
data = data['data']
|
| 121 |
+
arr = np.array(data)
|
| 122 |
+
if arr.ndim == 3:
|
| 123 |
+
arr = arr[..., 0]
|
| 124 |
+
mask = arr.astype(np.float32) > 0
|
| 125 |
+
h, w = shape
|
| 126 |
+
if mask.shape != (h, w):
|
| 127 |
+
raise ValueError(
|
| 128 |
+
f'LISA mask/RGB raster mismatch for {path}: mask={mask.shape}, rgb={(h, w)}. '
|
| 129 |
+
'Refusing to resize because the LISA input must share the normalized display-oriented grid.'
|
| 130 |
+
)
|
| 131 |
+
return mask
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def lisa_server_reachable(server_url: str, timeout: float = 2.0) -> bool:
|
| 135 |
+
try:
|
| 136 |
+
with urlopen(server_url, timeout=timeout):
|
| 137 |
+
return True
|
| 138 |
+
except Exception:
|
| 139 |
+
return False
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def call_lisa(server_url: str, image_path: str, prompt: str, out_prefix: Path, shape: tuple[int, int]) -> np.ndarray:
|
| 143 |
+
try:
|
| 144 |
+
from gradio_client import Client
|
| 145 |
+
except ImportError as exc:
|
| 146 |
+
raise RuntimeError('gradio_client is not installed in this environment') from exc
|
| 147 |
+
|
| 148 |
+
client = Client(server_url)
|
| 149 |
+
result = client.predict(prompt, image_path, api_name='/predict')
|
| 150 |
+
if not isinstance(result, (tuple, list)) or len(result) < 2:
|
| 151 |
+
raise RuntimeError(f'Unexpected LISA response: {result!r}')
|
| 152 |
+
|
| 153 |
+
rendered_path = Path(result[0])
|
| 154 |
+
json_path = Path(result[1])
|
| 155 |
+
copied_rendered = out_prefix.with_suffix('.png')
|
| 156 |
+
copied_json = out_prefix.with_suffix('.json')
|
| 157 |
+
if rendered_path.exists():
|
| 158 |
+
shutil.copyfile(rendered_path, copied_rendered)
|
| 159 |
+
shutil.copyfile(json_path, copied_json)
|
| 160 |
+
return mask_from_lisa_json(copied_json, shape)
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def largest_components(mask: np.ndarray, keep: int = 3, min_area: int = 64) -> np.ndarray:
|
| 164 |
+
num, labels, stats, _ = cv2.connectedComponentsWithStats(mask.astype(np.uint8), connectivity=8)
|
| 165 |
+
if num <= 1:
|
| 166 |
+
return mask.astype(bool)
|
| 167 |
+
areas = [(idx, stats[idx, cv2.CC_STAT_AREA]) for idx in range(1, num)]
|
| 168 |
+
areas = [(idx, area) for idx, area in areas if area >= min_area]
|
| 169 |
+
areas.sort(key=lambda item: item[1], reverse=True)
|
| 170 |
+
kept = np.zeros_like(mask, dtype=bool)
|
| 171 |
+
for idx, _ in areas[:keep]:
|
| 172 |
+
kept |= labels == idx
|
| 173 |
+
return kept
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def heuristic_target_mask(rgb: np.ndarray, preset: str) -> np.ndarray:
|
| 177 |
+
h, w = rgb.shape[:2]
|
| 178 |
+
gray = cv2.cvtColor(rgb, cv2.COLOR_RGB2GRAY)
|
| 179 |
+
edges = cv2.Canny(cv2.GaussianBlur(gray, (5, 5), 0), 60, 160)
|
| 180 |
+
lower = np.zeros((h, w), dtype=np.uint8)
|
| 181 |
+
lower[int(0.12 * h):, :] = 1
|
| 182 |
+
if preset == 'stairs':
|
| 183 |
+
horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (max(25, w // 18), 3))
|
| 184 |
+
horizontal = cv2.morphologyEx(edges, cv2.MORPH_CLOSE, horizontal_kernel)
|
| 185 |
+
band = cv2.dilate(horizontal, cv2.getStructuringElement(cv2.MORPH_RECT, (max(35, w // 14), max(9, h // 80))))
|
| 186 |
+
mask = (band > 0) & (lower > 0)
|
| 187 |
+
if mask.sum() < 0.03 * h * w:
|
| 188 |
+
mask = lower.astype(bool)
|
| 189 |
+
return mask
|
| 190 |
+
return lower.astype(bool)
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def heuristic_obstacle_mask(rgb: np.ndarray, boxes: list[tuple[float, float, float, float]], target_mask: np.ndarray) -> np.ndarray:
|
| 194 |
+
h, w = target_mask.shape
|
| 195 |
+
if boxes:
|
| 196 |
+
return boxes_to_mask(boxes, (h, w))
|
| 197 |
+
# Conservative central occluder prior for one-off experiments. It is not a
|
| 198 |
+
# replacement for a real LISA/Grounded-SAM obstacle mask.
|
| 199 |
+
return boxes_to_mask([(0.36, 0.32, 0.68, 0.88)], (h, w)) & cv2.dilate(
|
| 200 |
+
target_mask.astype(np.uint8),
|
| 201 |
+
cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (max(31, w // 24), max(31, h // 24))),
|
| 202 |
+
).astype(bool)
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
def row_span_amodal_mask(visible_mask: np.ndarray, obstacle_mask: np.ndarray, fallback_mask: np.ndarray) -> np.ndarray:
|
| 206 |
+
h, w = visible_mask.shape
|
| 207 |
+
rows = np.where(visible_mask.any(axis=1))[0]
|
| 208 |
+
if rows.size < 2:
|
| 209 |
+
return fallback_mask
|
| 210 |
+
|
| 211 |
+
lefts = np.full(h, np.nan, dtype=np.float32)
|
| 212 |
+
rights = np.full(h, np.nan, dtype=np.float32)
|
| 213 |
+
for y in rows:
|
| 214 |
+
xs = np.where(visible_mask[y])[0]
|
| 215 |
+
if xs.size:
|
| 216 |
+
lefts[y] = np.percentile(xs, 3)
|
| 217 |
+
rights[y] = np.percentile(xs, 97)
|
| 218 |
+
|
| 219 |
+
known = np.where(np.isfinite(lefts) & np.isfinite(rights))[0]
|
| 220 |
+
y_all = np.arange(h)
|
| 221 |
+
left_interp = np.interp(y_all, known, lefts[known])
|
| 222 |
+
right_interp = np.interp(y_all, known, rights[known])
|
| 223 |
+
|
| 224 |
+
margin = max(20, int(0.04 * w))
|
| 225 |
+
top = max(0, int(known.min() - 0.04 * h))
|
| 226 |
+
bottom = min(h, int(known.max() + 0.08 * h))
|
| 227 |
+
amodal = np.zeros((h, w), dtype=bool)
|
| 228 |
+
for y in range(top, bottom):
|
| 229 |
+
left = int(np.clip(left_interp[y] - margin, 0, w - 1))
|
| 230 |
+
right = int(np.clip(right_interp[y] + margin, left + 1, w))
|
| 231 |
+
amodal[y, left:right] = True
|
| 232 |
+
|
| 233 |
+
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (max(21, w // 48), max(21, h // 48)))
|
| 234 |
+
amodal = cv2.morphologyEx(amodal.astype(np.uint8), cv2.MORPH_CLOSE, kernel).astype(bool)
|
| 235 |
+
amodal |= visible_mask
|
| 236 |
+
return amodal
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def reviewed_geometry_support_prior(
|
| 240 |
+
reviewed_amodal: np.ndarray,
|
| 241 |
+
target_visible: np.ndarray,
|
| 242 |
+
obstacle_mask: np.ndarray,
|
| 243 |
+
*,
|
| 244 |
+
envelope_margin_fraction: float = 0.012,
|
| 245 |
+
maximum_hull_expansion_ratio: float = 2.0,
|
| 246 |
+
) -> tuple[np.ndarray, dict[str, Any]]:
|
| 247 |
+
"""Bridge reviewed occluder gaps without replacing the reviewed mask.
|
| 248 |
+
|
| 249 |
+
Accessibility annotations can be semantically correct yet retain a person-
|
| 250 |
+
shaped notch in the support surface. Raw triangulation then preserves that
|
| 251 |
+
notch as a hole. This helper builds a *geometry-only hypothesis*:
|
| 252 |
+
|
| 253 |
+
1. retain complete obstacle components that intersect the reviewed hidden
|
| 254 |
+
target;
|
| 255 |
+
2. form small component bounding envelopes, matching the 2D removal policy;
|
| 256 |
+
3. add only envelope pixels that lie inside a robust convex support hull.
|
| 257 |
+
|
| 258 |
+
The hull is derived from non-trivial reviewed components, so isolated mask
|
| 259 |
+
speckles cannot stretch it across the image. Highly concave/ambiguous
|
| 260 |
+
targets fail closed and keep the reviewed mask unchanged.
|
| 261 |
+
"""
|
| 262 |
+
|
| 263 |
+
amodal = np.asarray(reviewed_amodal, dtype=bool)
|
| 264 |
+
visible = np.asarray(target_visible, dtype=bool)
|
| 265 |
+
obstacle = np.asarray(obstacle_mask, dtype=bool)
|
| 266 |
+
if amodal.ndim != 2 or visible.ndim != 2 or obstacle.ndim != 2:
|
| 267 |
+
raise ValueError("reviewed amodal, visible, and obstacle masks must be 2D")
|
| 268 |
+
if amodal.shape != visible.shape or amodal.shape != obstacle.shape:
|
| 269 |
+
raise ValueError("reviewed amodal, visible, and obstacle masks must align")
|
| 270 |
+
if envelope_margin_fraction < 0:
|
| 271 |
+
raise ValueError("envelope_margin_fraction must be non-negative")
|
| 272 |
+
if maximum_hull_expansion_ratio < 1:
|
| 273 |
+
raise ValueError("maximum_hull_expansion_ratio must be at least 1")
|
| 274 |
+
|
| 275 |
+
reviewed = amodal | visible
|
| 276 |
+
hidden = reviewed & ~visible
|
| 277 |
+
visual_removal, removal_stats = build_visual_removal_mask(hidden, obstacle)
|
| 278 |
+
completion_envelope, envelope_stats = build_completion_envelope(
|
| 279 |
+
visual_removal,
|
| 280 |
+
obstacle,
|
| 281 |
+
margin_fraction=envelope_margin_fraction,
|
| 282 |
+
)
|
| 283 |
+
|
| 284 |
+
reviewed_pixels = int(reviewed.sum())
|
| 285 |
+
base_metadata: dict[str, Any] = {
|
| 286 |
+
"policy": (
|
| 287 |
+
"reviewed_mask_plus_occluder_envelope_inside_robust_convex_support_hull"
|
| 288 |
+
),
|
| 289 |
+
"reviewed_pixel_count": reviewed_pixels,
|
| 290 |
+
"reviewed_hidden_pixel_count": int(hidden.sum()),
|
| 291 |
+
"visual_removal_pixel_count": int(visual_removal.sum()),
|
| 292 |
+
"completion_envelope_pixel_count": int(completion_envelope.sum()),
|
| 293 |
+
"envelope_margin_fraction": float(envelope_margin_fraction),
|
| 294 |
+
"maximum_hull_expansion_ratio": float(maximum_hull_expansion_ratio),
|
| 295 |
+
"obstacle_components_retained": int(
|
| 296 |
+
removal_stats["obstacle_components_retained"]
|
| 297 |
+
),
|
| 298 |
+
"envelope_component_count": int(envelope_stats["component_count"]),
|
| 299 |
+
}
|
| 300 |
+
if reviewed_pixels == 0:
|
| 301 |
+
return reviewed.copy(), {
|
| 302 |
+
**base_metadata,
|
| 303 |
+
"status": "unchanged_empty_reviewed_mask",
|
| 304 |
+
"hull_source_pixel_count": 0,
|
| 305 |
+
"hull_pixel_count": 0,
|
| 306 |
+
"hull_expansion_ratio": None,
|
| 307 |
+
"added_support_pixel_count": 0,
|
| 308 |
+
}
|
| 309 |
+
|
| 310 |
+
count, labels, component_stats, _ = cv2.connectedComponentsWithStats(
|
| 311 |
+
reviewed.astype(np.uint8),
|
| 312 |
+
connectivity=8,
|
| 313 |
+
)
|
| 314 |
+
minimum_component_area = max(32, int(round(reviewed_pixels * 0.002)))
|
| 315 |
+
hull_source = np.zeros_like(reviewed)
|
| 316 |
+
retained_component_count = 0
|
| 317 |
+
for label in range(1, count):
|
| 318 |
+
area = int(component_stats[label, cv2.CC_STAT_AREA])
|
| 319 |
+
if area >= minimum_component_area:
|
| 320 |
+
hull_source |= labels == label
|
| 321 |
+
retained_component_count += 1
|
| 322 |
+
if not hull_source.any():
|
| 323 |
+
hull_source = reviewed.copy()
|
| 324 |
+
retained_component_count = max(0, count - 1)
|
| 325 |
+
|
| 326 |
+
ys, xs = np.where(hull_source)
|
| 327 |
+
points = np.column_stack([xs, ys]).astype(np.int32)
|
| 328 |
+
hull = np.zeros_like(reviewed, dtype=np.uint8)
|
| 329 |
+
if points.shape[0] >= 3:
|
| 330 |
+
cv2.fillConvexPoly(hull, cv2.convexHull(points), 1)
|
| 331 |
+
else:
|
| 332 |
+
hull[hull_source] = 1
|
| 333 |
+
hull_mask = hull.astype(bool)
|
| 334 |
+
hull_source_pixels = int(hull_source.sum())
|
| 335 |
+
hull_pixels = int(hull_mask.sum())
|
| 336 |
+
hull_ratio = hull_pixels / max(hull_source_pixels, 1)
|
| 337 |
+
|
| 338 |
+
metadata = {
|
| 339 |
+
**base_metadata,
|
| 340 |
+
"minimum_hull_component_area": minimum_component_area,
|
| 341 |
+
"hull_source_component_count": retained_component_count,
|
| 342 |
+
"hull_source_pixel_count": hull_source_pixels,
|
| 343 |
+
"hull_source_fraction_of_reviewed": round(
|
| 344 |
+
hull_source_pixels / reviewed_pixels,
|
| 345 |
+
8,
|
| 346 |
+
),
|
| 347 |
+
"hull_pixel_count": hull_pixels,
|
| 348 |
+
"hull_expansion_ratio": round(hull_ratio, 8),
|
| 349 |
+
}
|
| 350 |
+
if hull_ratio > maximum_hull_expansion_ratio:
|
| 351 |
+
return reviewed.copy(), {
|
| 352 |
+
**metadata,
|
| 353 |
+
"status": "unchanged_ambiguous_hull_expansion",
|
| 354 |
+
"added_support_pixel_count": 0,
|
| 355 |
+
}
|
| 356 |
+
|
| 357 |
+
added_support = completion_envelope & hull_mask & ~reviewed
|
| 358 |
+
support_prior = reviewed | added_support
|
| 359 |
+
return support_prior, {
|
| 360 |
+
**metadata,
|
| 361 |
+
"status": (
|
| 362 |
+
"augmented_occluded_support_hypothesis"
|
| 363 |
+
if added_support.any()
|
| 364 |
+
else "unchanged_no_supported_gap"
|
| 365 |
+
),
|
| 366 |
+
"added_support_pixel_count": int(added_support.sum()),
|
| 367 |
+
"support_prior_pixel_count": int(support_prior.sum()),
|
| 368 |
+
"support_prior_is_metric_truth": False,
|
| 369 |
+
"human_review_required": True,
|
| 370 |
+
}
|
| 371 |
+
|
| 372 |
+
|
| 373 |
+
def maybe_refine_masks_with_sam(
|
| 374 |
+
args,
|
| 375 |
+
rgb: np.ndarray,
|
| 376 |
+
target_visible: np.ndarray,
|
| 377 |
+
obstacle: np.ndarray,
|
| 378 |
+
output_dir: Path,
|
| 379 |
+
) -> tuple[np.ndarray, np.ndarray, str | None]:
|
| 380 |
+
if not args.sam_refine:
|
| 381 |
+
return target_visible, obstacle, None
|
| 382 |
+
if not args.sam_checkpoint:
|
| 383 |
+
message = 'SAM refinement requested but --sam-checkpoint was not provided'
|
| 384 |
+
if args.sam_fallback == 'error':
|
| 385 |
+
raise ValueError(message)
|
| 386 |
+
print(f'{message}; keeping coarse masks', file=sys.stderr)
|
| 387 |
+
return target_visible, obstacle, message
|
| 388 |
+
|
| 389 |
+
pre_target = output_dir / 'pre_sam_target_visible_mask.png'
|
| 390 |
+
pre_obstacle = output_dir / 'pre_sam_obstacle_mask.png'
|
| 391 |
+
save_mask(pre_target, target_visible)
|
| 392 |
+
save_mask(pre_obstacle, obstacle)
|
| 393 |
+
|
| 394 |
+
sam_output_dir = output_dir / 'sam_refine'
|
| 395 |
+
cmd = [
|
| 396 |
+
sys.executable,
|
| 397 |
+
'-m',
|
| 398 |
+
'accessibilityamodal.sam_refinement',
|
| 399 |
+
'--image', args.image,
|
| 400 |
+
'--target-mask', str(pre_target),
|
| 401 |
+
'--obstacle-mask', str(pre_obstacle),
|
| 402 |
+
'--output-dir', str(sam_output_dir),
|
| 403 |
+
'--sam-repo', args.sam_repo,
|
| 404 |
+
'--sam-checkpoint', args.sam_checkpoint,
|
| 405 |
+
'--sam-model-type', args.sam_model_type,
|
| 406 |
+
'--device', args.sam_device,
|
| 407 |
+
'--keep-target-components', str(args.keep_target_components),
|
| 408 |
+
'--keep-obstacle-components', str(args.keep_obstacle_components),
|
| 409 |
+
]
|
| 410 |
+
print('Running:', ' '.join(cmd))
|
| 411 |
+
try:
|
| 412 |
+
subprocess.run(cmd, check=True)
|
| 413 |
+
except Exception as exc:
|
| 414 |
+
if args.sam_fallback == 'error':
|
| 415 |
+
raise
|
| 416 |
+
message = f'SAM refinement failed; keeping coarse masks: {exc}'
|
| 417 |
+
print(message, file=sys.stderr)
|
| 418 |
+
return target_visible, obstacle, message
|
| 419 |
+
|
| 420 |
+
refined_target = read_mask(sam_output_dir / 'target_visible_mask_sam.png', rgb.shape[:2])
|
| 421 |
+
refined_obstacle = read_mask(sam_output_dir / 'obstacle_mask_sam.png', rgb.shape[:2])
|
| 422 |
+
return refined_target, refined_obstacle, f'SAM: {args.sam_model_type}'
|
| 423 |
+
|
| 424 |
+
|
| 425 |
+
def write_three_value_mask(
|
| 426 |
+
path: Path, target_visible: np.ndarray, amodal_mask: np.ndarray
|
| 427 |
+
) -> None:
|
| 428 |
+
mask = np.full(amodal_mask.shape, 255, dtype=np.uint8)
|
| 429 |
+
hidden = amodal_mask & ~target_visible
|
| 430 |
+
mask[target_visible] = 188
|
| 431 |
+
mask[hidden] = 0
|
| 432 |
+
Image.fromarray(mask).save(path)
|
| 433 |
+
|
| 434 |
+
|
| 435 |
+
def overlay(rgb: np.ndarray, masks: list[tuple[np.ndarray, tuple[int, int, int], float]]) -> np.ndarray:
|
| 436 |
+
out = rgb.astype(np.float32).copy()
|
| 437 |
+
for mask, color, alpha in masks:
|
| 438 |
+
if mask.any():
|
| 439 |
+
out[mask] = out[mask] * (1.0 - alpha) + np.array(color, dtype=np.float32) * alpha
|
| 440 |
+
return np.clip(out, 0, 255).astype(np.uint8)
|
| 441 |
+
|
| 442 |
+
|
| 443 |
+
def build_masks(args) -> dict[str, Path]:
|
| 444 |
+
output_dir = Path(args.output_dir)
|
| 445 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 446 |
+
rgb = read_rgb(args.image)
|
| 447 |
+
shape = rgb.shape[:2]
|
| 448 |
+
preset = ACCESSIBILITY_PRESETS[args.preset]
|
| 449 |
+
target_prompt = args.target_prompt or preset['target_prompt']
|
| 450 |
+
obstacle_prompt = args.obstacle_prompt or preset['obstacle_prompt']
|
| 451 |
+
lisa_image_path = output_dir / 'lisa_input_display_oriented.png'
|
| 452 |
+
if args.use_lisa:
|
| 453 |
+
Image.fromarray(rgb).save(lisa_image_path)
|
| 454 |
+
|
| 455 |
+
if args.target_visible_mask:
|
| 456 |
+
target_visible = read_mask(args.target_visible_mask, shape)
|
| 457 |
+
target_source = args.target_visible_mask
|
| 458 |
+
elif args.use_lisa and lisa_server_reachable(args.lisa_server_url):
|
| 459 |
+
target_visible = call_lisa(args.lisa_server_url, str(lisa_image_path), target_prompt, output_dir / 'lisa_target_visible', shape)
|
| 460 |
+
target_source = f'LISA: {target_prompt}'
|
| 461 |
+
else:
|
| 462 |
+
if args.use_lisa and args.lisa_fallback == 'error':
|
| 463 |
+
raise RuntimeError(f'LISA server is not reachable: {args.lisa_server_url}')
|
| 464 |
+
target_visible = heuristic_target_mask(rgb, args.preset)
|
| 465 |
+
target_source = 'heuristic target mask'
|
| 466 |
+
|
| 467 |
+
if not args.target_visible_mask:
|
| 468 |
+
target_visible = largest_components(
|
| 469 |
+
target_visible,
|
| 470 |
+
keep=args.keep_target_components,
|
| 471 |
+
min_area=max(64, shape[0] * shape[1] // 20000),
|
| 472 |
+
)
|
| 473 |
+
|
| 474 |
+
if args.obstacle_mask:
|
| 475 |
+
obstacle = read_mask(args.obstacle_mask, shape)
|
| 476 |
+
obstacle_source = args.obstacle_mask
|
| 477 |
+
elif args.use_lisa and lisa_server_reachable(args.lisa_server_url):
|
| 478 |
+
try:
|
| 479 |
+
obstacle = call_lisa(args.lisa_server_url, str(lisa_image_path), obstacle_prompt, output_dir / 'lisa_obstacle', shape)
|
| 480 |
+
obstacle_source = f'LISA: {obstacle_prompt}'
|
| 481 |
+
except Exception as exc:
|
| 482 |
+
if args.lisa_fallback == 'error':
|
| 483 |
+
raise
|
| 484 |
+
print(f'LISA obstacle segmentation failed, falling back to heuristic obstacle mask: {exc}', file=sys.stderr)
|
| 485 |
+
obstacle = heuristic_obstacle_mask(rgb, args.occlusion_box, target_visible)
|
| 486 |
+
obstacle_source = 'heuristic obstacle mask after LISA failure'
|
| 487 |
+
else:
|
| 488 |
+
if args.use_lisa and args.lisa_fallback == 'error':
|
| 489 |
+
raise RuntimeError(f'LISA server is not reachable: {args.lisa_server_url}')
|
| 490 |
+
obstacle = heuristic_obstacle_mask(rgb, args.occlusion_box, target_visible)
|
| 491 |
+
obstacle_source = 'heuristic obstacle mask'
|
| 492 |
+
|
| 493 |
+
if not args.obstacle_mask:
|
| 494 |
+
obstacle = largest_components(
|
| 495 |
+
obstacle,
|
| 496 |
+
keep=args.keep_obstacle_components,
|
| 497 |
+
min_area=max(64, shape[0] * shape[1] // 30000),
|
| 498 |
+
)
|
| 499 |
+
target_visible, obstacle, sam_source = maybe_refine_masks_with_sam(args, rgb, target_visible, obstacle, output_dir)
|
| 500 |
+
if sam_source:
|
| 501 |
+
target_source = f'{target_source} + {sam_source}'
|
| 502 |
+
obstacle_source = f'{obstacle_source} + {sam_source}'
|
| 503 |
+
# Occluders are never part of the support surface. This guard also fixes
|
| 504 |
+
# broad heuristic/LISA target masks that accidentally include a person,
|
| 505 |
+
# vehicle, railing, or clutter region.
|
| 506 |
+
target_visible &= ~obstacle
|
| 507 |
+
|
| 508 |
+
if args.target_amodal_mask:
|
| 509 |
+
provided_amodal = read_mask(args.target_amodal_mask, shape)
|
| 510 |
+
reviewed_amodal = provided_amodal | target_visible
|
| 511 |
+
amodal_support_prior, support_prior_metadata = (
|
| 512 |
+
reviewed_geometry_support_prior(
|
| 513 |
+
reviewed_amodal,
|
| 514 |
+
target_visible,
|
| 515 |
+
obstacle,
|
| 516 |
+
)
|
| 517 |
+
)
|
| 518 |
+
amodal = amodal_support_prior.copy()
|
| 519 |
+
hidden = amodal_support_prior & ~target_visible
|
| 520 |
+
amodal_source = args.target_amodal_mask
|
| 521 |
+
completion_policy = (
|
| 522 |
+
"reviewed target mask plus auditable occluder-constrained "
|
| 523 |
+
"geometry support hypothesis"
|
| 524 |
+
)
|
| 525 |
+
else:
|
| 526 |
+
fallback_target = heuristic_target_mask(rgb, args.preset)
|
| 527 |
+
# Row-span completion is only a structural support prior. Restrict hidden
|
| 528 |
+
# completion to observed obstacle pixels so disjoint stairs/walkways are not
|
| 529 |
+
# connected across unoccluded image regions.
|
| 530 |
+
amodal_support_prior = row_span_amodal_mask(
|
| 531 |
+
target_visible, obstacle, fallback_target
|
| 532 |
+
)
|
| 533 |
+
hidden = obstacle & amodal_support_prior & ~target_visible
|
| 534 |
+
amodal = target_visible | hidden
|
| 535 |
+
reviewed_amodal = amodal.copy()
|
| 536 |
+
amodal_source = "obstacle-constrained perspective baseline"
|
| 537 |
+
completion_policy = "obstacle-constrained row-span structural support prior"
|
| 538 |
+
support_prior_metadata = {
|
| 539 |
+
"policy": completion_policy,
|
| 540 |
+
"status": "heuristic_obstacle_constrained_support_prior",
|
| 541 |
+
"reviewed_pixel_count": int(amodal.sum()),
|
| 542 |
+
"added_support_pixel_count": 0,
|
| 543 |
+
"support_prior_pixel_count": int(amodal_support_prior.sum()),
|
| 544 |
+
"support_prior_is_metric_truth": False,
|
| 545 |
+
"human_review_required": True,
|
| 546 |
+
}
|
| 547 |
+
|
| 548 |
+
visual_3d_hidden = reviewed_amodal & ~target_visible
|
| 549 |
+
paths = {
|
| 550 |
+
'target_visible': output_dir / 'target_visible_mask.png',
|
| 551 |
+
'obstacle': output_dir / 'obstacle_mask.png',
|
| 552 |
+
'amodal': output_dir / 'amodal_accessibility_mask.png',
|
| 553 |
+
'reviewed_amodal': output_dir / 'reviewed_amodal_source_mask.png',
|
| 554 |
+
'hidden': output_dir / 'hidden_completion_mask.png',
|
| 555 |
+
'support_prior': output_dir / 'amodal_support_prior.png',
|
| 556 |
+
'visual_3d_condition': output_dir / 'amodal3d_three_value_mask.png',
|
| 557 |
+
'visual_3d_condition_compatibility': (
|
| 558 |
+
output_dir / 'accessibilityamodal_condition_mask.png'
|
| 559 |
+
),
|
| 560 |
+
'overlay': output_dir / 'mask_debug_overlay.png',
|
| 561 |
+
'manifest': output_dir / 'mask_manifest.json',
|
| 562 |
+
}
|
| 563 |
+
save_mask(paths['target_visible'], target_visible)
|
| 564 |
+
save_mask(paths['obstacle'], obstacle)
|
| 565 |
+
save_mask(paths['amodal'], amodal)
|
| 566 |
+
save_mask(paths['reviewed_amodal'], reviewed_amodal)
|
| 567 |
+
save_mask(paths['hidden'], hidden)
|
| 568 |
+
save_mask(paths['support_prior'], amodal_support_prior)
|
| 569 |
+
# Visual Accessibility3D conditioning must reflect only the reviewed support.
|
| 570 |
+
# The expanded support prior remains a geometry-only hypothesis consumed by
|
| 571 |
+
# reconstruct_steps and must not leak into the visual three-value mask.
|
| 572 |
+
write_three_value_mask(
|
| 573 |
+
paths['visual_3d_condition'],
|
| 574 |
+
target_visible,
|
| 575 |
+
reviewed_amodal,
|
| 576 |
+
)
|
| 577 |
+
shutil.copyfile(
|
| 578 |
+
paths['visual_3d_condition'],
|
| 579 |
+
paths['visual_3d_condition_compatibility'],
|
| 580 |
+
)
|
| 581 |
+
debug = overlay(rgb, [
|
| 582 |
+
(amodal, (0, 150, 255), 0.30),
|
| 583 |
+
(target_visible, (0, 220, 80), 0.45),
|
| 584 |
+
(hidden, (255, 40, 40), 0.65),
|
| 585 |
+
])
|
| 586 |
+
Image.fromarray(debug).save(paths['overlay'])
|
| 587 |
+
|
| 588 |
+
manifest = {
|
| 589 |
+
'sample_id': args.sample_id,
|
| 590 |
+
'image': args.image,
|
| 591 |
+
'preset': args.preset,
|
| 592 |
+
'source_preset': getattr(args, 'source_preset', args.preset),
|
| 593 |
+
'category_override_applied': getattr(args, 'category_override_applied', False),
|
| 594 |
+
'category': 'walkway' if args.preset == 'walkable' else args.preset,
|
| 595 |
+
'target_prompt': target_prompt,
|
| 596 |
+
'obstacle_prompt': obstacle_prompt,
|
| 597 |
+
'target_source': target_source,
|
| 598 |
+
'obstacle_source': obstacle_source,
|
| 599 |
+
'amodal_source': amodal_source,
|
| 600 |
+
'completion_policy': completion_policy,
|
| 601 |
+
'target_visible_pixel_count': int(target_visible.sum()),
|
| 602 |
+
'obstacle_pixel_count': int(obstacle.sum()),
|
| 603 |
+
'target_obstacle_overlap_pixel_count': int((target_visible & obstacle).sum()),
|
| 604 |
+
'reviewed_amodal_pixel_count': int(reviewed_amodal.sum()),
|
| 605 |
+
'reviewed_hidden_pixel_count': int(visual_3d_hidden.sum()),
|
| 606 |
+
'geometry_support_amodal_pixel_count': int(amodal.sum()),
|
| 607 |
+
'geometry_support_hidden_pixel_count': int(hidden.sum()),
|
| 608 |
+
# Compatibility counts retained for existing manifest consumers.
|
| 609 |
+
'hidden_pixel_count': int(hidden.sum()),
|
| 610 |
+
'amodal_pixel_count': int(amodal.sum()),
|
| 611 |
+
'geometry_support_prior': support_prior_metadata,
|
| 612 |
+
'mask_roles': {
|
| 613 |
+
'visual_3d_condition': {
|
| 614 |
+
'path': str(paths['visual_3d_condition']),
|
| 615 |
+
'compatibility_path': str(
|
| 616 |
+
paths['visual_3d_condition_compatibility']
|
| 617 |
+
),
|
| 618 |
+
'role': 'strict_reviewed_amodal_condition_for_amodal3d',
|
| 619 |
+
'source_mask': 'reviewed_amodal',
|
| 620 |
+
'uses_geometry_support_prior': False,
|
| 621 |
+
'pixel_values': {
|
| 622 |
+
'background': 255,
|
| 623 |
+
'visible_target': 188,
|
| 624 |
+
'hidden_target': 0,
|
| 625 |
+
},
|
| 626 |
+
'amodal_pixel_count': int(reviewed_amodal.sum()),
|
| 627 |
+
'hidden_pixel_count': int(visual_3d_hidden.sum()),
|
| 628 |
+
},
|
| 629 |
+
'geometry_support_prior': {
|
| 630 |
+
'path': str(paths['support_prior']),
|
| 631 |
+
'role': 'geometry_only_support_hypothesis',
|
| 632 |
+
'source_mask': 'reviewed_amodal',
|
| 633 |
+
'added_pixel_count': int(
|
| 634 |
+
(amodal_support_prior & ~reviewed_amodal).sum()
|
| 635 |
+
),
|
| 636 |
+
'amodal_pixel_count': int(amodal_support_prior.sum()),
|
| 637 |
+
'hidden_pixel_count': int(
|
| 638 |
+
(amodal_support_prior & ~target_visible).sum()
|
| 639 |
+
),
|
| 640 |
+
},
|
| 641 |
+
'reconstruct_steps_amodal_input': {
|
| 642 |
+
'path': str(paths['amodal']),
|
| 643 |
+
'role': 'geometry_support_input_for_reconstruct_steps',
|
| 644 |
+
'source_mask': 'amodal_support_prior',
|
| 645 |
+
'uses_geometry_support_prior': True,
|
| 646 |
+
},
|
| 647 |
+
},
|
| 648 |
+
'outputs': {key: str(value) for key, value in paths.items() if key != 'manifest'},
|
| 649 |
+
'external_repos_reviewed': {
|
| 650 |
+
'amodal': 'external backend; path supplied by user',
|
| 651 |
+
'Amodal-Wild': 'external backend; path supplied by user',
|
| 652 |
+
'diffusion-vas': 'external backend; path supplied by user',
|
| 653 |
+
'pix2gestalt': 'external backend; path supplied by user',
|
| 654 |
+
},
|
| 655 |
+
}
|
| 656 |
+
paths['manifest'].write_text(json.dumps(manifest, indent=2, ensure_ascii=False), encoding='utf-8')
|
| 657 |
+
return paths
|
| 658 |
+
|
| 659 |
+
|
| 660 |
+
def prepare_depth(args) -> tuple[str | None, float, str]:
|
| 661 |
+
if args.depth:
|
| 662 |
+
return args.depth, args.depth_scale, f'user depth: {args.depth}'
|
| 663 |
+
if not args.estimate_depth:
|
| 664 |
+
synthetic_kind = (
|
| 665 |
+
'synthetic perspective stair prior'
|
| 666 |
+
if args.preset == 'stairs'
|
| 667 |
+
else 'synthetic continuous perspective prior'
|
| 668 |
+
)
|
| 669 |
+
return None, args.depth_scale, synthetic_kind
|
| 670 |
+
|
| 671 |
+
output_dir = Path(args.output_dir)
|
| 672 |
+
depth_path = output_dir / 'estimated_depth.npy'
|
| 673 |
+
depth_vis_path = output_dir / 'estimated_depth_vis.png'
|
| 674 |
+
depth_manifest_path = output_dir / 'depth_manifest.json'
|
| 675 |
+
cmd = [
|
| 676 |
+
sys.executable,
|
| 677 |
+
'-m',
|
| 678 |
+
'accessibilityamodal.depth',
|
| 679 |
+
'--image', args.image,
|
| 680 |
+
'--output-depth', str(depth_path),
|
| 681 |
+
'--output-vis', str(depth_vis_path),
|
| 682 |
+
'--manifest', str(depth_manifest_path),
|
| 683 |
+
'--engine', args.depth_engine,
|
| 684 |
+
'--near', str(args.depth_near),
|
| 685 |
+
'--far', str(args.depth_far),
|
| 686 |
+
'--max-size', str(args.depth_max_size),
|
| 687 |
+
]
|
| 688 |
+
if args.depth_engine == 'transformers':
|
| 689 |
+
cmd.extend(['--model', args.depth_model, '--device', args.depth_device])
|
| 690 |
+
elif args.depth_engine == 'depth_anything_v2':
|
| 691 |
+
cmd.extend([
|
| 692 |
+
'--device', args.depth_device,
|
| 693 |
+
'--depth-anything-repo', args.depth_anything_repo,
|
| 694 |
+
'--encoder', args.depth_encoder,
|
| 695 |
+
'--input-size', str(args.depth_input_size),
|
| 696 |
+
])
|
| 697 |
+
if args.depth_checkpoint:
|
| 698 |
+
cmd.extend(['--checkpoint', args.depth_checkpoint])
|
| 699 |
+
if args.metric_depth:
|
| 700 |
+
cmd.extend(['--metric-depth', '--max-depth', str(args.max_depth)])
|
| 701 |
+
elif args.depth_engine == 'vggt':
|
| 702 |
+
cmd.extend([
|
| 703 |
+
'--device', args.depth_device,
|
| 704 |
+
'--vggt-repo', args.vggt_repo,
|
| 705 |
+
'--vggt-model', args.vggt_model,
|
| 706 |
+
'--vggt-input-size', str(args.vggt_input_size),
|
| 707 |
+
'--output-conf', str(output_dir / 'vggt_depth_conf.npy'),
|
| 708 |
+
'--output-conf-vis', str(output_dir / 'vggt_depth_conf.png'),
|
| 709 |
+
'--output-raw-depth', str(output_dir / 'vggt_raw_depth.npy'),
|
| 710 |
+
'--output-world-points', str(output_dir / 'vggt_world_points.npy'),
|
| 711 |
+
'--output-point-conf', str(output_dir / 'vggt_world_points_conf.npy'),
|
| 712 |
+
'--output-point-cloud', str(output_dir / 'vggt_point_cloud.ply'),
|
| 713 |
+
'--point-conf-threshold', str(args.vggt_point_conf_threshold),
|
| 714 |
+
'--max-point-cloud-points', str(args.vggt_max_point_cloud_points),
|
| 715 |
+
])
|
| 716 |
+
if args.vggt_checkpoint:
|
| 717 |
+
cmd.extend(['--vggt-checkpoint', args.vggt_checkpoint])
|
| 718 |
+
if args.vggt_keep_raw_depth:
|
| 719 |
+
cmd.append('--vggt-keep-raw-depth')
|
| 720 |
+
if args.invert_depth:
|
| 721 |
+
cmd.append('--invert-depth')
|
| 722 |
+
|
| 723 |
+
print('Running:', ' '.join(cmd))
|
| 724 |
+
try:
|
| 725 |
+
subprocess.run(cmd, check=True)
|
| 726 |
+
except Exception as exc:
|
| 727 |
+
if args.depth_fallback == 'error':
|
| 728 |
+
raise
|
| 729 |
+
print(f'Depth estimation failed, falling back to reconstruct_steps synthetic depth: {exc}', file=sys.stderr)
|
| 730 |
+
return None, args.depth_scale, f'depth estimation failed; synthetic fallback: {exc}'
|
| 731 |
+
depth_kind = f'estimated depth: {args.depth_engine}'
|
| 732 |
+
if args.metric_depth:
|
| 733 |
+
depth_kind += ' metric'
|
| 734 |
+
return str(depth_path), 1.0, depth_kind
|
| 735 |
+
|
| 736 |
+
|
| 737 |
+
def run_reconstruct(args, masks: dict[str, Path], depth_path: str | None, depth_scale: float) -> None:
|
| 738 |
+
if args.skip_reconstruct:
|
| 739 |
+
return
|
| 740 |
+
category = 'walkway' if args.preset == 'walkable' else args.preset
|
| 741 |
+
geometry_mode = (
|
| 742 |
+
'stairs' if args.preset == 'stairs'
|
| 743 |
+
else 'ramp' if args.preset == 'ramp'
|
| 744 |
+
else 'walkable'
|
| 745 |
+
)
|
| 746 |
+
cmd = [
|
| 747 |
+
sys.executable,
|
| 748 |
+
'-m',
|
| 749 |
+
'accessibilityamodal.reconstruct',
|
| 750 |
+
'--image', args.image,
|
| 751 |
+
'--obstacle-mask', str(masks['obstacle']),
|
| 752 |
+
'--amodal-mask', str(masks['amodal']),
|
| 753 |
+
'--visible-mask', str(masks['target_visible']),
|
| 754 |
+
'--geometry-mode', geometry_mode,
|
| 755 |
+
'--category', category,
|
| 756 |
+
'--output-dir', str(Path(args.output_dir) / 'geometry'),
|
| 757 |
+
]
|
| 758 |
+
if args.sample_id:
|
| 759 |
+
cmd.extend(['--sample-id', args.sample_id])
|
| 760 |
+
if args.reference_image:
|
| 761 |
+
cmd.extend(['--reference-image', args.reference_image])
|
| 762 |
+
completed_rgb = getattr(args, 'completed_rgb', None)
|
| 763 |
+
if completed_rgb:
|
| 764 |
+
cmd.extend(['--completed-rgb', completed_rgb])
|
| 765 |
+
if depth_path:
|
| 766 |
+
cmd.extend(['--depth', depth_path, '--depth-scale', str(depth_scale)])
|
| 767 |
+
if args.fx is not None:
|
| 768 |
+
cmd.extend(['--fx', str(args.fx)])
|
| 769 |
+
if args.fy is not None:
|
| 770 |
+
cmd.extend(['--fy', str(args.fy)])
|
| 771 |
+
if args.cx is not None:
|
| 772 |
+
cmd.extend(['--cx', str(args.cx)])
|
| 773 |
+
if args.cy is not None:
|
| 774 |
+
cmd.extend(['--cy', str(args.cy)])
|
| 775 |
+
print('Running:', ' '.join(cmd))
|
| 776 |
+
subprocess.run(cmd, check=True)
|
| 777 |
+
|
| 778 |
+
|
| 779 |
+
def write_pipeline_manifest(args, masks: dict[str, Path], depth_source: str, depth_path: str | None) -> None:
|
| 780 |
+
output_dir = Path(args.output_dir)
|
| 781 |
+
manifest = {
|
| 782 |
+
'sample_id': args.sample_id,
|
| 783 |
+
'image': args.image,
|
| 784 |
+
'reference_image': args.reference_image,
|
| 785 |
+
'completed_rgb': args.completed_rgb,
|
| 786 |
+
'preset': args.preset,
|
| 787 |
+
'source_preset': getattr(args, 'source_preset', args.preset),
|
| 788 |
+
'category_override_applied': getattr(args, 'category_override_applied', False),
|
| 789 |
+
'category': 'walkway' if args.preset == 'walkable' else args.preset,
|
| 790 |
+
'use_lisa': args.use_lisa,
|
| 791 |
+
'lisa_server_url': args.lisa_server_url if args.use_lisa else None,
|
| 792 |
+
'depth_source': depth_source,
|
| 793 |
+
'depth_path': depth_path,
|
| 794 |
+
'reconstruct_output_dir': None if args.skip_reconstruct else str(output_dir / 'geometry'),
|
| 795 |
+
'mask_outputs': {key: str(value) for key, value in masks.items()},
|
| 796 |
+
'note': 'Depth from monocular/heuristic sources is relative unless calibrated metric depth is provided.',
|
| 797 |
+
}
|
| 798 |
+
(output_dir / 'pipeline_manifest.json').write_text(json.dumps(manifest, indent=2, ensure_ascii=False), encoding='utf-8')
|
| 799 |
+
|
| 800 |
+
|
| 801 |
+
def build_parser():
|
| 802 |
+
parser = argparse.ArgumentParser(description='Build accessibility masks from text prompts and run 3D geometry completion.')
|
| 803 |
+
parser.add_argument('--image', default='./3R/2.jpg')
|
| 804 |
+
parser.add_argument('--sample-id', default=None)
|
| 805 |
+
parser.add_argument('--reference-image', default=None)
|
| 806 |
+
parser.add_argument(
|
| 807 |
+
'--completed-rgb',
|
| 808 |
+
default=None,
|
| 809 |
+
help='Aligned 2D completion used only to texture the reviewed hidden target region.',
|
| 810 |
+
)
|
| 811 |
+
parser.add_argument('--preset', choices=sorted(ACCESSIBILITY_PRESETS), default='stairs')
|
| 812 |
+
parser.add_argument(
|
| 813 |
+
'--category-overrides',
|
| 814 |
+
default=str(DEFAULT_CATEGORY_OVERRIDES_PATH),
|
| 815 |
+
help='Human-reviewed sample category overrides.',
|
| 816 |
+
)
|
| 817 |
+
parser.add_argument('--target-prompt', default=None)
|
| 818 |
+
parser.add_argument('--obstacle-prompt', default=None)
|
| 819 |
+
parser.add_argument('--target-visible-mask', default=None, help='Use an existing visible target mask instead of LISA/heuristic.')
|
| 820 |
+
parser.add_argument('--target-amodal-mask', default=None, help='Use a reviewed amodal target mask and bypass automatic mask completion.')
|
| 821 |
+
parser.add_argument('--obstacle-mask', default=None, help='Use an existing obstacle mask instead of LISA/heuristic.')
|
| 822 |
+
parser.add_argument('--occlusion-box', action='append', type=parse_box, default=[], help='Fallback obstacle box, normalized or pixel x1,y1,x2,y2.')
|
| 823 |
+
parser.add_argument('--use-lisa', action='store_true', help='Call a running LISA Gradio server for text-guided masks.')
|
| 824 |
+
parser.add_argument('--lisa-server-url', default='http://127.0.0.1:7860/')
|
| 825 |
+
parser.add_argument('--lisa-fallback', choices=['heuristic', 'error'], default='heuristic', help='What to do when a requested LISA mask cannot be obtained.')
|
| 826 |
+
parser.add_argument('--output-dir', default='./output/accessibility_pipeline/')
|
| 827 |
+
parser.add_argument('--keep-target-components', type=int, default=6)
|
| 828 |
+
parser.add_argument('--keep-obstacle-components', type=int, default=4)
|
| 829 |
+
parser.add_argument('--depth', default=None)
|
| 830 |
+
parser.add_argument('--depth-scale', type=float, default=1.0)
|
| 831 |
+
parser.add_argument('--estimate-depth', action='store_true', help='Generate a local relative depth map before reconstruction.')
|
| 832 |
+
parser.add_argument('--depth-engine', choices=['heuristic', 'transformers', 'depth_anything_v2', 'vggt'], default='heuristic', help='Local depth generator used with --estimate-depth.')
|
| 833 |
+
parser.add_argument('--depth-model', default='depth-anything/Depth-Anything-V2-Small-hf', help='Transformers depth model id or local path.')
|
| 834 |
+
parser.add_argument('--depth-device', default='auto', help='auto, cpu, cuda, cuda:0, or a transformers device string.')
|
| 835 |
+
parser.add_argument('--depth-anything-repo', default='../diffusion-vas/models/Depth_Anything_V2')
|
| 836 |
+
parser.add_argument('--depth-checkpoint', default=None, help='Local Depth Anything V2 .pth checkpoint.')
|
| 837 |
+
parser.add_argument('--depth-encoder', choices=['vits', 'vitb', 'vitl', 'vitg'], default='vitl')
|
| 838 |
+
parser.add_argument('--metric-depth', action='store_true', help='Use a metric Depth Anything V2 checkpoint and keep metric units.')
|
| 839 |
+
parser.add_argument('--max-depth', type=float, default=20.0, help='Metric depth max range in meters.')
|
| 840 |
+
parser.add_argument('--depth-input-size', type=int, default=518, help='Depth Anything V2 inference input size.')
|
| 841 |
+
parser.add_argument('--vggt-repo', default='vggt', help='Local facebookresearch/VGGT checkout.')
|
| 842 |
+
parser.add_argument('--vggt-model', default='facebook/VGGT-1B', help='Hugging Face model id or local VGGT model directory.')
|
| 843 |
+
parser.add_argument('--vggt-checkpoint', default=None, help='Optional local VGGT model.pt checkpoint.')
|
| 844 |
+
parser.add_argument('--vggt-input-size', type=int, default=518)
|
| 845 |
+
parser.add_argument('--vggt-keep-raw-depth', action='store_true', help='Do not near/far normalize VGGT depth before reconstruction.')
|
| 846 |
+
parser.add_argument('--vggt-point-conf-threshold', type=float, default=1.0)
|
| 847 |
+
parser.add_argument('--vggt-max-point-cloud-points', type=int, default=120000)
|
| 848 |
+
parser.add_argument('--depth-near', type=float, default=1.0)
|
| 849 |
+
parser.add_argument('--depth-far', type=float, default=6.0)
|
| 850 |
+
parser.add_argument('--depth-max-size', type=int, default=1280, help='Resize longest side for local depth generation. Use 0 to keep original size.')
|
| 851 |
+
parser.add_argument('--invert-depth', action='store_true', help='Invert estimated relative depth before normalization.')
|
| 852 |
+
parser.add_argument('--depth-fallback', choices=['synthetic', 'error'], default='synthetic', help='What to do if --estimate-depth fails.')
|
| 853 |
+
parser.add_argument('--fx', type=float, default=None)
|
| 854 |
+
parser.add_argument('--fy', type=float, default=None)
|
| 855 |
+
parser.add_argument('--cx', type=float, default=None)
|
| 856 |
+
parser.add_argument('--cy', type=float, default=None)
|
| 857 |
+
parser.add_argument('--sam-refine', action='store_true', help='Refine target/obstacle masks with Segment Anything box prompts.')
|
| 858 |
+
parser.add_argument('--sam-repo', default='../amodal/segment-anything')
|
| 859 |
+
parser.add_argument('--sam-checkpoint', default=None)
|
| 860 |
+
parser.add_argument('--sam-model-type', choices=['vit_h', 'vit_l', 'vit_b', 'default'], default='vit_h')
|
| 861 |
+
parser.add_argument('--sam-device', default='auto')
|
| 862 |
+
parser.add_argument('--sam-fallback', choices=['coarse', 'error'], default='coarse')
|
| 863 |
+
parser.add_argument('--skip-reconstruct', action='store_true')
|
| 864 |
+
return parser
|
| 865 |
+
|
| 866 |
+
|
| 867 |
+
def main():
|
| 868 |
+
args = build_parser().parse_args()
|
| 869 |
+
args.source_preset = args.preset
|
| 870 |
+
args.preset = resolve_reviewed_category(
|
| 871 |
+
args.sample_id or '',
|
| 872 |
+
args.preset,
|
| 873 |
+
load_reviewed_category_overrides(args.category_overrides),
|
| 874 |
+
)
|
| 875 |
+
if args.preset not in ACCESSIBILITY_PRESETS:
|
| 876 |
+
raise ValueError(f'Unsupported reviewed category override: {args.preset}')
|
| 877 |
+
args.category_override_applied = args.preset != args.source_preset
|
| 878 |
+
masks = build_masks(args)
|
| 879 |
+
depth_path, depth_scale, depth_source = prepare_depth(args)
|
| 880 |
+
write_pipeline_manifest(args, masks, depth_source, depth_path)
|
| 881 |
+
run_reconstruct(args, masks, depth_path, depth_scale)
|
| 882 |
+
print(f'Wrote mask outputs to {args.output_dir}')
|
| 883 |
+
|
| 884 |
+
|
| 885 |
+
if __name__ == '__main__':
|
| 886 |
+
main()
|
accessibilityamodal/reconstruct.py
ADDED
|
@@ -0,0 +1,1641 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Category-aware geometry-first depth completion for accessibility surfaces.
|
| 2 |
+
|
| 3 |
+
This script is intentionally separate from inference.py. Accessibility3D is a learned
|
| 4 |
+
single-object generator; stair/ramp/sidewalk completion needs explicit scene
|
| 5 |
+
geometry. The pipeline here is:
|
| 6 |
+
|
| 7 |
+
1. Load RGB image plus optional obstacle mask, amodal stair mask, and depth map.
|
| 8 |
+
2. Use stair bands only for stairs; fit a continuous surface for walkways and
|
| 9 |
+
ramps.
|
| 10 |
+
3. Extrapolate visible target depth only into the reviewed hidden region.
|
| 11 |
+
4. Export completed depth, confidence, point cloud, mesh, and debug overlays.
|
| 12 |
+
|
| 13 |
+
Mask convention:
|
| 14 |
+
- obstacle mask: white/nonzero marks the object hiding the stair.
|
| 15 |
+
- amodal mask: white/nonzero marks the whole stair/ramp target region, including
|
| 16 |
+
the occluded part.
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
from __future__ import annotations
|
| 20 |
+
|
| 21 |
+
import argparse
|
| 22 |
+
import json
|
| 23 |
+
from pathlib import Path
|
| 24 |
+
from typing import Iterable
|
| 25 |
+
|
| 26 |
+
import cv2
|
| 27 |
+
import numpy as np
|
| 28 |
+
from PIL import Image, ImageOps
|
| 29 |
+
|
| 30 |
+
from accessibilityamodal.geometry_analysis import build_accessibility_geometry_analysis
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
DEFAULT_OBSTACLE_BOX = (0.36, 0.32, 0.68, 0.88)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def parse_box(text: str) -> tuple[float, float, float, float]:
|
| 37 |
+
values = tuple(float(v) for v in text.split(','))
|
| 38 |
+
if len(values) != 4:
|
| 39 |
+
raise argparse.ArgumentTypeError('box must be x1,y1,x2,y2')
|
| 40 |
+
x1, y1, x2, y2 = values
|
| 41 |
+
if x2 <= x1 or y2 <= y1:
|
| 42 |
+
raise argparse.ArgumentTypeError('box must satisfy x2>x1 and y2>y1')
|
| 43 |
+
return values
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def read_rgb(path: str | Path, max_size: int | None = None) -> np.ndarray:
|
| 47 |
+
image = ImageOps.exif_transpose(Image.open(path)).convert('RGB')
|
| 48 |
+
if max_size and max(image.size) > max_size:
|
| 49 |
+
scale = max_size / max(image.size)
|
| 50 |
+
new_size = (round(image.size[0] * scale), round(image.size[1] * scale))
|
| 51 |
+
image = image.resize(new_size, Image.Resampling.LANCZOS)
|
| 52 |
+
return np.array(image)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def display_shape(path: str | Path) -> tuple[int, int]:
|
| 56 |
+
"""Return the canonical EXIF-normalized source HxW without resampling it."""
|
| 57 |
+
image = ImageOps.exif_transpose(Image.open(path)).convert('RGB')
|
| 58 |
+
return image.height, image.width
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _aspect_ratio_matches(source_shape: tuple[int, int], target_shape: tuple[int, int]) -> bool:
|
| 62 |
+
source_h, source_w = source_shape
|
| 63 |
+
target_h, target_w = target_shape
|
| 64 |
+
return abs(source_w * target_h - target_w * source_h) <= max(source_w, target_w)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def resize_mask(mask: np.ndarray, shape: tuple[int, int]) -> np.ndarray:
|
| 68 |
+
h, w = shape
|
| 69 |
+
if mask.shape[:2] != (h, w):
|
| 70 |
+
if not _aspect_ratio_matches(mask.shape[:2], (h, w)):
|
| 71 |
+
raise ValueError(
|
| 72 |
+
f'Mask/RGB raster mismatch: mask={mask.shape[:2]}, rgb={(h, w)}. '
|
| 73 |
+
'Refusing to resize across incompatible aspect ratios because this usually indicates EXIF-orientation misalignment.'
|
| 74 |
+
)
|
| 75 |
+
mask = cv2.resize(mask.astype(np.uint8), (w, h), interpolation=cv2.INTER_NEAREST)
|
| 76 |
+
return mask > 0
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def read_mask(path: str | Path, shape: tuple[int, int]) -> np.ndarray:
|
| 80 |
+
mask = np.array(ImageOps.exif_transpose(Image.open(path)).convert('L'))
|
| 81 |
+
return resize_mask(mask > 127, shape)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def read_source_grid_mask(
|
| 85 |
+
path: str | Path,
|
| 86 |
+
source_shape: tuple[int, int],
|
| 87 |
+
geometry_shape: tuple[int, int],
|
| 88 |
+
) -> np.ndarray:
|
| 89 |
+
"""Read a source-grid mask, then perform the one known shared downscale.
|
| 90 |
+
|
| 91 |
+
The normal pipeline creates target/obstacle masks on the normalized source
|
| 92 |
+
RGB grid. Requiring that exact grid here prevents a stale same-aspect mask
|
| 93 |
+
from being silently accepted merely because it can be resized.
|
| 94 |
+
"""
|
| 95 |
+
mask = np.array(ImageOps.exif_transpose(Image.open(path)).convert('L')) > 127
|
| 96 |
+
if mask.shape != source_shape:
|
| 97 |
+
raise ValueError(
|
| 98 |
+
f'Source-grid mask mismatch for {path}: mask={mask.shape}, source_rgb={source_shape}. '
|
| 99 |
+
'Expected a mask drawn on the canonical EXIF-normalized source grid before geometry downscaling.'
|
| 100 |
+
)
|
| 101 |
+
return resize_mask(mask, geometry_shape)
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def read_source_grid_rgb(
|
| 105 |
+
path: str | Path,
|
| 106 |
+
source_shape: tuple[int, int],
|
| 107 |
+
geometry_shape: tuple[int, int],
|
| 108 |
+
) -> np.ndarray:
|
| 109 |
+
"""Read an RGB completion on the exact source grid, then downscale once.
|
| 110 |
+
|
| 111 |
+
A generated completion is allowed to provide texture only when it is
|
| 112 |
+
aligned with the canonical, EXIF-normalized source image. This avoids
|
| 113 |
+
silently painting a mesh from a stale or rotated completion.
|
| 114 |
+
"""
|
| 115 |
+
image = ImageOps.exif_transpose(Image.open(path)).convert('RGB')
|
| 116 |
+
if (image.height, image.width) != source_shape:
|
| 117 |
+
raise ValueError(
|
| 118 |
+
f'Source-grid completed RGB mismatch for {path}: '
|
| 119 |
+
f'completed_rgb={(image.height, image.width)}, source_rgb={source_shape}. '
|
| 120 |
+
'Expected a completion on the canonical EXIF-normalized source grid.'
|
| 121 |
+
)
|
| 122 |
+
target_h, target_w = geometry_shape
|
| 123 |
+
if image.size != (target_w, target_h):
|
| 124 |
+
image = image.resize((target_w, target_h), Image.Resampling.LANCZOS)
|
| 125 |
+
return np.asarray(image)
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def boxes_to_mask(boxes: Iterable[tuple[float, float, float, float]], shape: tuple[int, int]) -> np.ndarray:
|
| 129 |
+
h, w = shape
|
| 130 |
+
mask = np.zeros((h, w), dtype=bool)
|
| 131 |
+
for x1, y1, x2, y2 in boxes:
|
| 132 |
+
if max(x1, y1, x2, y2) <= 1.0:
|
| 133 |
+
left, top, right, bottom = x1 * w, y1 * h, x2 * w, y2 * h
|
| 134 |
+
else:
|
| 135 |
+
left, top, right, bottom = x1, y1, x2, y2
|
| 136 |
+
left = int(np.clip(round(left), 0, w - 1))
|
| 137 |
+
right = int(np.clip(round(right), left + 1, w))
|
| 138 |
+
top = int(np.clip(round(top), 0, h - 1))
|
| 139 |
+
bottom = int(np.clip(round(bottom), top + 1, h))
|
| 140 |
+
mask[top:bottom, left:right] = True
|
| 141 |
+
return mask
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def default_amodal_mask(shape: tuple[int, int]) -> np.ndarray:
|
| 145 |
+
h, w = shape
|
| 146 |
+
mask = np.zeros((h, w), dtype=bool)
|
| 147 |
+
top = int(h * 0.08)
|
| 148 |
+
mask[top:, :] = True
|
| 149 |
+
return mask
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def save_mask(path: Path, mask: np.ndarray) -> None:
|
| 153 |
+
Image.fromarray((mask.astype(np.uint8) * 255)).save(path)
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def visualize_depth(depth: np.ndarray, valid: np.ndarray) -> np.ndarray:
|
| 157 |
+
vis = np.zeros_like(depth, dtype=np.float32)
|
| 158 |
+
values = depth[valid & np.isfinite(depth) & (depth > 0)]
|
| 159 |
+
if values.size == 0:
|
| 160 |
+
return np.zeros((*depth.shape, 3), dtype=np.uint8)
|
| 161 |
+
lo, hi = np.percentile(values, [2, 98])
|
| 162 |
+
if hi <= lo:
|
| 163 |
+
hi = lo + 1.0
|
| 164 |
+
vis = np.clip((depth - lo) / (hi - lo), 0, 1)
|
| 165 |
+
vis[~valid] = 0
|
| 166 |
+
colored = cv2.applyColorMap((vis * 255).astype(np.uint8), cv2.COLORMAP_TURBO)
|
| 167 |
+
colored[~valid] = 0
|
| 168 |
+
return cv2.cvtColor(colored, cv2.COLOR_BGR2RGB)
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def line_endpoints_on_mask(
|
| 172 |
+
y_center: float,
|
| 173 |
+
slope: float,
|
| 174 |
+
mask: np.ndarray,
|
| 175 |
+
) -> tuple[tuple[int, int], tuple[int, int]] | None:
|
| 176 |
+
h, w = mask.shape
|
| 177 |
+
x_center = (w - 1) / 2.0
|
| 178 |
+
xs = np.arange(w, dtype=np.float32)
|
| 179 |
+
ys = np.rint(y_center + slope * (xs - x_center)).astype(np.int32)
|
| 180 |
+
in_frame = (ys >= 0) & (ys < h)
|
| 181 |
+
if not np.any(in_frame):
|
| 182 |
+
return None
|
| 183 |
+
valid_xs = xs[in_frame].astype(np.int32)
|
| 184 |
+
valid_ys = ys[in_frame]
|
| 185 |
+
on_mask = mask[valid_ys, valid_xs]
|
| 186 |
+
if np.any(on_mask):
|
| 187 |
+
valid_xs = valid_xs[on_mask]
|
| 188 |
+
valid_ys = valid_ys[on_mask]
|
| 189 |
+
left_index = int(np.argmin(valid_xs))
|
| 190 |
+
right_index = int(np.argmax(valid_xs))
|
| 191 |
+
return (
|
| 192 |
+
(int(valid_xs[left_index]), int(valid_ys[left_index])),
|
| 193 |
+
(int(valid_xs[right_index]), int(valid_ys[right_index])),
|
| 194 |
+
)
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def save_overlay(
|
| 198 |
+
path: Path,
|
| 199 |
+
rgb: np.ndarray,
|
| 200 |
+
obstacle_mask: np.ndarray,
|
| 201 |
+
amodal_mask: np.ndarray,
|
| 202 |
+
edges_y: list[int],
|
| 203 |
+
edge_slope: float = 0.0,
|
| 204 |
+
) -> None:
|
| 205 |
+
overlay = rgb.copy()
|
| 206 |
+
overlay[amodal_mask] = (0.65 * overlay[amodal_mask] + 0.35 * np.array([0, 160, 255])).astype(np.uint8)
|
| 207 |
+
overlay[obstacle_mask] = (0.45 * overlay[obstacle_mask] + 0.55 * np.array([255, 60, 20])).astype(np.uint8)
|
| 208 |
+
for y in edges_y:
|
| 209 |
+
y = int(np.clip(y, 0, overlay.shape[0] - 1))
|
| 210 |
+
endpoints = line_endpoints_on_mask(float(y), edge_slope, amodal_mask)
|
| 211 |
+
if endpoints is None:
|
| 212 |
+
endpoints = ((0, y), (overlay.shape[1] - 1, y))
|
| 213 |
+
cv2.line(overlay, endpoints[0], endpoints[1], (80, 255, 80), 2)
|
| 214 |
+
Image.fromarray(overlay).save(path)
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def read_depth(path: str | Path, shape: tuple[int, int], depth_scale: float) -> np.ndarray:
|
| 218 |
+
path = Path(path)
|
| 219 |
+
if path.suffix.lower() == '.npy':
|
| 220 |
+
depth = np.load(path).astype(np.float32)
|
| 221 |
+
elif path.suffix.lower() == '.npz':
|
| 222 |
+
data = np.load(path)
|
| 223 |
+
depth = data[data.files[0]].astype(np.float32)
|
| 224 |
+
else:
|
| 225 |
+
raw = cv2.imread(str(path), cv2.IMREAD_UNCHANGED)
|
| 226 |
+
if raw is None:
|
| 227 |
+
raise FileNotFoundError(path)
|
| 228 |
+
if raw.ndim == 3:
|
| 229 |
+
raw = cv2.cvtColor(raw, cv2.COLOR_BGR2GRAY)
|
| 230 |
+
depth = raw.astype(np.float32)
|
| 231 |
+
h, w = shape
|
| 232 |
+
if depth.shape[:2] != (h, w):
|
| 233 |
+
if not _aspect_ratio_matches(depth.shape[:2], (h, w)):
|
| 234 |
+
raise ValueError(
|
| 235 |
+
f'Depth/RGB raster mismatch: depth={depth.shape[:2]}, rgb={(h, w)}. '
|
| 236 |
+
'Refusing to resize across incompatible aspect ratios because this usually indicates orientation or source mismatch.'
|
| 237 |
+
)
|
| 238 |
+
depth = cv2.resize(depth, (w, h), interpolation=cv2.INTER_LINEAR)
|
| 239 |
+
depth *= depth_scale
|
| 240 |
+
return depth
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def detect_stair_edge_model(
|
| 244 |
+
rgb: np.ndarray,
|
| 245 |
+
roi: np.ndarray,
|
| 246 |
+
max_edges: int,
|
| 247 |
+
min_gap_ratio: float = 0.025,
|
| 248 |
+
min_line_ratio: float = 0.10,
|
| 249 |
+
maximum_angle_degrees: float = 25.0,
|
| 250 |
+
hough_threshold_ratio: float = 0.02,
|
| 251 |
+
max_line_gap_ratio: float = 0.04,
|
| 252 |
+
) -> tuple[list[int], float]:
|
| 253 |
+
h, w = roi.shape
|
| 254 |
+
x_center = (w - 1) / 2.0
|
| 255 |
+
gray = cv2.cvtColor(rgb, cv2.COLOR_RGB2GRAY)
|
| 256 |
+
gray = cv2.GaussianBlur(gray, (5, 5), 0)
|
| 257 |
+
median = float(np.median(gray[roi])) if np.any(roi) else float(np.median(gray))
|
| 258 |
+
lower = int(max(20, 0.66 * median))
|
| 259 |
+
upper = int(min(220, 1.33 * median + 20))
|
| 260 |
+
edges = cv2.Canny(gray, lower, upper)
|
| 261 |
+
edges[~roi] = 0
|
| 262 |
+
min_len = max(25, int(w * min_line_ratio))
|
| 263 |
+
lines = cv2.HoughLinesP(
|
| 264 |
+
edges,
|
| 265 |
+
rho=1,
|
| 266 |
+
theta=np.pi / 180,
|
| 267 |
+
threshold=max(20, int(w * hough_threshold_ratio)),
|
| 268 |
+
minLineLength=min_len,
|
| 269 |
+
maxLineGap=max(12, int(w * max_line_gap_ratio)),
|
| 270 |
+
)
|
| 271 |
+
if lines is None:
|
| 272 |
+
return [], 0.0
|
| 273 |
+
|
| 274 |
+
candidates: list[tuple[float, float, float]] = []
|
| 275 |
+
for line in lines[:, 0, :]:
|
| 276 |
+
x1, y1, x2, y2 = line.astype(float)
|
| 277 |
+
dx = x2 - x1
|
| 278 |
+
dy = y2 - y1
|
| 279 |
+
length = float(np.hypot(dx, dy))
|
| 280 |
+
if length < min_len:
|
| 281 |
+
continue
|
| 282 |
+
if abs(dx) < 1e-6:
|
| 283 |
+
continue
|
| 284 |
+
angle = abs(np.degrees(np.arctan2(dy, dx)))
|
| 285 |
+
angle = min(angle, abs(180 - angle))
|
| 286 |
+
if angle > maximum_angle_degrees:
|
| 287 |
+
continue
|
| 288 |
+
slope = dy / dx
|
| 289 |
+
y_at_center = y1 + slope * (x_center - x1)
|
| 290 |
+
if not (0.06 * h <= y_at_center <= 0.97 * h):
|
| 291 |
+
continue
|
| 292 |
+
candidates.append((y_at_center, slope, length))
|
| 293 |
+
|
| 294 |
+
if not candidates:
|
| 295 |
+
return [], 0.0
|
| 296 |
+
|
| 297 |
+
candidates.sort(key=lambda item: item[0])
|
| 298 |
+
min_gap = max(12, int(h * min_gap_ratio))
|
| 299 |
+
clusters: list[list[tuple[float, float, float]]] = []
|
| 300 |
+
for item in candidates:
|
| 301 |
+
if not clusters or item[0] - clusters[-1][-1][0] > min_gap:
|
| 302 |
+
clusters.append([item])
|
| 303 |
+
else:
|
| 304 |
+
clusters[-1].append(item)
|
| 305 |
+
|
| 306 |
+
merged: list[tuple[int, float, float]] = []
|
| 307 |
+
for cluster in clusters:
|
| 308 |
+
weights = np.array([item[2] for item in cluster], dtype=np.float32)
|
| 309 |
+
ys = np.array([item[0] for item in cluster], dtype=np.float32)
|
| 310 |
+
slopes = np.array([item[1] for item in cluster], dtype=np.float32)
|
| 311 |
+
merged.append((
|
| 312 |
+
int(round(float(np.average(ys, weights=weights)))),
|
| 313 |
+
float(np.average(slopes, weights=weights)),
|
| 314 |
+
float(weights.sum()),
|
| 315 |
+
))
|
| 316 |
+
|
| 317 |
+
merged.sort(key=lambda item: item[2], reverse=True)
|
| 318 |
+
selected = sorted(merged[:max_edges], key=lambda item: item[0])
|
| 319 |
+
if not selected:
|
| 320 |
+
return [], 0.0
|
| 321 |
+
weights = np.array([item[2] for item in selected], dtype=np.float32)
|
| 322 |
+
slopes = np.array([item[1] for item in selected], dtype=np.float32)
|
| 323 |
+
maximum_slope = float(np.tan(np.radians(maximum_angle_degrees)))
|
| 324 |
+
dominant_slope = float(np.clip(np.average(slopes, weights=weights), -maximum_slope, maximum_slope))
|
| 325 |
+
return [int(item[0]) for item in selected], dominant_slope
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
def detect_horizontal_edges(
|
| 329 |
+
rgb: np.ndarray,
|
| 330 |
+
roi: np.ndarray,
|
| 331 |
+
max_edges: int,
|
| 332 |
+
min_gap_ratio: float = 0.025,
|
| 333 |
+
min_line_ratio: float = 0.10,
|
| 334 |
+
maximum_angle_degrees: float = 25.0,
|
| 335 |
+
hough_threshold_ratio: float = 0.02,
|
| 336 |
+
max_line_gap_ratio: float = 0.04,
|
| 337 |
+
) -> list[int]:
|
| 338 |
+
edges_y, _ = detect_stair_edge_model(
|
| 339 |
+
rgb,
|
| 340 |
+
roi,
|
| 341 |
+
max_edges,
|
| 342 |
+
min_gap_ratio=min_gap_ratio,
|
| 343 |
+
min_line_ratio=min_line_ratio,
|
| 344 |
+
maximum_angle_degrees=maximum_angle_degrees,
|
| 345 |
+
hough_threshold_ratio=hough_threshold_ratio,
|
| 346 |
+
max_line_gap_ratio=max_line_gap_ratio,
|
| 347 |
+
)
|
| 348 |
+
return edges_y
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
def detect_horizontal_gradient_peaks(
|
| 352 |
+
rgb: np.ndarray,
|
| 353 |
+
roi: np.ndarray,
|
| 354 |
+
max_edges: int,
|
| 355 |
+
min_gap_ratio: float = 0.018,
|
| 356 |
+
) -> list[int]:
|
| 357 |
+
"""Recover stair tread rows when short perspective lines defeat Hough voting.
|
| 358 |
+
|
| 359 |
+
This detector is deliberately a secondary fallback: it aggregates vertical
|
| 360 |
+
image gradients only inside the reviewed visible stair mask and is used only
|
| 361 |
+
when the line detector found too few edges.
|
| 362 |
+
"""
|
| 363 |
+
h, w = roi.shape
|
| 364 |
+
gray = cv2.cvtColor(rgb, cv2.COLOR_RGB2GRAY)
|
| 365 |
+
gray = cv2.GaussianBlur(gray, (5, 5), 0)
|
| 366 |
+
vertical_gradient = np.abs(cv2.Sobel(gray, cv2.CV_32F, 0, 1, ksize=3))
|
| 367 |
+
scores = np.zeros(h, dtype=np.float32)
|
| 368 |
+
widths = roi.sum(axis=1)
|
| 369 |
+
minimum_row_pixels = max(8, int(w * 0.012))
|
| 370 |
+
for y in range(h):
|
| 371 |
+
values = vertical_gradient[y, roi[y]]
|
| 372 |
+
if values.size < minimum_row_pixels:
|
| 373 |
+
continue
|
| 374 |
+
strongest_count = max(1, int(values.size * 0.30))
|
| 375 |
+
scores[y] = float(np.partition(values, -strongest_count)[-strongest_count:].mean())
|
| 376 |
+
scores = cv2.GaussianBlur(scores[:, None], (1, 7), 0)[:, 0]
|
| 377 |
+
eligible = widths >= minimum_row_pixels
|
| 378 |
+
valid_scores = scores[eligible]
|
| 379 |
+
if valid_scores.size == 0 or float(valid_scores.max()) <= 0:
|
| 380 |
+
return []
|
| 381 |
+
threshold = max(
|
| 382 |
+
float(np.percentile(valid_scores, 60)),
|
| 383 |
+
float(np.median(valid_scores) + 0.25 * np.std(valid_scores)),
|
| 384 |
+
)
|
| 385 |
+
candidates = [
|
| 386 |
+
y
|
| 387 |
+
for y in range(2, h - 2)
|
| 388 |
+
if eligible[y]
|
| 389 |
+
and scores[y] >= threshold
|
| 390 |
+
and scores[y] >= float(scores[y - 2 : y + 3].max())
|
| 391 |
+
]
|
| 392 |
+
minimum_gap = max(8, int(h * min_gap_ratio))
|
| 393 |
+
selected: list[int] = []
|
| 394 |
+
for y in sorted(candidates, key=lambda row: float(scores[row]), reverse=True):
|
| 395 |
+
if all(abs(y - other) > minimum_gap for other in selected):
|
| 396 |
+
selected.append(y)
|
| 397 |
+
if len(selected) >= max_edges:
|
| 398 |
+
break
|
| 399 |
+
return sorted(selected)
|
| 400 |
+
|
| 401 |
+
|
| 402 |
+
def detect_depth_gradient_stair_edges(
|
| 403 |
+
depth: np.ndarray,
|
| 404 |
+
roi: np.ndarray,
|
| 405 |
+
max_edges: int,
|
| 406 |
+
min_gap_ratio: float = 0.020,
|
| 407 |
+
) -> list[int]:
|
| 408 |
+
"""Detect stair edges from depth map vertical gradient peaks.
|
| 409 |
+
|
| 410 |
+
This is a tertiary fallback for scenes where RGB Hough lines are
|
| 411 |
+
insufficient (e.g. distant/crowded stairs). Depth maps from
|
| 412 |
+
Depth Anything V2 often show clear discontinuities at step boundaries.
|
| 413 |
+
"""
|
| 414 |
+
h, w = roi.shape
|
| 415 |
+
if not np.any(roi) or not np.any(np.isfinite(depth[roi])):
|
| 416 |
+
return []
|
| 417 |
+
depth_f = depth.astype(np.float32).copy()
|
| 418 |
+
depth_f[~np.isfinite(depth_f)] = 0.0
|
| 419 |
+
depth_smooth = cv2.GaussianBlur(depth_f, (7, 7), 0)
|
| 420 |
+
vertical_grad = np.abs(cv2.Sobel(depth_smooth, cv2.CV_32F, 0, 1, ksize=5))
|
| 421 |
+
scores = np.zeros(h, dtype=np.float32)
|
| 422 |
+
widths = roi.sum(axis=1)
|
| 423 |
+
min_row_px = max(6, int(w * 0.01))
|
| 424 |
+
for y in range(h):
|
| 425 |
+
vals = vertical_grad[y, roi[y]]
|
| 426 |
+
if vals.size < min_row_px:
|
| 427 |
+
continue
|
| 428 |
+
top_count = max(1, int(vals.size * 0.25))
|
| 429 |
+
scores[y] = float(np.partition(vals, -top_count)[-top_count:].mean())
|
| 430 |
+
scores = cv2.GaussianBlur(scores[:, None], (1, 9), 0)[:, 0]
|
| 431 |
+
eligible = widths >= min_row_px
|
| 432 |
+
valid_scores = scores[eligible]
|
| 433 |
+
if valid_scores.size == 0 or float(valid_scores.max()) <= 0:
|
| 434 |
+
return []
|
| 435 |
+
threshold = max(
|
| 436 |
+
float(np.percentile(valid_scores, 65)),
|
| 437 |
+
float(np.median(valid_scores) + 0.3 * np.std(valid_scores)),
|
| 438 |
+
)
|
| 439 |
+
candidates = [
|
| 440 |
+
y for y in range(2, h - 2)
|
| 441 |
+
if eligible[y]
|
| 442 |
+
and scores[y] >= threshold
|
| 443 |
+
and scores[y] >= float(scores[y - 2 : y + 3].max())
|
| 444 |
+
]
|
| 445 |
+
min_gap = max(8, int(h * min_gap_ratio))
|
| 446 |
+
selected: list[int] = []
|
| 447 |
+
for y in sorted(candidates, key=lambda row: float(scores[row]), reverse=True):
|
| 448 |
+
if all(abs(y - other) > min_gap for other in selected):
|
| 449 |
+
selected.append(y)
|
| 450 |
+
if len(selected) >= max_edges:
|
| 451 |
+
break
|
| 452 |
+
return sorted(selected)
|
| 453 |
+
|
| 454 |
+
|
| 455 |
+
def stair_edge_coverage_ratio(edges_y: list[int], mask: np.ndarray) -> float:
|
| 456 |
+
rows = np.where(mask)[0]
|
| 457 |
+
if len(edges_y) < 2 or rows.size == 0:
|
| 458 |
+
return 0.0
|
| 459 |
+
target_span = max(int(rows.max() - rows.min()), 1)
|
| 460 |
+
return float((max(edges_y) - min(edges_y)) / target_span)
|
| 461 |
+
|
| 462 |
+
|
| 463 |
+
def fallback_edges(mask: np.ndarray, count: int) -> list[int]:
|
| 464 |
+
ys = np.where(mask)[0]
|
| 465 |
+
h = mask.shape[0]
|
| 466 |
+
if ys.size == 0:
|
| 467 |
+
top, bottom = int(0.15 * h), int(0.92 * h)
|
| 468 |
+
else:
|
| 469 |
+
top, bottom = int(np.percentile(ys, 8)), int(np.percentile(ys, 96))
|
| 470 |
+
if bottom <= top:
|
| 471 |
+
return []
|
| 472 |
+
return [int(round(v)) for v in np.linspace(top, bottom, count + 2)[1:-1]]
|
| 473 |
+
|
| 474 |
+
|
| 475 |
+
def merge_edge_positions(edges: Iterable[int], min_gap: int) -> list[int]:
|
| 476 |
+
values = sorted(int(v) for v in edges)
|
| 477 |
+
if not values:
|
| 478 |
+
return []
|
| 479 |
+
clusters: list[list[int]] = []
|
| 480 |
+
for value in values:
|
| 481 |
+
if not clusters or value - clusters[-1][-1] > min_gap:
|
| 482 |
+
clusters.append([value])
|
| 483 |
+
else:
|
| 484 |
+
clusters[-1].append(value)
|
| 485 |
+
return [int(round(float(np.mean(cluster)))) for cluster in clusters]
|
| 486 |
+
|
| 487 |
+
|
| 488 |
+
def make_step_prior_depth(shape: tuple[int, int], edges_y: list[int], near: float, far: float) -> np.ndarray:
|
| 489 |
+
h, w = shape
|
| 490 |
+
yy = np.linspace(0, 1, h, dtype=np.float32)[:, None]
|
| 491 |
+
base = near + (1.0 - yy) * (far - near)
|
| 492 |
+
depth = np.repeat(base, w, axis=1)
|
| 493 |
+
|
| 494 |
+
boundaries = [0] + sorted(int(y) for y in edges_y if 0 < y < h - 1) + [h]
|
| 495 |
+
if len(boundaries) > 2:
|
| 496 |
+
n_bands = len(boundaries) - 1
|
| 497 |
+
for band_idx, (top, bottom) in enumerate(zip(boundaries[:-1], boundaries[1:])):
|
| 498 |
+
band_height = max(1, bottom - top)
|
| 499 |
+
local = np.linspace(0, 1, band_height, dtype=np.float32)[:, None]
|
| 500 |
+
# Top bands are farther away; lower bands are closer. Keeping a
|
| 501 |
+
# slight within-band slope avoids fully flat cardboard strips.
|
| 502 |
+
band_far = far - (far - near) * band_idx / n_bands
|
| 503 |
+
band_near = far - (far - near) * (band_idx + 0.72) / n_bands
|
| 504 |
+
depth[top:bottom, :] = band_far + local * (band_near - band_far)
|
| 505 |
+
return depth.astype(np.float32)
|
| 506 |
+
|
| 507 |
+
|
| 508 |
+
def make_continuous_prior_depth(shape: tuple[int, int], near: float, far: float) -> np.ndarray:
|
| 509 |
+
h, w = shape
|
| 510 |
+
yy = np.linspace(0.0, 1.0, h, dtype=np.float32)[:, None]
|
| 511 |
+
depth = far - yy * (far - near)
|
| 512 |
+
return np.repeat(depth, w, axis=1).astype(np.float32)
|
| 513 |
+
|
| 514 |
+
|
| 515 |
+
def camera_intrinsics(w: int, h: int, fx: float | None, fy: float | None, cx: float | None, cy: float | None):
|
| 516 |
+
default_f = float(max(w, h))
|
| 517 |
+
fx = default_f if fx is None else fx
|
| 518 |
+
fy = default_f if fy is None else fy
|
| 519 |
+
cx = (w - 1) / 2.0 if cx is None else cx
|
| 520 |
+
cy = (h - 1) / 2.0 if cy is None else cy
|
| 521 |
+
return fx, fy, cx, cy
|
| 522 |
+
|
| 523 |
+
|
| 524 |
+
def pixels_to_points(depth: np.ndarray, fx: float, fy: float, cx: float, cy: float):
|
| 525 |
+
h, w = depth.shape
|
| 526 |
+
xs, ys = np.meshgrid(np.arange(w, dtype=np.float32), np.arange(h, dtype=np.float32))
|
| 527 |
+
z = depth.astype(np.float32)
|
| 528 |
+
x = (xs - cx) * z / fx
|
| 529 |
+
y = (ys - cy) * z / fy
|
| 530 |
+
return np.stack([x, y, z], axis=-1)
|
| 531 |
+
|
| 532 |
+
|
| 533 |
+
def fit_plane(points: np.ndarray):
|
| 534 |
+
if points.shape[0] < 50:
|
| 535 |
+
return None
|
| 536 |
+
centroid = points.mean(axis=0)
|
| 537 |
+
centered = points - centroid
|
| 538 |
+
try:
|
| 539 |
+
_, _, vh = np.linalg.svd(centered, full_matrices=False)
|
| 540 |
+
except np.linalg.LinAlgError:
|
| 541 |
+
return None
|
| 542 |
+
normal = vh[-1]
|
| 543 |
+
norm = np.linalg.norm(normal)
|
| 544 |
+
if norm < 1e-6:
|
| 545 |
+
return None
|
| 546 |
+
normal = normal / norm
|
| 547 |
+
d = -float(np.dot(normal, centroid))
|
| 548 |
+
return normal.astype(np.float32), d
|
| 549 |
+
|
| 550 |
+
|
| 551 |
+
def intersect_plane_for_pixels(shape: tuple[int, int], plane, fx: float, fy: float, cx: float, cy: float) -> np.ndarray:
|
| 552 |
+
h, w = shape
|
| 553 |
+
normal, d = plane
|
| 554 |
+
xs, ys = np.meshgrid(np.arange(w, dtype=np.float32), np.arange(h, dtype=np.float32))
|
| 555 |
+
rays = np.stack([(xs - cx) / fx, (ys - cy) / fy, np.ones((h, w), dtype=np.float32)], axis=-1)
|
| 556 |
+
denom = rays @ normal
|
| 557 |
+
with np.errstate(divide='ignore', invalid='ignore'):
|
| 558 |
+
t = -d / denom
|
| 559 |
+
t[~np.isfinite(t)] = 0
|
| 560 |
+
t[t <= 0] = 0
|
| 561 |
+
return t.astype(np.float32)
|
| 562 |
+
|
| 563 |
+
|
| 564 |
+
def inpaint_depth(depth: np.ndarray, fill_mask: np.ndarray, valid: np.ndarray) -> np.ndarray:
|
| 565 |
+
values = depth[valid & np.isfinite(depth) & (depth > 0)]
|
| 566 |
+
if values.size == 0:
|
| 567 |
+
return depth.copy()
|
| 568 |
+
lo, hi = np.percentile(values, [1, 99])
|
| 569 |
+
if hi <= lo:
|
| 570 |
+
hi = lo + 1.0
|
| 571 |
+
normalized = np.clip((depth - lo) / (hi - lo), 0, 1)
|
| 572 |
+
normalized[~np.isfinite(normalized)] = float(np.median(normalized[valid]))
|
| 573 |
+
filled = cv2.inpaint((normalized * 255).astype(np.uint8), fill_mask.astype(np.uint8) * 255, 5, cv2.INPAINT_TELEA)
|
| 574 |
+
return filled.astype(np.float32) / 255.0 * (hi - lo) + lo
|
| 575 |
+
|
| 576 |
+
|
| 577 |
+
def inpaint_rgb(rgb: np.ndarray, fill_mask: np.ndarray, radius: float = 5.0) -> np.ndarray:
|
| 578 |
+
if not np.any(fill_mask):
|
| 579 |
+
return rgb.copy()
|
| 580 |
+
bgr = cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR)
|
| 581 |
+
inpainted = cv2.inpaint(bgr, fill_mask.astype(np.uint8) * 255, radius, cv2.INPAINT_TELEA)
|
| 582 |
+
return cv2.cvtColor(inpainted, cv2.COLOR_BGR2RGB)
|
| 583 |
+
|
| 584 |
+
|
| 585 |
+
def stair_band_mask(
|
| 586 |
+
shape: tuple[int, int],
|
| 587 |
+
top: float,
|
| 588 |
+
bottom: float,
|
| 589 |
+
stair_edge_slope: float = 0.0,
|
| 590 |
+
) -> np.ndarray:
|
| 591 |
+
"""Return one tread/riser band aligned with the observed image perspective.
|
| 592 |
+
|
| 593 |
+
``edges_y`` are measured at the image centre. A non-zero slope means that
|
| 594 |
+
the same edge moves vertically across the frame, so horizontal image rows
|
| 595 |
+
would mix different physical steps. This helper preserves the legacy
|
| 596 |
+
horizontal behaviour when the slope is zero.
|
| 597 |
+
"""
|
| 598 |
+
h, w = shape
|
| 599 |
+
ys, xs = np.indices((h, w), dtype=np.float32)
|
| 600 |
+
center_x = (w - 1) * 0.5
|
| 601 |
+
centerline_y = ys - float(stair_edge_slope) * (xs - center_x)
|
| 602 |
+
return (centerline_y >= float(top)) & (centerline_y < float(bottom))
|
| 603 |
+
|
| 604 |
+
|
| 605 |
+
def complete_depth_by_planes(
|
| 606 |
+
depth: np.ndarray,
|
| 607 |
+
obstacle_mask: np.ndarray,
|
| 608 |
+
amodal_mask: np.ndarray,
|
| 609 |
+
edges_y: list[int],
|
| 610 |
+
fx: float,
|
| 611 |
+
fy: float,
|
| 612 |
+
cx: float,
|
| 613 |
+
cy: float,
|
| 614 |
+
min_fit_points: int,
|
| 615 |
+
stair_edge_slope: float = 0.0,
|
| 616 |
+
):
|
| 617 |
+
h, w = depth.shape
|
| 618 |
+
completed = depth.copy()
|
| 619 |
+
visible = amodal_mask & ~obstacle_mask & np.isfinite(depth) & (depth > 0)
|
| 620 |
+
inpainted = inpaint_depth(depth, obstacle_mask & amodal_mask, visible)
|
| 621 |
+
points = pixels_to_points(depth, fx, fy, cx, cy)
|
| 622 |
+
confidence = np.zeros(depth.shape, dtype=np.float32)
|
| 623 |
+
distance_to_visible = cv2.distanceTransform((~visible).astype(np.uint8), cv2.DIST_L2, 5)
|
| 624 |
+
plane_blend_scale = max(8.0, 0.025 * float(np.hypot(*depth.shape)))
|
| 625 |
+
plane_weight = np.clip(distance_to_visible / plane_blend_scale, 0.0, 0.85)
|
| 626 |
+
|
| 627 |
+
boundaries = [0] + sorted(int(y) for y in edges_y if 0 < y < h - 1) + [h]
|
| 628 |
+
fitted_planes = []
|
| 629 |
+
for top, bottom in zip(boundaries[:-1], boundaries[1:]):
|
| 630 |
+
band = stair_band_mask((h, w), top, bottom, stair_edge_slope)
|
| 631 |
+
fit_mask = band & visible
|
| 632 |
+
fill_mask = band & obstacle_mask & amodal_mask
|
| 633 |
+
plane = None
|
| 634 |
+
if int(fit_mask.sum()) >= min_fit_points:
|
| 635 |
+
sample = points[fit_mask]
|
| 636 |
+
if sample.shape[0] > 30000:
|
| 637 |
+
sample = sample[np.linspace(0, sample.shape[0] - 1, 30000).astype(np.int64)]
|
| 638 |
+
plane = fit_plane(sample)
|
| 639 |
+
if plane is not None:
|
| 640 |
+
plane_depth = intersect_plane_for_pixels((h, w), plane, fx, fy, cx, cy)
|
| 641 |
+
ok = fill_mask & (plane_depth > 0) & np.isfinite(plane_depth)
|
| 642 |
+
completed[ok] = (
|
| 643 |
+
plane_weight[ok] * plane_depth[ok]
|
| 644 |
+
+ (1.0 - plane_weight[ok]) * inpainted[ok]
|
| 645 |
+
)
|
| 646 |
+
missing = fill_mask & ~ok
|
| 647 |
+
completed[missing] = inpainted[missing]
|
| 648 |
+
residual = np.abs(sample @ plane[0] + plane[1])
|
| 649 |
+
scale = max(float(np.median(sample[:, 2])), 1e-6)
|
| 650 |
+
quality = float(np.exp(-12.0 * np.median(residual) / scale))
|
| 651 |
+
confidence[ok] = np.clip(quality, 0.35, 0.95)
|
| 652 |
+
confidence[missing] = 0.25
|
| 653 |
+
mode = 'plane_diagonal' if abs(stair_edge_slope) > 1e-6 else 'plane'
|
| 654 |
+
fitted_planes.append((top, bottom, int(fit_mask.sum()), mode))
|
| 655 |
+
else:
|
| 656 |
+
completed[fill_mask] = inpainted[fill_mask]
|
| 657 |
+
confidence[fill_mask] = 0.20
|
| 658 |
+
mode = 'inpaint_fallback_diagonal' if abs(stair_edge_slope) > 1e-6 else 'inpaint_fallback'
|
| 659 |
+
fitted_planes.append((top, bottom, int(fit_mask.sum()), mode))
|
| 660 |
+
|
| 661 |
+
return completed, fitted_planes, confidence
|
| 662 |
+
|
| 663 |
+
|
| 664 |
+
def robust_fit_plane(points: np.ndarray, min_points: int):
|
| 665 |
+
if points.shape[0] < min_points:
|
| 666 |
+
return None, 0.0, 0
|
| 667 |
+
sample = points
|
| 668 |
+
if sample.shape[0] > 50000:
|
| 669 |
+
sample = sample[np.linspace(0, sample.shape[0] - 1, 50000).astype(np.int64)]
|
| 670 |
+
plane = None
|
| 671 |
+
for _ in range(4):
|
| 672 |
+
plane = fit_plane(sample)
|
| 673 |
+
if plane is None:
|
| 674 |
+
return None, 0.0, int(sample.shape[0])
|
| 675 |
+
residual = np.abs(sample @ plane[0] + plane[1])
|
| 676 |
+
median = float(np.median(residual))
|
| 677 |
+
mad = float(np.median(np.abs(residual - median)))
|
| 678 |
+
threshold = max(median + 2.5 * max(mad, 1e-6), float(np.percentile(residual, 70)))
|
| 679 |
+
retained = sample[residual <= threshold]
|
| 680 |
+
if retained.shape[0] < min_points or retained.shape[0] >= sample.shape[0] * 0.98:
|
| 681 |
+
break
|
| 682 |
+
sample = retained
|
| 683 |
+
if plane is None:
|
| 684 |
+
return None, 0.0, int(sample.shape[0])
|
| 685 |
+
residual = np.abs(sample @ plane[0] + plane[1])
|
| 686 |
+
scale = max(float(np.median(sample[:, 2])), 1e-6)
|
| 687 |
+
quality = float(np.exp(-10.0 * float(np.median(residual)) / scale))
|
| 688 |
+
return plane, float(np.clip(quality, 0.0, 1.0)), int(sample.shape[0])
|
| 689 |
+
|
| 690 |
+
|
| 691 |
+
def complete_depth_continuous_surface(
|
| 692 |
+
depth: np.ndarray,
|
| 693 |
+
completion_mask: np.ndarray,
|
| 694 |
+
amodal_mask: np.ndarray,
|
| 695 |
+
fx: float,
|
| 696 |
+
fy: float,
|
| 697 |
+
cx: float,
|
| 698 |
+
cy: float,
|
| 699 |
+
min_fit_points: int,
|
| 700 |
+
):
|
| 701 |
+
completed = depth.copy()
|
| 702 |
+
confidence = np.zeros(depth.shape, dtype=np.float32)
|
| 703 |
+
visible = amodal_mask & ~completion_mask & np.isfinite(depth) & (depth > 0)
|
| 704 |
+
inpainted = inpaint_depth(depth, completion_mask, visible)
|
| 705 |
+
points = pixels_to_points(depth, fx, fy, cx, cy)
|
| 706 |
+
plane, quality, sample_count = robust_fit_plane(points[visible], min_fit_points)
|
| 707 |
+
method = 'edge_aware_inpaint'
|
| 708 |
+
if plane is None:
|
| 709 |
+
completed[completion_mask] = inpainted[completion_mask]
|
| 710 |
+
distance_to_visible = cv2.distanceTransform((~visible).astype(np.uint8), cv2.DIST_L2, 5)
|
| 711 |
+
distance_scale = max(8.0, 0.06 * float(np.hypot(*depth.shape)))
|
| 712 |
+
distance_confidence = np.exp(-distance_to_visible / distance_scale)
|
| 713 |
+
confidence[completion_mask] = np.clip(
|
| 714 |
+
0.20 + 0.25 * distance_confidence[completion_mask],
|
| 715 |
+
0.20,
|
| 716 |
+
0.45,
|
| 717 |
+
)
|
| 718 |
+
return completed, [(0, depth.shape[0], sample_count, method)], confidence
|
| 719 |
+
|
| 720 |
+
plane_depth = intersect_plane_for_pixels(depth.shape, plane, fx, fy, cx, cy)
|
| 721 |
+
visible_values = depth[visible]
|
| 722 |
+
low, high = np.percentile(visible_values, [1, 99])
|
| 723 |
+
plausible = (
|
| 724 |
+
completion_mask
|
| 725 |
+
& np.isfinite(plane_depth)
|
| 726 |
+
& (plane_depth > max(1e-6, 0.50 * low))
|
| 727 |
+
& (plane_depth < 1.50 * high)
|
| 728 |
+
)
|
| 729 |
+
# Keep local boundary detail from inpainting while the robust plane supplies
|
| 730 |
+
# a stable global surface through larger occluders.
|
| 731 |
+
distance_to_visible = cv2.distanceTransform((~visible).astype(np.uint8), cv2.DIST_L2, 5)
|
| 732 |
+
plane_blend_scale = max(8.0, 0.025 * float(np.hypot(*depth.shape)))
|
| 733 |
+
plane_weight = np.clip(distance_to_visible / plane_blend_scale, 0.0, 0.85)
|
| 734 |
+
completed[plausible] = (
|
| 735 |
+
plane_weight[plausible] * plane_depth[plausible]
|
| 736 |
+
+ (1.0 - plane_weight[plausible]) * inpainted[plausible]
|
| 737 |
+
)
|
| 738 |
+
fallback = completion_mask & ~plausible
|
| 739 |
+
completed[fallback] = inpainted[fallback]
|
| 740 |
+
|
| 741 |
+
distance_scale = max(8.0, 0.06 * float(np.hypot(*depth.shape)))
|
| 742 |
+
distance_confidence = np.exp(-distance_to_visible / distance_scale)
|
| 743 |
+
confidence[plausible] = np.clip(quality * distance_confidence[plausible], 0.25, 0.95)
|
| 744 |
+
confidence[fallback] = np.clip(
|
| 745 |
+
0.20 + 0.35 * distance_confidence[fallback],
|
| 746 |
+
0.20,
|
| 747 |
+
0.55,
|
| 748 |
+
)
|
| 749 |
+
method = 'robust_continuous_plane_plus_edge_aware_inpaint'
|
| 750 |
+
return completed, [(0, depth.shape[0], sample_count, method)], confidence
|
| 751 |
+
|
| 752 |
+
|
| 753 |
+
def complete_depth_generic_inpaint(
|
| 754 |
+
depth: np.ndarray, completion_mask: np.ndarray, amodal_mask: np.ndarray
|
| 755 |
+
):
|
| 756 |
+
visible = amodal_mask & ~completion_mask & np.isfinite(depth) & (depth > 0)
|
| 757 |
+
completed = depth.copy()
|
| 758 |
+
inpainted = inpaint_depth(depth, completion_mask, visible)
|
| 759 |
+
completed[completion_mask] = inpainted[completion_mask]
|
| 760 |
+
confidence = np.zeros(depth.shape, dtype=np.float32)
|
| 761 |
+
confidence[completion_mask] = 0.20
|
| 762 |
+
return completed, [(0, depth.shape[0], int(visible.sum()), 'edge_aware_inpaint')], confidence
|
| 763 |
+
|
| 764 |
+
|
| 765 |
+
def enforce_hidden_boundary_continuity(
|
| 766 |
+
completed: np.ndarray,
|
| 767 |
+
source_depth: np.ndarray,
|
| 768 |
+
visible: np.ndarray,
|
| 769 |
+
hidden: np.ndarray,
|
| 770 |
+
radius: int,
|
| 771 |
+
anchor_power: float = 0.45,
|
| 772 |
+
) -> np.ndarray:
|
| 773 |
+
if radius <= 0 or not np.any(hidden) or not np.any(visible):
|
| 774 |
+
return completed
|
| 775 |
+
size = radius * 2 + 1
|
| 776 |
+
sums = cv2.boxFilter(
|
| 777 |
+
np.where(visible, source_depth, 0.0).astype(np.float32),
|
| 778 |
+
-1,
|
| 779 |
+
(size, size),
|
| 780 |
+
normalize=False,
|
| 781 |
+
borderType=cv2.BORDER_CONSTANT,
|
| 782 |
+
)
|
| 783 |
+
counts = cv2.boxFilter(
|
| 784 |
+
visible.astype(np.float32),
|
| 785 |
+
-1,
|
| 786 |
+
(size, size),
|
| 787 |
+
normalize=False,
|
| 788 |
+
borderType=cv2.BORDER_CONSTANT,
|
| 789 |
+
)
|
| 790 |
+
boundary = hidden & (counts > 0)
|
| 791 |
+
if not np.any(boundary):
|
| 792 |
+
return completed
|
| 793 |
+
neighbor_depth = np.zeros_like(completed, dtype=np.float32)
|
| 794 |
+
neighbor_depth[boundary] = sums[boundary] / counts[boundary]
|
| 795 |
+
distance = cv2.distanceTransform((~visible).astype(np.uint8), cv2.DIST_L2, 5)
|
| 796 |
+
raw_anchor = 1.0 - np.clip((distance - 1.0) / max(float(radius), 1.0), 0.0, 1.0)
|
| 797 |
+
anchor_weight = np.power(raw_anchor, max(float(anchor_power), 1e-3))
|
| 798 |
+
output = completed.copy()
|
| 799 |
+
output[boundary] = (
|
| 800 |
+
(1.0 - anchor_weight[boundary]) * completed[boundary]
|
| 801 |
+
+ anchor_weight[boundary] * neighbor_depth[boundary]
|
| 802 |
+
)
|
| 803 |
+
return output
|
| 804 |
+
|
| 805 |
+
|
| 806 |
+
DEPTH_PLAUSIBILITY_POLICIES = {
|
| 807 |
+
# Stairs legitimately span several depth layers, so retain a wider target
|
| 808 |
+
# envelope. A plane/ray intersection must still stay inside the robust
|
| 809 |
+
# numerical domain observed in the source depth map.
|
| 810 |
+
'stairs': {
|
| 811 |
+
'target_span_margin': 1.00,
|
| 812 |
+
'maximum_correction_ratio': 0.05,
|
| 813 |
+
},
|
| 814 |
+
# A reviewed walkable surface or ramp should be locally continuous. The
|
| 815 |
+
# margin remains wide enough for perspective extrapolation already accepted
|
| 816 |
+
# by complete_depth_continuous_surface.
|
| 817 |
+
'walkable': {
|
| 818 |
+
'target_span_margin': 0.75,
|
| 819 |
+
'maximum_correction_ratio': 0.02,
|
| 820 |
+
},
|
| 821 |
+
'ramp': {
|
| 822 |
+
'target_span_margin': 0.75,
|
| 823 |
+
'maximum_correction_ratio': 0.02,
|
| 824 |
+
},
|
| 825 |
+
# Generic mode has the weakest category prior, but it may not invent an
|
| 826 |
+
# unbounded scale outside the observed depth domain.
|
| 827 |
+
'generic': {
|
| 828 |
+
'target_span_margin': 1.50,
|
| 829 |
+
'maximum_correction_ratio': 0.05,
|
| 830 |
+
},
|
| 831 |
+
}
|
| 832 |
+
|
| 833 |
+
|
| 834 |
+
def enforce_hidden_depth_plausibility(
|
| 835 |
+
completed: np.ndarray,
|
| 836 |
+
source_depth: np.ndarray,
|
| 837 |
+
visible: np.ndarray,
|
| 838 |
+
hidden: np.ndarray,
|
| 839 |
+
geometry_mode: str,
|
| 840 |
+
) -> tuple[np.ndarray, np.ndarray, dict]:
|
| 841 |
+
"""Reject numerically implausible hidden depths and return an audit record.
|
| 842 |
+
|
| 843 |
+
Plane/ray intersections can explode when a fitted plane is nearly parallel
|
| 844 |
+
to a camera ray. This guard is deliberately conservative:
|
| 845 |
+
|
| 846 |
+
* only hidden pixels may be changed;
|
| 847 |
+
* the accepted range is inferred from the observed source-depth scale, not
|
| 848 |
+
assumed to be metric;
|
| 849 |
+
* category-specific target margins allow layered stairs more variation than
|
| 850 |
+
continuous walkable surfaces and ramps;
|
| 851 |
+
* rejected values use bounded edge-aware inpainting instead of clipping a
|
| 852 |
+
singular plane to a hard wall.
|
| 853 |
+
|
| 854 |
+
The returned correction mask is suitable for lowering confidence and for a
|
| 855 |
+
review overlay. A large correction ratio is marked as withheld in the
|
| 856 |
+
audit rather than silently presented as a trustworthy geometry candidate.
|
| 857 |
+
"""
|
| 858 |
+
if completed.shape != source_depth.shape:
|
| 859 |
+
raise ValueError(
|
| 860 |
+
f'Completed/source depth shape mismatch: {completed.shape} != {source_depth.shape}'
|
| 861 |
+
)
|
| 862 |
+
if visible.shape != completed.shape or hidden.shape != completed.shape:
|
| 863 |
+
raise ValueError('Visible/hidden masks must share the completed-depth raster')
|
| 864 |
+
if geometry_mode not in DEPTH_PLAUSIBILITY_POLICIES:
|
| 865 |
+
raise ValueError(f'Unsupported geometry mode for depth plausibility: {geometry_mode!r}')
|
| 866 |
+
|
| 867 |
+
hidden = hidden.astype(bool)
|
| 868 |
+
visible = visible.astype(bool) & ~hidden
|
| 869 |
+
policy = DEPTH_PLAUSIBILITY_POLICIES[geometry_mode]
|
| 870 |
+
hidden_count = int(hidden.sum())
|
| 871 |
+
empty_corrections = np.zeros(completed.shape, dtype=bool)
|
| 872 |
+
if hidden_count == 0:
|
| 873 |
+
return completed.copy(), empty_corrections, {
|
| 874 |
+
'policy': f'{geometry_mode}_observed_depth_domain_v1',
|
| 875 |
+
'status': 'not_applicable_empty_hidden_region',
|
| 876 |
+
'candidate_eligible_for_review': True,
|
| 877 |
+
'hidden_pixel_count': 0,
|
| 878 |
+
'corrected_pixel_count': 0,
|
| 879 |
+
'corrected_ratio': 0.0,
|
| 880 |
+
'maximum_correction_ratio': float(policy['maximum_correction_ratio']),
|
| 881 |
+
'source_depth_is_treated_as_metric_truth': False,
|
| 882 |
+
}
|
| 883 |
+
|
| 884 |
+
source_valid = np.isfinite(source_depth) & (source_depth > 0)
|
| 885 |
+
source_values = source_depth[source_valid]
|
| 886 |
+
visible_valid = visible & source_valid
|
| 887 |
+
visible_values = source_depth[visible_valid]
|
| 888 |
+
if source_values.size == 0 or visible_values.size == 0:
|
| 889 |
+
invalid = hidden & (
|
| 890 |
+
~np.isfinite(completed)
|
| 891 |
+
| (completed <= 0)
|
| 892 |
+
)
|
| 893 |
+
return completed.copy(), invalid, {
|
| 894 |
+
'policy': f'{geometry_mode}_observed_depth_domain_v1',
|
| 895 |
+
'status': 'withheld_no_visible_depth_support',
|
| 896 |
+
'candidate_eligible_for_review': False,
|
| 897 |
+
'hidden_pixel_count': hidden_count,
|
| 898 |
+
'corrected_pixel_count': 0,
|
| 899 |
+
'corrected_ratio': 0.0,
|
| 900 |
+
'maximum_correction_ratio': float(policy['maximum_correction_ratio']),
|
| 901 |
+
'source_depth_is_treated_as_metric_truth': False,
|
| 902 |
+
'reason': 'No positive finite visible target depth was available for a bounded fallback.',
|
| 903 |
+
}
|
| 904 |
+
|
| 905 |
+
source_q001, source_q999 = np.percentile(source_values, [0.1, 99.9])
|
| 906 |
+
source_span = max(
|
| 907 |
+
float(source_q999 - source_q001),
|
| 908 |
+
0.05 * abs(float(np.median(source_values))),
|
| 909 |
+
1e-6,
|
| 910 |
+
)
|
| 911 |
+
# Ignore isolated source-map outliers while never extending beyond the
|
| 912 |
+
# finite positive range actually emitted by the depth estimator.
|
| 913 |
+
source_lower = max(
|
| 914 |
+
float(source_values.min()),
|
| 915 |
+
float(source_q001 - 0.10 * source_span),
|
| 916 |
+
)
|
| 917 |
+
source_upper = min(
|
| 918 |
+
float(source_values.max()),
|
| 919 |
+
float(source_q999 + 0.10 * source_span),
|
| 920 |
+
)
|
| 921 |
+
|
| 922 |
+
target_q01, target_q99 = np.percentile(visible_values, [1.0, 99.0])
|
| 923 |
+
target_span = max(
|
| 924 |
+
float(target_q99 - target_q01),
|
| 925 |
+
0.05 * abs(float(np.median(visible_values))),
|
| 926 |
+
1e-6,
|
| 927 |
+
)
|
| 928 |
+
target_margin = float(policy['target_span_margin']) * target_span
|
| 929 |
+
accepted_lower = max(source_lower, float(target_q01 - target_margin))
|
| 930 |
+
accepted_upper = min(source_upper, float(target_q99 + target_margin))
|
| 931 |
+
if not np.isfinite(accepted_lower) or not np.isfinite(accepted_upper) or accepted_upper < accepted_lower:
|
| 932 |
+
accepted_lower, accepted_upper = source_lower, source_upper
|
| 933 |
+
|
| 934 |
+
corrections = hidden & (
|
| 935 |
+
~np.isfinite(completed)
|
| 936 |
+
| (completed < accepted_lower)
|
| 937 |
+
| (completed > accepted_upper)
|
| 938 |
+
)
|
| 939 |
+
corrected_count = int(corrections.sum())
|
| 940 |
+
output = completed.copy()
|
| 941 |
+
if corrected_count:
|
| 942 |
+
fallback = inpaint_depth(source_depth, hidden, visible_valid)
|
| 943 |
+
fallback = np.nan_to_num(
|
| 944 |
+
fallback,
|
| 945 |
+
nan=float(np.median(visible_values)),
|
| 946 |
+
posinf=accepted_upper,
|
| 947 |
+
neginf=accepted_lower,
|
| 948 |
+
)
|
| 949 |
+
fallback = np.clip(fallback, accepted_lower, accepted_upper)
|
| 950 |
+
output[corrections] = fallback[corrections]
|
| 951 |
+
|
| 952 |
+
# Preserve source depth bit-for-bit outside the reviewed hidden region.
|
| 953 |
+
output[~hidden] = source_depth[~hidden]
|
| 954 |
+
corrected_ratio = float(corrected_count / hidden_count)
|
| 955 |
+
maximum_correction_ratio = float(policy['maximum_correction_ratio'])
|
| 956 |
+
candidate_eligible = bool(
|
| 957 |
+
corrected_ratio <= maximum_correction_ratio
|
| 958 |
+
and np.all(np.isfinite(output[hidden]))
|
| 959 |
+
and np.all(output[hidden] >= accepted_lower)
|
| 960 |
+
and np.all(output[hidden] <= accepted_upper)
|
| 961 |
+
)
|
| 962 |
+
if not candidate_eligible:
|
| 963 |
+
status = 'withheld_excessive_depth_corrections'
|
| 964 |
+
elif corrected_count:
|
| 965 |
+
status = 'corrected_sparse_depth_outliers'
|
| 966 |
+
else:
|
| 967 |
+
status = 'within_observed_depth_domain'
|
| 968 |
+
audit = {
|
| 969 |
+
'policy': f'{geometry_mode}_observed_depth_domain_v1',
|
| 970 |
+
'status': status,
|
| 971 |
+
'candidate_eligible_for_review': candidate_eligible,
|
| 972 |
+
'hidden_pixel_count': hidden_count,
|
| 973 |
+
'corrected_pixel_count': corrected_count,
|
| 974 |
+
'corrected_ratio': corrected_ratio,
|
| 975 |
+
'maximum_correction_ratio': maximum_correction_ratio,
|
| 976 |
+
'accepted_depth_lower': float(accepted_lower),
|
| 977 |
+
'accepted_depth_upper': float(accepted_upper),
|
| 978 |
+
'observed_source_depth_min': float(source_values.min()),
|
| 979 |
+
'observed_source_depth_max': float(source_values.max()),
|
| 980 |
+
'visible_target_depth_p01': float(target_q01),
|
| 981 |
+
'visible_target_depth_p99': float(target_q99),
|
| 982 |
+
'source_depth_is_treated_as_metric_truth': False,
|
| 983 |
+
}
|
| 984 |
+
return output, corrections, audit
|
| 985 |
+
|
| 986 |
+
|
| 987 |
+
def write_point_cloud_ply(path: Path, points: np.ndarray, colors: np.ndarray, mask: np.ndarray, stride: int) -> int:
|
| 988 |
+
sampled = np.zeros(mask.shape, dtype=bool)
|
| 989 |
+
sampled[::stride, ::stride] = True
|
| 990 |
+
use = mask & sampled & np.all(np.isfinite(points), axis=-1) & (points[..., 2] > 0)
|
| 991 |
+
pts = points[use]
|
| 992 |
+
cols = colors[use]
|
| 993 |
+
with open(path, 'w', encoding='ascii') as f:
|
| 994 |
+
f.write('ply\nformat ascii 1.0\n')
|
| 995 |
+
f.write(f'element vertex {len(pts)}\n')
|
| 996 |
+
f.write('property float x\nproperty float y\nproperty float z\n')
|
| 997 |
+
f.write('property uchar red\nproperty uchar green\nproperty uchar blue\n')
|
| 998 |
+
f.write('end_header\n')
|
| 999 |
+
for p, c in zip(pts, cols):
|
| 1000 |
+
f.write(f'{p[0]:.6f} {p[1]:.6f} {p[2]:.6f} {int(c[0])} {int(c[1])} {int(c[2])}\n')
|
| 1001 |
+
return int(len(pts))
|
| 1002 |
+
|
| 1003 |
+
|
| 1004 |
+
def write_mesh_ply(path: Path, points: np.ndarray, colors: np.ndarray, mask: np.ndarray, stride: int, max_depth_jump: float) -> tuple[int, int]:
|
| 1005 |
+
h, w = mask.shape
|
| 1006 |
+
ys = np.arange(0, h, stride)
|
| 1007 |
+
xs = np.arange(0, w, stride)
|
| 1008 |
+
vertex_id = -np.ones((len(ys), len(xs)), dtype=np.int64)
|
| 1009 |
+
vertices = []
|
| 1010 |
+
vertex_colors = []
|
| 1011 |
+
for iy, y in enumerate(ys):
|
| 1012 |
+
for ix, x in enumerate(xs):
|
| 1013 |
+
if mask[y, x] and np.isfinite(points[y, x]).all() and points[y, x, 2] > 0:
|
| 1014 |
+
vertex_id[iy, ix] = len(vertices)
|
| 1015 |
+
vertices.append(points[y, x])
|
| 1016 |
+
vertex_colors.append(colors[y, x])
|
| 1017 |
+
|
| 1018 |
+
faces = []
|
| 1019 |
+
for iy in range(len(ys) - 1):
|
| 1020 |
+
for ix in range(len(xs) - 1):
|
| 1021 |
+
ids = [vertex_id[iy, ix], vertex_id[iy, ix + 1], vertex_id[iy + 1, ix], vertex_id[iy + 1, ix + 1]]
|
| 1022 |
+
if min(ids) < 0:
|
| 1023 |
+
continue
|
| 1024 |
+
z = np.array([vertices[i][2] for i in ids], dtype=np.float32)
|
| 1025 |
+
if float(z.max() - z.min()) > max_depth_jump:
|
| 1026 |
+
continue
|
| 1027 |
+
faces.append((ids[0], ids[2], ids[1]))
|
| 1028 |
+
faces.append((ids[1], ids[2], ids[3]))
|
| 1029 |
+
|
| 1030 |
+
with open(path, 'w', encoding='ascii') as f:
|
| 1031 |
+
f.write('ply\nformat ascii 1.0\n')
|
| 1032 |
+
f.write(f'element vertex {len(vertices)}\n')
|
| 1033 |
+
f.write('property float x\nproperty float y\nproperty float z\n')
|
| 1034 |
+
f.write('property uchar red\nproperty uchar green\nproperty uchar blue\n')
|
| 1035 |
+
f.write(f'element face {len(faces)}\n')
|
| 1036 |
+
f.write('property list uchar int vertex_indices\n')
|
| 1037 |
+
f.write('end_header\n')
|
| 1038 |
+
for p, c in zip(vertices, vertex_colors):
|
| 1039 |
+
f.write(f'{p[0]:.6f} {p[1]:.6f} {p[2]:.6f} {int(c[0])} {int(c[1])} {int(c[2])}\n')
|
| 1040 |
+
for face in faces:
|
| 1041 |
+
f.write(f'3 {face[0]} {face[1]} {face[2]}\n')
|
| 1042 |
+
return int(len(vertices)), int(len(faces))
|
| 1043 |
+
|
| 1044 |
+
|
| 1045 |
+
CATEGORY_GEOMETRY_MODES = {
|
| 1046 |
+
'stairs': 'stairs',
|
| 1047 |
+
'ramp': 'ramp',
|
| 1048 |
+
'walkway': 'walkable',
|
| 1049 |
+
'walkable': 'walkable',
|
| 1050 |
+
'curb_cut': 'walkable',
|
| 1051 |
+
'raised_curb': 'walkable',
|
| 1052 |
+
'tactile_paving': 'walkable',
|
| 1053 |
+
'unknown': 'generic',
|
| 1054 |
+
}
|
| 1055 |
+
|
| 1056 |
+
|
| 1057 |
+
def resolve_geometry_mode(category, requested_mode):
|
| 1058 |
+
"""Infer a safe mode from category and reject contradictory geometry priors."""
|
| 1059 |
+
if category is None:
|
| 1060 |
+
return requested_mode or 'stairs'
|
| 1061 |
+
expected_mode = CATEGORY_GEOMETRY_MODES[category]
|
| 1062 |
+
if requested_mode is not None and requested_mode != expected_mode:
|
| 1063 |
+
raise ValueError(
|
| 1064 |
+
f'Category {category!r} requires geometry mode {expected_mode!r}; '
|
| 1065 |
+
f'got {requested_mode!r}'
|
| 1066 |
+
)
|
| 1067 |
+
return expected_mode
|
| 1068 |
+
|
| 1069 |
+
|
| 1070 |
+
def run(args):
|
| 1071 |
+
if not args.image:
|
| 1072 |
+
raise ValueError("--image is required")
|
| 1073 |
+
args.geometry_mode = resolve_geometry_mode(
|
| 1074 |
+
getattr(args, 'category', None),
|
| 1075 |
+
getattr(args, 'geometry_mode', None),
|
| 1076 |
+
)
|
| 1077 |
+
output_dir = Path(args.output_dir)
|
| 1078 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 1079 |
+
|
| 1080 |
+
source_shape = display_shape(args.image)
|
| 1081 |
+
rgb = read_rgb(args.image, args.max_size)
|
| 1082 |
+
h, w = rgb.shape[:2]
|
| 1083 |
+
shape = (h, w)
|
| 1084 |
+
|
| 1085 |
+
if args.amodal_mask:
|
| 1086 |
+
amodal_mask = read_source_grid_mask(args.amodal_mask, source_shape, shape)
|
| 1087 |
+
amodal_source = args.amodal_mask
|
| 1088 |
+
else:
|
| 1089 |
+
amodal_mask = default_amodal_mask(shape)
|
| 1090 |
+
amodal_source = 'default lower-scene mask'
|
| 1091 |
+
|
| 1092 |
+
if args.obstacle_mask:
|
| 1093 |
+
obstacle_mask = read_source_grid_mask(args.obstacle_mask, source_shape, shape)
|
| 1094 |
+
obstacle_source = args.obstacle_mask
|
| 1095 |
+
else:
|
| 1096 |
+
boxes = args.occlusion_box
|
| 1097 |
+
if not boxes and not args.no_default_obstacle:
|
| 1098 |
+
boxes = [DEFAULT_OBSTACLE_BOX]
|
| 1099 |
+
obstacle_mask = boxes_to_mask(boxes, shape) if boxes else np.zeros(shape, dtype=bool)
|
| 1100 |
+
obstacle_source = 'occlusion boxes' if boxes else 'empty obstacle mask'
|
| 1101 |
+
|
| 1102 |
+
obstacle_mask &= amodal_mask
|
| 1103 |
+
if args.visible_mask:
|
| 1104 |
+
visible_mask = read_source_grid_mask(args.visible_mask, source_shape, shape) & amodal_mask & ~obstacle_mask
|
| 1105 |
+
visible_source = args.visible_mask
|
| 1106 |
+
completion_mask = amodal_mask & ~visible_mask
|
| 1107 |
+
else:
|
| 1108 |
+
visible_mask = amodal_mask & ~obstacle_mask
|
| 1109 |
+
visible_source = 'amodal mask minus obstacle mask'
|
| 1110 |
+
completion_mask = obstacle_mask & amodal_mask
|
| 1111 |
+
|
| 1112 |
+
geometry_mode = args.geometry_mode
|
| 1113 |
+
reference_edges: list[int] = []
|
| 1114 |
+
target_edges: list[int] = []
|
| 1115 |
+
edges_y: list[int] = []
|
| 1116 |
+
reference_edge_slope = 0.0
|
| 1117 |
+
target_edge_slope = 0.0
|
| 1118 |
+
stair_edge_slope = 0.0
|
| 1119 |
+
edge_source = 'not_applicable_for_continuous_surface'
|
| 1120 |
+
used_regular_fallback_edges = False
|
| 1121 |
+
used_perspective_edge_expansion = False
|
| 1122 |
+
used_gradient_edge_expansion = False
|
| 1123 |
+
used_depth_gradient_edge_expansion = False
|
| 1124 |
+
stair_edge_confidence = 1.0 if geometry_mode != 'stairs' else 0.0
|
| 1125 |
+
stair_edge_coverage = 1.0 if geometry_mode != 'stairs' else 0.0
|
| 1126 |
+
if geometry_mode == 'stairs':
|
| 1127 |
+
reference_rgb = None
|
| 1128 |
+
if args.stair_edge_y:
|
| 1129 |
+
edges_y = sorted(set(int(y) for y in args.stair_edge_y if 0 < int(y) < h - 1))
|
| 1130 |
+
stair_edge_slope = float(np.tan(np.radians(args.stair_edge_angle_degrees)))
|
| 1131 |
+
edge_source = 'reviewed stair edge y positions'
|
| 1132 |
+
stair_edge_confidence = 1.0
|
| 1133 |
+
stair_edge_coverage = stair_edge_coverage_ratio(edges_y, amodal_mask)
|
| 1134 |
+
elif args.reference_image and Path(args.reference_image).exists():
|
| 1135 |
+
reference_rgb = read_rgb(args.reference_image, max_size=max(h, w))
|
| 1136 |
+
if reference_rgb.shape[:2] != shape:
|
| 1137 |
+
if not _aspect_ratio_matches(reference_rgb.shape[:2], shape):
|
| 1138 |
+
raise ValueError(
|
| 1139 |
+
f'Reference RGB/target raster mismatch: reference={reference_rgb.shape[:2]}, target={shape}. '
|
| 1140 |
+
'Refusing to resize across incompatible aspect ratios because the reference supplies stair-edge evidence.'
|
| 1141 |
+
)
|
| 1142 |
+
reference_rgb = cv2.resize(reference_rgb, (w, h), interpolation=cv2.INTER_AREA)
|
| 1143 |
+
reference_edges, reference_edge_slope = detect_stair_edge_model(
|
| 1144 |
+
reference_rgb,
|
| 1145 |
+
amodal_mask,
|
| 1146 |
+
args.max_step_edges,
|
| 1147 |
+
)
|
| 1148 |
+
|
| 1149 |
+
if not args.stair_edge_y:
|
| 1150 |
+
target_edges, target_edge_slope = detect_stair_edge_model(rgb, visible_mask, args.max_step_edges)
|
| 1151 |
+
if reference_edges and len(reference_edges) >= len(target_edges):
|
| 1152 |
+
edges_y = reference_edges
|
| 1153 |
+
stair_edge_slope = reference_edge_slope
|
| 1154 |
+
edge_source = args.reference_image
|
| 1155 |
+
elif target_edges:
|
| 1156 |
+
edges_y = target_edges
|
| 1157 |
+
stair_edge_slope = target_edge_slope
|
| 1158 |
+
edge_source = args.image
|
| 1159 |
+
else:
|
| 1160 |
+
edge_source = 'regular fallback edges'
|
| 1161 |
+
|
| 1162 |
+
initial_coverage = stair_edge_coverage_ratio(edges_y, amodal_mask)
|
| 1163 |
+
if initial_coverage < args.minimum_auto_stair_edge_coverage:
|
| 1164 |
+
relaxed_target, relaxed_target_slope = detect_stair_edge_model(
|
| 1165 |
+
rgb,
|
| 1166 |
+
visible_mask,
|
| 1167 |
+
args.max_step_edges,
|
| 1168 |
+
min_gap_ratio=0.025,
|
| 1169 |
+
min_line_ratio=0.08,
|
| 1170 |
+
maximum_angle_degrees=30.0,
|
| 1171 |
+
hough_threshold_ratio=0.015,
|
| 1172 |
+
max_line_gap_ratio=0.05,
|
| 1173 |
+
)
|
| 1174 |
+
relaxed_reference = []
|
| 1175 |
+
relaxed_reference_slope = 0.0
|
| 1176 |
+
if reference_rgb is not None:
|
| 1177 |
+
relaxed_reference, relaxed_reference_slope = detect_stair_edge_model(
|
| 1178 |
+
reference_rgb,
|
| 1179 |
+
amodal_mask,
|
| 1180 |
+
args.max_step_edges,
|
| 1181 |
+
min_gap_ratio=0.025,
|
| 1182 |
+
min_line_ratio=0.08,
|
| 1183 |
+
maximum_angle_degrees=30.0,
|
| 1184 |
+
hough_threshold_ratio=0.015,
|
| 1185 |
+
max_line_gap_ratio=0.05,
|
| 1186 |
+
)
|
| 1187 |
+
expanded = merge_edge_positions(
|
| 1188 |
+
edges_y + relaxed_target + relaxed_reference,
|
| 1189 |
+
max(8, int(h * 0.018)),
|
| 1190 |
+
)
|
| 1191 |
+
expanded_coverage = stair_edge_coverage_ratio(expanded, amodal_mask)
|
| 1192 |
+
if expanded_coverage > initial_coverage:
|
| 1193 |
+
edges_y = expanded[: args.max_step_edges]
|
| 1194 |
+
if abs(stair_edge_slope) < 1e-6:
|
| 1195 |
+
stair_edge_slope = relaxed_target_slope or relaxed_reference_slope
|
| 1196 |
+
edge_source = f'{edge_source} + perspective-tolerant expansion'
|
| 1197 |
+
used_perspective_edge_expansion = True
|
| 1198 |
+
|
| 1199 |
+
if len(edges_y) < args.minimum_detected_step_edges:
|
| 1200 |
+
gradient_edges = detect_horizontal_gradient_peaks(
|
| 1201 |
+
rgb,
|
| 1202 |
+
visible_mask,
|
| 1203 |
+
args.max_step_edges,
|
| 1204 |
+
)
|
| 1205 |
+
expanded = merge_edge_positions(
|
| 1206 |
+
edges_y + gradient_edges,
|
| 1207 |
+
max(8, int(h * 0.018)),
|
| 1208 |
+
)
|
| 1209 |
+
if len(expanded) > len(edges_y):
|
| 1210 |
+
edges_y = expanded[: args.max_step_edges]
|
| 1211 |
+
edge_source = f'{edge_source} + visible-mask row-gradient expansion'
|
| 1212 |
+
used_gradient_edge_expansion = True
|
| 1213 |
+
|
| 1214 |
+
fallback = fallback_edges(amodal_mask, args.fallback_step_edges)
|
| 1215 |
+
if len(edges_y) < args.minimum_detected_step_edges:
|
| 1216 |
+
edges_y = merge_edge_positions(edges_y + fallback, max(10, int(h * 0.035)))
|
| 1217 |
+
edge_source = (
|
| 1218 |
+
f'{edge_source} + regular fallback edges'
|
| 1219 |
+
if edge_source != 'regular fallback edges'
|
| 1220 |
+
else edge_source
|
| 1221 |
+
)
|
| 1222 |
+
used_regular_fallback_edges = True
|
| 1223 |
+
stair_edge_confidence = 0.25
|
| 1224 |
+
else:
|
| 1225 |
+
stair_edge_coverage = stair_edge_coverage_ratio(edges_y, amodal_mask)
|
| 1226 |
+
count_score = min(len(edges_y) / 5.0, 1.0)
|
| 1227 |
+
coverage_score = min(
|
| 1228 |
+
stair_edge_coverage / max(args.minimum_auto_stair_edge_coverage, 1e-6),
|
| 1229 |
+
1.0,
|
| 1230 |
+
)
|
| 1231 |
+
stair_edge_confidence = float(
|
| 1232 |
+
np.clip(0.30 + 0.25 * count_score + 0.35 * coverage_score, 0.0, 0.90)
|
| 1233 |
+
)
|
| 1234 |
+
if used_gradient_edge_expansion:
|
| 1235 |
+
stair_edge_confidence = min(stair_edge_confidence, 0.75)
|
| 1236 |
+
if not edges_y:
|
| 1237 |
+
edges_y = fallback
|
| 1238 |
+
used_regular_fallback_edges = bool(edges_y)
|
| 1239 |
+
stair_edge_coverage = stair_edge_coverage_ratio(edges_y, amodal_mask)
|
| 1240 |
+
if len(edges_y) < args.minimum_detected_step_edges:
|
| 1241 |
+
stair_edge_slope = 0.0
|
| 1242 |
+
|
| 1243 |
+
if args.depth:
|
| 1244 |
+
depth = read_depth(args.depth, shape, args.depth_scale)
|
| 1245 |
+
depth_source = args.depth
|
| 1246 |
+
else:
|
| 1247 |
+
if geometry_mode == 'stairs':
|
| 1248 |
+
depth = make_step_prior_depth(shape, edges_y, args.synthetic_near, args.synthetic_far)
|
| 1249 |
+
depth_source = 'synthetic perspective stair prior'
|
| 1250 |
+
else:
|
| 1251 |
+
depth = make_continuous_prior_depth(shape, args.synthetic_near, args.synthetic_far)
|
| 1252 |
+
depth_source = 'synthetic continuous perspective prior'
|
| 1253 |
+
|
| 1254 |
+
# --- Post-depth stair edge expansion using depth gradient ---
|
| 1255 |
+
# When a real depth map is available and RGB-based coverage is still low,
|
| 1256 |
+
# try to find additional stair edges from depth vertical gradient peaks.
|
| 1257 |
+
if (
|
| 1258 |
+
geometry_mode == 'stairs'
|
| 1259 |
+
and args.depth
|
| 1260 |
+
and not args.stair_edge_y
|
| 1261 |
+
and stair_edge_coverage < args.minimum_auto_stair_edge_coverage
|
| 1262 |
+
):
|
| 1263 |
+
depth_edges = detect_depth_gradient_stair_edges(
|
| 1264 |
+
depth, visible_mask, args.max_step_edges
|
| 1265 |
+
)
|
| 1266 |
+
if depth_edges:
|
| 1267 |
+
expanded = merge_edge_positions(
|
| 1268 |
+
edges_y + depth_edges,
|
| 1269 |
+
max(8, int(h * 0.018)),
|
| 1270 |
+
)
|
| 1271 |
+
expanded_coverage = stair_edge_coverage_ratio(expanded, amodal_mask)
|
| 1272 |
+
if expanded_coverage > stair_edge_coverage:
|
| 1273 |
+
edges_y = expanded[: args.max_step_edges]
|
| 1274 |
+
stair_edge_coverage = expanded_coverage
|
| 1275 |
+
edge_source = f'{edge_source} + depth-gradient expansion'
|
| 1276 |
+
# Slightly lower confidence since depth-gradient edges are
|
| 1277 |
+
# a secondary signal, but higher than regular fallback.
|
| 1278 |
+
count_score = min(len(edges_y) / 5.0, 1.0)
|
| 1279 |
+
coverage_score = min(
|
| 1280 |
+
stair_edge_coverage / max(args.minimum_auto_stair_edge_coverage, 1e-6),
|
| 1281 |
+
1.0,
|
| 1282 |
+
)
|
| 1283 |
+
stair_edge_confidence = float(
|
| 1284 |
+
np.clip(0.25 + 0.20 * count_score + 0.35 * coverage_score, 0.0, 0.85)
|
| 1285 |
+
)
|
| 1286 |
+
used_depth_gradient_edge_expansion = True
|
| 1287 |
+
|
| 1288 |
+
# --- Late fallback to regular fallback edges if coverage is still low ---
|
| 1289 |
+
if (
|
| 1290 |
+
geometry_mode == 'stairs'
|
| 1291 |
+
and not args.stair_edge_y
|
| 1292 |
+
and (len(edges_y) < args.minimum_detected_step_edges or stair_edge_coverage < args.minimum_auto_stair_edge_coverage)
|
| 1293 |
+
):
|
| 1294 |
+
fallback = fallback_edges(amodal_mask, args.fallback_step_edges)
|
| 1295 |
+
expanded = merge_edge_positions(edges_y + fallback, max(8, int(h * 0.012)))
|
| 1296 |
+
expanded_coverage = stair_edge_coverage_ratio(expanded, amodal_mask)
|
| 1297 |
+
if expanded_coverage > stair_edge_coverage:
|
| 1298 |
+
edges_y = expanded[: args.max_step_edges]
|
| 1299 |
+
stair_edge_coverage = expanded_coverage
|
| 1300 |
+
edge_source = f'{edge_source} + late fallback expansion'
|
| 1301 |
+
used_regular_fallback_edges = True
|
| 1302 |
+
stair_edge_confidence = 0.25
|
| 1303 |
+
|
| 1304 |
+
fx, fy, cx, cy = camera_intrinsics(w, h, args.fx, args.fy, args.cx, args.cy)
|
| 1305 |
+
if geometry_mode == 'stairs':
|
| 1306 |
+
completed_depth, completion_log, confidence = complete_depth_by_planes(
|
| 1307 |
+
depth,
|
| 1308 |
+
completion_mask,
|
| 1309 |
+
amodal_mask,
|
| 1310 |
+
edges_y,
|
| 1311 |
+
fx,
|
| 1312 |
+
fy,
|
| 1313 |
+
cx,
|
| 1314 |
+
cy,
|
| 1315 |
+
args.min_fit_points,
|
| 1316 |
+
stair_edge_slope=stair_edge_slope,
|
| 1317 |
+
)
|
| 1318 |
+
if used_regular_fallback_edges:
|
| 1319 |
+
confidence[completion_mask] *= 0.45
|
| 1320 |
+
completion_method = 'piecewise_stair_planes_with_low_confidence_fallback_edges'
|
| 1321 |
+
else:
|
| 1322 |
+
confidence[completion_mask] *= stair_edge_confidence
|
| 1323 |
+
completion_method = 'piecewise_stair_planes'
|
| 1324 |
+
elif geometry_mode in {'walkable', 'ramp'}:
|
| 1325 |
+
completed_depth, completion_log, confidence = complete_depth_continuous_surface(
|
| 1326 |
+
depth,
|
| 1327 |
+
completion_mask,
|
| 1328 |
+
amodal_mask,
|
| 1329 |
+
fx,
|
| 1330 |
+
fy,
|
| 1331 |
+
cx,
|
| 1332 |
+
cy,
|
| 1333 |
+
args.min_fit_points,
|
| 1334 |
+
)
|
| 1335 |
+
completion_method = completion_log[0][3]
|
| 1336 |
+
else:
|
| 1337 |
+
completed_depth, completion_log, confidence = complete_depth_generic_inpaint(
|
| 1338 |
+
depth, completion_mask, amodal_mask
|
| 1339 |
+
)
|
| 1340 |
+
completion_method = 'edge_aware_inpaint'
|
| 1341 |
+
|
| 1342 |
+
completed_depth = enforce_hidden_boundary_continuity(
|
| 1343 |
+
completed_depth,
|
| 1344 |
+
depth,
|
| 1345 |
+
visible_mask,
|
| 1346 |
+
completion_mask,
|
| 1347 |
+
args.boundary_blend_radius,
|
| 1348 |
+
args.boundary_anchor_power,
|
| 1349 |
+
)
|
| 1350 |
+
completed_depth, depth_correction_mask, depth_plausibility = (
|
| 1351 |
+
enforce_hidden_depth_plausibility(
|
| 1352 |
+
completed_depth,
|
| 1353 |
+
depth,
|
| 1354 |
+
visible_mask,
|
| 1355 |
+
completion_mask,
|
| 1356 |
+
geometry_mode,
|
| 1357 |
+
)
|
| 1358 |
+
)
|
| 1359 |
+
confidence[depth_correction_mask] = np.minimum(
|
| 1360 |
+
confidence[depth_correction_mask],
|
| 1361 |
+
0.15,
|
| 1362 |
+
)
|
| 1363 |
+
# Completion is strictly confined to the reviewed hidden region. The
|
| 1364 |
+
# plausibility guard already enforces this; retain the assignment as a
|
| 1365 |
+
# visible invariant immediately before geometry/texture export.
|
| 1366 |
+
completed_depth[~completion_mask] = depth[~completion_mask]
|
| 1367 |
+
completed_region = completion_mask
|
| 1368 |
+
if args.completed_rgb:
|
| 1369 |
+
provided_completed_rgb = read_source_grid_rgb(
|
| 1370 |
+
args.completed_rgb,
|
| 1371 |
+
source_shape,
|
| 1372 |
+
shape,
|
| 1373 |
+
)
|
| 1374 |
+
clean_completed_colors = rgb.copy()
|
| 1375 |
+
clean_completed_colors[completed_region] = provided_completed_rgb[completed_region]
|
| 1376 |
+
color_completion_method = 'aligned_provided_rgb_completion'
|
| 1377 |
+
completed_rgb_source = str(args.completed_rgb)
|
| 1378 |
+
else:
|
| 1379 |
+
clean_completed_colors = inpaint_rgb(rgb, completed_region, radius=5.0)
|
| 1380 |
+
color_completion_method = 'opencv_telea_fallback'
|
| 1381 |
+
completed_rgb_source = None
|
| 1382 |
+
overlay_completed_colors = rgb.copy()
|
| 1383 |
+
overlay_completed_colors[completed_region] = (
|
| 1384 |
+
0.35 * overlay_completed_colors[completed_region] + 0.65 * np.array([255, 90, 30])
|
| 1385 |
+
).astype(np.uint8)
|
| 1386 |
+
|
| 1387 |
+
points = pixels_to_points(completed_depth, fx, fy, cx, cy)
|
| 1388 |
+
valid_3d = amodal_mask & np.isfinite(completed_depth) & (completed_depth > 0)
|
| 1389 |
+
|
| 1390 |
+
save_mask(output_dir / 'obstacle_mask.png', obstacle_mask)
|
| 1391 |
+
save_mask(output_dir / 'amodal_stair_mask.png', amodal_mask)
|
| 1392 |
+
save_mask(output_dir / 'amodal_target_mask.png', amodal_mask)
|
| 1393 |
+
save_mask(output_dir / 'target_visible_mask.png', visible_mask)
|
| 1394 |
+
save_mask(output_dir / 'hidden_completion_mask.png', completion_mask)
|
| 1395 |
+
save_mask(output_dir / 'depth_plausibility_corrections.png', depth_correction_mask)
|
| 1396 |
+
save_overlay(output_dir / 'debug_overlay.png', rgb, obstacle_mask, amodal_mask, edges_y, stair_edge_slope)
|
| 1397 |
+
Image.fromarray(visualize_depth(depth, valid_3d)).save(output_dir / 'input_depth_vis.png')
|
| 1398 |
+
Image.fromarray(visualize_depth(completed_depth, valid_3d)).save(output_dir / 'completed_depth_vis.png')
|
| 1399 |
+
hidden_depth = np.zeros_like(completed_depth, dtype=np.float32)
|
| 1400 |
+
hidden_depth[completion_mask] = completed_depth[completion_mask]
|
| 1401 |
+
completion_delta = np.zeros_like(completed_depth, dtype=np.float32)
|
| 1402 |
+
completion_delta[completion_mask] = np.abs(completed_depth[completion_mask] - depth[completion_mask])
|
| 1403 |
+
Image.fromarray(visualize_depth(completed_depth, completion_mask)).save(
|
| 1404 |
+
output_dir / 'hidden_completed_depth_vis.png'
|
| 1405 |
+
)
|
| 1406 |
+
Image.fromarray(visualize_depth(completion_delta, completion_mask)).save(
|
| 1407 |
+
output_dir / 'completion_delta_vis.png'
|
| 1408 |
+
)
|
| 1409 |
+
Image.fromarray(np.clip(confidence * 255.0, 0, 255).astype(np.uint8)).save(
|
| 1410 |
+
output_dir / 'completion_confidence.png'
|
| 1411 |
+
)
|
| 1412 |
+
Image.fromarray(clean_completed_colors).save(output_dir / 'completed_rgb_inpaint.png')
|
| 1413 |
+
Image.fromarray(overlay_completed_colors).save(output_dir / 'completed_region_overlay.png')
|
| 1414 |
+
np.save(output_dir / 'completed_depth.npy', completed_depth.astype(np.float32))
|
| 1415 |
+
np.save(output_dir / 'hidden_completed_depth.npy', hidden_depth)
|
| 1416 |
+
np.save(output_dir / 'completion_delta.npy', completion_delta)
|
| 1417 |
+
np.save(output_dir / 'completion_confidence.npy', confidence.astype(np.float32))
|
| 1418 |
+
|
| 1419 |
+
point_count = write_point_cloud_ply(output_dir / 'completed_point_cloud.ply', points, clean_completed_colors, valid_3d, args.point_stride)
|
| 1420 |
+
depth_values = completed_depth[valid_3d]
|
| 1421 |
+
auto_jump = max(0.03, 0.08 * float(np.percentile(depth_values, 95) - np.percentile(depth_values, 5))) if depth_values.size else 0.2
|
| 1422 |
+
mesh_jump = args.max_depth_jump if args.max_depth_jump is not None else auto_jump
|
| 1423 |
+
vertex_count, face_count = write_mesh_ply(
|
| 1424 |
+
output_dir / 'completed_mesh.ply',
|
| 1425 |
+
points,
|
| 1426 |
+
clean_completed_colors,
|
| 1427 |
+
valid_3d,
|
| 1428 |
+
args.mesh_stride,
|
| 1429 |
+
mesh_jump,
|
| 1430 |
+
)
|
| 1431 |
+
|
| 1432 |
+
visible_unchanged = bool(np.array_equal(completed_depth[~completion_mask], depth[~completion_mask]))
|
| 1433 |
+
finite_completed = bool(np.all(np.isfinite(completed_depth[amodal_mask])))
|
| 1434 |
+
confidence_values = confidence[completion_mask]
|
| 1435 |
+
mean_confidence = float(confidence_values.mean()) if confidence_values.size else 1.0
|
| 1436 |
+
report_title = {
|
| 1437 |
+
'stairs': 'Stair Geometry Completion Report',
|
| 1438 |
+
'walkable': 'Walkable Surface Geometry Completion Report',
|
| 1439 |
+
'ramp': 'Ramp Geometry Completion Report',
|
| 1440 |
+
'generic': 'Generic Surface Geometry Completion Report',
|
| 1441 |
+
}[geometry_mode]
|
| 1442 |
+
report = [
|
| 1443 |
+
f'# {report_title}',
|
| 1444 |
+
'',
|
| 1445 |
+
f'- geometry mode: `{geometry_mode}`',
|
| 1446 |
+
f'- completion method: `{completion_method}`',
|
| 1447 |
+
f'- image: `{args.image}`',
|
| 1448 |
+
f'- reference image used for edges: `{edge_source}`',
|
| 1449 |
+
f'- obstacle mask source: `{obstacle_source}`',
|
| 1450 |
+
f'- amodal mask source: `{amodal_source}`',
|
| 1451 |
+
f'- visible mask source: `{visible_source}`',
|
| 1452 |
+
f'- depth source: `{depth_source}`',
|
| 1453 |
+
f'- resolution used: `{w}x{h}`',
|
| 1454 |
+
f'- intrinsics: `fx={fx:.3f}, fy={fy:.3f}, cx={cx:.3f}, cy={cy:.3f}`',
|
| 1455 |
+
f'- stair edge y positions: `{edges_y}`',
|
| 1456 |
+
f'- stair edge slope: `{stair_edge_slope:.6f}`',
|
| 1457 |
+
f'- stair edge angle degrees: `{np.degrees(np.arctan(stair_edge_slope)):.6f}`',
|
| 1458 |
+
f'- used regular fallback stair edges: `{used_regular_fallback_edges}`',
|
| 1459 |
+
f'- used perspective-tolerant edge expansion: `{used_perspective_edge_expansion}`',
|
| 1460 |
+
f'- used visible-mask row-gradient edge expansion: `{used_gradient_edge_expansion}`',
|
| 1461 |
+
f'- stair edge confidence: `{stair_edge_confidence:.6f}`',
|
| 1462 |
+
f'- stair edge vertical coverage: `{stair_edge_coverage:.6f}`',
|
| 1463 |
+
f'- visible depth unchanged outside hidden region: `{visible_unchanged}`',
|
| 1464 |
+
f'- completed target depth finite: `{finite_completed}`',
|
| 1465 |
+
f'- boundary blend radius: `{args.boundary_blend_radius}`',
|
| 1466 |
+
f'- boundary anchor power: `{args.boundary_anchor_power:.6f}`',
|
| 1467 |
+
f'- mean hidden completion confidence: `{mean_confidence:.6f}`',
|
| 1468 |
+
f'- depth plausibility status: `{depth_plausibility["status"]}`',
|
| 1469 |
+
f'- depth plausibility candidate eligible for review: `{depth_plausibility["candidate_eligible_for_review"]}`',
|
| 1470 |
+
f'- corrected implausible hidden depths: `{depth_plausibility["corrected_pixel_count"]}/{depth_plausibility["hidden_pixel_count"]}`',
|
| 1471 |
+
f'- accepted observed-scale depth interval: `{depth_plausibility.get("accepted_depth_lower", "unavailable")} .. {depth_plausibility.get("accepted_depth_upper", "unavailable")}`',
|
| 1472 |
+
f'- point cloud vertices: `{point_count}`',
|
| 1473 |
+
f'- mesh vertices/faces: `{vertex_count}/{face_count}`',
|
| 1474 |
+
'- clean RGB completion: `completed_rgb_inpaint.png`',
|
| 1475 |
+
'',
|
| 1476 |
+
'## Completion Regions',
|
| 1477 |
+
'',
|
| 1478 |
+
]
|
| 1479 |
+
for top, bottom, samples, mode in completion_log:
|
| 1480 |
+
report.append(f'- y `{top}:{bottom}` visible samples `{samples}` -> `{mode}`')
|
| 1481 |
+
prior_note = (
|
| 1482 |
+
'- If no depth map is provided, stairs use a synthetic step prior; other modes use a continuous perspective prior. Both are debug-only.'
|
| 1483 |
+
)
|
| 1484 |
+
report.extend([
|
| 1485 |
+
'',
|
| 1486 |
+
'## Notes',
|
| 1487 |
+
'',
|
| 1488 |
+
prior_note,
|
| 1489 |
+
'- Structured accessibility geometry constraints are written to `accessibility_geometry_analysis.json`.',
|
| 1490 |
+
'- If monocular or heuristic depth is provided, the output is still relative unless calibrated metric depth is used.',
|
| 1491 |
+
'- Hidden-depth plausibility limits are inferred from the observed source-depth scale; they are not metric safety thresholds.',
|
| 1492 |
+
'- Replace the default obstacle box with a real suitcase/occluder mask for meaningful completion.',
|
| 1493 |
+
'- For path planning, provide calibrated intrinsics and metric depth or LiDAR points; otherwise units are arbitrary.',
|
| 1494 |
+
])
|
| 1495 |
+
(output_dir / 'run_report.md').write_text('\n'.join(report), encoding='utf-8')
|
| 1496 |
+
analysis = build_accessibility_geometry_analysis(
|
| 1497 |
+
sample_id=args.sample_id,
|
| 1498 |
+
category=args.category,
|
| 1499 |
+
geometry_mode=geometry_mode,
|
| 1500 |
+
visible_mask=visible_mask,
|
| 1501 |
+
amodal_mask=amodal_mask,
|
| 1502 |
+
hidden_mask=completion_mask,
|
| 1503 |
+
obstacle_mask=obstacle_mask,
|
| 1504 |
+
depth=depth,
|
| 1505 |
+
completed_depth=completed_depth,
|
| 1506 |
+
confidence=confidence,
|
| 1507 |
+
completion_method=completion_method,
|
| 1508 |
+
edges_y=edges_y,
|
| 1509 |
+
stair_edge_slope=stair_edge_slope,
|
| 1510 |
+
edge_source=edge_source,
|
| 1511 |
+
stair_edge_confidence=stair_edge_confidence,
|
| 1512 |
+
stair_edge_coverage=stair_edge_coverage,
|
| 1513 |
+
used_regular_fallback_edges=used_regular_fallback_edges,
|
| 1514 |
+
visible_depth_unchanged=visible_unchanged,
|
| 1515 |
+
completed_target_depth_finite=finite_completed,
|
| 1516 |
+
point_cloud_vertices=point_count,
|
| 1517 |
+
mesh_vertices=vertex_count,
|
| 1518 |
+
mesh_faces=face_count,
|
| 1519 |
+
)
|
| 1520 |
+
analysis_path = output_dir / 'accessibility_geometry_analysis.json'
|
| 1521 |
+
analysis_path.write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding='utf-8')
|
| 1522 |
+
manifest = {
|
| 1523 |
+
'geometry_mode': geometry_mode,
|
| 1524 |
+
'category': analysis['category'],
|
| 1525 |
+
'sample_id': args.sample_id,
|
| 1526 |
+
'completion_method': completion_method,
|
| 1527 |
+
'color_completion_method': color_completion_method,
|
| 1528 |
+
'completed_rgb_source': completed_rgb_source,
|
| 1529 |
+
'completion_log': [
|
| 1530 |
+
{'top': int(top), 'bottom': int(bottom), 'visible_samples': int(samples), 'mode': mode}
|
| 1531 |
+
for top, bottom, samples, mode in completion_log
|
| 1532 |
+
],
|
| 1533 |
+
'depth_source': depth_source,
|
| 1534 |
+
'used_regular_fallback_stair_edges': used_regular_fallback_edges,
|
| 1535 |
+
'used_perspective_edge_expansion': used_perspective_edge_expansion,
|
| 1536 |
+
'used_gradient_edge_expansion': used_gradient_edge_expansion,
|
| 1537 |
+
'used_depth_gradient_edge_expansion': used_depth_gradient_edge_expansion,
|
| 1538 |
+
'stair_edge_confidence': stair_edge_confidence,
|
| 1539 |
+
'stair_edge_coverage': stair_edge_coverage,
|
| 1540 |
+
'stair_edges_y': edges_y,
|
| 1541 |
+
'stair_edge_slope': stair_edge_slope,
|
| 1542 |
+
'stair_edge_angle_degrees': float(np.degrees(np.arctan(stair_edge_slope))),
|
| 1543 |
+
'visible_depth_unchanged_outside_hidden': visible_unchanged,
|
| 1544 |
+
'completed_target_depth_finite': finite_completed,
|
| 1545 |
+
'visible_pixel_count': int(visible_mask.sum()),
|
| 1546 |
+
'amodal_pixel_count': int(amodal_mask.sum()),
|
| 1547 |
+
'hidden_pixel_count': int(completion_mask.sum()),
|
| 1548 |
+
'obstacle_pixel_count': int(obstacle_mask.sum()),
|
| 1549 |
+
'obstacle_target_overlap_pixel_count': int((obstacle_mask & amodal_mask).sum()),
|
| 1550 |
+
'obstacle_visible_overlap_ratio': float((obstacle_mask & visible_mask).sum() / max(int(visible_mask.sum()), 1)),
|
| 1551 |
+
'hidden_changed_ratio': float(((np.abs(completed_depth - depth) > 1e-5) & completion_mask).sum() / max(int(completion_mask.sum()), 1)),
|
| 1552 |
+
'boundary_blend_radius': args.boundary_blend_radius,
|
| 1553 |
+
'boundary_anchor_power': args.boundary_anchor_power,
|
| 1554 |
+
'mean_hidden_completion_confidence': mean_confidence,
|
| 1555 |
+
'depth_plausibility': depth_plausibility,
|
| 1556 |
+
'depth_plausibility_corrections': str(output_dir / 'depth_plausibility_corrections.png'),
|
| 1557 |
+
'geometry_candidate_eligible_for_review': depth_plausibility['candidate_eligible_for_review'],
|
| 1558 |
+
'point_cloud_vertices': point_count,
|
| 1559 |
+
'mesh_vertices': vertex_count,
|
| 1560 |
+
'mesh_faces': face_count,
|
| 1561 |
+
'completed_rgb_inpaint': str(output_dir / 'completed_rgb_inpaint.png'),
|
| 1562 |
+
'accessibility_geometry_analysis': str(analysis_path),
|
| 1563 |
+
'depth_is_metric_truth': False,
|
| 1564 |
+
}
|
| 1565 |
+
(output_dir / 'geometry_manifest.json').write_text(
|
| 1566 |
+
json.dumps(manifest, indent=2, ensure_ascii=False), encoding='utf-8'
|
| 1567 |
+
)
|
| 1568 |
+
|
| 1569 |
+
print(f'Wrote outputs to {output_dir}')
|
| 1570 |
+
print(f'Geometry mode: {geometry_mode}')
|
| 1571 |
+
print(f'Completion method: {completion_method}')
|
| 1572 |
+
print(f'Point cloud vertices: {point_count}')
|
| 1573 |
+
print(f'Mesh vertices/faces: {vertex_count}/{face_count}')
|
| 1574 |
+
|
| 1575 |
+
|
| 1576 |
+
def build_parser():
|
| 1577 |
+
parser = argparse.ArgumentParser(description='Category-aware 3D amodal completion for accessibility surfaces.')
|
| 1578 |
+
parser.add_argument('--image', default=None, help='Occluded RGB image (required when running reconstruction).')
|
| 1579 |
+
parser.add_argument('--sample-id', default=None, help='Optional stable sample id written to JSON outputs.')
|
| 1580 |
+
parser.add_argument(
|
| 1581 |
+
'--category',
|
| 1582 |
+
choices=['stairs', 'ramp', 'walkway', 'walkable', 'curb_cut', 'raised_curb', 'tactile_paving', 'unknown'],
|
| 1583 |
+
default=None,
|
| 1584 |
+
help='Accessibility target category. Defaults to the category implied by --geometry-mode.',
|
| 1585 |
+
)
|
| 1586 |
+
parser.add_argument('--reference-image', default=None, help='Optional clean/similar stair image used to estimate stair edge layout.')
|
| 1587 |
+
parser.add_argument(
|
| 1588 |
+
'--completed-rgb',
|
| 1589 |
+
default=None,
|
| 1590 |
+
help=(
|
| 1591 |
+
'Optional display-oriented RGB completion on the exact source grid. '
|
| 1592 |
+
'Only hidden target pixels are used as mesh colors; when omitted, '
|
| 1593 |
+
'OpenCV Telea is retained as a debug fallback.'
|
| 1594 |
+
),
|
| 1595 |
+
)
|
| 1596 |
+
parser.add_argument('--depth', default=None, help='Optional depth map: .npy, .npz, 16-bit png, or grayscale image.')
|
| 1597 |
+
parser.add_argument('--depth-scale', type=float, default=1.0, help='Multiplier applied to loaded depth values.')
|
| 1598 |
+
parser.add_argument('--obstacle-mask', default=None, help='Binary mask of the occluding object, white/nonzero is obstacle.')
|
| 1599 |
+
parser.add_argument('--amodal-mask', default=None, help='Binary mask of the full target region, including occluded area.')
|
| 1600 |
+
parser.add_argument('--visible-mask', default=None, help='Reviewed visible target mask. Hidden completion is amodal minus visible.')
|
| 1601 |
+
parser.add_argument(
|
| 1602 |
+
'--geometry-mode',
|
| 1603 |
+
choices=['stairs', 'walkable', 'ramp', 'generic'],
|
| 1604 |
+
default=None,
|
| 1605 |
+
help='Geometry prior. Defaults to the category-compatible mode; contradictory category/mode pairs are rejected.',
|
| 1606 |
+
)
|
| 1607 |
+
parser.add_argument('--occlusion-box', action='append', type=parse_box, default=[], help='Obstacle box x1,y1,x2,y2. Coordinates can be normalized or pixels. Repeatable.')
|
| 1608 |
+
parser.add_argument('--no-default-obstacle', action='store_true', help='Do not use the built-in approximate suitcase box when no mask/box is given.')
|
| 1609 |
+
parser.add_argument('--output-dir', default='./output/stair_geometry/', help='Output directory.')
|
| 1610 |
+
parser.add_argument('--max-size', type=int, default=1280, help='Resize longest image side before processing. Use 0 to keep original size.')
|
| 1611 |
+
parser.add_argument('--max-step-edges', type=int, default=9)
|
| 1612 |
+
parser.add_argument('--fallback-step-edges', type=int, default=5)
|
| 1613 |
+
parser.add_argument('--minimum-detected-step-edges', type=int, default=2)
|
| 1614 |
+
parser.add_argument('--minimum-auto-stair-edge-coverage', type=float, default=0.35)
|
| 1615 |
+
parser.add_argument('--stair-edge-y', action='append', type=int, default=[], help='Reviewed stair tread-edge y coordinate. Repeatable; disables regular fallback edges.')
|
| 1616 |
+
parser.add_argument('--stair-edge-angle-degrees', type=float, default=0.0, help='Reviewed tread-edge angle for --stair-edge-y; 0 means horizontal.')
|
| 1617 |
+
parser.add_argument('--synthetic-near', type=float, default=1.0)
|
| 1618 |
+
parser.add_argument('--synthetic-far', type=float, default=6.0)
|
| 1619 |
+
parser.add_argument('--fx', type=float, default=None)
|
| 1620 |
+
parser.add_argument('--fy', type=float, default=None)
|
| 1621 |
+
parser.add_argument('--cx', type=float, default=None)
|
| 1622 |
+
parser.add_argument('--cy', type=float, default=None)
|
| 1623 |
+
parser.add_argument('--min-fit-points', type=int, default=800)
|
| 1624 |
+
parser.add_argument('--boundary-blend-radius', type=int, default=12)
|
| 1625 |
+
parser.add_argument(
|
| 1626 |
+
'--boundary-anchor-power',
|
| 1627 |
+
type=float,
|
| 1628 |
+
default=0.45,
|
| 1629 |
+
help='Power applied to hidden-boundary anchoring weights; lower values preserve visible-depth continuity over a wider band.',
|
| 1630 |
+
)
|
| 1631 |
+
parser.add_argument('--point-stride', type=int, default=3)
|
| 1632 |
+
parser.add_argument('--mesh-stride', type=int, default=5)
|
| 1633 |
+
parser.add_argument('--max-depth-jump', type=float, default=None, help='Max depth discontinuity allowed when making mesh faces.')
|
| 1634 |
+
return parser
|
| 1635 |
+
|
| 1636 |
+
|
| 1637 |
+
if __name__ == '__main__':
|
| 1638 |
+
args = build_parser().parse_args()
|
| 1639 |
+
if args.max_size == 0:
|
| 1640 |
+
args.max_size = None
|
| 1641 |
+
run(args)
|
accessibilityamodal/sam_refinement.py
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Refine coarse target/obstacle masks with Segment Anything box prompts."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
import json
|
| 7 |
+
import sys
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
import cv2
|
| 11 |
+
import numpy as np
|
| 12 |
+
from PIL import Image, ImageOps
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def read_rgb(path: str | Path) -> np.ndarray:
|
| 16 |
+
return np.array(ImageOps.exif_transpose(Image.open(path)).convert('RGB'))
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def read_mask(path: str | Path, shape: tuple[int, int]) -> np.ndarray:
|
| 20 |
+
mask = np.array(ImageOps.exif_transpose(Image.open(path)).convert('L')) > 127
|
| 21 |
+
h, w = shape
|
| 22 |
+
if mask.shape != (h, w):
|
| 23 |
+
raise ValueError(
|
| 24 |
+
f'Mask/RGB raster mismatch for {path}: mask={mask.shape}, rgb={(h, w)}. '
|
| 25 |
+
'Refusing to resize because this can hide EXIF-orientation misalignment.'
|
| 26 |
+
)
|
| 27 |
+
return mask
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def save_mask(path: str | Path, mask: np.ndarray) -> None:
|
| 31 |
+
Image.fromarray((mask.astype(np.uint8) * 255)).save(path)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def component_boxes(mask: np.ndarray, keep: int, min_area: int, pad: int) -> np.ndarray:
|
| 35 |
+
num, labels, stats, _ = cv2.connectedComponentsWithStats(mask.astype(np.uint8), connectivity=8)
|
| 36 |
+
boxes = []
|
| 37 |
+
areas = []
|
| 38 |
+
h, w = mask.shape
|
| 39 |
+
for idx in range(1, num):
|
| 40 |
+
area = int(stats[idx, cv2.CC_STAT_AREA])
|
| 41 |
+
if area < min_area:
|
| 42 |
+
continue
|
| 43 |
+
x = int(stats[idx, cv2.CC_STAT_LEFT])
|
| 44 |
+
y = int(stats[idx, cv2.CC_STAT_TOP])
|
| 45 |
+
bw = int(stats[idx, cv2.CC_STAT_WIDTH])
|
| 46 |
+
bh = int(stats[idx, cv2.CC_STAT_HEIGHT])
|
| 47 |
+
boxes.append([max(0, x - pad), max(0, y - pad), min(w - 1, x + bw + pad), min(h - 1, y + bh + pad)])
|
| 48 |
+
areas.append(area)
|
| 49 |
+
if not boxes:
|
| 50 |
+
return np.empty((0, 4), dtype=np.float32)
|
| 51 |
+
order = np.argsort(np.array(areas))[::-1][:keep]
|
| 52 |
+
return np.array([boxes[i] for i in order], dtype=np.float32)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def refine_one_mask(predictor, mask: np.ndarray, keep: int, min_area: int, pad: int) -> np.ndarray:
|
| 56 |
+
boxes = component_boxes(mask, keep=keep, min_area=min_area, pad=pad)
|
| 57 |
+
if boxes.size == 0:
|
| 58 |
+
return mask
|
| 59 |
+
|
| 60 |
+
import torch
|
| 61 |
+
|
| 62 |
+
transformed = predictor.transform.apply_boxes_torch(
|
| 63 |
+
torch.as_tensor(boxes, dtype=torch.float32, device=predictor.device),
|
| 64 |
+
mask.shape,
|
| 65 |
+
)
|
| 66 |
+
masks, scores, _ = predictor.predict_torch(
|
| 67 |
+
point_coords=None,
|
| 68 |
+
point_labels=None,
|
| 69 |
+
boxes=transformed,
|
| 70 |
+
multimask_output=True,
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
refined = np.zeros_like(mask, dtype=bool)
|
| 74 |
+
masks_np = masks.detach().cpu().numpy()
|
| 75 |
+
scores_np = scores.detach().cpu().numpy()
|
| 76 |
+
for i in range(masks_np.shape[0]):
|
| 77 |
+
best = int(np.argmax(scores_np[i]))
|
| 78 |
+
refined |= masks_np[i, best].astype(bool)
|
| 79 |
+
return refined
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def overlay(rgb: np.ndarray, masks: list[tuple[np.ndarray, tuple[int, int, int], float]]) -> np.ndarray:
|
| 83 |
+
out = rgb.astype(np.float32).copy()
|
| 84 |
+
for mask, color, alpha in masks:
|
| 85 |
+
if mask.any():
|
| 86 |
+
out[mask] = out[mask] * (1.0 - alpha) + np.array(color, dtype=np.float32) * alpha
|
| 87 |
+
return np.clip(out, 0, 255).astype(np.uint8)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def build_parser() -> argparse.ArgumentParser:
|
| 91 |
+
parser = argparse.ArgumentParser(description='Refine binary masks with SAM using mask-derived box prompts.')
|
| 92 |
+
parser.add_argument('--image', required=True)
|
| 93 |
+
parser.add_argument('--target-mask', required=True)
|
| 94 |
+
parser.add_argument('--obstacle-mask', required=True)
|
| 95 |
+
parser.add_argument('--output-dir', required=True)
|
| 96 |
+
parser.add_argument('--sam-repo', default='../amodal/segment-anything', help='Path containing the segment_anything package.')
|
| 97 |
+
parser.add_argument('--sam-checkpoint', required=True)
|
| 98 |
+
parser.add_argument('--sam-model-type', choices=['vit_h', 'vit_l', 'vit_b', 'default'], default='vit_h')
|
| 99 |
+
parser.add_argument('--device', default='auto')
|
| 100 |
+
parser.add_argument('--keep-target-components', type=int, default=8)
|
| 101 |
+
parser.add_argument('--keep-obstacle-components', type=int, default=4)
|
| 102 |
+
parser.add_argument('--min-area', type=int, default=64)
|
| 103 |
+
return parser
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def main() -> None:
|
| 107 |
+
args = build_parser().parse_args()
|
| 108 |
+
rgb = read_rgb(args.image)
|
| 109 |
+
shape = rgb.shape[:2]
|
| 110 |
+
target = read_mask(args.target_mask, shape)
|
| 111 |
+
obstacle = read_mask(args.obstacle_mask, shape)
|
| 112 |
+
|
| 113 |
+
sam_repo = Path(args.sam_repo).resolve()
|
| 114 |
+
checkpoint = Path(args.sam_checkpoint).resolve()
|
| 115 |
+
if not checkpoint.exists():
|
| 116 |
+
raise FileNotFoundError(f'SAM checkpoint not found: {checkpoint}')
|
| 117 |
+
if not sam_repo.exists():
|
| 118 |
+
raise FileNotFoundError(f'SAM repo not found: {sam_repo}')
|
| 119 |
+
|
| 120 |
+
sys.path.insert(0, str(sam_repo))
|
| 121 |
+
import torch
|
| 122 |
+
from segment_anything import SamPredictor, sam_model_registry
|
| 123 |
+
|
| 124 |
+
if args.device == 'auto':
|
| 125 |
+
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
| 126 |
+
else:
|
| 127 |
+
device = args.device
|
| 128 |
+
sam = sam_model_registry[args.sam_model_type](checkpoint=str(checkpoint)).to(device=device)
|
| 129 |
+
predictor = SamPredictor(sam)
|
| 130 |
+
predictor.set_image(rgb)
|
| 131 |
+
|
| 132 |
+
refined_target = refine_one_mask(
|
| 133 |
+
predictor,
|
| 134 |
+
target,
|
| 135 |
+
keep=args.keep_target_components,
|
| 136 |
+
min_area=args.min_area,
|
| 137 |
+
pad=args.box_pad,
|
| 138 |
+
)
|
| 139 |
+
refined_obstacle = refine_one_mask(
|
| 140 |
+
predictor,
|
| 141 |
+
obstacle,
|
| 142 |
+
keep=args.keep_obstacle_components,
|
| 143 |
+
min_area=args.min_area,
|
| 144 |
+
pad=args.box_pad,
|
| 145 |
+
)
|
| 146 |
+
|
| 147 |
+
output_dir = Path(args.output_dir)
|
| 148 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 149 |
+
target_path = output_dir / 'target_visible_mask_sam.png'
|
| 150 |
+
obstacle_path = output_dir / 'obstacle_mask_sam.png'
|
| 151 |
+
overlay_path = output_dir / 'sam_refine_overlay.png'
|
| 152 |
+
manifest_path = output_dir / 'sam_refine_manifest.json'
|
| 153 |
+
save_mask(target_path, refined_target)
|
| 154 |
+
save_mask(obstacle_path, refined_obstacle)
|
| 155 |
+
Image.fromarray(overlay(rgb, [
|
| 156 |
+
(refined_target, (0, 220, 80), 0.45),
|
| 157 |
+
(refined_obstacle, (255, 60, 20), 0.55),
|
| 158 |
+
])).save(overlay_path)
|
| 159 |
+
manifest_path.write_text(json.dumps({
|
| 160 |
+
'image': args.image,
|
| 161 |
+
'sam_repo': str(sam_repo),
|
| 162 |
+
'sam_checkpoint': str(checkpoint),
|
| 163 |
+
'sam_model_type': args.sam_model_type,
|
| 164 |
+
'device': device,
|
| 165 |
+
'target_output': str(target_path),
|
| 166 |
+
'obstacle_output': str(obstacle_path),
|
| 167 |
+
'overlay': str(overlay_path),
|
| 168 |
+
}, indent=2), encoding='utf-8')
|
| 169 |
+
print(f'Wrote SAM-refined masks to {output_dir}')
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
if __name__ == '__main__':
|
| 173 |
+
main()
|
accessibilityamodal/verification.py
ADDED
|
@@ -0,0 +1,764 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Auditable quality gates for accessibility 3D completion candidates.
|
| 2 |
+
|
| 3 |
+
This module deliberately records compact, inspectable evidence instead of
|
| 4 |
+
using an opaque free-form reasoning step. It is a *reconstruction quality*
|
| 5 |
+
gate, not a certification of accessibility, safety, or metric navigation.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import json
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from typing import Any
|
| 13 |
+
|
| 14 |
+
import cv2
|
| 15 |
+
import numpy as np
|
| 16 |
+
from PIL import Image
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
DECISION_ORDER = {"accept": 0, "manual_review": 1, "reject": 2}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
CATEGORY_POLICIES: dict[str, dict[str, float]] = {
|
| 23 |
+
"stairs": {
|
| 24 |
+
"min_visible_image_ratio": 0.008,
|
| 25 |
+
"min_visible_amodal_ratio": 0.15,
|
| 26 |
+
"max_hidden_visible_ratio": 4.0,
|
| 27 |
+
"max_hidden_amodal_ratio": 0.75,
|
| 28 |
+
"max_obstacle_amodal_ratio": 0.75,
|
| 29 |
+
},
|
| 30 |
+
"ramp": {
|
| 31 |
+
"min_visible_image_ratio": 0.006,
|
| 32 |
+
"min_visible_amodal_ratio": 0.12,
|
| 33 |
+
"max_hidden_visible_ratio": 3.5,
|
| 34 |
+
"max_hidden_amodal_ratio": 0.72,
|
| 35 |
+
"max_obstacle_amodal_ratio": 0.72,
|
| 36 |
+
},
|
| 37 |
+
"walkway": {
|
| 38 |
+
"min_visible_image_ratio": 0.008,
|
| 39 |
+
"min_visible_amodal_ratio": 0.12,
|
| 40 |
+
"max_hidden_visible_ratio": 3.5,
|
| 41 |
+
"max_hidden_amodal_ratio": 0.72,
|
| 42 |
+
"max_obstacle_amodal_ratio": 0.72,
|
| 43 |
+
},
|
| 44 |
+
"curb_cut": {
|
| 45 |
+
"min_visible_image_ratio": 0.004,
|
| 46 |
+
"min_visible_amodal_ratio": 0.10,
|
| 47 |
+
"max_hidden_visible_ratio": 3.0,
|
| 48 |
+
"max_hidden_amodal_ratio": 0.70,
|
| 49 |
+
"max_obstacle_amodal_ratio": 0.70,
|
| 50 |
+
},
|
| 51 |
+
"tactile_paving": {
|
| 52 |
+
"min_visible_image_ratio": 0.002,
|
| 53 |
+
"min_visible_amodal_ratio": 0.10,
|
| 54 |
+
"max_hidden_visible_ratio": 3.0,
|
| 55 |
+
"max_hidden_amodal_ratio": 0.70,
|
| 56 |
+
"max_obstacle_amodal_ratio": 0.70,
|
| 57 |
+
},
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _round(value: float, digits: int = 6) -> float:
|
| 62 |
+
return round(float(value), digits)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _ratio(numerator: int | float, denominator: int | float) -> float:
|
| 66 |
+
return float(numerator) / float(denominator) if denominator else 0.0
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _policy(category: str) -> dict[str, float]:
|
| 70 |
+
return CATEGORY_POLICIES.get(category, CATEGORY_POLICIES["walkway"])
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _component_stats(mask: np.ndarray) -> dict[str, Any]:
|
| 74 |
+
count, _, stats, _ = cv2.connectedComponentsWithStats(mask.astype(np.uint8), 8)
|
| 75 |
+
areas = stats[1:, cv2.CC_STAT_AREA] if count > 1 else np.empty((0,), dtype=np.int32)
|
| 76 |
+
total = int(mask.sum())
|
| 77 |
+
largest = int(areas.max()) if areas.size else 0
|
| 78 |
+
return {
|
| 79 |
+
"component_count": int(len(areas)),
|
| 80 |
+
"largest_component_pixels": largest,
|
| 81 |
+
"largest_component_fraction": _round(_ratio(largest, total)),
|
| 82 |
+
}
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def _trace(
|
| 86 |
+
rule_id: str,
|
| 87 |
+
decision: str,
|
| 88 |
+
observed: Any,
|
| 89 |
+
expected: Any,
|
| 90 |
+
explanation: str,
|
| 91 |
+
) -> dict[str, Any]:
|
| 92 |
+
return {
|
| 93 |
+
"rule_id": rule_id,
|
| 94 |
+
"decision": decision,
|
| 95 |
+
"observed": observed,
|
| 96 |
+
"expected": expected,
|
| 97 |
+
"explanation": explanation,
|
| 98 |
+
}
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def _worst_decision(*decisions: str) -> str:
|
| 102 |
+
return max(decisions, key=lambda value: DECISION_ORDER[value])
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def evaluate_mask_preflight(
|
| 106 |
+
*,
|
| 107 |
+
category: str,
|
| 108 |
+
visible_mask: np.ndarray,
|
| 109 |
+
amodal_mask: np.ndarray,
|
| 110 |
+
hidden_mask: np.ndarray,
|
| 111 |
+
obstacle_mask: np.ndarray,
|
| 112 |
+
sam3_quality: dict[str, Any] | None = None,
|
| 113 |
+
reviewed_visible_metadata: dict[str, Any] | None = None,
|
| 114 |
+
) -> dict[str, Any]:
|
| 115 |
+
"""Evaluate whether masks have enough observed support for automatic 3D."""
|
| 116 |
+
policy = _policy(category)
|
| 117 |
+
input_shapes = {
|
| 118 |
+
"visible": list(visible_mask.shape),
|
| 119 |
+
"amodal": list(amodal_mask.shape),
|
| 120 |
+
"hidden": list(hidden_mask.shape),
|
| 121 |
+
"obstacle": list(obstacle_mask.shape),
|
| 122 |
+
}
|
| 123 |
+
if not (
|
| 124 |
+
visible_mask.shape == amodal_mask.shape == hidden_mask.shape == obstacle_mask.shape
|
| 125 |
+
):
|
| 126 |
+
return {
|
| 127 |
+
"decision": "reject",
|
| 128 |
+
"policy": policy,
|
| 129 |
+
"measurements": {"input_mask_shapes": input_shapes},
|
| 130 |
+
"audit_trace": [
|
| 131 |
+
_trace(
|
| 132 |
+
"mask_shape_consistency",
|
| 133 |
+
"reject",
|
| 134 |
+
input_shapes,
|
| 135 |
+
"all masks have exactly the same HxW shape",
|
| 136 |
+
"A gate must not resample mismatched masks, because that can silently change the claimed visible/hidden support.",
|
| 137 |
+
)
|
| 138 |
+
],
|
| 139 |
+
}
|
| 140 |
+
image_area = int(visible_mask.size)
|
| 141 |
+
visible_pixels = int(visible_mask.sum())
|
| 142 |
+
amodal_pixels = int(amodal_mask.sum())
|
| 143 |
+
hidden_pixels = int(hidden_mask.sum())
|
| 144 |
+
obstacle_pixels = int(obstacle_mask.sum())
|
| 145 |
+
obstacle_amodal_pixels = int((obstacle_mask & amodal_mask).sum())
|
| 146 |
+
visible_amodal_ratio = _ratio(visible_pixels, amodal_pixels)
|
| 147 |
+
hidden_visible_ratio = _ratio(hidden_pixels, max(visible_pixels, 1))
|
| 148 |
+
hidden_amodal_ratio = _ratio(hidden_pixels, amodal_pixels)
|
| 149 |
+
obstacle_amodal_ratio = _ratio(obstacle_amodal_pixels, amodal_pixels)
|
| 150 |
+
trace: list[dict[str, Any]] = []
|
| 151 |
+
|
| 152 |
+
sam_status = str((sam3_quality or {}).get("review_status") or "missing")
|
| 153 |
+
reviewed_visible_approved = bool(
|
| 154 |
+
reviewed_visible_metadata
|
| 155 |
+
and str(reviewed_visible_metadata.get("source_kind", "")) == "human_reviewed_visible_mask_only"
|
| 156 |
+
and str(reviewed_visible_metadata.get("review_status", "")) == "approved"
|
| 157 |
+
and bool(reviewed_visible_metadata.get("review_is_human", False))
|
| 158 |
+
and bool(reviewed_visible_metadata.get("visible_confirmed", False))
|
| 159 |
+
and str(reviewed_visible_metadata.get("annotator", "")).strip()
|
| 160 |
+
)
|
| 161 |
+
if reviewed_visible_metadata is not None:
|
| 162 |
+
trace.append(
|
| 163 |
+
_trace(
|
| 164 |
+
"reviewed_visible_mask_status",
|
| 165 |
+
"accept" if reviewed_visible_approved else "reject",
|
| 166 |
+
{
|
| 167 |
+
"source_kind": reviewed_visible_metadata.get("source_kind"),
|
| 168 |
+
"review_status": reviewed_visible_metadata.get("review_status"),
|
| 169 |
+
"review_is_human": bool(reviewed_visible_metadata.get("review_is_human", False)),
|
| 170 |
+
"visible_confirmed": bool(reviewed_visible_metadata.get("visible_confirmed", False)),
|
| 171 |
+
"annotator_present": bool(str(reviewed_visible_metadata.get("annotator", "")).strip()),
|
| 172 |
+
},
|
| 173 |
+
"approved, human-reviewed, visible-confirmed workspace with an annotator",
|
| 174 |
+
"A reviewed visible boundary can replace the SAM3 proposal only with explicit human approval and provenance."
|
| 175 |
+
if reviewed_visible_approved
|
| 176 |
+
else "The reviewed-visible workspace lacks the required approval provenance.",
|
| 177 |
+
)
|
| 178 |
+
)
|
| 179 |
+
if sam3_quality is not None and not reviewed_visible_approved:
|
| 180 |
+
category_verification = dict(sam3_quality.get("category_verification") or {})
|
| 181 |
+
selected_category = str(sam3_quality.get("selected_category") or "")
|
| 182 |
+
best_category = str(category_verification.get("best_scored_category") or "")
|
| 183 |
+
category_disagreement = bool(category_verification.get("category_disagreement", False))
|
| 184 |
+
category_ok = (
|
| 185 |
+
(not selected_category or selected_category == category)
|
| 186 |
+
and (not best_category or best_category == category)
|
| 187 |
+
and not category_disagreement
|
| 188 |
+
)
|
| 189 |
+
trace.append(
|
| 190 |
+
_trace(
|
| 191 |
+
"category_semantic_consistency",
|
| 192 |
+
"accept" if category_ok else "manual_review",
|
| 193 |
+
{
|
| 194 |
+
"requested_category": category,
|
| 195 |
+
"selected_category": selected_category or None,
|
| 196 |
+
"best_scored_category": best_category or None,
|
| 197 |
+
"category_disagreement": category_disagreement,
|
| 198 |
+
},
|
| 199 |
+
"selected and best-scored category agree with the requested category",
|
| 200 |
+
"The prompt evidence does not consistently identify the requested stairs/ramp/walkway structure; review category and support boundary."
|
| 201 |
+
if not category_ok
|
| 202 |
+
else "Prompt-category evidence is consistent with the requested support structure.",
|
| 203 |
+
)
|
| 204 |
+
)
|
| 205 |
+
if reviewed_visible_approved:
|
| 206 |
+
trace.append(
|
| 207 |
+
_trace(
|
| 208 |
+
"sam3_proposal_status",
|
| 209 |
+
"accept",
|
| 210 |
+
sam_status,
|
| 211 |
+
"recorded only; superseded by approved reviewed-visible mask",
|
| 212 |
+
"The original SAM3 proposal is retained for audit but does not override a separately approved visible target boundary.",
|
| 213 |
+
)
|
| 214 |
+
)
|
| 215 |
+
elif sam_status in {"reject_or_reprompt", "error"}:
|
| 216 |
+
trace.append(
|
| 217 |
+
_trace(
|
| 218 |
+
"sam3_review_status",
|
| 219 |
+
"reject",
|
| 220 |
+
sam_status,
|
| 221 |
+
"candidate_accept_after_visual_review or manual_review",
|
| 222 |
+
"The segmentation stage itself rejected this proposal; it must not be sent to automatic 3D generation.",
|
| 223 |
+
)
|
| 224 |
+
)
|
| 225 |
+
elif sam_status == "manual_review":
|
| 226 |
+
trace.append(
|
| 227 |
+
_trace(
|
| 228 |
+
"sam3_review_status",
|
| 229 |
+
"manual_review",
|
| 230 |
+
sam_status,
|
| 231 |
+
"candidate_accept_after_visual_review",
|
| 232 |
+
"The segmentation proposal needs human mask review before any 3D result is used.",
|
| 233 |
+
)
|
| 234 |
+
)
|
| 235 |
+
else:
|
| 236 |
+
trace.append(
|
| 237 |
+
_trace(
|
| 238 |
+
"sam3_review_status",
|
| 239 |
+
"accept" if sam_status == "candidate_accept_after_visual_review" else "manual_review",
|
| 240 |
+
sam_status,
|
| 241 |
+
"candidate_accept_after_visual_review",
|
| 242 |
+
"SAM3 status is recorded as evidence; automatic masks remain proposals rather than ground truth.",
|
| 243 |
+
)
|
| 244 |
+
)
|
| 245 |
+
|
| 246 |
+
visible_image_ratio = _ratio(visible_pixels, image_area)
|
| 247 |
+
for rule_id, observed, threshold, comparator, explanation in (
|
| 248 |
+
(
|
| 249 |
+
"minimum_observed_support",
|
| 250 |
+
visible_image_ratio,
|
| 251 |
+
policy["min_visible_image_ratio"],
|
| 252 |
+
">=",
|
| 253 |
+
"The observed target footprint is too small to support a stable scene-level reconstruction.",
|
| 254 |
+
),
|
| 255 |
+
(
|
| 256 |
+
"visible_over_amodal_support",
|
| 257 |
+
visible_amodal_ratio,
|
| 258 |
+
policy["min_visible_amodal_ratio"],
|
| 259 |
+
">=",
|
| 260 |
+
"Most of the alleged support is hidden, so the completion would be driven by hallucinated geometry rather than observed structure.",
|
| 261 |
+
),
|
| 262 |
+
(
|
| 263 |
+
"hidden_over_visible_support",
|
| 264 |
+
hidden_visible_ratio,
|
| 265 |
+
policy["max_hidden_visible_ratio"],
|
| 266 |
+
"<=",
|
| 267 |
+
"The hidden region is disproportionate to observed support and commonly indicates wall/railing leakage into the amodal mask.",
|
| 268 |
+
),
|
| 269 |
+
(
|
| 270 |
+
"hidden_over_amodal_support",
|
| 271 |
+
hidden_amodal_ratio,
|
| 272 |
+
policy["max_hidden_amodal_ratio"],
|
| 273 |
+
"<=",
|
| 274 |
+
"Automatic 3D must retain enough visible surface evidence rather than invent nearly the whole target.",
|
| 275 |
+
),
|
| 276 |
+
(
|
| 277 |
+
"obstacle_dominates_target",
|
| 278 |
+
obstacle_amodal_ratio,
|
| 279 |
+
policy["max_obstacle_amodal_ratio"],
|
| 280 |
+
"<=",
|
| 281 |
+
"An obstacle that occupies most of the alleged target is likely a side wall, railing, or segmentation leak rather than a compact occluder.",
|
| 282 |
+
),
|
| 283 |
+
):
|
| 284 |
+
passed = observed >= threshold if comparator == ">=" else observed <= threshold
|
| 285 |
+
trace.append(
|
| 286 |
+
_trace(
|
| 287 |
+
rule_id,
|
| 288 |
+
"accept" if passed else "reject",
|
| 289 |
+
_round(observed),
|
| 290 |
+
f"{comparator} {_round(threshold)}",
|
| 291 |
+
"Observed support satisfies the guard." if passed else explanation,
|
| 292 |
+
)
|
| 293 |
+
)
|
| 294 |
+
|
| 295 |
+
components = _component_stats(visible_mask)
|
| 296 |
+
fragmented = (
|
| 297 |
+
components["component_count"] >= 8
|
| 298 |
+
and components["largest_component_fraction"] < 0.35
|
| 299 |
+
)
|
| 300 |
+
trace.append(
|
| 301 |
+
_trace(
|
| 302 |
+
"visible_support_connectedness",
|
| 303 |
+
"manual_review" if fragmented else "accept",
|
| 304 |
+
components,
|
| 305 |
+
"largest component >= 35% when >= 8 components",
|
| 306 |
+
"Fragmented stair treads can be valid, but need a reviewed support boundary before surface completion."
|
| 307 |
+
if fragmented
|
| 308 |
+
else "Observed support is sufficiently connected for automatic processing.",
|
| 309 |
+
)
|
| 310 |
+
)
|
| 311 |
+
|
| 312 |
+
decision = _worst_decision(*(row["decision"] for row in trace))
|
| 313 |
+
return {
|
| 314 |
+
"decision": decision,
|
| 315 |
+
"policy": policy,
|
| 316 |
+
"measurements": {
|
| 317 |
+
"image_pixels": image_area,
|
| 318 |
+
"visible_pixels": visible_pixels,
|
| 319 |
+
"amodal_pixels": amodal_pixels,
|
| 320 |
+
"hidden_pixels": hidden_pixels,
|
| 321 |
+
"obstacle_pixels": obstacle_pixels,
|
| 322 |
+
"visible_image_ratio": _round(visible_image_ratio),
|
| 323 |
+
"visible_amodal_ratio": _round(visible_amodal_ratio),
|
| 324 |
+
"hidden_visible_ratio": _round(hidden_visible_ratio),
|
| 325 |
+
"hidden_amodal_ratio": _round(hidden_amodal_ratio),
|
| 326 |
+
"obstacle_amodal_ratio": _round(obstacle_amodal_ratio),
|
| 327 |
+
**components,
|
| 328 |
+
},
|
| 329 |
+
"audit_trace": trace,
|
| 330 |
+
}
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
def evaluate_geometry_structure(
|
| 334 |
+
category: str,
|
| 335 |
+
geometry_manifest: dict[str, Any] | None,
|
| 336 |
+
) -> dict[str, Any]:
|
| 337 |
+
"""Check deterministic evidence emitted by accessibilityamodal.reconstruct."""
|
| 338 |
+
if not geometry_manifest:
|
| 339 |
+
return {
|
| 340 |
+
"decision": "manual_review",
|
| 341 |
+
"measurements": {},
|
| 342 |
+
"audit_trace": [
|
| 343 |
+
_trace(
|
| 344 |
+
"geometry_manifest_present",
|
| 345 |
+
"manual_review",
|
| 346 |
+
False,
|
| 347 |
+
True,
|
| 348 |
+
"No deterministic geometry manifest is available yet.",
|
| 349 |
+
)
|
| 350 |
+
],
|
| 351 |
+
}
|
| 352 |
+
trace: list[dict[str, Any]] = []
|
| 353 |
+
expected_mode = {
|
| 354 |
+
"stairs": "stairs",
|
| 355 |
+
"ramp": "ramp",
|
| 356 |
+
"walkway": "walkable",
|
| 357 |
+
"curb_cut": "walkable",
|
| 358 |
+
"tactile_paving": "walkable",
|
| 359 |
+
}.get(category)
|
| 360 |
+
observed_mode = str(geometry_manifest.get("geometry_mode") or "")
|
| 361 |
+
trace.append(
|
| 362 |
+
_trace(
|
| 363 |
+
"geometry_category_mode_consistency",
|
| 364 |
+
"accept" if observed_mode == expected_mode else "manual_review",
|
| 365 |
+
{"requested_category": category, "geometry_mode": observed_mode or None},
|
| 366 |
+
f"geometry_mode={expected_mode}",
|
| 367 |
+
"The deterministic geometry branch does not match the requested accessibility structure."
|
| 368 |
+
if observed_mode != expected_mode
|
| 369 |
+
else "The deterministic geometry branch matches the requested structure.",
|
| 370 |
+
)
|
| 371 |
+
)
|
| 372 |
+
log = list(geometry_manifest.get("completion_log") or [])
|
| 373 |
+
plane_bands = sum(
|
| 374 |
+
1 for row in log if str(row.get("mode") or "").startswith("plane")
|
| 375 |
+
)
|
| 376 |
+
fallback_bands = sum(
|
| 377 |
+
1 for row in log if "fallback" in str(row.get("mode") or ""))
|
| 378 |
+
fallback_ratio = _ratio(fallback_bands, max(len(log), 1))
|
| 379 |
+
trace.append(
|
| 380 |
+
_trace(
|
| 381 |
+
"completion_band_support",
|
| 382 |
+
"reject" if fallback_ratio > 0.25 else "accept",
|
| 383 |
+
{"band_count": len(log), "plane_bands": plane_bands, "fallback_bands": fallback_bands, "fallback_ratio": _round(fallback_ratio)},
|
| 384 |
+
"fallback_ratio <= 0.25",
|
| 385 |
+
"Too many stair bands lacked observed points and fell back to image inpainting."
|
| 386 |
+
if fallback_ratio > 0.25
|
| 387 |
+
else "Most completion bands are supported by fitted geometry.",
|
| 388 |
+
)
|
| 389 |
+
)
|
| 390 |
+
hidden_pixels = int(geometry_manifest.get("hidden_pixel_count") or 0)
|
| 391 |
+
confidence = float(geometry_manifest.get("mean_hidden_completion_confidence") or 0.0)
|
| 392 |
+
hidden_confidence_decision = "accept" if hidden_pixels == 0 else "manual_review" if confidence < 0.50 else "accept"
|
| 393 |
+
trace.append(
|
| 394 |
+
_trace(
|
| 395 |
+
"hidden_geometry_confidence",
|
| 396 |
+
hidden_confidence_decision,
|
| 397 |
+
"not_applicable_no_hidden_region" if hidden_pixels == 0 else _round(confidence),
|
| 398 |
+
"not applicable when hidden_pixel_count=0; otherwise >= 0.50",
|
| 399 |
+
"No target surface was extrapolated, so hidden-depth confidence is not applicable."
|
| 400 |
+
if hidden_pixels == 0
|
| 401 |
+
else "Hidden depth confidence is low; do not treat the result as a publishable or navigation-ready surface."
|
| 402 |
+
if hidden_confidence_decision == "manual_review"
|
| 403 |
+
else "Hidden completion confidence meets the local visualization threshold.",
|
| 404 |
+
)
|
| 405 |
+
)
|
| 406 |
+
faces = int(geometry_manifest.get("mesh_faces") or 0)
|
| 407 |
+
trace.append(
|
| 408 |
+
_trace(
|
| 409 |
+
"mesh_nonempty",
|
| 410 |
+
"reject" if faces <= 0 else "accept",
|
| 411 |
+
faces,
|
| 412 |
+
"> 0",
|
| 413 |
+
"No mesh faces were produced." if faces <= 0 else "Mesh contains triangle faces.",
|
| 414 |
+
)
|
| 415 |
+
)
|
| 416 |
+
if category == "stairs":
|
| 417 |
+
edges = list(geometry_manifest.get("stair_edges_y") or [])
|
| 418 |
+
edge_confidence = float(geometry_manifest.get("stair_edge_confidence") or 0.0)
|
| 419 |
+
trace.append(
|
| 420 |
+
_trace(
|
| 421 |
+
"stair_repetition_evidence",
|
| 422 |
+
"manual_review" if len(edges) < 3 or edge_confidence < 0.45 else "accept",
|
| 423 |
+
{"edge_count": len(edges), "edge_confidence": _round(edge_confidence), "edge_slope": _round(float(geometry_manifest.get("stair_edge_slope") or 0.0))},
|
| 424 |
+
"at least 3 edges and confidence >= 0.45",
|
| 425 |
+
"Too little repeated tread evidence is available for a reliable stair regularization."
|
| 426 |
+
if len(edges) < 3 or edge_confidence < 0.45
|
| 427 |
+
else "Repeated stair-edge evidence supports a segmented stair prior.",
|
| 428 |
+
)
|
| 429 |
+
)
|
| 430 |
+
decision = _worst_decision(*(row["decision"] for row in trace))
|
| 431 |
+
return {
|
| 432 |
+
"decision": decision,
|
| 433 |
+
"measurements": {
|
| 434 |
+
"completion_band_count": len(log),
|
| 435 |
+
"plane_band_count": plane_bands,
|
| 436 |
+
"fallback_band_count": fallback_bands,
|
| 437 |
+
"fallback_band_ratio": _round(fallback_ratio),
|
| 438 |
+
"hidden_pixel_count": hidden_pixels,
|
| 439 |
+
"mean_hidden_completion_confidence": _round(confidence),
|
| 440 |
+
"mesh_faces": faces,
|
| 441 |
+
},
|
| 442 |
+
"audit_trace": trace,
|
| 443 |
+
}
|
| 444 |
+
|
| 445 |
+
|
| 446 |
+
def _foreground_measurement(path: Path) -> dict[str, float]:
|
| 447 |
+
image = np.asarray(Image.open(path).convert("RGB"))
|
| 448 |
+
foreground = np.any(image < 245, axis=2)
|
| 449 |
+
ys, xs = np.where(foreground)
|
| 450 |
+
if xs.size == 0:
|
| 451 |
+
return {"coverage": 0.0, "bbox_width_ratio": 0.0, "bbox_height_ratio": 0.0, "bbox_aspect_ratio": 0.0}
|
| 452 |
+
width_ratio = _ratio(int(xs.max() - xs.min() + 1), image.shape[1])
|
| 453 |
+
height_ratio = _ratio(int(ys.max() - ys.min() + 1), image.shape[0])
|
| 454 |
+
return {
|
| 455 |
+
"coverage": _round(float(foreground.mean())),
|
| 456 |
+
"bbox_width_ratio": _round(width_ratio),
|
| 457 |
+
"bbox_height_ratio": _round(height_ratio),
|
| 458 |
+
"bbox_aspect_ratio": _round(min(width_ratio, height_ratio) / max(width_ratio, height_ratio, 1e-6)),
|
| 459 |
+
}
|
| 460 |
+
|
| 461 |
+
|
| 462 |
+
def evaluate_full_gpu_render_contract(learned_dir: Path) -> dict[str, Any]:
|
| 463 |
+
"""Require evidence that both learned representations were rasterized on CUDA."""
|
| 464 |
+
|
| 465 |
+
manifest_path = learned_dir / "manifest.json"
|
| 466 |
+
required_files = (
|
| 467 |
+
"sample_gaussian.gif",
|
| 468 |
+
"sample_mesh.gif",
|
| 469 |
+
"sample_multi.gif",
|
| 470 |
+
"multiview_contact_sheet.jpg",
|
| 471 |
+
"mesh.ply",
|
| 472 |
+
)
|
| 473 |
+
missing_files = [
|
| 474 |
+
name
|
| 475 |
+
for name in required_files
|
| 476 |
+
if not (learned_dir / name).is_file()
|
| 477 |
+
or (learned_dir / name).stat().st_size <= 0
|
| 478 |
+
]
|
| 479 |
+
payload: dict[str, Any] = {}
|
| 480 |
+
manifest_error = None
|
| 481 |
+
try:
|
| 482 |
+
value = json.loads(manifest_path.read_text(encoding="utf-8"))
|
| 483 |
+
if not isinstance(value, dict):
|
| 484 |
+
raise ValueError("manifest root is not an object")
|
| 485 |
+
payload = value
|
| 486 |
+
except Exception as exc:
|
| 487 |
+
manifest_error = f"{type(exc).__name__}: {exc}"
|
| 488 |
+
|
| 489 |
+
renderer = payload.get("gpu_renderer_runtime")
|
| 490 |
+
validation = payload.get("render_validation")
|
| 491 |
+
gaussian = renderer.get("gaussian") if isinstance(renderer, dict) else None
|
| 492 |
+
dense_mesh = (
|
| 493 |
+
renderer.get("dense_mesh") if isinstance(renderer, dict) else None
|
| 494 |
+
)
|
| 495 |
+
nviews = int(payload.get("nviews") or 0)
|
| 496 |
+
gaussian_views = sorted(learned_dir.glob("*_gs.png"))
|
| 497 |
+
mesh_views = sorted(learned_dir.glob("*_mesh.png"))
|
| 498 |
+
contract_ok = (
|
| 499 |
+
manifest_error is None
|
| 500 |
+
and not missing_files
|
| 501 |
+
and isinstance(renderer, dict)
|
| 502 |
+
and isinstance(gaussian, dict)
|
| 503 |
+
and gaussian.get("available") is True
|
| 504 |
+
and gaussian.get("required") is True
|
| 505 |
+
and gaussian.get("device") == "cuda"
|
| 506 |
+
and gaussian.get("runtime_import_succeeded") is True
|
| 507 |
+
and isinstance(dense_mesh, dict)
|
| 508 |
+
and dense_mesh.get("available") is True
|
| 509 |
+
and dense_mesh.get("required") is True
|
| 510 |
+
and dense_mesh.get("device") == "cuda"
|
| 511 |
+
and dense_mesh.get("runtime_import_succeeded") is True
|
| 512 |
+
and dense_mesh.get("cuda_context_preflight_succeeded") is True
|
| 513 |
+
and renderer.get("cpu_render_fallback_allowed") is False
|
| 514 |
+
and renderer.get("gaussian_only_debug_mode") is False
|
| 515 |
+
and isinstance(validation, dict)
|
| 516 |
+
and validation.get("validated") is True
|
| 517 |
+
and validation.get("dense_mesh_rendered_on_gpu") is True
|
| 518 |
+
and validation.get("cpu_render_fallback_used") is False
|
| 519 |
+
and int(validation.get("mesh_face_count") or 0) > 0
|
| 520 |
+
and nviews > 0
|
| 521 |
+
and len(gaussian_views) == nviews
|
| 522 |
+
and len(mesh_views) == nviews
|
| 523 |
+
)
|
| 524 |
+
return {
|
| 525 |
+
"decision": "accept" if contract_ok else "reject",
|
| 526 |
+
"measurements": {
|
| 527 |
+
"manifest_error": manifest_error,
|
| 528 |
+
"missing_files": missing_files,
|
| 529 |
+
"declared_view_count": nviews,
|
| 530 |
+
"gaussian_view_count": len(gaussian_views),
|
| 531 |
+
"mesh_view_count": len(mesh_views),
|
| 532 |
+
"gaussian_cuda": bool(
|
| 533 |
+
isinstance(gaussian, dict)
|
| 534 |
+
and gaussian.get("device") == "cuda"
|
| 535 |
+
),
|
| 536 |
+
"dense_mesh_cuda": bool(
|
| 537 |
+
isinstance(dense_mesh, dict)
|
| 538 |
+
and dense_mesh.get("device") == "cuda"
|
| 539 |
+
),
|
| 540 |
+
"cpu_render_fallback_used": (
|
| 541 |
+
validation.get("cpu_render_fallback_used")
|
| 542 |
+
if isinstance(validation, dict)
|
| 543 |
+
else None
|
| 544 |
+
),
|
| 545 |
+
},
|
| 546 |
+
"audit_trace": [
|
| 547 |
+
_trace(
|
| 548 |
+
"full_gpu_renderer_contract",
|
| 549 |
+
"accept" if contract_ok else "reject",
|
| 550 |
+
{
|
| 551 |
+
"manifest_present": manifest_path.is_file(),
|
| 552 |
+
"missing_files": missing_files,
|
| 553 |
+
"gaussian_views": len(gaussian_views),
|
| 554 |
+
"mesh_views": len(mesh_views),
|
| 555 |
+
"declared_views": nviews,
|
| 556 |
+
},
|
| 557 |
+
"CUDA Gaussian and nvdiffrast mesh renders, no CPU fallback",
|
| 558 |
+
(
|
| 559 |
+
"The learned result proves both CUDA render paths and a "
|
| 560 |
+
"validated dense triangle mesh."
|
| 561 |
+
if contract_ok
|
| 562 |
+
else "The learned result is incomplete or does not prove the required CUDA Gaussian + dense-mesh render contract."
|
| 563 |
+
),
|
| 564 |
+
)
|
| 565 |
+
],
|
| 566 |
+
}
|
| 567 |
+
|
| 568 |
+
|
| 569 |
+
def evaluate_learned_multiview(
|
| 570 |
+
category: str,
|
| 571 |
+
learned_dir: Path | None,
|
| 572 |
+
) -> dict[str, Any]:
|
| 573 |
+
"""Check learned visual candidates without treating renderer coordinates as metric geometry."""
|
| 574 |
+
if learned_dir is None or not learned_dir.is_dir():
|
| 575 |
+
return {
|
| 576 |
+
"decision": "manual_review",
|
| 577 |
+
"measurements": {},
|
| 578 |
+
"audit_trace": [
|
| 579 |
+
_trace("learned_multiview_present", "manual_review", False, True, "No learned Accessibility3D multiview result is available.")
|
| 580 |
+
],
|
| 581 |
+
}
|
| 582 |
+
views = sorted(learned_dir.glob("*_gs.png"))
|
| 583 |
+
if not views:
|
| 584 |
+
return {
|
| 585 |
+
"decision": "reject",
|
| 586 |
+
"measurements": {},
|
| 587 |
+
"audit_trace": [
|
| 588 |
+
_trace("learned_multiview_present", "reject", False, True, "The learned 3D result has no rendered multiview evidence.")
|
| 589 |
+
],
|
| 590 |
+
}
|
| 591 |
+
measures = [_foreground_measurement(path) for path in views]
|
| 592 |
+
gpu_contract = evaluate_full_gpu_render_contract(learned_dir)
|
| 593 |
+
median_coverage = float(np.median([row["coverage"] for row in measures]))
|
| 594 |
+
median_aspect = float(np.median([row["bbox_aspect_ratio"] for row in measures]))
|
| 595 |
+
coverage_decision = (
|
| 596 |
+
"reject" if median_coverage < 0.05 else "manual_review" if median_coverage < 0.12 else "accept"
|
| 597 |
+
)
|
| 598 |
+
aspect_decision = (
|
| 599 |
+
"reject" if median_aspect < 0.10 else "manual_review" if median_aspect < 0.30 else "accept"
|
| 600 |
+
)
|
| 601 |
+
trace = [
|
| 602 |
+
*gpu_contract["audit_trace"],
|
| 603 |
+
_trace(
|
| 604 |
+
"learned_multiview_evidence_type",
|
| 605 |
+
"accept",
|
| 606 |
+
"Gaussian-splat raster previews (*_gs.png)",
|
| 607 |
+
"visual reconstruction evidence only",
|
| 608 |
+
"These views can expose a collapsed visual candidate but do not establish physical dimensions or navigability.",
|
| 609 |
+
),
|
| 610 |
+
_trace(
|
| 611 |
+
"multiview_foreground_coverage",
|
| 612 |
+
coverage_decision,
|
| 613 |
+
_round(median_coverage),
|
| 614 |
+
">= 0.12",
|
| 615 |
+
"The learned object occupies almost none of the rendered views and is likely an empty or fragmentary result."
|
| 616 |
+
if coverage_decision == "reject"
|
| 617 |
+
else "The object is small in the rendered views; inspect its requested scale and framing before use."
|
| 618 |
+
if coverage_decision == "manual_review"
|
| 619 |
+
else "The rendered object has adequate view coverage.",
|
| 620 |
+
),
|
| 621 |
+
_trace(
|
| 622 |
+
"multiview_silhouette_thickness",
|
| 623 |
+
aspect_decision,
|
| 624 |
+
_round(median_aspect),
|
| 625 |
+
">= 0.30",
|
| 626 |
+
"Most views are nearly one-dimensional, which is inconsistent with a usable support-surface candidate."
|
| 627 |
+
if aspect_decision == "reject"
|
| 628 |
+
else "The views are elongated; this can be valid for a long ramp or stair flight, but needs human geometry review."
|
| 629 |
+
if aspect_decision == "manual_review"
|
| 630 |
+
else "Rendered silhouettes have a plausible two-dimensional extent.",
|
| 631 |
+
),
|
| 632 |
+
]
|
| 633 |
+
mesh_measurements: dict[str, Any] = {}
|
| 634 |
+
mesh_path = learned_dir / "mesh.ply"
|
| 635 |
+
if mesh_path.is_file():
|
| 636 |
+
try:
|
| 637 |
+
import trimesh
|
| 638 |
+
|
| 639 |
+
mesh = trimesh.load(mesh_path, process=False)
|
| 640 |
+
extent = np.asarray(mesh.bounds[1] - mesh.bounds[0], dtype=np.float64)
|
| 641 |
+
minmax_ratio = float(extent.min() / max(float(extent.max()), 1e-6))
|
| 642 |
+
mesh_measurements = {
|
| 643 |
+
"vertices": int(len(mesh.vertices)),
|
| 644 |
+
"faces": int(len(mesh.faces)),
|
| 645 |
+
"extent": [_round(value) for value in extent],
|
| 646 |
+
"minmax_extent_ratio": _round(minmax_ratio),
|
| 647 |
+
"watertight": bool(mesh.is_watertight),
|
| 648 |
+
}
|
| 649 |
+
extent_decision = (
|
| 650 |
+
"reject" if minmax_ratio < 0.02 else "manual_review" if minmax_ratio < 0.08 else "accept"
|
| 651 |
+
)
|
| 652 |
+
trace.append(
|
| 653 |
+
_trace(
|
| 654 |
+
"mesh_near_degeneracy",
|
| 655 |
+
extent_decision,
|
| 656 |
+
_round(minmax_ratio),
|
| 657 |
+
">= 0.08 (manual review below); < 0.02 rejects",
|
| 658 |
+
"The learned mesh is almost flat in its own normalized coordinates, consistent with a degenerate fragment."
|
| 659 |
+
if extent_decision == "reject"
|
| 660 |
+
else "The learned mesh is elongated; normalized extents alone cannot distinguish a valid long ramp/staircase from a fragment."
|
| 661 |
+
if extent_decision == "manual_review"
|
| 662 |
+
else "No near-zero mesh axis was detected. This is not a physical-scale check.",
|
| 663 |
+
)
|
| 664 |
+
)
|
| 665 |
+
except Exception as exc: # pragma: no cover - optional mesh parsing
|
| 666 |
+
trace.append(_trace("mesh_near_degeneracy", "manual_review", "unavailable", ">= 0.08", f"Could not inspect learned mesh: {type(exc).__name__}"))
|
| 667 |
+
decision = _worst_decision(*(row["decision"] for row in trace))
|
| 668 |
+
return {
|
| 669 |
+
"decision": decision,
|
| 670 |
+
"measurements": {
|
| 671 |
+
"view_count": len(measures),
|
| 672 |
+
"median_foreground_coverage": _round(median_coverage),
|
| 673 |
+
"median_silhouette_aspect": _round(median_aspect),
|
| 674 |
+
"views": measures,
|
| 675 |
+
"mesh": mesh_measurements,
|
| 676 |
+
"gpu_render_contract": gpu_contract["measurements"],
|
| 677 |
+
},
|
| 678 |
+
"audit_trace": trace,
|
| 679 |
+
}
|
| 680 |
+
|
| 681 |
+
|
| 682 |
+
def mobility_interpretation(category: str, decision: str) -> dict[str, Any]:
|
| 683 |
+
"""Return conservative, non-metric audience-specific interpretation."""
|
| 684 |
+
withheld = decision != "accept"
|
| 685 |
+
common = "withheld pending mask/geometry review" if withheld else "requires calibrated clearance and surface survey"
|
| 686 |
+
if category == "stairs":
|
| 687 |
+
return {
|
| 688 |
+
"pedestrian": common,
|
| 689 |
+
"blind_or_low_vision_pedestrian": "requires surveyed handrail, edge, tactile, lighting, and obstacle information; monocular 3D is insufficient",
|
| 690 |
+
"wheelchair": "stairs are not an accessible route; require a separately verified ramp/lift/alternate route",
|
| 691 |
+
"robot_or_robot_dog": "requires metric riser/tread, friction, width, and local obstacle sensing; do not execute from this visual model alone",
|
| 692 |
+
}
|
| 693 |
+
return {
|
| 694 |
+
"pedestrian": common,
|
| 695 |
+
"blind_or_low_vision_pedestrian": "requires surveyed tactile/edge/obstacle information; monocular 3D is insufficient",
|
| 696 |
+
"wheelchair": common,
|
| 697 |
+
"robot_or_robot_dog": "requires metric slope, clearance, friction, and local obstacle sensing; do not execute from this visual model alone",
|
| 698 |
+
}
|
| 699 |
+
|
| 700 |
+
|
| 701 |
+
def build_verification(
|
| 702 |
+
*,
|
| 703 |
+
sample_id: str,
|
| 704 |
+
category: str,
|
| 705 |
+
visible_mask: np.ndarray,
|
| 706 |
+
amodal_mask: np.ndarray,
|
| 707 |
+
hidden_mask: np.ndarray,
|
| 708 |
+
obstacle_mask: np.ndarray,
|
| 709 |
+
sam3_quality: dict[str, Any] | None = None,
|
| 710 |
+
reviewed_visible_metadata: dict[str, Any] | None = None,
|
| 711 |
+
geometry_manifest: dict[str, Any] | None = None,
|
| 712 |
+
learned_dir: Path | None = None,
|
| 713 |
+
) -> dict[str, Any]:
|
| 714 |
+
"""Build one JSON-ready, auditable accessibility 3D verification record."""
|
| 715 |
+
preflight = evaluate_mask_preflight(
|
| 716 |
+
category=category,
|
| 717 |
+
visible_mask=visible_mask,
|
| 718 |
+
amodal_mask=amodal_mask,
|
| 719 |
+
hidden_mask=hidden_mask,
|
| 720 |
+
obstacle_mask=obstacle_mask,
|
| 721 |
+
sam3_quality=sam3_quality,
|
| 722 |
+
reviewed_visible_metadata=reviewed_visible_metadata,
|
| 723 |
+
)
|
| 724 |
+
geometry = evaluate_geometry_structure(category, geometry_manifest)
|
| 725 |
+
learned = evaluate_learned_multiview(category, learned_dir)
|
| 726 |
+
overall = _worst_decision(preflight["decision"], geometry["decision"], learned["decision"])
|
| 727 |
+
if geometry_manifest is None and learned_dir is None:
|
| 728 |
+
overall = preflight["decision"]
|
| 729 |
+
trace = [
|
| 730 |
+
*preflight["audit_trace"],
|
| 731 |
+
*geometry["audit_trace"],
|
| 732 |
+
*learned["audit_trace"],
|
| 733 |
+
]
|
| 734 |
+
return {
|
| 735 |
+
"schema_version": 1,
|
| 736 |
+
"sample_id": sample_id,
|
| 737 |
+
"category": category,
|
| 738 |
+
"decision": overall,
|
| 739 |
+
"gate": {
|
| 740 |
+
"allow_automatic_geometry": preflight["decision"] == "accept",
|
| 741 |
+
"allow_learned_object_3d": (
|
| 742 |
+
preflight["decision"] == "accept"
|
| 743 |
+
and geometry["decision"] == "accept"
|
| 744 |
+
and learned["decision"] == "accept"
|
| 745 |
+
),
|
| 746 |
+
"allow_publication": False,
|
| 747 |
+
"publication_note": "Human license/privacy review and calibrated geometry validation remain required.",
|
| 748 |
+
},
|
| 749 |
+
"mask_preflight": preflight,
|
| 750 |
+
"geometry_structure": geometry,
|
| 751 |
+
"learned_multiview": learned,
|
| 752 |
+
"mobility_interpretation": mobility_interpretation(category, overall),
|
| 753 |
+
"audit_trace": trace,
|
| 754 |
+
"recommended_action": (
|
| 755 |
+
"re_prompt_or_review_masks_before_3d" if preflight["decision"] != "accept"
|
| 756 |
+
else "review_geometry_before_release" if overall != "accept"
|
| 757 |
+
else "keep_as_local_visual_candidate_not_navigation_truth"
|
| 758 |
+
),
|
| 759 |
+
"limitations": [
|
| 760 |
+
"This is a structured reconstruction-quality gate, not chain-of-thought or a safety certification.",
|
| 761 |
+
"Monocular/depth-model geometry is not calibrated metric truth unless separately calibrated.",
|
| 762 |
+
"Do not use this record alone to control a pedestrian aid, wheelchair, robot, or robot dog.",
|
| 763 |
+
],
|
| 764 |
+
}
|
accessibilityamodal/visual_completion.py
ADDED
|
@@ -0,0 +1,493 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Mask policy, prompts, and quality gates for visual accessibility completion.
|
| 2 |
+
|
| 3 |
+
The geometry target and the visual removal region have deliberately different
|
| 4 |
+
roles. ``hidden`` remains the geometry target. For visual inpainting, an
|
| 5 |
+
entire foreground obstacle instance is removed when any pixel in its
|
| 6 |
+
8-connected component intersects ``hidden``. Nearby people or objects that do
|
| 7 |
+
not intersect ``hidden`` remain untouched.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import math
|
| 13 |
+
from typing import Any
|
| 14 |
+
|
| 15 |
+
import cv2
|
| 16 |
+
import numpy as np
|
| 17 |
+
from PIL import Image
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
PROMPTS = {
|
| 21 |
+
"stairs": (
|
| 22 |
+
"photorealistic continuation of the same staircase behind the removed foreground "
|
| 23 |
+
"occluder, continuous stair treads aligned with the visible steps, same perspective, "
|
| 24 |
+
"same material and texture, same lighting and exposure, empty completed surface"
|
| 25 |
+
),
|
| 26 |
+
"ramp": (
|
| 27 |
+
"photorealistic continuation of the same accessible ramp behind the removed foreground "
|
| 28 |
+
"occluder, continuous sloped walking surface, same perspective, same material and "
|
| 29 |
+
"texture, same lighting and exposure, empty completed surface"
|
| 30 |
+
),
|
| 31 |
+
"curb_cut": (
|
| 32 |
+
"photorealistic continuation of the same curb cut and pavement transition behind the "
|
| 33 |
+
"removed foreground occluder, same perspective, same material and texture, same lighting "
|
| 34 |
+
"and exposure, continuous empty completed surface"
|
| 35 |
+
),
|
| 36 |
+
"raised_curb": (
|
| 37 |
+
"photorealistic continuation of the same raised curb behind the removed foreground "
|
| 38 |
+
"occluder, one continuous level curb top and straight curb face, same perspective, "
|
| 39 |
+
"same material and texture, same lighting and exposure, no added steps"
|
| 40 |
+
),
|
| 41 |
+
"tactile_paving": (
|
| 42 |
+
"photorealistic continuation of the same tactile paving path behind the removed "
|
| 43 |
+
"foreground occluder, regularly aligned tactile pattern, same perspective, same material "
|
| 44 |
+
"and texture, same lighting and exposure, continuous empty completed surface"
|
| 45 |
+
),
|
| 46 |
+
"walkway": (
|
| 47 |
+
"photorealistic continuation of the same accessible pedestrian walkway behind the "
|
| 48 |
+
"removed foreground occluder, continuous walking surface, same perspective, same material "
|
| 49 |
+
"and texture, same lighting and exposure, empty completed surface"
|
| 50 |
+
),
|
| 51 |
+
}
|
| 52 |
+
|
| 53 |
+
NEGATIVE_PROMPT = (
|
| 54 |
+
"person, human, legs, pedestrian, bicycle, wheel, wheelchair, stroller, walker, cart, "
|
| 55 |
+
"luggage, bag, backpack, cane, obstacle, vehicle, animal, fog, haze, blur, melted "
|
| 56 |
+
"geometry, warped stairs, broken steps, duplicate steps, misaligned edges, text, "
|
| 57 |
+
"watermark, illustration"
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
FOG_HAZE_MAX_SHARPNESS_RATIO = 0.65
|
| 61 |
+
FOG_HAZE_MAX_CONTRAST_RATIO = 0.85
|
| 62 |
+
OVER_SHARP_MIN_SHARPNESS_RATIO = 1.9
|
| 63 |
+
OVER_SHARP_MIN_SEAM_PENALTY = 1.45
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _binary_array(mask: np.ndarray, name: str) -> np.ndarray:
|
| 67 |
+
array = np.asarray(mask)
|
| 68 |
+
if array.ndim != 2:
|
| 69 |
+
raise ValueError(f"{name} must be a 2D mask, got shape={array.shape}")
|
| 70 |
+
return array.astype(bool, copy=False)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def derive_hidden_mask(target_amodal: np.ndarray, target_visible: np.ndarray) -> np.ndarray:
|
| 74 |
+
"""Derive the geometry hidden target without changing either input mask."""
|
| 75 |
+
|
| 76 |
+
amodal = _binary_array(target_amodal, "target_amodal")
|
| 77 |
+
visible = _binary_array(target_visible, "target_visible")
|
| 78 |
+
if amodal.shape != visible.shape:
|
| 79 |
+
raise ValueError("Target amodal and visible masks must have the same shape")
|
| 80 |
+
return amodal & ~visible
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def mask_statistics(mask: np.ndarray) -> dict[str, int | float]:
|
| 84 |
+
"""Return path-free mask statistics suitable for a portable manifest."""
|
| 85 |
+
|
| 86 |
+
binary = _binary_array(mask, "mask")
|
| 87 |
+
height, width = binary.shape
|
| 88 |
+
pixels = int(binary.sum())
|
| 89 |
+
image_pixels = int(binary.size)
|
| 90 |
+
return {
|
| 91 |
+
"width": int(width),
|
| 92 |
+
"height": int(height),
|
| 93 |
+
"image_pixels": image_pixels,
|
| 94 |
+
"pixel_count": pixels,
|
| 95 |
+
"image_fraction": round(pixels / image_pixels, 8) if image_pixels else 0.0,
|
| 96 |
+
}
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def _retain_components_occluding_hidden(
|
| 100 |
+
detected_obstacles: np.ndarray,
|
| 101 |
+
hidden: np.ndarray,
|
| 102 |
+
) -> tuple[np.ndarray, int, int]:
|
| 103 |
+
"""Equivalent policy to filter_accessibility_occluding_obstacles."""
|
| 104 |
+
|
| 105 |
+
if detected_obstacles.shape != hidden.shape:
|
| 106 |
+
raise ValueError("Obstacle and hidden masks must have the same shape")
|
| 107 |
+
count, labels = cv2.connectedComponents(
|
| 108 |
+
detected_obstacles.astype(np.uint8),
|
| 109 |
+
connectivity=8,
|
| 110 |
+
)
|
| 111 |
+
keep_labels = np.unique(labels[hidden & detected_obstacles])
|
| 112 |
+
keep_labels = keep_labels[keep_labels != 0]
|
| 113 |
+
return np.isin(labels, keep_labels), int(count - 1), int(len(keep_labels))
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def build_visual_removal_mask(
|
| 117 |
+
hidden: np.ndarray,
|
| 118 |
+
obstacle: np.ndarray | None = None,
|
| 119 |
+
) -> tuple[np.ndarray, dict[str, Any]]:
|
| 120 |
+
"""Build the visual inpaint mask while preserving ``hidden`` for geometry.
|
| 121 |
+
|
| 122 |
+
When ``obstacle`` is omitted, the returned visual mask is exactly ``hidden``
|
| 123 |
+
for backward compatibility.
|
| 124 |
+
"""
|
| 125 |
+
|
| 126 |
+
geometry_hidden = _binary_array(hidden, "hidden")
|
| 127 |
+
if obstacle is None:
|
| 128 |
+
detected = np.zeros_like(geometry_hidden)
|
| 129 |
+
retained = np.zeros_like(geometry_hidden)
|
| 130 |
+
before_components = 0
|
| 131 |
+
retained_components = 0
|
| 132 |
+
else:
|
| 133 |
+
detected = _binary_array(obstacle, "obstacle")
|
| 134 |
+
if detected.shape != geometry_hidden.shape:
|
| 135 |
+
raise ValueError("Obstacle and hidden masks must have the same shape")
|
| 136 |
+
retained, before_components, retained_components = (
|
| 137 |
+
_retain_components_occluding_hidden(detected, geometry_hidden)
|
| 138 |
+
)
|
| 139 |
+
|
| 140 |
+
visual_removal = geometry_hidden | retained
|
| 141 |
+
stats = {
|
| 142 |
+
"policy": (
|
| 143 |
+
"hidden_union_full_8_connected_obstacle_components_intersecting_hidden"
|
| 144 |
+
if obstacle is not None
|
| 145 |
+
else "legacy_hidden_only"
|
| 146 |
+
),
|
| 147 |
+
"obstacle_mask_provided": obstacle is not None,
|
| 148 |
+
"geometry_hidden_unchanged": True,
|
| 149 |
+
"geometry_hidden": mask_statistics(geometry_hidden),
|
| 150 |
+
"obstacle_input": mask_statistics(detected),
|
| 151 |
+
"obstacle_components_input": before_components,
|
| 152 |
+
"obstacle_retained": mask_statistics(retained),
|
| 153 |
+
"obstacle_components_retained": retained_components,
|
| 154 |
+
"non_occluding_obstacle_pixels_excluded": int((detected & ~retained).sum()),
|
| 155 |
+
"visual_removal": mask_statistics(visual_removal),
|
| 156 |
+
"visual_extra_pixels_beyond_hidden": int((visual_removal & ~geometry_hidden).sum()),
|
| 157 |
+
}
|
| 158 |
+
return visual_removal, stats
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
def build_completion_envelope(
|
| 162 |
+
visual_removal: np.ndarray,
|
| 163 |
+
obstacle: np.ndarray | None,
|
| 164 |
+
*,
|
| 165 |
+
margin_fraction: float = 0.022,
|
| 166 |
+
) -> tuple[np.ndarray, dict[str, Any]]:
|
| 167 |
+
"""Fill retained foreground-instance boxes before generative completion.
|
| 168 |
+
|
| 169 |
+
A person mask often excludes a carried bag, walker, bicycle frame, or the
|
| 170 |
+
small gaps between limbs. Inpainting only the segmentation silhouette can
|
| 171 |
+
therefore preserve or regenerate those objects. This appearance-only mask
|
| 172 |
+
fills the bounding box of each retained obstacle component and adds a small
|
| 173 |
+
image-relative margin. Geometry continues to use the unchanged hidden
|
| 174 |
+
target; non-occluding obstacle pixels are protected by the caller.
|
| 175 |
+
"""
|
| 176 |
+
|
| 177 |
+
removal = _binary_array(visual_removal, "visual_removal")
|
| 178 |
+
if margin_fraction < 0:
|
| 179 |
+
raise ValueError("margin_fraction must be non-negative")
|
| 180 |
+
if obstacle is None:
|
| 181 |
+
return removal.copy(), {
|
| 182 |
+
"policy": "visual_removal_without_obstacle_envelope",
|
| 183 |
+
"component_count": 0,
|
| 184 |
+
"margin_pixels": 0,
|
| 185 |
+
"extra_pixels": 0,
|
| 186 |
+
"completion_envelope": mask_statistics(removal),
|
| 187 |
+
}
|
| 188 |
+
|
| 189 |
+
detected = _binary_array(obstacle, "obstacle")
|
| 190 |
+
if detected.shape != removal.shape:
|
| 191 |
+
raise ValueError("Obstacle and visual removal masks must have the same shape")
|
| 192 |
+
retained_obstacle = detected & removal
|
| 193 |
+
count, labels, stats, _ = cv2.connectedComponentsWithStats(
|
| 194 |
+
retained_obstacle.astype(np.uint8),
|
| 195 |
+
connectivity=8,
|
| 196 |
+
)
|
| 197 |
+
height, width = removal.shape
|
| 198 |
+
margin = int(round(min(height, width) * margin_fraction))
|
| 199 |
+
envelope = removal.copy()
|
| 200 |
+
boxes: list[dict[str, int]] = []
|
| 201 |
+
for label in range(1, count):
|
| 202 |
+
x, y, box_width, box_height, area = (
|
| 203 |
+
int(value) for value in stats[label]
|
| 204 |
+
)
|
| 205 |
+
if area <= 0:
|
| 206 |
+
continue
|
| 207 |
+
x1 = max(0, x - margin)
|
| 208 |
+
y1 = max(0, y - margin)
|
| 209 |
+
x2 = min(width, x + box_width + margin)
|
| 210 |
+
y2 = min(height, y + box_height + margin)
|
| 211 |
+
envelope[y1:y2, x1:x2] = True
|
| 212 |
+
boxes.append(
|
| 213 |
+
{
|
| 214 |
+
"x1": x1,
|
| 215 |
+
"y1": y1,
|
| 216 |
+
"x2": x2,
|
| 217 |
+
"y2": y2,
|
| 218 |
+
"source_component_pixels": area,
|
| 219 |
+
}
|
| 220 |
+
)
|
| 221 |
+
return envelope, {
|
| 222 |
+
"policy": "retained_obstacle_component_bounding_envelopes",
|
| 223 |
+
"component_count": len(boxes),
|
| 224 |
+
"margin_pixels": margin,
|
| 225 |
+
"boxes": boxes,
|
| 226 |
+
"extra_pixels": int((envelope & ~removal).sum()),
|
| 227 |
+
"completion_envelope": mask_statistics(envelope),
|
| 228 |
+
}
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def quality_flags_for_metrics(
|
| 232 |
+
*,
|
| 233 |
+
sharpness_ratio: float,
|
| 234 |
+
contrast_ratio: float,
|
| 235 |
+
seam_penalty: float,
|
| 236 |
+
) -> list[str]:
|
| 237 |
+
"""Return deterministic visual-risk flags for candidate metrics."""
|
| 238 |
+
|
| 239 |
+
flags: list[str] = []
|
| 240 |
+
if (
|
| 241 |
+
sharpness_ratio < FOG_HAZE_MAX_SHARPNESS_RATIO
|
| 242 |
+
and contrast_ratio < FOG_HAZE_MAX_CONTRAST_RATIO
|
| 243 |
+
):
|
| 244 |
+
flags.append("fog_haze")
|
| 245 |
+
if (
|
| 246 |
+
sharpness_ratio > OVER_SHARP_MIN_SHARPNESS_RATIO
|
| 247 |
+
and seam_penalty > OVER_SHARP_MIN_SEAM_PENALTY
|
| 248 |
+
):
|
| 249 |
+
flags.append("over_sharp_foreground_artifact")
|
| 250 |
+
return flags
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
def candidate_quality(
|
| 254 |
+
image: Image.Image,
|
| 255 |
+
mask: Image.Image,
|
| 256 |
+
category: str,
|
| 257 |
+
) -> dict[str, Any]:
|
| 258 |
+
"""Score a visual candidate and attach conservative review-gate signals."""
|
| 259 |
+
|
| 260 |
+
rgb = np.asarray(image.convert("RGB"), dtype=np.uint8)
|
| 261 |
+
gray = cv2.cvtColor(rgb, cv2.COLOR_RGB2GRAY).astype(np.float32)
|
| 262 |
+
binary = (np.asarray(mask.convert("L")) > 127).astype(np.uint8)
|
| 263 |
+
if binary.shape != gray.shape:
|
| 264 |
+
raise ValueError(
|
| 265 |
+
f"Candidate/mask raster mismatch: image={gray.shape}, mask={binary.shape}"
|
| 266 |
+
)
|
| 267 |
+
if not binary.any():
|
| 268 |
+
raise ValueError("Candidate quality requires a nonempty visual removal mask")
|
| 269 |
+
|
| 270 |
+
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (17, 17))
|
| 271 |
+
outer = (cv2.dilate(binary, kernel) > 0) & ~(binary > 0)
|
| 272 |
+
inner = (binary > 0) & ~(cv2.erode(binary, kernel) > 0)
|
| 273 |
+
interior = cv2.erode(binary, np.ones((5, 5), np.uint8)) > 0
|
| 274 |
+
if not interior.any():
|
| 275 |
+
interior = binary > 0
|
| 276 |
+
if not outer.any():
|
| 277 |
+
outer = ~(binary > 0)
|
| 278 |
+
if not outer.any():
|
| 279 |
+
outer = np.ones_like(binary, dtype=bool)
|
| 280 |
+
|
| 281 |
+
gx = np.abs(cv2.Sobel(gray, cv2.CV_32F, 1, 0, ksize=3))
|
| 282 |
+
gy = np.abs(cv2.Sobel(gray, cv2.CV_32F, 0, 1, ksize=3))
|
| 283 |
+
laplacian = np.abs(cv2.Laplacian(gray, cv2.CV_32F, ksize=3))
|
| 284 |
+
|
| 285 |
+
eps = 1e-6
|
| 286 |
+
reference_sharpness = float(np.mean(laplacian[outer])) + eps
|
| 287 |
+
sharpness_ratio = float(np.mean(laplacian[interior])) / reference_sharpness
|
| 288 |
+
reference_contrast = float(np.std(gray[outer])) + eps
|
| 289 |
+
contrast_ratio = float(np.std(gray[interior])) / reference_contrast
|
| 290 |
+
seam_energy = float(np.mean((gx + gy)[inner])) if inner.any() else 0.0
|
| 291 |
+
reference_edge = float(np.mean((gx + gy)[outer])) + eps
|
| 292 |
+
seam_penalty = seam_energy / reference_edge
|
| 293 |
+
|
| 294 |
+
horizontal_fraction = float(np.mean(gy[interior])) / (
|
| 295 |
+
float(np.mean(gx[interior]) + np.mean(gy[interior])) + eps
|
| 296 |
+
)
|
| 297 |
+
structure_bonus = horizontal_fraction if category == "stairs" else 0.5
|
| 298 |
+
sharp_term = min(sharpness_ratio, 1.8) / 1.8
|
| 299 |
+
contrast_term = min(contrast_ratio, 1.5) / 1.5
|
| 300 |
+
seam_term = math.exp(-max(0.0, seam_penalty - 1.0))
|
| 301 |
+
score = (
|
| 302 |
+
0.38 * sharp_term
|
| 303 |
+
+ 0.24 * contrast_term
|
| 304 |
+
+ 0.23 * structure_bonus
|
| 305 |
+
+ 0.15 * seam_term
|
| 306 |
+
)
|
| 307 |
+
quality_flags = quality_flags_for_metrics(
|
| 308 |
+
sharpness_ratio=sharpness_ratio,
|
| 309 |
+
contrast_ratio=contrast_ratio,
|
| 310 |
+
seam_penalty=seam_penalty,
|
| 311 |
+
)
|
| 312 |
+
|
| 313 |
+
# A flagged candidate must rank below every unflagged candidate. Keeping
|
| 314 |
+
# the unbounded negative value preserves useful ordering when all
|
| 315 |
+
# candidates are risky and must be withheld for review.
|
| 316 |
+
gate_score = float(score) - float(len(quality_flags))
|
| 317 |
+
return {
|
| 318 |
+
"score": round(float(score), 6),
|
| 319 |
+
"gate_score": round(gate_score, 6),
|
| 320 |
+
"quality_flags": quality_flags,
|
| 321 |
+
"review_required": bool(quality_flags),
|
| 322 |
+
"sharpness_ratio": round(sharpness_ratio, 6),
|
| 323 |
+
"contrast_ratio": round(contrast_ratio, 6),
|
| 324 |
+
"horizontal_edge_fraction": round(horizontal_fraction, 6),
|
| 325 |
+
"seam_penalty": round(seam_penalty, 6),
|
| 326 |
+
}
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
def candidate_clutter_metrics(
|
| 330 |
+
image: Image.Image,
|
| 331 |
+
original: Image.Image,
|
| 332 |
+
completion_envelope: np.ndarray,
|
| 333 |
+
target_amodal: np.ndarray | None,
|
| 334 |
+
) -> dict[str, Any]:
|
| 335 |
+
"""Measure new line/edge clutter outside the modeled support target."""
|
| 336 |
+
|
| 337 |
+
envelope = _binary_array(completion_envelope, "completion_envelope")
|
| 338 |
+
if target_amodal is None:
|
| 339 |
+
return {
|
| 340 |
+
"available": False,
|
| 341 |
+
"reason": "target_amodal_unavailable",
|
| 342 |
+
}
|
| 343 |
+
target = _binary_array(target_amodal, "target_amodal")
|
| 344 |
+
if target.shape != envelope.shape:
|
| 345 |
+
raise ValueError("Target amodal and completion envelope must have the same shape")
|
| 346 |
+
|
| 347 |
+
candidate_rgb = np.asarray(image.convert("RGB"), dtype=np.uint8)
|
| 348 |
+
original_rgb = np.asarray(original.convert("RGB"), dtype=np.uint8)
|
| 349 |
+
if candidate_rgb.shape[:2] != envelope.shape:
|
| 350 |
+
raise ValueError("Candidate and completion envelope must have the same shape")
|
| 351 |
+
if original_rgb.shape != candidate_rgb.shape:
|
| 352 |
+
raise ValueError("Original and candidate RGB rasters must have the same shape")
|
| 353 |
+
|
| 354 |
+
outside_target = envelope & ~target
|
| 355 |
+
minimum_pixels = max(256, int(round(0.001 * outside_target.size)))
|
| 356 |
+
outside_pixels = int(outside_target.sum())
|
| 357 |
+
if outside_pixels < minimum_pixels:
|
| 358 |
+
return {
|
| 359 |
+
"available": False,
|
| 360 |
+
"reason": "outside_target_clutter_unavailable",
|
| 361 |
+
"outside_target_pixels": outside_pixels,
|
| 362 |
+
"minimum_pixels": minimum_pixels,
|
| 363 |
+
}
|
| 364 |
+
|
| 365 |
+
original_gray = cv2.cvtColor(original_rgb, cv2.COLOR_RGB2GRAY)
|
| 366 |
+
candidate_gray = cv2.cvtColor(candidate_rgb, cv2.COLOR_RGB2GRAY)
|
| 367 |
+
median_gray = float(np.median(original_gray))
|
| 368 |
+
canny_low = max(20, int(0.5 * median_gray))
|
| 369 |
+
canny_high = max(canny_low + 1, min(220, int(1.2 * median_gray)))
|
| 370 |
+
edges = cv2.Canny(
|
| 371 |
+
candidate_gray,
|
| 372 |
+
canny_low,
|
| 373 |
+
canny_high,
|
| 374 |
+
L2gradient=True,
|
| 375 |
+
) > 0
|
| 376 |
+
outside_edges = edges & outside_target
|
| 377 |
+
edge_density = float(outside_edges.sum()) / outside_pixels
|
| 378 |
+
|
| 379 |
+
height, width = outside_target.shape
|
| 380 |
+
minimum_dimension = min(height, width)
|
| 381 |
+
minimum_line_length = max(12, int(round(0.015 * minimum_dimension)))
|
| 382 |
+
maximum_line_gap = max(4, int(round(0.005 * minimum_dimension)))
|
| 383 |
+
lines = cv2.HoughLinesP(
|
| 384 |
+
outside_edges.astype(np.uint8) * 255,
|
| 385 |
+
1,
|
| 386 |
+
np.pi / 180.0,
|
| 387 |
+
threshold=minimum_line_length,
|
| 388 |
+
minLineLength=minimum_line_length,
|
| 389 |
+
maxLineGap=maximum_line_gap,
|
| 390 |
+
)
|
| 391 |
+
total_line_length = 0.0
|
| 392 |
+
line_count = 0
|
| 393 |
+
if lines is not None:
|
| 394 |
+
line_count = int(len(lines))
|
| 395 |
+
for line in lines[:, 0, :]:
|
| 396 |
+
x1, y1, x2, y2 = (int(value) for value in line)
|
| 397 |
+
total_line_length += math.hypot(x2 - x1, y2 - y1)
|
| 398 |
+
line_density_per_1000 = 1000.0 * total_line_length / outside_pixels
|
| 399 |
+
return {
|
| 400 |
+
"available": True,
|
| 401 |
+
"outside_target_pixels": outside_pixels,
|
| 402 |
+
"minimum_pixels": minimum_pixels,
|
| 403 |
+
"canny_low": canny_low,
|
| 404 |
+
"canny_high": canny_high,
|
| 405 |
+
"edge_density": round(edge_density, 8),
|
| 406 |
+
"hough_line_count": line_count,
|
| 407 |
+
"hough_minimum_line_length": minimum_line_length,
|
| 408 |
+
"hough_maximum_line_gap": maximum_line_gap,
|
| 409 |
+
"line_density_per_1000_pixels": round(line_density_per_1000, 8),
|
| 410 |
+
}
|
| 411 |
+
|
| 412 |
+
|
| 413 |
+
def _average_tie_percentile_ranks(values: list[float]) -> list[float]:
|
| 414 |
+
if len(values) <= 1:
|
| 415 |
+
return [0.0] * len(values)
|
| 416 |
+
denominator = len(values) - 1
|
| 417 |
+
ranks: list[float] = []
|
| 418 |
+
for value in values:
|
| 419 |
+
lower = sum(other < value for other in values)
|
| 420 |
+
equal_other = sum(other == value for other in values) - 1
|
| 421 |
+
ranks.append((lower + 0.5 * equal_other) / denominator)
|
| 422 |
+
return ranks
|
| 423 |
+
|
| 424 |
+
|
| 425 |
+
def apply_selection_clutter_penalty(
|
| 426 |
+
candidates: list[dict[str, Any]],
|
| 427 |
+
*,
|
| 428 |
+
maximum_penalty: float = 0.15,
|
| 429 |
+
) -> bool:
|
| 430 |
+
"""Add a cohort-relative clutter term used only to rank candidates."""
|
| 431 |
+
|
| 432 |
+
if maximum_penalty < 0:
|
| 433 |
+
raise ValueError("maximum_penalty must be non-negative")
|
| 434 |
+
if not candidates:
|
| 435 |
+
return False
|
| 436 |
+
metrics = [row["quality"].get("clutter_metrics", {}) for row in candidates]
|
| 437 |
+
if not all(metric.get("available") is True for metric in metrics):
|
| 438 |
+
for row in candidates:
|
| 439 |
+
quality = row["quality"]
|
| 440 |
+
base = float(quality.get("gate_score", quality.get("score", 0.0)))
|
| 441 |
+
quality["selection_score"] = round(base, 6)
|
| 442 |
+
quality["clutter_selection_penalty"] = None
|
| 443 |
+
quality["clutter_selection_note"] = (
|
| 444 |
+
"unavailable_fallback_to_absolute_quality_score"
|
| 445 |
+
)
|
| 446 |
+
return False
|
| 447 |
+
|
| 448 |
+
edge_ranks = _average_tie_percentile_ranks(
|
| 449 |
+
[float(metric["edge_density"]) for metric in metrics]
|
| 450 |
+
)
|
| 451 |
+
line_ranks = _average_tie_percentile_ranks(
|
| 452 |
+
[float(metric["line_density_per_1000_pixels"]) for metric in metrics]
|
| 453 |
+
)
|
| 454 |
+
for row, edge_rank, line_rank in zip(candidates, edge_ranks, line_ranks):
|
| 455 |
+
quality = row["quality"]
|
| 456 |
+
base = float(quality.get("gate_score", quality.get("score", 0.0)))
|
| 457 |
+
clutter_rank = 0.5 * (edge_rank + line_rank)
|
| 458 |
+
penalty = maximum_penalty * clutter_rank
|
| 459 |
+
quality["clutter_edge_percentile_rank"] = round(edge_rank, 6)
|
| 460 |
+
quality["clutter_line_percentile_rank"] = round(line_rank, 6)
|
| 461 |
+
quality["clutter_rank"] = round(clutter_rank, 6)
|
| 462 |
+
quality["clutter_selection_penalty"] = round(penalty, 6)
|
| 463 |
+
quality["selection_score"] = round(base - penalty, 6)
|
| 464 |
+
quality["clutter_selection_note"] = (
|
| 465 |
+
"cohort_relative_ranking_only_not_an_acceptance_or_passability_gate"
|
| 466 |
+
)
|
| 467 |
+
return True
|
| 468 |
+
|
| 469 |
+
|
| 470 |
+
def select_candidate(candidates: list[dict[str, Any]]) -> tuple[dict[str, Any], str]:
|
| 471 |
+
"""Select by gate score and withhold publication when every row is risky."""
|
| 472 |
+
|
| 473 |
+
if not candidates:
|
| 474 |
+
raise ValueError("At least one completion candidate is required")
|
| 475 |
+
|
| 476 |
+
def rank_key(row: dict[str, Any]) -> tuple[float, float, float, int]:
|
| 477 |
+
quality = row["quality"]
|
| 478 |
+
return (
|
| 479 |
+
float(
|
| 480 |
+
quality.get(
|
| 481 |
+
"selection_score",
|
| 482 |
+
quality.get("gate_score", quality.get("score", 0.0)),
|
| 483 |
+
)
|
| 484 |
+
),
|
| 485 |
+
float(quality.get("gate_score", quality.get("score", 0.0))),
|
| 486 |
+
float(quality.get("score", 0.0)),
|
| 487 |
+
-int(row.get("index", 0)),
|
| 488 |
+
)
|
| 489 |
+
|
| 490 |
+
selected = max(candidates, key=rank_key)
|
| 491 |
+
all_risky = all(bool(row["quality"].get("quality_flags", [])) for row in candidates)
|
| 492 |
+
status = "withheld_needs_review" if all_risky else "candidate_selected_for_review"
|
| 493 |
+
return selected, status
|
accesspath.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""AccessPath command-line entry point for the full 2D/3D pipeline."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import subprocess
|
| 8 |
+
import sys
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
ROOT = Path(__file__).resolve().parent
|
| 13 |
+
TOOLS = {
|
| 14 |
+
"2d": ROOT / "tools" / "accessibility_2d_completion.py",
|
| 15 |
+
"3d": ROOT / "tools" / "accessibility_3d_completion.py",
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def main() -> int:
|
| 20 |
+
parser = argparse.ArgumentParser(
|
| 21 |
+
description="Run the AccessPath pipeline or one of its completion backends."
|
| 22 |
+
)
|
| 23 |
+
parser.add_argument(
|
| 24 |
+
"mode",
|
| 25 |
+
choices=("pipeline", *TOOLS),
|
| 26 |
+
help="Full Slurm pipeline, or a direct 2D/3D backend.",
|
| 27 |
+
)
|
| 28 |
+
parser.add_argument(
|
| 29 |
+
"args",
|
| 30 |
+
nargs=argparse.REMAINDER,
|
| 31 |
+
help="Arguments forwarded to the selected backend; prefix them with --.",
|
| 32 |
+
)
|
| 33 |
+
parsed = parser.parse_args()
|
| 34 |
+
forwarded = parsed.args[1:] if parsed.args[:1] == ["--"] else parsed.args
|
| 35 |
+
if parsed.mode == "pipeline":
|
| 36 |
+
from accesspath3r.cli import main as pipeline_main
|
| 37 |
+
|
| 38 |
+
sys.argv = [str(ROOT / "accesspath.py"), *forwarded]
|
| 39 |
+
return pipeline_main()
|
| 40 |
+
return subprocess.run([sys.executable, str(TOOLS[parsed.mode]), *forwarded], check=False).returncode
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
if __name__ == "__main__":
|
| 44 |
+
raise SystemExit(main())
|
accesspath3r/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Accessibility3R: accessibility-scene 2D and 3D amodal completion."""
|
| 2 |
+
|
| 3 |
+
__version__ = "0.2.0"
|
accesspath3r/cli.py
ADDED
|
@@ -0,0 +1,809 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""AccessibilityAmodal one-command mask, 2D completion, and 3D inference.
|
| 3 |
+
|
| 4 |
+
This is a lightweight login-node launcher. GPU work is submitted to Slurm via
|
| 5 |
+
``slurm/run_accesspath_demo.sbatch``; this process can wait for the
|
| 6 |
+
job and report the persistent result directory.
|
| 7 |
+
|
| 8 |
+
The third-party Accessibility3D model is accessed only through the project adapter
|
| 9 |
+
``tools/accessibility_3d_completion.py``.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import argparse
|
| 15 |
+
import json
|
| 16 |
+
import os
|
| 17 |
+
import re
|
| 18 |
+
import shutil
|
| 19 |
+
import subprocess
|
| 20 |
+
import sys
|
| 21 |
+
import time
|
| 22 |
+
from datetime import datetime
|
| 23 |
+
from pathlib import Path
|
| 24 |
+
|
| 25 |
+
from accesspath3r.privacy import public_path, sanitize_text
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
| 29 |
+
SBATCH_SCRIPT = PROJECT_ROOT / "slurm" / "run_accesspath_demo.sbatch"
|
| 30 |
+
PRIMARY_3D_DIRNAME = "accessibility3d"
|
| 31 |
+
DEFAULT_PROMPT_CONFIG = (
|
| 32 |
+
PROJECT_ROOT / "configs" / "accessibility_mask_prompts_e5_v2.json"
|
| 33 |
+
)
|
| 34 |
+
STRUCTURAL_STAIRS_PROMPT_CONFIG = (
|
| 35 |
+
PROJECT_ROOT / "configs" / "accessibility_stairs_structural_occlusion_prompts.json"
|
| 36 |
+
)
|
| 37 |
+
OUTDOOR_STAIRS_NO_OCCLUSION_PROMPT_CONFIG = (
|
| 38 |
+
PROJECT_ROOT / "configs" / "accessibility_stairs_outdoor_no_occlusion_prompts.json"
|
| 39 |
+
)
|
| 40 |
+
CATEGORIES = ("curb_cut", "ramp", "stairs", "tactile_paving", "walkway")
|
| 41 |
+
GEOMETRY_ALLOWED_2D_STATUSES = {
|
| 42 |
+
"candidate_selected_for_review",
|
| 43 |
+
"skipped_empty_removal_mask",
|
| 44 |
+
}
|
| 45 |
+
TERMINAL_STATES = {
|
| 46 |
+
"BOOT_FAIL",
|
| 47 |
+
"CANCELLED",
|
| 48 |
+
"COMPLETED",
|
| 49 |
+
"DEADLINE",
|
| 50 |
+
"FAILED",
|
| 51 |
+
"NODE_FAIL",
|
| 52 |
+
"OUT_OF_MEMORY",
|
| 53 |
+
"PREEMPTED",
|
| 54 |
+
"REVOKED",
|
| 55 |
+
"TIMEOUT",
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def safe_sample_id(value: str) -> str:
|
| 60 |
+
result = re.sub(r"[^A-Za-z0-9_-]+", "_", value).strip("_")
|
| 61 |
+
return result or "single_image"
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def normalized_state(value: str) -> str:
|
| 65 |
+
return value.strip().split()[0].split("+")[0] if value.strip() else ""
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def run_command(command: list[str]) -> subprocess.CompletedProcess[str]:
|
| 69 |
+
return subprocess.run(
|
| 70 |
+
command,
|
| 71 |
+
cwd=PROJECT_ROOT,
|
| 72 |
+
check=False,
|
| 73 |
+
capture_output=True,
|
| 74 |
+
text=True,
|
| 75 |
+
)
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def query_job_state(job_id: str) -> str | None:
|
| 79 |
+
queued = run_command(["squeue", "-h", "-j", job_id, "-o", "%T"])
|
| 80 |
+
if queued.returncode == 0 and queued.stdout.strip():
|
| 81 |
+
return normalized_state(queued.stdout.splitlines()[0])
|
| 82 |
+
|
| 83 |
+
accounting = run_command(
|
| 84 |
+
[
|
| 85 |
+
"sacct",
|
| 86 |
+
"-n",
|
| 87 |
+
"-P",
|
| 88 |
+
"-j",
|
| 89 |
+
job_id,
|
| 90 |
+
"--format=JobIDRaw,State,ExitCode",
|
| 91 |
+
]
|
| 92 |
+
)
|
| 93 |
+
if accounting.returncode != 0:
|
| 94 |
+
return None
|
| 95 |
+
for line in accounting.stdout.splitlines():
|
| 96 |
+
fields = line.split("|")
|
| 97 |
+
if len(fields) >= 2 and fields[0] == job_id:
|
| 98 |
+
return normalized_state(fields[1])
|
| 99 |
+
return None
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def format_clock(seconds: float) -> str:
|
| 103 |
+
seconds = max(0, int(round(seconds)))
|
| 104 |
+
hours, remainder = divmod(seconds, 3600)
|
| 105 |
+
minutes, seconds = divmod(remainder, 60)
|
| 106 |
+
if hours:
|
| 107 |
+
return f"{hours:d}:{minutes:02d}:{seconds:02d}"
|
| 108 |
+
return f"{minutes:02d}:{seconds:02d}"
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def parse_timestamp(value: object) -> float | None:
|
| 112 |
+
if not isinstance(value, str):
|
| 113 |
+
return None
|
| 114 |
+
try:
|
| 115 |
+
return datetime.fromisoformat(value).timestamp()
|
| 116 |
+
except ValueError:
|
| 117 |
+
return None
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def read_progress(output_dir: Path) -> dict | None:
|
| 121 |
+
try:
|
| 122 |
+
value = json.loads((output_dir / "progress.json").read_text(encoding="utf-8"))
|
| 123 |
+
except (FileNotFoundError, json.JSONDecodeError, OSError):
|
| 124 |
+
return None
|
| 125 |
+
return value if isinstance(value, dict) else None
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def infer_progress_from_artifacts(
|
| 129 |
+
output_dir: Path,
|
| 130 |
+
sample_id: str,
|
| 131 |
+
include_generative_3d: bool,
|
| 132 |
+
) -> dict | None:
|
| 133 |
+
"""Provide useful progress for jobs launched before progress.json existed."""
|
| 134 |
+
presentation_manifest = (
|
| 135 |
+
output_dir
|
| 136 |
+
/ "04_3d_completion"
|
| 137 |
+
/ "geometry"
|
| 138 |
+
/ "presentation_models"
|
| 139 |
+
/ "manifest.json"
|
| 140 |
+
)
|
| 141 |
+
legacy_variants_manifest = (
|
| 142 |
+
output_dir
|
| 143 |
+
/ "04_3d_completion"
|
| 144 |
+
/ "geometry"
|
| 145 |
+
/ "3d_variants"
|
| 146 |
+
/ "manifest.json"
|
| 147 |
+
)
|
| 148 |
+
if not presentation_manifest.is_file() and legacy_variants_manifest.is_file():
|
| 149 |
+
presentation_manifest = legacy_variants_manifest
|
| 150 |
+
learned_manifest = output_dir / PRIMARY_3D_DIRNAME / "manifest.json"
|
| 151 |
+
current_compatibility_manifest = (
|
| 152 |
+
output_dir / "04_3d_completion" / "visual_candidate" / "manifest.json"
|
| 153 |
+
)
|
| 154 |
+
if not learned_manifest.is_file() and current_compatibility_manifest.is_file():
|
| 155 |
+
learned_manifest = current_compatibility_manifest
|
| 156 |
+
final_marker = learned_manifest if include_generative_3d else presentation_manifest
|
| 157 |
+
if final_marker.is_file():
|
| 158 |
+
return {
|
| 159 |
+
"status": "completed",
|
| 160 |
+
"percent": 100.0,
|
| 161 |
+
"stage_label": "All outputs completed",
|
| 162 |
+
}
|
| 163 |
+
milestones = (
|
| 164 |
+
(
|
| 165 |
+
learned_manifest,
|
| 166 |
+
96.0,
|
| 167 |
+
"Accessibility3D CUDA Gaussian rotation and dense triangle mesh",
|
| 168 |
+
),
|
| 169 |
+
(
|
| 170 |
+
presentation_manifest,
|
| 171 |
+
74.0,
|
| 172 |
+
"Diagnostic category-constrained geometry",
|
| 173 |
+
),
|
| 174 |
+
(
|
| 175 |
+
output_dir / "04_3d_completion" / "geometry" / "completed_mesh_turntable_slow.gif",
|
| 176 |
+
44.0,
|
| 177 |
+
"Continuous-surface and solid 3D views",
|
| 178 |
+
),
|
| 179 |
+
(
|
| 180 |
+
output_dir / "04_3d_completion" / "geometry" / "geometry_manifest.json",
|
| 181 |
+
43.0,
|
| 182 |
+
"Basic 3D turntable preview",
|
| 183 |
+
),
|
| 184 |
+
(
|
| 185 |
+
output_dir / "03_2d_completion" / "completed_rgb_selected.png",
|
| 186 |
+
34.0,
|
| 187 |
+
"Depth and 3D geometry reconstruction",
|
| 188 |
+
),
|
| 189 |
+
(
|
| 190 |
+
output_dir / "02_masks" / "metadata.json",
|
| 191 |
+
20.0,
|
| 192 |
+
"GPU generative 2D completion and candidate selection",
|
| 193 |
+
),
|
| 194 |
+
(
|
| 195 |
+
output_dir / "01_sam3" / "summary.json",
|
| 196 |
+
16.0,
|
| 197 |
+
"Visible, hidden, amodal, and obstacle masks",
|
| 198 |
+
),
|
| 199 |
+
)
|
| 200 |
+
for marker, percent, label in milestones:
|
| 201 |
+
if marker.is_file():
|
| 202 |
+
return {"status": "running", "percent": percent, "stage_label": label}
|
| 203 |
+
return None
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def interpolated_percent(progress: dict, now: float) -> float:
|
| 207 |
+
try:
|
| 208 |
+
percent = float(progress.get("percent", 0.0))
|
| 209 |
+
except (TypeError, ValueError):
|
| 210 |
+
percent = 0.0
|
| 211 |
+
if progress.get("status") != "running":
|
| 212 |
+
return max(0.0, min(100.0, percent))
|
| 213 |
+
try:
|
| 214 |
+
end_percent = float(progress.get("stage_end_percent", percent))
|
| 215 |
+
expected = float(progress.get("stage_expected_seconds", 0.0))
|
| 216 |
+
except (TypeError, ValueError):
|
| 217 |
+
return max(0.0, min(99.0, percent))
|
| 218 |
+
stage_started = parse_timestamp(progress.get("stage_started_at"))
|
| 219 |
+
if stage_started is None or expected <= 0 or end_percent <= percent:
|
| 220 |
+
return max(0.0, min(99.0, percent))
|
| 221 |
+
fraction = max(0.0, min(0.98, (now - stage_started) / expected))
|
| 222 |
+
interpolated = percent + (end_percent - percent) * fraction
|
| 223 |
+
return max(0.0, min(99.0, interpolated))
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
def estimated_remaining_seconds(
|
| 227 |
+
progress: dict | None,
|
| 228 |
+
now: float,
|
| 229 |
+
elapsed: float,
|
| 230 |
+
base_total: float,
|
| 231 |
+
) -> float:
|
| 232 |
+
"""Keep ETA conservative when a stage or an earlier stage runs long."""
|
| 233 |
+
adjusted_total = max(1.0, base_total)
|
| 234 |
+
if not progress or progress.get("status") != "running":
|
| 235 |
+
return max(0.0, adjusted_total - elapsed)
|
| 236 |
+
overall_started = parse_timestamp(progress.get("started_at"))
|
| 237 |
+
stage_started = parse_timestamp(progress.get("stage_started_at"))
|
| 238 |
+
try:
|
| 239 |
+
start_percent = float(progress.get("percent", 0.0))
|
| 240 |
+
stage_expected = float(progress.get("stage_expected_seconds", 0.0))
|
| 241 |
+
except (TypeError, ValueError):
|
| 242 |
+
return max(0.0, adjusted_total - elapsed)
|
| 243 |
+
if overall_started is not None and stage_started is not None:
|
| 244 |
+
planned_stage_start = max(0.0, start_percent) / 100.0 * base_total
|
| 245 |
+
actual_stage_start = max(0.0, stage_started - overall_started)
|
| 246 |
+
adjusted_total += max(0.0, actual_stage_start - planned_stage_start)
|
| 247 |
+
adjusted_total += max(0.0, now - stage_started - stage_expected)
|
| 248 |
+
return max(0.0, adjusted_total - elapsed)
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
class ProgressDisplay:
|
| 252 |
+
def __init__(self, job_id: str) -> None:
|
| 253 |
+
self.job_id = job_id
|
| 254 |
+
self.is_tty = sys.stdout.isatty()
|
| 255 |
+
self.last_key: tuple[str, str] | None = None
|
| 256 |
+
self.last_width = 0
|
| 257 |
+
|
| 258 |
+
def update(
|
| 259 |
+
self,
|
| 260 |
+
percent: float,
|
| 261 |
+
label: str,
|
| 262 |
+
elapsed: float,
|
| 263 |
+
eta: float | None,
|
| 264 |
+
state: str,
|
| 265 |
+
final: bool = False,
|
| 266 |
+
) -> None:
|
| 267 |
+
width = 28
|
| 268 |
+
filled = min(width, max(0, int(round(width * percent / 100.0))))
|
| 269 |
+
bar = "#" * filled + "-" * (width - filled)
|
| 270 |
+
if state == "PENDING":
|
| 271 |
+
timing = f"queue {format_clock(elapsed)} | GPU ETA ~{format_clock(eta or 0)}"
|
| 272 |
+
elif state == "FAILED":
|
| 273 |
+
timing = f"elapsed {format_clock(elapsed)} | failed"
|
| 274 |
+
elif eta is None:
|
| 275 |
+
timing = f"elapsed {format_clock(elapsed)} | ETA calculating"
|
| 276 |
+
elif eta <= 0 and not final:
|
| 277 |
+
timing = f"elapsed {format_clock(elapsed)} | finishing"
|
| 278 |
+
else:
|
| 279 |
+
timing = f"elapsed {format_clock(elapsed)} | ETA ~{format_clock(eta)}"
|
| 280 |
+
line = (
|
| 281 |
+
f"[Slurm {self.job_id}] [{bar}] {percent:5.1f}% | "
|
| 282 |
+
f"{label} | {timing}"
|
| 283 |
+
)
|
| 284 |
+
key = (state, label)
|
| 285 |
+
if self.is_tty:
|
| 286 |
+
padding = " " * max(0, self.last_width - len(line))
|
| 287 |
+
print(f"\r{line}{padding}", end="\n" if final else "", flush=True)
|
| 288 |
+
self.last_width = len(line)
|
| 289 |
+
elif key != self.last_key or final:
|
| 290 |
+
print(line, flush=True)
|
| 291 |
+
self.last_key = key
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
def wait_for_job(
|
| 295 |
+
job_id: str,
|
| 296 |
+
poll_seconds: float,
|
| 297 |
+
output_dir: Path,
|
| 298 |
+
sample_id: str,
|
| 299 |
+
include_generative_3d: bool,
|
| 300 |
+
estimated_seconds: float,
|
| 301 |
+
) -> str:
|
| 302 |
+
missing_polls = 0
|
| 303 |
+
submitted_at = time.monotonic()
|
| 304 |
+
running_at: float | None = None
|
| 305 |
+
display = ProgressDisplay(job_id)
|
| 306 |
+
while True:
|
| 307 |
+
state = query_job_state(job_id)
|
| 308 |
+
now_wall = time.time()
|
| 309 |
+
now_monotonic = time.monotonic()
|
| 310 |
+
progress = read_progress(output_dir) or infer_progress_from_artifacts(
|
| 311 |
+
output_dir,
|
| 312 |
+
sample_id,
|
| 313 |
+
include_generative_3d,
|
| 314 |
+
)
|
| 315 |
+
|
| 316 |
+
if state:
|
| 317 |
+
missing_polls = 0
|
| 318 |
+
if state == "RUNNING" and running_at is None:
|
| 319 |
+
running_at = now_monotonic
|
| 320 |
+
else:
|
| 321 |
+
# sacct can lag briefly after a job leaves squeue.
|
| 322 |
+
missing_polls += 1
|
| 323 |
+
if missing_polls >= 12 and not progress:
|
| 324 |
+
raise RuntimeError(
|
| 325 |
+
"Unable to read the job state from squeue or sacct. "
|
| 326 |
+
f"Check manually with: squeue -j {job_id}"
|
| 327 |
+
)
|
| 328 |
+
|
| 329 |
+
progress_status = progress.get("status") if progress else None
|
| 330 |
+
if progress_status == "completed":
|
| 331 |
+
runtime_started = parse_timestamp(progress.get("started_at"))
|
| 332 |
+
elapsed = now_wall - runtime_started if runtime_started else 0.0
|
| 333 |
+
display.update(100.0, "All outputs completed", elapsed, 0.0, "COMPLETED", final=True)
|
| 334 |
+
return "COMPLETED"
|
| 335 |
+
if progress_status == "failed":
|
| 336 |
+
runtime_started = parse_timestamp(progress.get("started_at"))
|
| 337 |
+
elapsed = now_wall - runtime_started if runtime_started else 0.0
|
| 338 |
+
display.update(
|
| 339 |
+
interpolated_percent(progress, now_wall),
|
| 340 |
+
str(progress.get("message") or progress.get("stage_label") or "GPU stage failed"),
|
| 341 |
+
elapsed,
|
| 342 |
+
None,
|
| 343 |
+
"FAILED",
|
| 344 |
+
final=True,
|
| 345 |
+
)
|
| 346 |
+
return "FAILED"
|
| 347 |
+
|
| 348 |
+
if state == "PENDING":
|
| 349 |
+
display.update(
|
| 350 |
+
0.0,
|
| 351 |
+
"Waiting for a GPU",
|
| 352 |
+
now_monotonic - submitted_at,
|
| 353 |
+
estimated_seconds,
|
| 354 |
+
state,
|
| 355 |
+
)
|
| 356 |
+
else:
|
| 357 |
+
if running_at is None:
|
| 358 |
+
running_at = now_monotonic
|
| 359 |
+
runtime_started = parse_timestamp(progress.get("started_at")) if progress else None
|
| 360 |
+
elapsed = (
|
| 361 |
+
max(0.0, now_wall - runtime_started)
|
| 362 |
+
if runtime_started is not None
|
| 363 |
+
else now_monotonic - running_at
|
| 364 |
+
)
|
| 365 |
+
total_estimate = estimated_seconds
|
| 366 |
+
if progress:
|
| 367 |
+
try:
|
| 368 |
+
total_estimate = float(progress.get("total_estimated_seconds", estimated_seconds))
|
| 369 |
+
except (TypeError, ValueError):
|
| 370 |
+
pass
|
| 371 |
+
percent = interpolated_percent(progress, now_wall) if progress else 0.0
|
| 372 |
+
label = (
|
| 373 |
+
str(progress.get("stage_label") or "Initializing GPU job")
|
| 374 |
+
if progress
|
| 375 |
+
else "Initializing GPU job"
|
| 376 |
+
)
|
| 377 |
+
display.update(
|
| 378 |
+
percent,
|
| 379 |
+
label,
|
| 380 |
+
elapsed,
|
| 381 |
+
estimated_remaining_seconds(progress, now_wall, elapsed, total_estimate),
|
| 382 |
+
state or "UNKNOWN",
|
| 383 |
+
final=bool(state in TERMINAL_STATES),
|
| 384 |
+
)
|
| 385 |
+
|
| 386 |
+
if state in TERMINAL_STATES:
|
| 387 |
+
return state
|
| 388 |
+
time.sleep(poll_seconds)
|
| 389 |
+
|
| 390 |
+
|
| 391 |
+
def estimate_runtime_seconds(args: argparse.Namespace) -> int:
|
| 392 |
+
# Broad SAM3 prompt evaluation and the presentation-model renderer scale
|
| 393 |
+
# with image size. These values are calibrated from the 1368x1824 9528
|
| 394 |
+
# run; GLB texture baking is accounted for separately because it dominated
|
| 395 |
+
# that job's final stage instead of behaving like the normal mesh export.
|
| 396 |
+
seconds = (5 if args.reviewed_visible_workspace else 150) + 13 + (24 if not args.no_depth else 6) + 3 + 420
|
| 397 |
+
if not args.no_2d:
|
| 398 |
+
seconds += 40
|
| 399 |
+
if args.fast_2d_baseline:
|
| 400 |
+
seconds += 12
|
| 401 |
+
if not args.no_generative_3d:
|
| 402 |
+
seconds += 75
|
| 403 |
+
if args.export_glb:
|
| 404 |
+
seconds += 540
|
| 405 |
+
return seconds
|
| 406 |
+
|
| 407 |
+
|
| 408 |
+
def build_parser() -> argparse.ArgumentParser:
|
| 409 |
+
parser = argparse.ArgumentParser(
|
| 410 |
+
description=(
|
| 411 |
+
"Submit one accessibility image to SAM3 mask inference, amodal/hidden "
|
| 412 |
+
"completion, 2D completion, Depth Anything geometry reconstruction, "
|
| 413 |
+
"and Accessibility3D CUDA Gaussian rendering with a dense triangle mesh."
|
| 414 |
+
)
|
| 415 |
+
)
|
| 416 |
+
parser.add_argument("image_pos", nargs="?", help="Input RGB image (positional form).")
|
| 417 |
+
parser.add_argument("--image", dest="image_opt", help="Input RGB image.")
|
| 418 |
+
parser.add_argument("--category", required=True, choices=CATEGORIES)
|
| 419 |
+
parser.add_argument("--sample-id", default=None)
|
| 420 |
+
parser.add_argument(
|
| 421 |
+
"--output-dir",
|
| 422 |
+
default=None,
|
| 423 |
+
help=(
|
| 424 |
+
"Persistent output directory. "
|
| 425 |
+
"Default: output/inference/<sample>_<time>."
|
| 426 |
+
),
|
| 427 |
+
)
|
| 428 |
+
parser.add_argument(
|
| 429 |
+
"--prompt-config",
|
| 430 |
+
default=None,
|
| 431 |
+
help=(
|
| 432 |
+
"Optional expert override for the mask prompt JSON. By default, a unified "
|
| 433 |
+
"target/occluder prompt bank is evaluated against the input image, so the "
|
| 434 |
+
"caller does not need to choose a scene-specific JSON."
|
| 435 |
+
),
|
| 436 |
+
)
|
| 437 |
+
parser.add_argument(
|
| 438 |
+
"--reviewed-visible-workspace",
|
| 439 |
+
default=None,
|
| 440 |
+
help=(
|
| 441 |
+
"A one-image workspace saved by serve_accessibility_visible_mask_review.py. "
|
| 442 |
+
"Requires explicit human approval; replaces the SAM3 visible-target proposal only."
|
| 443 |
+
),
|
| 444 |
+
)
|
| 445 |
+
parser.add_argument(
|
| 446 |
+
"--structural-occluder",
|
| 447 |
+
action="store_true",
|
| 448 |
+
help=(
|
| 449 |
+
"For stairs occluded by a pillar, newel post, banister, or railing. "
|
| 450 |
+
"Selects the dedicated structural-occlusion prompt config."
|
| 451 |
+
),
|
| 452 |
+
)
|
| 453 |
+
parser.add_argument(
|
| 454 |
+
"--no-target-occluder",
|
| 455 |
+
action="store_true",
|
| 456 |
+
help=(
|
| 457 |
+
"For a visible outdoor stair flight with no target-surface occluder. "
|
| 458 |
+
"Uses prompts that exclude permanent handrails and side walls from the obstacle mask."
|
| 459 |
+
),
|
| 460 |
+
)
|
| 461 |
+
parser.add_argument(
|
| 462 |
+
"--sd-2d",
|
| 463 |
+
action="store_true",
|
| 464 |
+
help="Legacy compatibility flag; GPU generative 2D completion is enabled by default.",
|
| 465 |
+
)
|
| 466 |
+
parser.add_argument(
|
| 467 |
+
"--fast-2d-baseline",
|
| 468 |
+
action="store_true",
|
| 469 |
+
help="Also produce the old OpenCV 2D baseline for comparison (not a primary result).",
|
| 470 |
+
)
|
| 471 |
+
parser.add_argument("--no-2d", action="store_true", help="Skip GPU generative 2D completion.")
|
| 472 |
+
parser.add_argument("--no-depth", action="store_true", help="Use debug perspective geometry instead of Depth Anything.")
|
| 473 |
+
parser.add_argument(
|
| 474 |
+
"--no-generative-3d",
|
| 475 |
+
action="store_true",
|
| 476 |
+
help="Skip the primary Accessibility3D CUDA rotation and dense learned mesh.",
|
| 477 |
+
)
|
| 478 |
+
parser.add_argument(
|
| 479 |
+
"--export-glb",
|
| 480 |
+
action="store_true",
|
| 481 |
+
help="Also export the Accessibility3D dense triangle result as mesh.glb.",
|
| 482 |
+
)
|
| 483 |
+
parser.add_argument("--allow-large-2d", action="store_true", help="Allow 2D completion masks larger than the normal safety gate.")
|
| 484 |
+
parser.add_argument("--no-wait", action="store_true", help="Submit and return immediately instead of waiting for results.")
|
| 485 |
+
parser.add_argument("--poll-seconds", type=float, default=5.0, help=argparse.SUPPRESS)
|
| 486 |
+
parser.add_argument("--dry-run", action="store_true", help="Validate and print the planned submission without submitting.")
|
| 487 |
+
return parser
|
| 488 |
+
|
| 489 |
+
|
| 490 |
+
def resolve_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> dict[str, str]:
|
| 491 |
+
if bool(args.image_pos) == bool(args.image_opt):
|
| 492 |
+
parser.error("provide exactly one image, either positional or with --image")
|
| 493 |
+
image = Path(args.image_opt or args.image_pos).expanduser().resolve()
|
| 494 |
+
if not image.is_file():
|
| 495 |
+
parser.error(
|
| 496 |
+
"input image does not exist: "
|
| 497 |
+
f"{public_path(image, project_root=PROJECT_ROOT)}"
|
| 498 |
+
)
|
| 499 |
+
if args.poll_seconds < 2:
|
| 500 |
+
parser.error("--poll-seconds must be at least 2")
|
| 501 |
+
if args.structural_occluder and args.category != "stairs":
|
| 502 |
+
parser.error("--structural-occluder currently applies only to --category stairs")
|
| 503 |
+
if args.no_target_occluder and args.category != "stairs":
|
| 504 |
+
parser.error("--no-target-occluder currently applies only to --category stairs")
|
| 505 |
+
if args.structural_occluder and args.no_target_occluder:
|
| 506 |
+
parser.error("--structural-occluder and --no-target-occluder are mutually exclusive")
|
| 507 |
+
if (args.structural_occluder or args.no_target_occluder) and args.prompt_config:
|
| 508 |
+
parser.error("choose one of --structural-occluder, --no-target-occluder, or --prompt-config")
|
| 509 |
+
reviewed_visible_workspace = None
|
| 510 |
+
if args.reviewed_visible_workspace:
|
| 511 |
+
reviewed_visible_workspace = Path(args.reviewed_visible_workspace).expanduser().resolve()
|
| 512 |
+
if not reviewed_visible_workspace.is_dir():
|
| 513 |
+
parser.error(
|
| 514 |
+
"reviewed-visible workspace does not exist: "
|
| 515 |
+
f"{public_path(reviewed_visible_workspace, project_root=PROJECT_ROOT)}"
|
| 516 |
+
)
|
| 517 |
+
|
| 518 |
+
if args.structural_occluder:
|
| 519 |
+
prompt_config = STRUCTURAL_STAIRS_PROMPT_CONFIG
|
| 520 |
+
elif args.no_target_occluder:
|
| 521 |
+
prompt_config = OUTDOOR_STAIRS_NO_OCCLUSION_PROMPT_CONFIG
|
| 522 |
+
elif args.prompt_config:
|
| 523 |
+
prompt_config = Path(args.prompt_config).expanduser().resolve()
|
| 524 |
+
else:
|
| 525 |
+
prompt_config = DEFAULT_PROMPT_CONFIG
|
| 526 |
+
if not prompt_config.is_file():
|
| 527 |
+
parser.error(
|
| 528 |
+
"prompt config does not exist: "
|
| 529 |
+
f"{public_path(prompt_config, project_root=PROJECT_ROOT)}"
|
| 530 |
+
)
|
| 531 |
+
if not SBATCH_SCRIPT.is_file():
|
| 532 |
+
parser.error(
|
| 533 |
+
"Slurm script does not exist: "
|
| 534 |
+
f"{public_path(SBATCH_SCRIPT, project_root=PROJECT_ROOT)}"
|
| 535 |
+
)
|
| 536 |
+
if shutil.which("sbatch") is None:
|
| 537 |
+
parser.error("sbatch is unavailable; run this command on the configured Slurm server")
|
| 538 |
+
|
| 539 |
+
sample_id = safe_sample_id(args.sample_id or image.stem)
|
| 540 |
+
if args.output_dir:
|
| 541 |
+
output_dir = Path(args.output_dir).expanduser().resolve()
|
| 542 |
+
else:
|
| 543 |
+
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
| 544 |
+
output_dir = PROJECT_ROOT / "output" / "inference" / f"{sample_id}_{timestamp}"
|
| 545 |
+
|
| 546 |
+
return {
|
| 547 |
+
"IMAGE": str(image),
|
| 548 |
+
"CATEGORY": args.category,
|
| 549 |
+
"SAMPLE_ID": sample_id,
|
| 550 |
+
"OUTPUT_DIR": str(output_dir),
|
| 551 |
+
"PROMPT_CONFIG": str(prompt_config),
|
| 552 |
+
"RUN_GPU_2D": "0" if args.no_2d else "1",
|
| 553 |
+
"RUN_FAST_2D": "1" if args.fast_2d_baseline else "0",
|
| 554 |
+
"ALLOW_LARGE_2D": "1" if args.allow_large_2d else "0",
|
| 555 |
+
"RUN_DEPTH": "0" if args.no_depth else "1",
|
| 556 |
+
"RUN_ACCESSIBILITYAMODAL_VISUAL_3D": "0" if args.no_generative_3d else "1",
|
| 557 |
+
"EXPORT_ACCESSIBILITYAMODAL_VISUAL_GLB": "1" if args.export_glb else "0",
|
| 558 |
+
"NO_TARGET_OCCLUDER": "1" if args.no_target_occluder else "0",
|
| 559 |
+
"REVIEWED_VISIBLE_WORKSPACE": str(reviewed_visible_workspace) if reviewed_visible_workspace else "",
|
| 560 |
+
}
|
| 561 |
+
|
| 562 |
+
|
| 563 |
+
def print_result_paths(
|
| 564 |
+
output_dir: Path,
|
| 565 |
+
sample_id: str,
|
| 566 |
+
include_gpu_2d: bool,
|
| 567 |
+
include_generative_3d: bool,
|
| 568 |
+
include_glb: bool,
|
| 569 |
+
reviewed_visible: bool = False,
|
| 570 |
+
) -> None:
|
| 571 |
+
def shown(path: Path) -> str:
|
| 572 |
+
return public_path(
|
| 573 |
+
path,
|
| 574 |
+
project_root=PROJECT_ROOT,
|
| 575 |
+
output_root=output_dir,
|
| 576 |
+
)
|
| 577 |
+
|
| 578 |
+
print("\nResults:")
|
| 579 |
+
print(f" output root: {shown(output_dir)}")
|
| 580 |
+
if include_gpu_2d:
|
| 581 |
+
print(
|
| 582 |
+
f" quick review: "
|
| 583 |
+
f"{shown(output_dir / '00_quick_review' / 'overview.jpg')}"
|
| 584 |
+
)
|
| 585 |
+
if reviewed_visible:
|
| 586 |
+
print(f" reviewed visible: {shown(output_dir / '01_reviewed_visible' / 'metadata.json')}")
|
| 587 |
+
else:
|
| 588 |
+
print(f" SAM3 overlay: {shown(output_dir / '01_sam3' / 'samples' / sample_id / 'overlay.png')}")
|
| 589 |
+
print(f" mask overlay: {shown(output_dir / '02_masks' / 'mask_overlay.png')}")
|
| 590 |
+
print(f" visible mask: {shown(output_dir / '02_masks' / 'target_visible.png')}")
|
| 591 |
+
print(f" hidden mask: {shown(output_dir / '02_masks' / 'hidden.png')}")
|
| 592 |
+
print(f" amodal mask: {shown(output_dir / '02_masks' / 'target_amodal.png')}")
|
| 593 |
+
print(f" obstacle mask: {shown(output_dir / '02_masks' / 'obstacle.png')}")
|
| 594 |
+
if include_gpu_2d:
|
| 595 |
+
print(f" GPU 2D selected: {shown(output_dir / '03_2d_completion' / 'completed_rgb_selected.png')}")
|
| 596 |
+
print(f" GPU 2D candidates: {shown(output_dir / '03_2d_completion' / 'candidate_comparison.jpg')}")
|
| 597 |
+
print(f" depth diagnostic: {shown(output_dir / '04_3d_completion' / 'geometry' / 'completed_depth_vis.png')}")
|
| 598 |
+
print(f" point-cloud debug: {shown(output_dir / '04_3d_completion' / 'geometry' / 'completed_point_cloud.ply')}")
|
| 599 |
+
print(f" geometry debug: {shown(output_dir / '04_3d_completion' / 'geometry' / 'completed_mesh.ply')}")
|
| 600 |
+
print(f" geometry fallback: {shown(output_dir / '04_3d_completion' / 'turntable' / 'turntable.gif')}")
|
| 601 |
+
print(
|
| 602 |
+
" presentation 3D: "
|
| 603 |
+
f"{shown(output_dir / '04_3d_completion' / 'geometry' / 'presentation_models')}"
|
| 604 |
+
)
|
| 605 |
+
if include_generative_3d:
|
| 606 |
+
visual_dir = output_dir / PRIMARY_3D_DIRNAME
|
| 607 |
+
print(f" quality checks: {shown(output_dir / '05_quality_checks' / 'verification.json')}")
|
| 608 |
+
print(f" primary 3D GIF: {shown(visual_dir / 'sample_gaussian.gif')}")
|
| 609 |
+
print(f" mesh normal diag: {shown(visual_dir / 'sample_mesh.gif')}")
|
| 610 |
+
print(f" primary 3D views: {shown(visual_dir / 'multiview_contact_sheet.jpg')}")
|
| 611 |
+
print(f" dense face mesh: {shown(visual_dir / 'mesh.ply')}")
|
| 612 |
+
if include_glb:
|
| 613 |
+
print(f" textured 3D GLB: {shown(visual_dir / 'mesh.glb')}")
|
| 614 |
+
|
| 615 |
+
|
| 616 |
+
def main(argv: list[str] | None = None) -> int:
|
| 617 |
+
parser = build_parser()
|
| 618 |
+
args = parser.parse_args(argv)
|
| 619 |
+
job_env = resolve_args(args, parser)
|
| 620 |
+
output_dir = Path(job_env["OUTPUT_DIR"])
|
| 621 |
+
sample_id = job_env["SAMPLE_ID"]
|
| 622 |
+
|
| 623 |
+
print("AccessibilityAmodal one-command inference")
|
| 624 |
+
print(
|
| 625 |
+
" image: "
|
| 626 |
+
f"{public_path(job_env['IMAGE'], project_root=PROJECT_ROOT)}"
|
| 627 |
+
)
|
| 628 |
+
print(f" category: {job_env['CATEGORY']}")
|
| 629 |
+
if job_env["REVIEWED_VISIBLE_WORKSPACE"]:
|
| 630 |
+
print(" visible mask: human-reviewed workspace (SAM3 proposal bypassed)")
|
| 631 |
+
print(" prompt config: not used after reviewed-mask validation")
|
| 632 |
+
else:
|
| 633 |
+
print(
|
| 634 |
+
" prompt config: "
|
| 635 |
+
f"{public_path(job_env['PROMPT_CONFIG'], project_root=PROJECT_ROOT)}"
|
| 636 |
+
)
|
| 637 |
+
print(
|
| 638 |
+
" output: "
|
| 639 |
+
f"{public_path(output_dir, project_root=PROJECT_ROOT, output_root=output_dir)}"
|
| 640 |
+
)
|
| 641 |
+
print(" GPU execution: Slurm, one GPU (5090 partition)")
|
| 642 |
+
print(
|
| 643 |
+
" 3D outputs: diagnostic depth/point cloud"
|
| 644 |
+
+ (
|
| 645 |
+
" + primary Accessibility3D CUDA Gaussian rotation/dense mesh"
|
| 646 |
+
if not args.no_generative_3d
|
| 647 |
+
else " + category-geometry fallback"
|
| 648 |
+
)
|
| 649 |
+
)
|
| 650 |
+
estimated_seconds = estimate_runtime_seconds(args)
|
| 651 |
+
print(
|
| 652 |
+
f" estimated time: about {format_clock(estimated_seconds)} after GPU starts "
|
| 653 |
+
"(Slurm queue excluded)"
|
| 654 |
+
)
|
| 655 |
+
|
| 656 |
+
command = ["sbatch", "--parsable", "--export=ALL", str(SBATCH_SCRIPT)]
|
| 657 |
+
if args.dry_run:
|
| 658 |
+
print("Dry run; no job submitted.")
|
| 659 |
+
shown_command = [*command[:-1], public_path(command[-1], project_root=PROJECT_ROOT)]
|
| 660 |
+
print(" " + " ".join(shown_command))
|
| 661 |
+
return 0
|
| 662 |
+
|
| 663 |
+
environment = os.environ.copy()
|
| 664 |
+
environment.update(job_env)
|
| 665 |
+
submitted = subprocess.run(
|
| 666 |
+
command,
|
| 667 |
+
cwd=PROJECT_ROOT,
|
| 668 |
+
env=environment,
|
| 669 |
+
check=False,
|
| 670 |
+
capture_output=True,
|
| 671 |
+
text=True,
|
| 672 |
+
)
|
| 673 |
+
if submitted.returncode != 0:
|
| 674 |
+
detail = submitted.stderr.strip() or submitted.stdout.strip()
|
| 675 |
+
raise RuntimeError(
|
| 676 |
+
"Slurm submission failed: "
|
| 677 |
+
+ sanitize_text(
|
| 678 |
+
detail,
|
| 679 |
+
project_root=PROJECT_ROOT,
|
| 680 |
+
output_root=output_dir,
|
| 681 |
+
sensitive_paths=(job_env["IMAGE"], job_env["PROMPT_CONFIG"]),
|
| 682 |
+
)
|
| 683 |
+
)
|
| 684 |
+
job_id = submitted.stdout.strip().split(";")[0]
|
| 685 |
+
if not job_id.isdigit():
|
| 686 |
+
raise RuntimeError(f"Could not parse Slurm job ID from: {submitted.stdout!r}")
|
| 687 |
+
|
| 688 |
+
print(f"Submitted Slurm job {job_id}")
|
| 689 |
+
print(f" stdout: Logs/slurm-{job_id}.out")
|
| 690 |
+
print(f" stderr: Logs/slurm-{job_id}.err")
|
| 691 |
+
if args.no_wait:
|
| 692 |
+
print(f"Monitor with: squeue -j {job_id}")
|
| 693 |
+
print_result_paths(
|
| 694 |
+
output_dir,
|
| 695 |
+
sample_id,
|
| 696 |
+
not args.no_2d,
|
| 697 |
+
not args.no_generative_3d,
|
| 698 |
+
args.export_glb,
|
| 699 |
+
bool(job_env["REVIEWED_VISIBLE_WORKSPACE"]),
|
| 700 |
+
)
|
| 701 |
+
return 0
|
| 702 |
+
|
| 703 |
+
try:
|
| 704 |
+
state = wait_for_job(
|
| 705 |
+
job_id,
|
| 706 |
+
args.poll_seconds,
|
| 707 |
+
output_dir,
|
| 708 |
+
sample_id,
|
| 709 |
+
not args.no_generative_3d,
|
| 710 |
+
estimated_seconds,
|
| 711 |
+
)
|
| 712 |
+
except KeyboardInterrupt:
|
| 713 |
+
print(
|
| 714 |
+
f"\nStopped waiting; Slurm job {job_id} is still managed separately. "
|
| 715 |
+
f"Cancel it only if needed with: scancel {job_id}",
|
| 716 |
+
file=sys.stderr,
|
| 717 |
+
)
|
| 718 |
+
return 130
|
| 719 |
+
|
| 720 |
+
if state != "COMPLETED":
|
| 721 |
+
raise RuntimeError(
|
| 722 |
+
f"Slurm job {job_id} ended with state {state}. "
|
| 723 |
+
f"Inspect Logs/slurm-{job_id}.err"
|
| 724 |
+
)
|
| 725 |
+
expected = output_dir / "02_masks" / "mask_overlay.png"
|
| 726 |
+
if not expected.is_file():
|
| 727 |
+
raise RuntimeError(
|
| 728 |
+
f"Slurm job {job_id} completed but the expected mask output is missing: "
|
| 729 |
+
f"{public_path(expected, project_root=PROJECT_ROOT, output_root=output_dir)}"
|
| 730 |
+
)
|
| 731 |
+
preflight_path = output_dir / "05_quality_checks" / "preflight.json"
|
| 732 |
+
if preflight_path.is_file():
|
| 733 |
+
try:
|
| 734 |
+
preflight = json.loads(preflight_path.read_text(encoding="utf-8"))
|
| 735 |
+
except (json.JSONDecodeError, OSError):
|
| 736 |
+
preflight = {}
|
| 737 |
+
if preflight.get("decision") != "accept":
|
| 738 |
+
print(
|
| 739 |
+
"\n3D generation was intentionally withheld because the mask/structure "
|
| 740 |
+
"preflight requires re-prompting or human review."
|
| 741 |
+
)
|
| 742 |
+
print(
|
| 743 |
+
" verification: "
|
| 744 |
+
f"{public_path(preflight_path, project_root=PROJECT_ROOT, output_root=output_dir)}"
|
| 745 |
+
)
|
| 746 |
+
print("Automatic masks are review candidates, not ground truth.")
|
| 747 |
+
return 0
|
| 748 |
+
completion_manifest_path = output_dir / "03_2d_completion" / "manifest.json"
|
| 749 |
+
if not args.no_2d:
|
| 750 |
+
if not completion_manifest_path.is_file():
|
| 751 |
+
raise RuntimeError(
|
| 752 |
+
"Slurm job completed but the 2D completion manifest is missing: "
|
| 753 |
+
f"{public_path(completion_manifest_path, project_root=PROJECT_ROOT, output_root=output_dir)}"
|
| 754 |
+
)
|
| 755 |
+
try:
|
| 756 |
+
completion_manifest = json.loads(
|
| 757 |
+
completion_manifest_path.read_text(encoding="utf-8")
|
| 758 |
+
)
|
| 759 |
+
except (json.JSONDecodeError, OSError) as exc:
|
| 760 |
+
raise RuntimeError("The 2D completion manifest is unreadable") from exc
|
| 761 |
+
completion_status = completion_manifest.get("status")
|
| 762 |
+
if completion_status not in GEOMETRY_ALLOWED_2D_STATUSES:
|
| 763 |
+
print(
|
| 764 |
+
"\nThe 2D result needs review and was not used to condition 3D: "
|
| 765 |
+
f"{completion_status!r}. Accessibility3D used the original RGB and strict "
|
| 766 |
+
"three-value mask instead."
|
| 767 |
+
)
|
| 768 |
+
comparison = output_dir / "03_2d_completion" / "candidate_comparison.jpg"
|
| 769 |
+
if comparison.is_file():
|
| 770 |
+
print(
|
| 771 |
+
" candidates: "
|
| 772 |
+
f"{public_path(comparison, project_root=PROJECT_ROOT, output_root=output_dir)}"
|
| 773 |
+
)
|
| 774 |
+
|
| 775 |
+
if not args.no_generative_3d:
|
| 776 |
+
verification_path = output_dir / "05_quality_checks" / "verification.json"
|
| 777 |
+
if not verification_path.is_file():
|
| 778 |
+
raise RuntimeError(
|
| 779 |
+
"Slurm job completed but the generated-3D verification report is missing: "
|
| 780 |
+
f"{public_path(verification_path, project_root=PROJECT_ROOT, output_root=output_dir)}"
|
| 781 |
+
)
|
| 782 |
+
try:
|
| 783 |
+
verification = json.loads(verification_path.read_text(encoding="utf-8"))
|
| 784 |
+
except (json.JSONDecodeError, OSError) as exc:
|
| 785 |
+
raise RuntimeError("Generated-3D verification report is unreadable") from exc
|
| 786 |
+
if verification.get("decision") != "accept":
|
| 787 |
+
print(
|
| 788 |
+
"\nThe Accessibility3D result remains available for human review, but its "
|
| 789 |
+
"structured verification did not accept automatic use."
|
| 790 |
+
)
|
| 791 |
+
print(
|
| 792 |
+
" verification: "
|
| 793 |
+
f"{public_path(verification_path, project_root=PROJECT_ROOT, output_root=output_dir)}"
|
| 794 |
+
)
|
| 795 |
+
|
| 796 |
+
print_result_paths(
|
| 797 |
+
output_dir,
|
| 798 |
+
sample_id,
|
| 799 |
+
not args.no_2d,
|
| 800 |
+
not args.no_generative_3d,
|
| 801 |
+
args.export_glb,
|
| 802 |
+
bool(job_env["REVIEWED_VISIBLE_WORKSPACE"]),
|
| 803 |
+
)
|
| 804 |
+
print("\nAutomatic masks are review candidates, not ground truth.")
|
| 805 |
+
return 0
|
| 806 |
+
|
| 807 |
+
|
| 808 |
+
if __name__ == "__main__":
|
| 809 |
+
raise SystemExit(main())
|
accesspath3r/privacy.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Privacy-safe formatting for console output, logs, and public artifacts."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import os
|
| 6 |
+
import re
|
| 7 |
+
import socket
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from typing import Iterable
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
EMAIL_PATTERN = re.compile(
|
| 13 |
+
r"(?<![A-Za-z0-9._%+-])[A-Za-z0-9._%+-]+@"
|
| 14 |
+
r"[A-Za-z0-9.-]+\.[A-Za-z]{2,}(?![A-Za-z0-9.-])"
|
| 15 |
+
)
|
| 16 |
+
HOME_PATH_PATTERN = re.compile(r"/(?:home|Users)/[^/\s:'\"]+")
|
| 17 |
+
DATA_USER_PATH_PATTERN = re.compile(r"/data\d*/userdata/[^/\s:'\"]+")
|
| 18 |
+
SECRET_QUERY_PATTERN = re.compile(
|
| 19 |
+
r"(?i)(\b(?:access[_-]?token|api[_-]?key|password|secret|token)=)"
|
| 20 |
+
r"[^&\s]+"
|
| 21 |
+
)
|
| 22 |
+
IPV4_PATTERN = re.compile(r"(?<![\d.])(?:\d{1,3}\.){3}\d{1,3}(?![\d.])")
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _resolved(path: str | os.PathLike[str] | Path) -> Path:
|
| 26 |
+
return Path(path).expanduser().resolve(strict=False)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def public_path(
|
| 30 |
+
path: str | os.PathLike[str] | Path,
|
| 31 |
+
*,
|
| 32 |
+
project_root: str | os.PathLike[str] | Path,
|
| 33 |
+
output_root: str | os.PathLike[str] | Path | None = None,
|
| 34 |
+
) -> str:
|
| 35 |
+
"""Return a useful path label without exposing a workstation/user prefix."""
|
| 36 |
+
resolved = _resolved(path)
|
| 37 |
+
project = _resolved(project_root)
|
| 38 |
+
try:
|
| 39 |
+
relative = resolved.relative_to(project)
|
| 40 |
+
return relative.as_posix() or "."
|
| 41 |
+
except ValueError:
|
| 42 |
+
pass
|
| 43 |
+
|
| 44 |
+
if output_root is not None:
|
| 45 |
+
output = _resolved(output_root)
|
| 46 |
+
try:
|
| 47 |
+
relative = resolved.relative_to(output)
|
| 48 |
+
suffix = relative.as_posix()
|
| 49 |
+
return "${OUTPUT_DIR}" + (f"/{suffix}" if suffix else "")
|
| 50 |
+
except ValueError:
|
| 51 |
+
pass
|
| 52 |
+
|
| 53 |
+
home_value = os.environ.get("HOME")
|
| 54 |
+
if home_value:
|
| 55 |
+
home = _resolved(home_value)
|
| 56 |
+
try:
|
| 57 |
+
relative = resolved.relative_to(home)
|
| 58 |
+
suffix = relative.as_posix()
|
| 59 |
+
return "~" + (f"/{suffix}" if suffix else "")
|
| 60 |
+
except ValueError:
|
| 61 |
+
pass
|
| 62 |
+
|
| 63 |
+
return f"<external>/{resolved.name}" if resolved.name else "<external>"
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _replacement_pairs(
|
| 67 |
+
*,
|
| 68 |
+
project_root: str | os.PathLike[str] | Path | None,
|
| 69 |
+
output_root: str | os.PathLike[str] | Path | None,
|
| 70 |
+
sensitive_paths: Iterable[str | os.PathLike[str] | Path],
|
| 71 |
+
) -> list[tuple[str, str]]:
|
| 72 |
+
pairs: list[tuple[str, str]] = []
|
| 73 |
+
|
| 74 |
+
def add(value: str | os.PathLike[str] | Path | None, label: str) -> None:
|
| 75 |
+
if value is None or not str(value).strip():
|
| 76 |
+
return
|
| 77 |
+
raw = str(value).rstrip("/")
|
| 78 |
+
# Relative values such as "." are meaningful CLI inputs but unsafe
|
| 79 |
+
# replacement needles ("." would rewrite every decimal and suffix).
|
| 80 |
+
if raw and Path(raw).expanduser().is_absolute():
|
| 81 |
+
pairs.append((raw, label))
|
| 82 |
+
try:
|
| 83 |
+
resolved = str(_resolved(value)).rstrip("/")
|
| 84 |
+
except (OSError, RuntimeError, ValueError):
|
| 85 |
+
return
|
| 86 |
+
if resolved and resolved not in {".", "/"} and resolved != raw:
|
| 87 |
+
pairs.append((resolved, label))
|
| 88 |
+
|
| 89 |
+
add(output_root, "${OUTPUT_DIR}")
|
| 90 |
+
add(project_root, "${PROJECT_ROOT}")
|
| 91 |
+
add(os.environ.get("HOME"), "${HOME}")
|
| 92 |
+
|
| 93 |
+
for index, path in enumerate(sensitive_paths, start=1):
|
| 94 |
+
add(path, f"${{PRIVATE_PATH_{index}}}")
|
| 95 |
+
|
| 96 |
+
# Replace longer prefixes first so OUTPUT_DIR wins over PROJECT_ROOT.
|
| 97 |
+
return sorted(set(pairs), key=lambda pair: len(pair[0]), reverse=True)
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def sanitize_text(
|
| 101 |
+
text: str,
|
| 102 |
+
*,
|
| 103 |
+
project_root: str | os.PathLike[str] | Path | None = None,
|
| 104 |
+
output_root: str | os.PathLike[str] | Path | None = None,
|
| 105 |
+
sensitive_paths: Iterable[str | os.PathLike[str] | Path] = (),
|
| 106 |
+
) -> str:
|
| 107 |
+
"""Redact local identity/path details while retaining diagnostic meaning."""
|
| 108 |
+
result = text
|
| 109 |
+
for value, label in _replacement_pairs(
|
| 110 |
+
project_root=project_root,
|
| 111 |
+
output_root=output_root,
|
| 112 |
+
sensitive_paths=sensitive_paths,
|
| 113 |
+
):
|
| 114 |
+
result = result.replace(value, label)
|
| 115 |
+
|
| 116 |
+
result = HOME_PATH_PATTERN.sub("${HOME}", result)
|
| 117 |
+
result = DATA_USER_PATH_PATTERN.sub("${HOME}", result)
|
| 118 |
+
result = EMAIL_PATTERN.sub("<redacted-email>", result)
|
| 119 |
+
result = SECRET_QUERY_PATTERN.sub(r"\1<redacted>", result)
|
| 120 |
+
|
| 121 |
+
def redact_ipv4(match: re.Match[str]) -> str:
|
| 122 |
+
value = match.group(0)
|
| 123 |
+
octets = value.split(".")
|
| 124 |
+
if any(int(octet) > 255 for octet in octets) or value.startswith("127."):
|
| 125 |
+
return value
|
| 126 |
+
return "<redacted-ip>"
|
| 127 |
+
|
| 128 |
+
result = IPV4_PATTERN.sub(redact_ipv4, result)
|
| 129 |
+
|
| 130 |
+
identity_values = {
|
| 131 |
+
os.environ.get("USER", ""),
|
| 132 |
+
os.environ.get("LOGNAME", ""),
|
| 133 |
+
os.environ.get("HOSTNAME", ""),
|
| 134 |
+
socket.gethostname(),
|
| 135 |
+
}
|
| 136 |
+
for value in sorted(identity_values, key=len, reverse=True):
|
| 137 |
+
# Avoid replacing common words such as "root" in ordinary diagnostics.
|
| 138 |
+
if len(value) < 4 or value.lower() in {"root", "user", "admin"}:
|
| 139 |
+
continue
|
| 140 |
+
result = re.sub(
|
| 141 |
+
rf"(?<![A-Za-z0-9_.-]){re.escape(value)}(?![A-Za-z0-9_.-])",
|
| 142 |
+
"<redacted-identity>",
|
| 143 |
+
result,
|
| 144 |
+
)
|
| 145 |
+
return result
|
checksums.sha256
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
1e8416b2bf9a40f8acbadd4b8022991ae2a4e31e2788eccbcd14a1b239f45f65 ./accessibilityamodal/category_overrides.py
|
| 2 |
+
f4d13e3551e673d463533afe15c261c737ae2f2306b2bf8ce5ae3b33d2d27060 ./accessibilityamodal/depth.py
|
| 3 |
+
a104312a59a92c36b64c4e1d0496888a53556f7196098209fc8995eec3b763e9 ./accessibilityamodal/geometry_analysis.py
|
| 4 |
+
ffd43f475c991eadf62c21930bc152ea4ae5224cbecededecd97b09e728ed8b3 ./accessibilityamodal/__init__.py
|
| 5 |
+
a088e86abdd7979ba7fc446d0ca14b44291277a1319368057916e095861207dd ./accessibilityamodal/pipeline.py
|
| 6 |
+
8e86aea92c34a8594d21f30a4d76f550e99beadedeb83f55772bc8402d487c03 ./accessibilityamodal/reconstruct.py
|
| 7 |
+
aecc62c1549d5743f1d581c07b647e88f66c1a42f86e85c2e0a98d992eb65880 ./accessibilityamodal/sam_refinement.py
|
| 8 |
+
7fb2b6fee108870792824b1e07677de38fdfcdf1a4e0032d58807750e4315100 ./accessibilityamodal/verification.py
|
| 9 |
+
69eade48cdd5137949c872e7af0ed3f62566ec4e1a51fdff3cd76b695381340b ./accessibilityamodal/visual_completion.py
|
| 10 |
+
0f8001502b13d74a4741dc4ba90c5eed5deb11c169cbb4b16c6f4ac2ddef874c ./accesspath3r/cli.py
|
| 11 |
+
3bb666ae061f470306b4641b7433d6a576907d0c1f56984f6dae6b4734cebd1b ./accesspath3r/__init__.py
|
| 12 |
+
cecfb61d0694188d371c36843a18e8cc597c41ff3e301abc617fd917ddba502c ./accesspath3r/privacy.py
|
| 13 |
+
c88c5f5745fd7bd5a2f0e820c6601fdc7adbc90d6c8eb2cacb1bc34825374424 ./accesspath.py
|
| 14 |
+
28f2a3b43403ecf179685aee9adafb65f5bc8a340314aa8d4cf36b8a1f00b7bc ./configs/accessibility_mask_prompts_e5_v2.json
|
| 15 |
+
9372af9c530a594e9355944829aca2723ad270fe95716caaeaa191e820b7c0e7 ./configs/accessibility_stairs_outdoor_no_occlusion_prompts.json
|
| 16 |
+
cac568cf2b332471ce5d2a3ceaae4f460882322d80b8f00c40c7c1c813bebf2a ./configs/accessibility_taxonomy.json
|
| 17 |
+
7b2f950655ee26db6f7134c4f75ee87548603132f69375dd163617252b5f6c61 ./docs/DEPENDENCIES_AND_WEIGHTS.md
|
| 18 |
+
69cb8637ad32704c9c30df9ad6d529350e205bd62aa3d3a8b66460cb3a326763 ./docs/REPRODUCIBLE_PIPELINE.md
|
| 19 |
+
1c93d6a102763a352f8e4bc17fd36946dbc8bdd8f7a98ef71644506184311ec1 ./docs/SOURCE_MANIFEST.md
|
| 20 |
+
a1be042986b949b43c9439e5a2e4bf06f8c44296f64f93231b9ebf5982c3e141 ./.gitignore
|
| 21 |
+
3204ccc802a3d1fd983b1157acc28788f32e202b0f90af0e6ce82e002415d1c0 ./README.md
|
| 22 |
+
2a3b1c478474d1cba76823f19dc506deb9c30915ca8633bd7937bddeb0a5e7dd ./requirements/runtime.txt
|
| 23 |
+
05b36802d29098a9ed1bb9e2f8391781f5ae70ccdad359241a961e2f9ee15bb1 ./slurm/run_accesspath_demo.sbatch
|
| 24 |
+
abf0e66126a0c36c88e4b27977c3233c56c6672e69579fe5e25a7a53010ec38d ./THIRD_PARTY_NOTICES.md
|
| 25 |
+
5cff25d3577e68bace79480268b531d3f1578e0b3f8f7ac5e41031b111a1e2a9 ./tools/accessibility_2d_completion.py
|
| 26 |
+
5f51907ecc5e3de1f86ce2649bcac0c584430a43b157f1eb9f612af831091570 ./tools/accessibility_3d_completion.py
|
| 27 |
+
1f525525bf318f32ba7773d3a8cf1f4be0cc5c146de72f4f47f099f1dcf1c9b5 ./tools/accessibility_3d_variants.py
|
| 28 |
+
adceb94323890674b9252f4349e1f6274d825584ff6556cd5bf65cc95b074f14 ./tools/accessibility_amodal_mask.py
|
| 29 |
+
acf495884dc154ff4d9e6c0969426b478c591cfd9b23e0b53cec3959780ce21d ./tools/accessibility_fast_2d_baseline.py
|
| 30 |
+
98f34cd0710206e11e82d383147a9df9d83bb4cedd37aea187c4185d3a170f38 ./tools/accessibility_mask_proposals.py
|
| 31 |
+
84fe82f2e16e296d0f79c640d54122a1d91c031d9ddd2c9037dae833fc0c9595 ./tools/audit_accessibility_3d_candidate.py
|
| 32 |
+
3702ac02c658dc70106890ffeea47260d34a7270298f34989eb749de73f06a1a ./tools/build_accessibility_review_bundle.py
|
| 33 |
+
9692cbff8beeea2ed800054a5cf6b1140352c011195d3778228c9875da1bceaa ./tools/build_accessibility_solid_mesh_showcase.py
|
| 34 |
+
056a87a9cf9dc6e46884e2539a546a7e86a4fe8dd6d735bcff5a9392146a726f ./tools/render_accessibility_turntable.py
|
| 35 |
+
f7480d500f9cd07efc3224b258f126ac6baa92d4a5a01373efaf01a6616ab1e6 ./tools/sanitize_log_stream.py
|
| 36 |
+
9bcff66ab4e0afa11ff4a4985bbdff7492e5e6ad68e6031cde420743cccba6e5 ./tools/sanitize_output_metadata.py
|
| 37 |
+
06130da821e8e2578bcd243c84cef03fc6c8e56f3727d44472bd0f70e4615bab ./tools/train_accessibility_amodal_adapter.py
|
| 38 |
+
9e2085f3fdefc2672da5c71547afe3e24a511aa9d4dda622f3bf0708dfdb15a5 ./tools/update_accessibility_inference_progress.py
|
| 39 |
+
96f20519960ad84050e5bfe84206c0f3ee78de27792ee1441c15c555a6165ab0 ./tools/validate_accessibility_reviewed_visible_workspace.py
|
configs/accessibility_mask_prompts_e5_v2.json
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "2.0",
|
| 3 |
+
"repair_basis": "Frozen before the E5 v2 rerun from pre-existing project repair and reannotation prompt inventories; v1 outputs are retained.",
|
| 4 |
+
"categories": {
|
| 5 |
+
"stairs": {
|
| 6 |
+
"prompts": [
|
| 7 |
+
"stair steps",
|
| 8 |
+
"outdoor staircase",
|
| 9 |
+
"concrete stair steps",
|
| 10 |
+
"stair treads",
|
| 11 |
+
"pedestrian stairway",
|
| 12 |
+
"steps at a building entrance"
|
| 13 |
+
],
|
| 14 |
+
"max_area_ratio": 0.75
|
| 15 |
+
},
|
| 16 |
+
"ramp": {
|
| 17 |
+
"prompts": [
|
| 18 |
+
"wheelchair ramp",
|
| 19 |
+
"wheelchair access ramp",
|
| 20 |
+
"accessible ramp with handrails",
|
| 21 |
+
"accessible sloped walking surface",
|
| 22 |
+
"sloped sidewalk ramp",
|
| 23 |
+
"pedestrian access ramp"
|
| 24 |
+
]
|
| 25 |
+
},
|
| 26 |
+
"tactile_paving": {
|
| 27 |
+
"prompts": [
|
| 28 |
+
"tactile walking surface indicator strip",
|
| 29 |
+
"raised bar tactile paving path",
|
| 30 |
+
"ribbed guidance tiles on pedestrian route",
|
| 31 |
+
"tactile guidance paving strip",
|
| 32 |
+
"tactile warning strip on sidewalk",
|
| 33 |
+
"truncated dome warning surface"
|
| 34 |
+
],
|
| 35 |
+
"max_area_ratio": 0.65
|
| 36 |
+
},
|
| 37 |
+
"curb_cut": {
|
| 38 |
+
"prompts": [
|
| 39 |
+
"curb ramp",
|
| 40 |
+
"dropped curb",
|
| 41 |
+
"sidewalk curb cut",
|
| 42 |
+
"pedestrian curb ramp"
|
| 43 |
+
]
|
| 44 |
+
},
|
| 45 |
+
"walkway": {
|
| 46 |
+
"prompts": [
|
| 47 |
+
"pedestrian sidewalk",
|
| 48 |
+
"walkable pedestrian path",
|
| 49 |
+
"accessible walkway",
|
| 50 |
+
"pavement area where pedestrians can walk"
|
| 51 |
+
],
|
| 52 |
+
"max_area_ratio": 0.92
|
| 53 |
+
}
|
| 54 |
+
},
|
| 55 |
+
"category_confusions": {
|
| 56 |
+
"stairs": ["ramp"],
|
| 57 |
+
"ramp": ["stairs", "walkway"],
|
| 58 |
+
"tactile_paving": ["walkway"],
|
| 59 |
+
"curb_cut": ["ramp", "walkway"],
|
| 60 |
+
"walkway": ["stairs", "ramp"]
|
| 61 |
+
},
|
| 62 |
+
"obstacle_prompts": [
|
| 63 |
+
"person",
|
| 64 |
+
"wheelchair",
|
| 65 |
+
"baby stroller",
|
| 66 |
+
"dog",
|
| 67 |
+
"car",
|
| 68 |
+
"van",
|
| 69 |
+
"truck",
|
| 70 |
+
"bicycle",
|
| 71 |
+
"motorcycle",
|
| 72 |
+
"scooter",
|
| 73 |
+
"suitcase",
|
| 74 |
+
"trash bin",
|
| 75 |
+
"chair",
|
| 76 |
+
"table",
|
| 77 |
+
"bench",
|
| 78 |
+
"tree trunk",
|
| 79 |
+
"street sign",
|
| 80 |
+
"utility cabinet",
|
| 81 |
+
"traffic cone",
|
| 82 |
+
"bollard",
|
| 83 |
+
"pole",
|
| 84 |
+
"planter",
|
| 85 |
+
"box or package",
|
| 86 |
+
"construction material"
|
| 87 |
+
],
|
| 88 |
+
"category_obstacle_prompts": {
|
| 89 |
+
"stairs": [
|
| 90 |
+
"object blocking the stairs",
|
| 91 |
+
"post blocking the stairs",
|
| 92 |
+
"railing post occluding the staircase"
|
| 93 |
+
],
|
| 94 |
+
"ramp": [
|
| 95 |
+
"object blocking the ramp",
|
| 96 |
+
"post blocking the ramp"
|
| 97 |
+
],
|
| 98 |
+
"curb_cut": [
|
| 99 |
+
"object blocking the curb ramp",
|
| 100 |
+
"street pole"
|
| 101 |
+
],
|
| 102 |
+
"tactile_paving": [
|
| 103 |
+
"object on the tactile paving",
|
| 104 |
+
"street furniture"
|
| 105 |
+
],
|
| 106 |
+
"walkway": [
|
| 107 |
+
"object blocking the walkway",
|
| 108 |
+
"street pole"
|
| 109 |
+
]
|
| 110 |
+
},
|
| 111 |
+
"obstacle_maximum_area_ratio": {
|
| 112 |
+
"default": 0.20,
|
| 113 |
+
"person": 0.45,
|
| 114 |
+
"car": 0.55,
|
| 115 |
+
"van": 0.55,
|
| 116 |
+
"truck": 0.60
|
| 117 |
+
},
|
| 118 |
+
"barrier_obstacle_prompts": [
|
| 119 |
+
"temporary pedestrian barrier",
|
| 120 |
+
"construction safety barrier",
|
| 121 |
+
"road work barrier"
|
| 122 |
+
],
|
| 123 |
+
"barrier_obstacle_backends": [
|
| 124 |
+
"sam3"
|
| 125 |
+
],
|
| 126 |
+
"quality_thresholds": {
|
| 127 |
+
"minimum_target_area_ratio": 0.005,
|
| 128 |
+
"maximum_target_area_ratio": 0.92,
|
| 129 |
+
"minimum_prompt_agreement": 0.45,
|
| 130 |
+
"minimum_model_confidence": 0.45,
|
| 131 |
+
"auto_accept_score": 0.72,
|
| 132 |
+
"manual_review_score": 0.45
|
| 133 |
+
}
|
| 134 |
+
}
|
configs/accessibility_stairs_outdoor_no_occlusion_prompts.json
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"categories": {
|
| 3 |
+
"stairs": {
|
| 4 |
+
"prompts": [
|
| 5 |
+
"outdoor stone staircase",
|
| 6 |
+
"historic masonry stair steps",
|
| 7 |
+
"visible walking treads of an outdoor staircase",
|
| 8 |
+
"pedestrian stairway steps, excluding handrails and side walls"
|
| 9 |
+
],
|
| 10 |
+
"max_area_ratio": 0.45
|
| 11 |
+
}
|
| 12 |
+
},
|
| 13 |
+
"category_confusions": {
|
| 14 |
+
"stairs": []
|
| 15 |
+
},
|
| 16 |
+
"obstacle_prompts": [],
|
| 17 |
+
"category_obstacle_prompts": {},
|
| 18 |
+
"barrier_obstacle_prompts": [],
|
| 19 |
+
"barrier_obstacle_backends": [],
|
| 20 |
+
"quality_thresholds": {
|
| 21 |
+
"minimum_target_area_ratio": 0.01,
|
| 22 |
+
"maximum_target_area_ratio": 0.45,
|
| 23 |
+
"minimum_prompt_agreement": 0.45,
|
| 24 |
+
"minimum_model_confidence": 0.45,
|
| 25 |
+
"auto_accept_score": 0.72,
|
| 26 |
+
"manual_review_score": 0.45
|
| 27 |
+
}
|
| 28 |
+
}
|
configs/accessibility_taxonomy.json
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"categories": {
|
| 3 |
+
"stairs": {
|
| 4 |
+
"display_name": "Stairs",
|
| 5 |
+
"geometry_model": "tread_riser_block",
|
| 6 |
+
"primary_users": ["mobility_impairment", "blind_or_low_vision"],
|
| 7 |
+
"surface_role": "level_change_barrier",
|
| 8 |
+
"risk_metrics": ["step_edge_continuity", "riser_height_consistency", "handrail_or_obstacle_clearance"],
|
| 9 |
+
"annotation_priority": ["target_visible", "target_amodal", "hidden", "obstacle", "stair_edges_y"],
|
| 10 |
+
"notes": "Use category-specific tread and riser geometry. Do not treat stairs as one smooth ramp."
|
| 11 |
+
},
|
| 12 |
+
"ramp": {
|
| 13 |
+
"display_name": "Accessible Ramp",
|
| 14 |
+
"geometry_model": "sloped_prism",
|
| 15 |
+
"primary_users": ["wheelchair_user", "walker_user", "mobility_impairment"],
|
| 16 |
+
"surface_role": "accessible_level_change",
|
| 17 |
+
"risk_metrics": ["slope", "cross_slope", "landing_continuity", "side_edge_clearance"],
|
| 18 |
+
"annotation_priority": ["target_visible", "target_amodal", "hidden", "obstacle", "ramp_edges"],
|
| 19 |
+
"notes": "Fit a continuous sloped surface and solidify as a ramp prism with side caps."
|
| 20 |
+
},
|
| 21 |
+
"tactile_paving": {
|
| 22 |
+
"display_name": "Tactile Paving",
|
| 23 |
+
"geometry_model": "surface_attachment_ribs_or_domes",
|
| 24 |
+
"primary_users": ["blind_or_low_vision"],
|
| 25 |
+
"surface_role": "navigation_cue",
|
| 26 |
+
"risk_metrics": ["pattern_direction", "pattern_periodicity", "warning_surface_coverage", "continuity_under_occlusion"],
|
| 27 |
+
"annotation_priority": ["target_visible", "target_amodal", "hidden", "obstacle", "pattern_type"],
|
| 28 |
+
"notes": "Reconstruct as shallow raised dots/ribs attached to a walkway or ramp base surface, not as a standalone thick body."
|
| 29 |
+
},
|
| 30 |
+
"curb_cut": {
|
| 31 |
+
"display_name": "Curb Cut",
|
| 32 |
+
"geometry_model": "curb_ramp_prism",
|
| 33 |
+
"primary_users": ["wheelchair_user", "walker_user", "mobility_impairment"],
|
| 34 |
+
"surface_role": "street_sidewalk_transition",
|
| 35 |
+
"risk_metrics": ["curb_transition_height", "ramp_slope", "gutter_alignment", "truncated_dome_presence"],
|
| 36 |
+
"annotation_priority": ["target_visible", "target_amodal", "hidden", "obstacle", "curb_edges"],
|
| 37 |
+
"notes": "Model as a local ramp transition between sidewalk and roadway, often with tactile warning surface."
|
| 38 |
+
},
|
| 39 |
+
"raised_curb": {
|
| 40 |
+
"display_name": "Raised Curb",
|
| 41 |
+
"geometry_model": "raised_curb_prism",
|
| 42 |
+
"primary_users": ["wheelchair_user", "walker_user", "blind_or_low_vision", "mobility_impairment"],
|
| 43 |
+
"surface_role": "non_walkable_level_change_barrier",
|
| 44 |
+
"risk_metrics": ["curb_height", "top_edge_continuity", "vertical_face_continuity", "route_clearance"],
|
| 45 |
+
"annotation_priority": ["target_visible", "target_amodal", "hidden", "obstacle", "curb_top_edge", "curb_base_edge"],
|
| 46 |
+
"notes": "Model as one continuous raised prism with a top surface and vertical face. Never infer repeated stair treads or a sawtooth profile."
|
| 47 |
+
},
|
| 48 |
+
"walkway": {
|
| 49 |
+
"display_name": "Walkway / Flat Ground",
|
| 50 |
+
"geometry_model": "flat_or_low_slope_slab",
|
| 51 |
+
"primary_users": ["wheelchair_user", "walker_user", "blind_or_low_vision"],
|
| 52 |
+
"surface_role": "travel_path",
|
| 53 |
+
"risk_metrics": ["surface_continuity", "clear_width", "obstacle_clearance", "roughness"],
|
| 54 |
+
"annotation_priority": ["target_visible", "target_amodal", "hidden", "obstacle"],
|
| 55 |
+
"notes": "Use a continuous slab or low-slope plane. Keep tactile paving as a separate overlay category when present."
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
"user_groups": {
|
| 59 |
+
"wheelchair_user": {
|
| 60 |
+
"requires": ["ramp", "curb_cut", "walkway"],
|
| 61 |
+
"high_risk_categories": ["stairs", "raised_curb", "curb_cut", "ramp"]
|
| 62 |
+
},
|
| 63 |
+
"walker_user": {
|
| 64 |
+
"requires": ["ramp", "curb_cut", "walkway"],
|
| 65 |
+
"high_risk_categories": ["stairs", "raised_curb", "walkway"]
|
| 66 |
+
},
|
| 67 |
+
"blind_or_low_vision": {
|
| 68 |
+
"requires": ["tactile_paving", "curb_cut", "stairs", "walkway"],
|
| 69 |
+
"high_risk_categories": ["tactile_paving", "stairs", "raised_curb", "curb_cut"]
|
| 70 |
+
},
|
| 71 |
+
"mobility_impairment": {
|
| 72 |
+
"requires": ["ramp", "curb_cut", "walkway", "stairs"],
|
| 73 |
+
"high_risk_categories": ["stairs", "raised_curb", "ramp", "curb_cut"]
|
| 74 |
+
}
|
| 75 |
+
},
|
| 76 |
+
"dataset_policy": {
|
| 77 |
+
"split_by_category": true,
|
| 78 |
+
"split_by_user_group": true,
|
| 79 |
+
"allow_multi_label": true,
|
| 80 |
+
"gold_for_training": false,
|
| 81 |
+
"automatic_category_is_ground_truth": false,
|
| 82 |
+
"automatic_mask_is_ground_truth": false
|
| 83 |
+
}
|
| 84 |
+
}
|
docs/DEPENDENCIES_AND_WEIGHTS.md
ADDED
|
@@ -0,0 +1,309 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Dependencies, attribution, and weight downloads
|
| 2 |
+
|
| 3 |
+
This guide is written for a first-time user. AccessPath is inspired by and
|
| 4 |
+
adapted from prior work on amodal completion, segmentation, geometry, and
|
| 5 |
+
visual 3D reconstruction. It explains both **what is actually used at runtime**
|
| 6 |
+
and **what is only cited or retained as an optional comparison**. Do not
|
| 7 |
+
download a model merely because its name occurs in a related-work note.
|
| 8 |
+
|
| 9 |
+
## 1. What AccessPath implements
|
| 10 |
+
|
| 11 |
+
AccessPath implements an accessibility-focused workflow inspired by these
|
| 12 |
+
research directions, while adding project-level adaptation around external
|
| 13 |
+
models:
|
| 14 |
+
|
| 15 |
+
1. prompt selection and target/obstacle mask proposal orchestration;
|
| 16 |
+
2. constrained visible/hidden/amodal mask construction and validation;
|
| 17 |
+
3. mask-restricted 2D inpainting candidate generation and selection;
|
| 18 |
+
4. depth/geometry diagnostics, quality gates, rendering, and review bundles;
|
| 19 |
+
5. an adapter that sends an RGB image plus a three-value mask to an external
|
| 20 |
+
visual-3D backend.
|
| 21 |
+
|
| 22 |
+
It does **not** claim authorship of SAM 3, Stable Diffusion, Depth Anything V2,
|
| 23 |
+
VGGT, Amodal3R, TRELLIS, pix2gestalt, Open-World AMODAL, or Amodal Completion
|
| 24 |
+
in the Wild. Where one of these is a runtime backend, its code, license, and
|
| 25 |
+
weights remain separate; where it is listed as related work, AccessPath does
|
| 26 |
+
not import or execute it.
|
| 27 |
+
|
| 28 |
+
## 2. Exact dependency status
|
| 29 |
+
|
| 30 |
+
| Method/model | Status in this repository | Where in AccessPath | What a user must do |
|
| 31 |
+
| --- | --- | --- | --- |
|
| 32 |
+
| [SAM 3](https://github.com/facebookresearch/sam3) | Used by the automatic mask-proposal stage. | `tools/accessibility_mask_proposals.py` | Install it in its own environment and request access to [`facebook/sam3`](https://huggingface.co/facebook/sam3). |
|
| 33 |
+
| [Stable Diffusion inpainting](https://huggingface.co/sd-legacy/stable-diffusion-inpainting) | Used by the default GPU 2D completion stage. | `tools/accessibility_2d_completion.py` | Download an inpainting checkpoint under its own terms. |
|
| 34 |
+
| [Depth Anything V2](https://github.com/DepthAnything/Depth-Anything-V2) | Used by the default full-pipeline depth/geometry diagnostic. | `accessibilityamodal/depth.py` | Install the official repository and download the selected checkpoint. |
|
| 35 |
+
| [Amodal3R](https://sm0kywu.github.io/Amodal3R/) | Used by the default visual 3D stage through a thin adapter. | `tools/accessibility_3d_completion.py` | Install the official runtime and obtain its model assets. |
|
| 36 |
+
| [TRELLIS image large](https://huggingface.co/microsoft/TRELLIS-image-large) | Upstream dependency reported by the Amodal3R model card. | External Amodal3R runtime | Obtain it only following the upstream Amodal3R instructions. |
|
| 37 |
+
| [VGGT](https://github.com/facebookresearch/vggt) | **Optional** depth/point-map engine; not selected by the default Slurm launcher. | `accessibilityamodal/depth.py` | Install/download only when running `--depth-engine vggt`. |
|
| 38 |
+
| [pix2gestalt](https://github.com/cvlab-columbia/pix2gestalt) | **Not executed.** Kept only as a related-work/comparison URL in a baseline manifest. | `tools/accessibility_fast_2d_baseline.py` | No installation or checkpoint is required for AccessPath. |
|
| 39 |
+
| [Open-World AMODAL](https://github.com/saraao/amodal) | **Not executed.** Kept only as a related-work/comparison URL. | `tools/accessibility_fast_2d_baseline.py` | No installation or checkpoint is required for AccessPath. |
|
| 40 |
+
| [Amodal Completion in the Wild](https://github.com/Championchess/Amodal-Completion-in-the-Wild) | **Not executed.** Mentioned as a compatible external research backend only. | `accessibilityamodal/pipeline.py` | No installation or checkpoint is required for AccessPath. |
|
| 41 |
+
|
| 42 |
+
Therefore, no pix2gestalt, Open-World AMODAL, or Amodal-Wild code/weights are
|
| 43 |
+
needed for the commands in this repository. Their links are present so readers
|
| 44 |
+
can distinguish the accessibility-adapted AccessPath workflow from related
|
| 45 |
+
amodal-completion work.
|
| 46 |
+
|
| 47 |
+
## 3. Before starting
|
| 48 |
+
|
| 49 |
+
You need Linux, Git, Python, a CUDA-capable GPU, and a Slurm installation for
|
| 50 |
+
the all-in-one `pipeline` command. The direct `2d` and `3d` commands can be
|
| 51 |
+
run outside Slurm, but still need a compatible CUDA environment.
|
| 52 |
+
|
| 53 |
+
Clone the anonymous release:
|
| 54 |
+
|
| 55 |
+
```bash
|
| 56 |
+
git clone https://huggingface.co/anonymous-accesspath/AccessPath
|
| 57 |
+
cd AccessPath
|
| 58 |
+
```
|
| 59 |
+
|
| 60 |
+
Create a general utility environment. Install the PyTorch build that matches
|
| 61 |
+
your CUDA driver using the [official PyTorch selector](https://pytorch.org/get-started/locally/),
|
| 62 |
+
then install the Python packages used by the project wrappers:
|
| 63 |
+
|
| 64 |
+
```bash
|
| 65 |
+
python -m venv .venv
|
| 66 |
+
source .venv/bin/activate
|
| 67 |
+
python -m pip install --upgrade pip
|
| 68 |
+
# Install a CUDA-compatible PyTorch build here, following pytorch.org.
|
| 69 |
+
python -m pip install -r requirements/runtime.txt
|
| 70 |
+
python -m pip install diffusers transformers accelerate safetensors huggingface_hub
|
| 71 |
+
```
|
| 72 |
+
|
| 73 |
+
Do not place downloaded checkpoints inside the Git clone. Keep them in a
|
| 74 |
+
separate local `models/` directory and pass their paths through environment
|
| 75 |
+
variables. This avoids accidentally committing large or license-restricted
|
| 76 |
+
files.
|
| 77 |
+
|
| 78 |
+
## 4. Download each runtime asset
|
| 79 |
+
|
| 80 |
+
### 4.1 SAM 3: mask proposals
|
| 81 |
+
|
| 82 |
+
SAM 3 is used only to propose a visible target mask and an obstacle mask. The
|
| 83 |
+
proposal is not ground truth; inspect it before using it for an experiment.
|
| 84 |
+
|
| 85 |
+
1. Read and accept the access terms on [`facebook/sam3`](https://huggingface.co/facebook/sam3).
|
| 86 |
+
2. Follow the official [SAM 3 installation guide](https://github.com/facebookresearch/sam3).
|
| 87 |
+
SAM 3 may require a newer Python/PyTorch/CUDA combination than the remaining
|
| 88 |
+
pipeline, so a dedicated environment is recommended.
|
| 89 |
+
3. Authenticate and download the official files:
|
| 90 |
+
|
| 91 |
+
```bash
|
| 92 |
+
hf auth login
|
| 93 |
+
hf download facebook/sam3 sam3.pt config.json --local-dir /path/to/models/sam3
|
| 94 |
+
```
|
| 95 |
+
|
| 96 |
+
4. Record the environment and paths for AccessPath:
|
| 97 |
+
|
| 98 |
+
```bash
|
| 99 |
+
export SAM3_PYTHON=/path/to/sam3-environment/bin/python
|
| 100 |
+
export SAM3_REPO=/path/to/sam3-source
|
| 101 |
+
export SAM3_CHECKPOINT=/path/to/models/sam3/sam3.pt
|
| 102 |
+
```
|
| 103 |
+
|
| 104 |
+
### 4.2 Stable Diffusion: 2D inpainting
|
| 105 |
+
|
| 106 |
+
The 2D stage calls Diffusers' inpainting pipeline. Obtain an inpainting model
|
| 107 |
+
from [`sd-legacy/stable-diffusion-inpainting`](https://huggingface.co/sd-legacy/stable-diffusion-inpainting)
|
| 108 |
+
or use another compatible checkpoint only after checking its model card and
|
| 109 |
+
license. A typical download is:
|
| 110 |
+
|
| 111 |
+
```bash
|
| 112 |
+
hf download sd-legacy/stable-diffusion-inpainting \
|
| 113 |
+
--local-dir /path/to/models/stable-diffusion-inpainting
|
| 114 |
+
export SD_MODEL=/path/to/models/stable-diffusion-inpainting
|
| 115 |
+
```
|
| 116 |
+
|
| 117 |
+
The 2D implementation changes only the reviewed completion envelope and saves
|
| 118 |
+
its selected result plus metadata under the requested output directory.
|
| 119 |
+
|
| 120 |
+
### 4.3 Depth Anything V2: depth and geometry diagnostics
|
| 121 |
+
|
| 122 |
+
The default full launcher expects the large indoor metric checkpoint:
|
| 123 |
+
[`Depth-Anything-V2-Metric-Hypersim-Large`](https://huggingface.co/depth-anything/Depth-Anything-V2-Metric-Hypersim-Large).
|
| 124 |
+
Install source code from the official [Depth Anything V2 repository](https://github.com/DepthAnything/Depth-Anything-V2), then download the exact checkpoint:
|
| 125 |
+
|
| 126 |
+
```bash
|
| 127 |
+
git clone https://github.com/DepthAnything/Depth-Anything-V2 /path/to/Depth-Anything-V2
|
| 128 |
+
hf download depth-anything/Depth-Anything-V2-Metric-Hypersim-Large \
|
| 129 |
+
depth_anything_v2_metric_hypersim_vitl.pth \
|
| 130 |
+
--local-dir /path/to/models/depth-anything-v2-metric-hypersim-large
|
| 131 |
+
|
| 132 |
+
export DEPTH_REPO=/path/to/Depth-Anything-V2
|
| 133 |
+
export DEPTH_CHECKPOINT=/path/to/models/depth-anything-v2-metric-hypersim-large/depth_anything_v2_metric_hypersim_vitl.pth
|
| 134 |
+
```
|
| 135 |
+
|
| 136 |
+
Depth output is a diagnostic estimate. It is not automatically calibrated
|
| 137 |
+
metric ground truth for the input camera and must not be treated as a
|
| 138 |
+
navigation-safety measurement.
|
| 139 |
+
|
| 140 |
+
### 4.4 Amodal3R and TRELLIS: visual 3D completion
|
| 141 |
+
|
| 142 |
+
AccessPath does not redistribute Amodal3R source code or weights. The adapter
|
| 143 |
+
requires an external Python environment where this import succeeds:
|
| 144 |
+
|
| 145 |
+
```bash
|
| 146 |
+
python -c "from amodal3d.pipelines import Amodal3RImageTo3DPipeline; print('Amodal3R runtime ready')"
|
| 147 |
+
```
|
| 148 |
+
|
| 149 |
+
Use the official [Amodal3R project page](https://sm0kywu.github.io/Amodal3R/),
|
| 150 |
+
[model card](https://huggingface.co/Sm0kyWu/Amodal3R), and
|
| 151 |
+
[paper](https://arxiv.org/abs/2503.13439) for installation and license terms.
|
| 152 |
+
The model card states that its implementation is built on TRELLIS and obtains
|
| 153 |
+
pretrained assets from [`microsoft/TRELLIS-image-large`](https://huggingface.co/microsoft/TRELLIS-image-large).
|
| 154 |
+
For a local snapshot of the published model card repository:
|
| 155 |
+
|
| 156 |
+
```bash
|
| 157 |
+
hf download Sm0kyWu/Amodal3R --local-dir /path/to/models/Amodal3R
|
| 158 |
+
```
|
| 159 |
+
|
| 160 |
+
Follow the upstream project instructions for any additional TRELLIS assets and
|
| 161 |
+
CUDA rasterizer dependencies. Then configure AccessPath:
|
| 162 |
+
|
| 163 |
+
```bash
|
| 164 |
+
export AMODAL3D_PYTHON=/path/to/amodal3r-environment/bin/python
|
| 165 |
+
export AMODAL3D_MODEL=/path/to/models/Amodal3R
|
| 166 |
+
export AMODAL3D_TORCH_HOME=/path/to/model-cache/torch
|
| 167 |
+
```
|
| 168 |
+
|
| 169 |
+
The 3D result is a learned visual reconstruction. It is not guaranteed to be
|
| 170 |
+
metric, watertight, scale-calibrated, or safe for path-planning decisions.
|
| 171 |
+
|
| 172 |
+
### 4.5 Optional VGGT geometry route
|
| 173 |
+
|
| 174 |
+
VGGT is implemented as an optional `--depth-engine vggt` route. It is useful
|
| 175 |
+
for comparing a geometry/point-map estimate, but it is **not** the default
|
| 176 |
+
AccessPath visual-3D output and it is not required for the default full run.
|
| 177 |
+
|
| 178 |
+
```bash
|
| 179 |
+
git clone https://github.com/facebookresearch/vggt /path/to/vggt
|
| 180 |
+
python -m pip install -r /path/to/vggt/requirements.txt
|
| 181 |
+
hf download facebook/VGGT-1B --local-dir /path/to/models/VGGT-1B
|
| 182 |
+
```
|
| 183 |
+
|
| 184 |
+
Then invoke the diagnostic engine with `--depth-engine vggt`, pass
|
| 185 |
+
`--vggt-repo /path/to/vggt`, and use `--vggt-model /path/to/models/VGGT-1B`.
|
| 186 |
+
See the [official VGGT repository](https://github.com/facebookresearch/vggt)
|
| 187 |
+
and [model page](https://huggingface.co/facebook/VGGT-1B) for current access,
|
| 188 |
+
license, and checkpoint conditions.
|
| 189 |
+
|
| 190 |
+
### 4.6 Project amodal-mask adapter checkpoint
|
| 191 |
+
|
| 192 |
+
`AMODAL_CHECKPOINT` is an AccessPath-trained mask-adapter checkpoint, not an
|
| 193 |
+
upstream public dependency. It is deliberately withheld during anonymous
|
| 194 |
+
review. Its release requires confirmation of training-data permissions,
|
| 195 |
+
base-model terms, privacy review, and the paper's release policy.
|
| 196 |
+
|
| 197 |
+
Without this checkpoint, the **all-in-one** `pipeline` command stops at its
|
| 198 |
+
preflight check. Readers can still run the direct 2D and visual-3D commands
|
| 199 |
+
with their own reviewed masks. This limitation is intentional and explicit;
|
| 200 |
+
the repository does not substitute a different checkpoint silently.
|
| 201 |
+
|
| 202 |
+
## 5. Run the three supported routes
|
| 203 |
+
|
| 204 |
+
### Route A — direct 2D completion
|
| 205 |
+
|
| 206 |
+
Prepare an RGB image and three aligned binary masks: visible target, complete
|
| 207 |
+
amodal target, and obstacle. All must have the same width and height as the
|
| 208 |
+
RGB image.
|
| 209 |
+
|
| 210 |
+
```bash
|
| 211 |
+
python accesspath.py 2d -- \
|
| 212 |
+
--image inputs/scene.jpg \
|
| 213 |
+
--target-visible-mask inputs/target_visible.png \
|
| 214 |
+
--target-amodal-mask inputs/target_amodal.png \
|
| 215 |
+
--obstacle-mask inputs/obstacle.png \
|
| 216 |
+
--category stairs \
|
| 217 |
+
--model "$SD_MODEL" \
|
| 218 |
+
--output-dir outputs/stairs_2d \
|
| 219 |
+
--device cuda
|
| 220 |
+
```
|
| 221 |
+
|
| 222 |
+
Inspect the selected RGB and the JSON manifest under `outputs/stairs_2d`.
|
| 223 |
+
|
| 224 |
+
### Route B — direct visual 3D completion
|
| 225 |
+
|
| 226 |
+
Create one three-value mask at the RGB resolution: `255` is background, `188`
|
| 227 |
+
is visible target, and `0` is hidden target. Then run:
|
| 228 |
+
|
| 229 |
+
```bash
|
| 230 |
+
AMODAL3D_TORCH_HOME="$AMODAL3D_TORCH_HOME" \
|
| 231 |
+
"$AMODAL3D_PYTHON" tools/accessibility_3d_completion.py \
|
| 232 |
+
--image inputs/scene.jpg \
|
| 233 |
+
--mask inputs/amodal_3value.png \
|
| 234 |
+
--model "$AMODAL3D_MODEL" \
|
| 235 |
+
--output-dir outputs/stairs_3d
|
| 236 |
+
```
|
| 237 |
+
|
| 238 |
+
Inspect the generated manifest and image/mesh assets. Do not interpret a
|
| 239 |
+
plausible-looking 3D asset as a physical measurement.
|
| 240 |
+
|
| 241 |
+
### Route C — full Slurm pipeline
|
| 242 |
+
|
| 243 |
+
After configuring every required variable above, submit a finite one-image job:
|
| 244 |
+
|
| 245 |
+
```bash
|
| 246 |
+
python accesspath.py pipeline -- \
|
| 247 |
+
--image inputs/scene.jpg \
|
| 248 |
+
--category stairs \
|
| 249 |
+
--output-dir outputs/stairs_full
|
| 250 |
+
```
|
| 251 |
+
|
| 252 |
+
The command submits one GPU job, waits by default, saves its results, and exits
|
| 253 |
+
when the job terminates. Add `--no-wait` to return after submission. The output
|
| 254 |
+
review order is documented in [REPRODUCIBLE_PIPELINE.md](REPRODUCIBLE_PIPELINE.md).
|
| 255 |
+
|
| 256 |
+
## 6. Citation and related-work links
|
| 257 |
+
|
| 258 |
+
Use the official paper/model pages below when citing an external dependency.
|
| 259 |
+
|
| 260 |
+
```bibtex
|
| 261 |
+
@article{wu2025amodal3r,
|
| 262 |
+
title={Amodal3R: Amodal 3D Reconstruction from Occluded 2D Images},
|
| 263 |
+
author={Wu, Tianhao and Zheng, Chuanxia and Guan, Frank and Vedaldi, Andrea and Cham, Tat-Jen},
|
| 264 |
+
journal={arXiv preprint arXiv:2503.13439},
|
| 265 |
+
year={2025}
|
| 266 |
+
}
|
| 267 |
+
|
| 268 |
+
@inproceedings{wang2025vggt,
|
| 269 |
+
title={VGGT: Visual Geometry Grounded Transformer},
|
| 270 |
+
author={Wang, Jianyuan and Chen, Minghao and Karaev, Nikita and Vedaldi, Andrea and Rupprecht, Christian and Novotny, David},
|
| 271 |
+
booktitle={CVPR},
|
| 272 |
+
year={2025}
|
| 273 |
+
}
|
| 274 |
+
|
| 275 |
+
@inproceedings{ozguroglu2024pix2gestalt,
|
| 276 |
+
title={pix2gestalt: Amodal Segmentation by Synthesizing Wholes},
|
| 277 |
+
author={Ozguroglu, Ege and others},
|
| 278 |
+
booktitle={CVPR},
|
| 279 |
+
year={2024}
|
| 280 |
+
}
|
| 281 |
+
|
| 282 |
+
@inproceedings{ao2025open,
|
| 283 |
+
title={Open-World Amodal Appearance Completion},
|
| 284 |
+
author={Ao, Jiayang and Jiang, Yanbei and Ke, Qiuhong and Ehinger, Krista A.},
|
| 285 |
+
booktitle={CVPR},
|
| 286 |
+
year={2025}
|
| 287 |
+
}
|
| 288 |
+
|
| 289 |
+
@inproceedings{zhan2024amodal,
|
| 290 |
+
title={Amodal Ground Truth and Completion in the Wild},
|
| 291 |
+
author={Zhan, Guanqi and Zheng, Chuanxia and Xie, Weidi and Zisserman, Andrew},
|
| 292 |
+
booktitle={CVPR},
|
| 293 |
+
year={2024}
|
| 294 |
+
}
|
| 295 |
+
|
| 296 |
+
@article{yang2024depthanythingv2,
|
| 297 |
+
title={Depth Anything V2},
|
| 298 |
+
author={Yang, Lihe and others},
|
| 299 |
+
journal={arXiv:2406.09414},
|
| 300 |
+
year={2024}
|
| 301 |
+
}
|
| 302 |
+
```
|
| 303 |
+
|
| 304 |
+
For convenience, the canonical links are: [Amodal3R paper](https://arxiv.org/abs/2503.13439),
|
| 305 |
+
[VGGT paper](https://arxiv.org/abs/2503.11651),
|
| 306 |
+
[pix2gestalt paper](https://arxiv.org/abs/2401.14398),
|
| 307 |
+
[Open-World AMODAL paper](https://arxiv.org/abs/2411.13019),
|
| 308 |
+
[Amodal Completion in the Wild paper](https://arxiv.org/abs/2312.17247), and
|
| 309 |
+
[Depth Anything V2 paper](https://arxiv.org/abs/2406.09414).
|
docs/REPRODUCIBLE_PIPELINE.md
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Reproducible AccessPath pipeline
|
| 2 |
+
|
| 3 |
+
`python accesspath.py pipeline` is the complete single-image execution path.
|
| 4 |
+
It is a finite Slurm GPU job: it submits one image, writes persistent outputs,
|
| 5 |
+
and releases the GPU allocation at completion or failure.
|
| 6 |
+
|
| 7 |
+
## Execution graph
|
| 8 |
+
|
| 9 |
+
```text
|
| 10 |
+
RGB image
|
| 11 |
+
-> SAM 3 target/obstacle proposals (or validated reviewed visible mask)
|
| 12 |
+
-> constrained hidden/amodal mask proposal
|
| 13 |
+
-> mask and geometry preflight
|
| 14 |
+
-> GPU 2D inpainting candidates and deterministic quality selection
|
| 15 |
+
-> depth/point cloud/category-geometry diagnostics
|
| 16 |
+
-> visual 3D completion through the upstream Amodal3R backend
|
| 17 |
+
-> verification, turntable, and compact review bundle
|
| 18 |
+
```
|
| 19 |
+
|
| 20 |
+
Automatic masks and generated completions are review candidates, not ground
|
| 21 |
+
truth, metric geometry, or navigation-safety labels.
|
| 22 |
+
|
| 23 |
+
## Required runtime assets
|
| 24 |
+
|
| 25 |
+
Set paths through environment variables rather than editing repository files:
|
| 26 |
+
|
| 27 |
+
```bash
|
| 28 |
+
export SAM3_PYTHON=/path/to/sam3/python
|
| 29 |
+
export SAM3_REPO=/path/to/sam3/source
|
| 30 |
+
export SAM3_CHECKPOINT=/path/to/sam3/checkpoint.pt
|
| 31 |
+
export AMODAL3D_PYTHON=/path/to/amodal3r/python
|
| 32 |
+
export AMODAL_CHECKPOINT=/path/to/accesspath-amodal-mask-adapter.pt
|
| 33 |
+
export DEPTH_REPO=/path/to/Depth-Anything-V2
|
| 34 |
+
export DEPTH_CHECKPOINT=/path/to/depth-checkpoint.pth
|
| 35 |
+
export SD_MODEL=/path/to/stable-diffusion-inpainting
|
| 36 |
+
export AMODAL3D_MODEL=Sm0kyWu/Amodal3R
|
| 37 |
+
```
|
| 38 |
+
|
| 39 |
+
The optional `REVIEWED_VISIBLE_WORKSPACE` route bypasses automatic mask
|
| 40 |
+
proposal only when a validated reviewed mask workspace is supplied. All
|
| 41 |
+
checkpoint paths above are local user assets and are intentionally absent from
|
| 42 |
+
the repository.
|
| 43 |
+
|
| 44 |
+
## Submit one image
|
| 45 |
+
|
| 46 |
+
```bash
|
| 47 |
+
python accesspath.py pipeline -- \
|
| 48 |
+
--image path/to/image.jpg \
|
| 49 |
+
--category stairs \
|
| 50 |
+
--output-dir output/stairs_demo
|
| 51 |
+
```
|
| 52 |
+
|
| 53 |
+
Add `--no-wait` to return after Slurm submission, `--no-2d` to skip inpainting,
|
| 54 |
+
or `--no-generative-3d` to skip the upstream visual-3D stage. The command
|
| 55 |
+
prints the job identifier and paths to persisted review artifacts.
|
| 56 |
+
|
| 57 |
+
## Output review order
|
| 58 |
+
|
| 59 |
+
1. `00_quick_review/overview.jpg` — compact original/2D/3D comparison.
|
| 60 |
+
2. `01_sam3/` or `01_reviewed_visible/` — target and obstacle mask evidence.
|
| 61 |
+
3. `02_masks/` — visible, hidden, amodal, and obstacle masks.
|
| 62 |
+
4. `03_2d_completion/` — candidates, selection manifest, and selected RGB.
|
| 63 |
+
5. `04_3d_completion/` — depth and geometry diagnostics.
|
| 64 |
+
6. `accessibility3d/` — upstream Amodal3R visual-3D assets.
|
| 65 |
+
7. `05_quality_checks/` — preflight and final verification reports.
|
| 66 |
+
|
| 67 |
+
The job wrapper uses portable project-relative paths and sanitizes persisted
|
| 68 |
+
logs and text metadata. It never attempts to anonymize people or sensitive
|
| 69 |
+
content visible in the input image.
|
docs/SOURCE_MANIFEST.md
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Source manifest
|
| 2 |
+
|
| 3 |
+
This file maps every execution stage to its implementation so anonymous
|
| 4 |
+
reviewers can trace the running pipeline without unrelated development files.
|
| 5 |
+
|
| 6 |
+
| Stage | Primary implementation |
|
| 7 |
+
| --- | --- |
|
| 8 |
+
| Command submission, polling, portable paths | `accesspath.py`, `accesspath3r/cli.py`, `accesspath3r/privacy.py` |
|
| 9 |
+
| Finite GPU allocation and stage orchestration | `slurm/run_accesspath_demo.sbatch` |
|
| 10 |
+
| Target/obstacle mask proposal | `tools/accessibility_mask_proposals.py`, `configs/accessibility_mask_prompts_e5_v2.json` |
|
| 11 |
+
| Reviewed-mask validation and amodal-mask proposal | `tools/validate_accessibility_reviewed_visible_workspace.py`, `tools/accessibility_amodal_mask.py`, `tools/train_accessibility_amodal_adapter.py` |
|
| 12 |
+
| GPU visual 2D completion | `tools/accessibility_2d_completion.py`, `accessibilityamodal/visual_completion.py` |
|
| 13 |
+
| Deterministic 2D comparison baseline | `tools/accessibility_fast_2d_baseline.py` |
|
| 14 |
+
| Depth, geometry, mask refinement, and checks | `accessibilityamodal/depth.py`, `accessibilityamodal/reconstruct.py`, `accessibilityamodal/geometry_analysis.py`, `accessibilityamodal/pipeline.py`, `accessibilityamodal/sam_refinement.py`, `accessibilityamodal/verification.py` |
|
| 15 |
+
| Visual 3D backend adapter | `tools/accessibility_3d_completion.py` |
|
| 16 |
+
| 3D diagnostics and rendering | `tools/accessibility_3d_variants.py`, `tools/build_accessibility_solid_mesh_showcase.py`, `tools/render_accessibility_turntable.py` |
|
| 17 |
+
| Quality reports, compact review bundle, and metadata sanitization | `tools/audit_accessibility_3d_candidate.py`, `tools/build_accessibility_review_bundle.py`, `tools/update_accessibility_inference_progress.py`, `tools/sanitize_log_stream.py`, `tools/sanitize_output_metadata.py` |
|
| 18 |
+
|
| 19 |
+
No upstream Amodal3R source, model checkpoint, source image, annotation mask,
|
| 20 |
+
training run, experiment log, local environment, or author-identifying file is
|
| 21 |
+
included. Upstream model attribution is retained in
|
| 22 |
+
[THIRD_PARTY_NOTICES.md](../THIRD_PARTY_NOTICES.md).
|
requirements/runtime.txt
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
numpy
|
| 2 |
+
pillow
|
| 3 |
+
opencv-python
|
| 4 |
+
imageio
|
| 5 |
+
trimesh
|
| 6 |
+
gradio_client
|
slurm/run_accesspath_demo.sbatch
ADDED
|
@@ -0,0 +1,573 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
#SBATCH -J acc-single-demo
|
| 3 |
+
#SBATCH -p 5090
|
| 4 |
+
#SBATCH -G 1
|
| 5 |
+
#SBATCH -c 8
|
| 6 |
+
#SBATCH --mem=64G
|
| 7 |
+
#SBATCH -t 04:00:00
|
| 8 |
+
#SBATCH -o Logs/slurm-%j.out
|
| 9 |
+
#SBATCH -e Logs/slurm-%j.err
|
| 10 |
+
|
| 11 |
+
set -euo pipefail
|
| 12 |
+
mkdir -p Logs
|
| 13 |
+
|
| 14 |
+
PROJECT_DIR="${PROJECT_DIR:-${SLURM_SUBMIT_DIR:-$PWD}}"
|
| 15 |
+
IMAGE="${IMAGE:?Set IMAGE=/absolute/or/project-relative/image.jpg}"
|
| 16 |
+
CATEGORY="${CATEGORY:-walkway}"
|
| 17 |
+
SAMPLE_ID="${SAMPLE_ID:-}"
|
| 18 |
+
OUTPUT_DIR="${OUTPUT_DIR:-$PROJECT_DIR/output/inference/$SLURM_JOB_ID}"
|
| 19 |
+
ACCESSIBILITY3D_DIR="$OUTPUT_DIR/accessibility3d"
|
| 20 |
+
RUN_GPU_2D="${RUN_GPU_2D:-${RUN_SD_2D:-1}}"
|
| 21 |
+
RUN_FAST_2D="${RUN_FAST_2D:-0}"
|
| 22 |
+
ALLOW_LARGE_2D="${ALLOW_LARGE_2D:-0}"
|
| 23 |
+
RUN_DEPTH="${RUN_DEPTH:-1}"
|
| 24 |
+
RUN_AMODAL3D_3D="${RUN_ACCESSIBILITYAMODAL_VISUAL_3D:-${RUN_AMODAL3D_3D:-1}}"
|
| 25 |
+
RUN_AMODAL3D_GLB="${EXPORT_ACCESSIBILITYAMODAL_VISUAL_GLB:-${RUN_AMODAL3D_GLB:-0}}"
|
| 26 |
+
NO_TARGET_OCCLUDER="${NO_TARGET_OCCLUDER:-0}"
|
| 27 |
+
REVIEWED_VISIBLE_WORKSPACE="${REVIEWED_VISIBLE_WORKSPACE:-}"
|
| 28 |
+
|
| 29 |
+
SAM3_PYTHON="${SAM3_PYTHON:-$PROJECT_DIR/.envs/sam3/bin/python}"
|
| 30 |
+
AMODAL3D_PYTHON="${AMODAL3D_PYTHON:-$PROJECT_DIR/.envs/amodal3d/bin/python}"
|
| 31 |
+
SAM3_REPO="${SAM3_REPO:-$PROJECT_DIR/repos/sam3}"
|
| 32 |
+
SAM3_CHECKPOINT="${SAM3_CHECKPOINT:-$PROJECT_DIR/weights/sam3/sam3.pt}"
|
| 33 |
+
AMODAL_CHECKPOINT="${AMODAL_CHECKPOINT:-$PROJECT_DIR/weights/accessibility_amodal_adapter/best.pt}"
|
| 34 |
+
DEPTH_REPO="${DEPTH_REPO:-$PROJECT_DIR/repos/Depth-Anything-V2}"
|
| 35 |
+
DEPTH_CHECKPOINT="${DEPTH_CHECKPOINT:-$PROJECT_DIR/weights/depth_anything_v2_metric_hypersim_vitl.pth}"
|
| 36 |
+
PROMPT_CONFIG="${PROMPT_CONFIG:-}"
|
| 37 |
+
if [[ -z "$PROMPT_CONFIG" ]]; then
|
| 38 |
+
if [[ "$NO_TARGET_OCCLUDER" == "1" && "$CATEGORY" == "stairs" ]]; then
|
| 39 |
+
PROMPT_CONFIG="$PROJECT_DIR/configs/accessibility_stairs_outdoor_no_occlusion_prompts.json"
|
| 40 |
+
else
|
| 41 |
+
# Unified prompt bank: SAM3 decides which target/occluder concepts are
|
| 42 |
+
# present in the image; callers do not need to classify the scene first.
|
| 43 |
+
PROMPT_CONFIG="$PROJECT_DIR/configs/accessibility_mask_prompts_e5_v2.json"
|
| 44 |
+
fi
|
| 45 |
+
fi
|
| 46 |
+
SD_MODEL="${SD_MODEL:-$PROJECT_DIR/weights/stable-diffusion-inpainting}"
|
| 47 |
+
AMODAL3D_MODEL="${AMODAL3D_MODEL:-$PROJECT_DIR/weights/Amodal3R}"
|
| 48 |
+
AMODAL3D_TORCH_HOME="${AMODAL3D_TORCH_HOME:-$PROJECT_DIR/weights/torch}"
|
| 49 |
+
LOG_SANITIZER="$PROJECT_DIR/tools/sanitize_log_stream.py"
|
| 50 |
+
METADATA_SANITIZER="$PROJECT_DIR/tools/sanitize_output_metadata.py"
|
| 51 |
+
VERIFICATION_TOOL="$PROJECT_DIR/tools/audit_accessibility_3d_candidate.py"
|
| 52 |
+
REVIEWED_VISIBLE_VALIDATOR="$PROJECT_DIR/tools/validate_accessibility_reviewed_visible_workspace.py"
|
| 53 |
+
GPU_TURNTABLE_TOOL="$PROJECT_DIR/tools/render_accessibility_turntable.py"
|
| 54 |
+
REVIEW_BUNDLE_TOOL="$PROJECT_DIR/tools/build_accessibility_review_bundle.py"
|
| 55 |
+
|
| 56 |
+
# Keep workstation usernames, home prefixes, and external paths out of the
|
| 57 |
+
# persistent Slurm logs. Python warnings and tracebacks pass through this too.
|
| 58 |
+
privacy_filter_args=(
|
| 59 |
+
--project-root "$PROJECT_DIR"
|
| 60 |
+
--output-root "$OUTPUT_DIR"
|
| 61 |
+
--sensitive-path "$IMAGE"
|
| 62 |
+
--sensitive-path "$PROMPT_CONFIG"
|
| 63 |
+
--sensitive-path "$SAM3_PYTHON"
|
| 64 |
+
--sensitive-path "$AMODAL3D_PYTHON"
|
| 65 |
+
--sensitive-path "$SAM3_REPO"
|
| 66 |
+
--sensitive-path "$SAM3_CHECKPOINT"
|
| 67 |
+
--sensitive-path "$AMODAL_CHECKPOINT"
|
| 68 |
+
--sensitive-path "$DEPTH_REPO"
|
| 69 |
+
--sensitive-path "$DEPTH_CHECKPOINT"
|
| 70 |
+
--sensitive-path "$SD_MODEL"
|
| 71 |
+
--sensitive-path "$AMODAL3D_MODEL"
|
| 72 |
+
--sensitive-path "$AMODAL3D_TORCH_HOME"
|
| 73 |
+
)
|
| 74 |
+
if [[ -n "$REVIEWED_VISIBLE_WORKSPACE" ]]; then
|
| 75 |
+
privacy_filter_args+=(--sensitive-path "$REVIEWED_VISIBLE_WORKSPACE")
|
| 76 |
+
fi
|
| 77 |
+
LOG_FILTER_PYTHON="$AMODAL3D_PYTHON"
|
| 78 |
+
if [[ ! -x "$LOG_FILTER_PYTHON" ]]; then
|
| 79 |
+
LOG_FILTER_PYTHON="$(command -v python3 || true)"
|
| 80 |
+
fi
|
| 81 |
+
if [[ -n "$LOG_FILTER_PYTHON" ]] && [[ -f "$LOG_SANITIZER" ]]; then
|
| 82 |
+
exec > >("$LOG_FILTER_PYTHON" "$LOG_SANITIZER" "${privacy_filter_args[@]}")
|
| 83 |
+
exec 2> >("$LOG_FILTER_PYTHON" "$LOG_SANITIZER" "${privacy_filter_args[@]}" >&2)
|
| 84 |
+
fi
|
| 85 |
+
|
| 86 |
+
module load cuda/13.0 || true
|
| 87 |
+
|
| 88 |
+
case "$CATEGORY" in
|
| 89 |
+
curb_cut|ramp|stairs|tactile_paving|walkway) ;;
|
| 90 |
+
*) echo "Unsupported CATEGORY=$CATEGORY" >&2; exit 2 ;;
|
| 91 |
+
esac
|
| 92 |
+
|
| 93 |
+
if ! cd "$PROJECT_DIR" 2>/dev/null; then
|
| 94 |
+
echo "Could not enter the configured project directory." >&2
|
| 95 |
+
exit 2
|
| 96 |
+
fi
|
| 97 |
+
[[ -f "$IMAGE" ]] || { echo "The configured input image was not found." >&2; exit 2; }
|
| 98 |
+
IMAGE="$(realpath "$IMAGE")"
|
| 99 |
+
if [[ -z "$SAMPLE_ID" ]]; then
|
| 100 |
+
SAMPLE_ID="$(basename "$IMAGE")"
|
| 101 |
+
SAMPLE_ID="${SAMPLE_ID%.*}"
|
| 102 |
+
SAMPLE_ID="${SAMPLE_ID//[^A-Za-z0-9_-]/_}"
|
| 103 |
+
fi
|
| 104 |
+
|
| 105 |
+
[[ -x "$AMODAL3D_PYTHON" ]] || { echo "The Accessibility3D Python environment is missing." >&2; exit 2; }
|
| 106 |
+
for tool_path in "$VERIFICATION_TOOL" "$GPU_TURNTABLE_TOOL" "$REVIEW_BUNDLE_TOOL"; do
|
| 107 |
+
[[ -f "$tool_path" ]] || { echo "A required AccessibilityAmodal tool is missing." >&2; exit 2; }
|
| 108 |
+
done
|
| 109 |
+
[[ -e "$AMODAL_CHECKPOINT" ]] || { echo "The amodal-mask runtime asset is missing." >&2; exit 2; }
|
| 110 |
+
if [[ -n "$REVIEWED_VISIBLE_WORKSPACE" ]]; then
|
| 111 |
+
[[ -d "$REVIEWED_VISIBLE_WORKSPACE" ]] || { echo "The reviewed-visible workspace is missing." >&2; exit 2; }
|
| 112 |
+
[[ -f "$REVIEWED_VISIBLE_VALIDATOR" ]] || { echo "The reviewed-visible validator is missing." >&2; exit 2; }
|
| 113 |
+
else
|
| 114 |
+
[[ -x "$SAM3_PYTHON" ]] || { echo "The SAM3 Python environment is missing." >&2; exit 2; }
|
| 115 |
+
for path in "$SAM3_REPO" "$SAM3_CHECKPOINT"; do
|
| 116 |
+
[[ -e "$path" ]] || { echo "A required SAM3 runtime asset is missing." >&2; exit 2; }
|
| 117 |
+
done
|
| 118 |
+
fi
|
| 119 |
+
if [[ "$RUN_DEPTH" == "1" ]]; then
|
| 120 |
+
for path in "$DEPTH_REPO" "$DEPTH_CHECKPOINT"; do
|
| 121 |
+
[[ -e "$path" ]] || { echo "A required depth runtime asset is missing." >&2; exit 2; }
|
| 122 |
+
done
|
| 123 |
+
fi
|
| 124 |
+
if [[ "$RUN_GPU_2D" == "1" ]] && [[ ! -e "$SD_MODEL" ]]; then
|
| 125 |
+
echo "The Stable Diffusion inpainting model is missing." >&2
|
| 126 |
+
exit 2
|
| 127 |
+
fi
|
| 128 |
+
if [[ "$RUN_AMODAL3D_3D" == "1" ]]; then
|
| 129 |
+
for path in \
|
| 130 |
+
"$AMODAL3D_MODEL/pipeline.json" \
|
| 131 |
+
"$AMODAL3D_TORCH_HOME/hub/facebookresearch_dinov2_main"; do
|
| 132 |
+
[[ -e "$path" ]] || { echo "A required Accessibility3D runtime asset is missing." >&2; exit 2; }
|
| 133 |
+
done
|
| 134 |
+
fi
|
| 135 |
+
|
| 136 |
+
export JOB_TMPDIR="$TMPDIR/job-$SLURM_JOB_ID"
|
| 137 |
+
export XDG_CACHE_HOME="$JOB_TMPDIR/xdg_cache"
|
| 138 |
+
export HF_HOME="$JOB_TMPDIR/huggingface"
|
| 139 |
+
export TORCH_HOME="$JOB_TMPDIR/torch"
|
| 140 |
+
export TRITON_CACHE_DIR="$JOB_TMPDIR/triton"
|
| 141 |
+
export MPLCONFIGDIR="$JOB_TMPDIR/matplotlib"
|
| 142 |
+
export PYTHONNOUSERSITE=1
|
| 143 |
+
mkdir -p "$JOB_TMPDIR" "$XDG_CACHE_HOME" "$HF_HOME" "$TORCH_HOME" \
|
| 144 |
+
"$TRITON_CACHE_DIR" "$MPLCONFIGDIR" "$OUTPUT_DIR"
|
| 145 |
+
|
| 146 |
+
# These weights are seconds measured from a representative full stairs run on
|
| 147 |
+
# the target GPU partition. They drive an approximate progress bar and ETA; actual
|
| 148 |
+
# image complexity, cold model caches, and cluster load can change the timing.
|
| 149 |
+
SAM3_WEIGHT=150
|
| 150 |
+
if [[ -n "$REVIEWED_VISIBLE_WORKSPACE" ]]; then
|
| 151 |
+
SAM3_WEIGHT=5
|
| 152 |
+
fi
|
| 153 |
+
MASK_WEIGHT=13
|
| 154 |
+
FAST_2D_WEIGHT=12
|
| 155 |
+
GPU_2D_WEIGHT=40
|
| 156 |
+
GEOMETRY_WEIGHT=24
|
| 157 |
+
if [[ "$RUN_DEPTH" != "1" ]]; then
|
| 158 |
+
GEOMETRY_WEIGHT=6
|
| 159 |
+
fi
|
| 160 |
+
TURNTABLE_WEIGHT=3
|
| 161 |
+
VARIANTS_WEIGHT=420
|
| 162 |
+
AMODAL3D_WEIGHT=75
|
| 163 |
+
if [[ "$RUN_AMODAL3D_GLB" == "1" ]]; then
|
| 164 |
+
# On the 1368x1824 9528 reference run, GPU UV/PBR baking added about nine
|
| 165 |
+
# minutes after the ordinary Gaussian and mesh renders had completed.
|
| 166 |
+
AMODAL3D_WEIGHT=615
|
| 167 |
+
fi
|
| 168 |
+
|
| 169 |
+
TOTAL_WEIGHT=$((SAM3_WEIGHT + MASK_WEIGHT + GEOMETRY_WEIGHT + TURNTABLE_WEIGHT + VARIANTS_WEIGHT))
|
| 170 |
+
if [[ "$RUN_FAST_2D" == "1" ]]; then
|
| 171 |
+
TOTAL_WEIGHT=$((TOTAL_WEIGHT + FAST_2D_WEIGHT))
|
| 172 |
+
fi
|
| 173 |
+
if [[ "$RUN_GPU_2D" == "1" ]]; then
|
| 174 |
+
TOTAL_WEIGHT=$((TOTAL_WEIGHT + GPU_2D_WEIGHT))
|
| 175 |
+
fi
|
| 176 |
+
if [[ "$RUN_AMODAL3D_3D" == "1" ]]; then
|
| 177 |
+
TOTAL_WEIGHT=$((TOTAL_WEIGHT + AMODAL3D_WEIGHT))
|
| 178 |
+
fi
|
| 179 |
+
|
| 180 |
+
PROGRESS_FILE="$OUTPUT_DIR/progress.json"
|
| 181 |
+
PROGRESS_WRITER="$PROJECT_DIR/tools/update_accessibility_inference_progress.py"
|
| 182 |
+
COMPLETED_WEIGHT=0
|
| 183 |
+
CURRENT_STAGE_KEY="initializing"
|
| 184 |
+
CURRENT_STAGE_LABEL="Initializing GPU job"
|
| 185 |
+
|
| 186 |
+
start_stage() {
|
| 187 |
+
local stage_key="$1"
|
| 188 |
+
local stage_label="$2"
|
| 189 |
+
local stage_weight="$3"
|
| 190 |
+
local start_percent end_percent
|
| 191 |
+
start_percent=$((COMPLETED_WEIGHT * 100 / TOTAL_WEIGHT))
|
| 192 |
+
end_percent=$(((COMPLETED_WEIGHT + stage_weight) * 100 / TOTAL_WEIGHT))
|
| 193 |
+
CURRENT_STAGE_KEY="$stage_key"
|
| 194 |
+
CURRENT_STAGE_LABEL="$stage_label"
|
| 195 |
+
"$AMODAL3D_PYTHON" "$PROGRESS_WRITER" \
|
| 196 |
+
--path "$PROGRESS_FILE" \
|
| 197 |
+
--status running \
|
| 198 |
+
--stage-key "$stage_key" \
|
| 199 |
+
--stage-label "$stage_label" \
|
| 200 |
+
--percent "$start_percent" \
|
| 201 |
+
--stage-end-percent "$end_percent" \
|
| 202 |
+
--stage-expected-seconds "$stage_weight" \
|
| 203 |
+
--total-estimated-seconds "$TOTAL_WEIGHT"
|
| 204 |
+
}
|
| 205 |
+
|
| 206 |
+
finish_stage() {
|
| 207 |
+
COMPLETED_WEIGHT=$((COMPLETED_WEIGHT + $1))
|
| 208 |
+
}
|
| 209 |
+
|
| 210 |
+
sanitize_output_metadata() {
|
| 211 |
+
if [[ -x "$AMODAL3D_PYTHON" ]] && [[ -f "$METADATA_SANITIZER" ]]; then
|
| 212 |
+
"$AMODAL3D_PYTHON" "$METADATA_SANITIZER" \
|
| 213 |
+
--root "$OUTPUT_DIR" \
|
| 214 |
+
--project-root "$PROJECT_DIR" \
|
| 215 |
+
--sensitive-path "$IMAGE" \
|
| 216 |
+
--sensitive-path "$PROMPT_CONFIG" \
|
| 217 |
+
--sensitive-path "$SAM3_PYTHON" \
|
| 218 |
+
--sensitive-path "$AMODAL3D_PYTHON" \
|
| 219 |
+
--sensitive-path "$SAM3_REPO" \
|
| 220 |
+
--sensitive-path "$SAM3_CHECKPOINT" \
|
| 221 |
+
--sensitive-path "$AMODAL_CHECKPOINT" \
|
| 222 |
+
--sensitive-path "$DEPTH_REPO" \
|
| 223 |
+
--sensitive-path "$DEPTH_CHECKPOINT" \
|
| 224 |
+
--sensitive-path "$SD_MODEL" \
|
| 225 |
+
--sensitive-path "$AMODAL3D_MODEL" \
|
| 226 |
+
--sensitive-path "$AMODAL3D_TORCH_HOME"
|
| 227 |
+
fi
|
| 228 |
+
}
|
| 229 |
+
|
| 230 |
+
record_exit() {
|
| 231 |
+
local exit_code=$?
|
| 232 |
+
if [[ "$exit_code" -ne 0 ]]; then
|
| 233 |
+
set +e
|
| 234 |
+
"$AMODAL3D_PYTHON" "$PROGRESS_WRITER" \
|
| 235 |
+
--path "$PROGRESS_FILE" \
|
| 236 |
+
--status failed \
|
| 237 |
+
--message "Stage failed: $CURRENT_STAGE_LABEL" \
|
| 238 |
+
--exit-code "$exit_code"
|
| 239 |
+
fi
|
| 240 |
+
sanitize_output_metadata
|
| 241 |
+
}
|
| 242 |
+
trap record_exit EXIT
|
| 243 |
+
|
| 244 |
+
echo "Image: $IMAGE"
|
| 245 |
+
echo "Category: $CATEGORY"
|
| 246 |
+
echo "No target occluder mode: $NO_TARGET_OCCLUDER"
|
| 247 |
+
echo "Reviewed visible workspace: ${REVIEWED_VISIBLE_WORKSPACE:-none}"
|
| 248 |
+
echo "Sample ID: $SAMPLE_ID"
|
| 249 |
+
echo "Output: $OUTPUT_DIR"
|
| 250 |
+
echo "Estimated GPU runtime: about $(( (TOTAL_WEIGHT + 59) / 60 )) minutes (queue time excluded)"
|
| 251 |
+
|
| 252 |
+
TARGET_VISIBLE_MASK=""
|
| 253 |
+
OBSTACLE_CANDIDATE_MASK=""
|
| 254 |
+
if [[ -n "$REVIEWED_VISIBLE_WORKSPACE" ]]; then
|
| 255 |
+
REVIEWED_VISIBLE_OUTPUT="$OUTPUT_DIR/01_reviewed_visible"
|
| 256 |
+
start_stage "reviewed_visible" "Validated human-reviewed visible target mask" "$SAM3_WEIGHT"
|
| 257 |
+
"$AMODAL3D_PYTHON" "$REVIEWED_VISIBLE_VALIDATOR" \
|
| 258 |
+
--workspace "$REVIEWED_VISIBLE_WORKSPACE" \
|
| 259 |
+
--image "$IMAGE" \
|
| 260 |
+
--sample-id "$SAMPLE_ID" \
|
| 261 |
+
--category "$CATEGORY" \
|
| 262 |
+
--output-dir "$REVIEWED_VISIBLE_OUTPUT"
|
| 263 |
+
finish_stage "$SAM3_WEIGHT"
|
| 264 |
+
TARGET_VISIBLE_MASK="$REVIEWED_VISIBLE_OUTPUT/target_visible.png"
|
| 265 |
+
OBSTACLE_CANDIDATE_MASK="$REVIEWED_VISIBLE_OUTPUT/obstacle.png"
|
| 266 |
+
else
|
| 267 |
+
SAM3_OUTPUT="$OUTPUT_DIR/01_sam3"
|
| 268 |
+
start_stage "sam3" "SAM3 target and obstacle masks" "$SAM3_WEIGHT"
|
| 269 |
+
"$SAM3_PYTHON" tools/accessibility_mask_proposals.py \
|
| 270 |
+
--image "$IMAGE" \
|
| 271 |
+
--single-sample-id "$SAMPLE_ID" \
|
| 272 |
+
--single-category "$CATEGORY" \
|
| 273 |
+
--prompt-config "$PROMPT_CONFIG" \
|
| 274 |
+
--output-dir "$SAM3_OUTPUT" \
|
| 275 |
+
--backend sam3 \
|
| 276 |
+
--device cuda \
|
| 277 |
+
--artifact-level standard \
|
| 278 |
+
--sam3-repo "$SAM3_REPO" \
|
| 279 |
+
--sam3-checkpoint "$SAM3_CHECKPOINT"
|
| 280 |
+
finish_stage "$SAM3_WEIGHT"
|
| 281 |
+
SAM3_SAMPLE="$SAM3_OUTPUT/samples/$SAMPLE_ID"
|
| 282 |
+
TARGET_VISIBLE_MASK="$SAM3_SAMPLE/target_visible_candidate.png"
|
| 283 |
+
OBSTACLE_CANDIDATE_MASK="$SAM3_SAMPLE/obstacle.png"
|
| 284 |
+
fi
|
| 285 |
+
|
| 286 |
+
MASK_OUTPUT="$OUTPUT_DIR/02_masks"
|
| 287 |
+
start_stage "amodal_masks" "Visible, hidden, amodal, and obstacle masks" "$MASK_WEIGHT"
|
| 288 |
+
"$AMODAL3D_PYTHON" tools/accessibility_amodal_mask.py \
|
| 289 |
+
--image "$IMAGE" \
|
| 290 |
+
--target-visible-mask "$TARGET_VISIBLE_MASK" \
|
| 291 |
+
--obstacle-candidate-mask "$OBSTACLE_CANDIDATE_MASK" \
|
| 292 |
+
--category "$CATEGORY" \
|
| 293 |
+
--checkpoint "$AMODAL_CHECKPOINT" \
|
| 294 |
+
--output-dir "$MASK_OUTPUT" \
|
| 295 |
+
--device cuda
|
| 296 |
+
finish_stage "$MASK_WEIGHT"
|
| 297 |
+
|
| 298 |
+
# Do not turn a rejected SAM3 proposal into a polished but physically unrelated
|
| 299 |
+
# 3D object. The verifier records the evidence in a persistent JSON report;
|
| 300 |
+
# a clean exit is intentional so the user can correct/re-prompt the masks.
|
| 301 |
+
PREFLIGHT_DIR="$OUTPUT_DIR/05_quality_checks"
|
| 302 |
+
PREFLIGHT_DECISION="$("$AMODAL3D_PYTHON" "$VERIFICATION_TOOL" \
|
| 303 |
+
--run-dir "$OUTPUT_DIR" \
|
| 304 |
+
--sample-id "$SAMPLE_ID" \
|
| 305 |
+
--category "$CATEGORY" \
|
| 306 |
+
--output "$PREFLIGHT_DIR/preflight.json" \
|
| 307 |
+
--preflight-only \
|
| 308 |
+
--print-decision)"
|
| 309 |
+
echo "3D preflight decision: $PREFLIGHT_DECISION"
|
| 310 |
+
if [[ "$PREFLIGHT_DECISION" != "accept" ]]; then
|
| 311 |
+
"$AMODAL3D_PYTHON" "$PROGRESS_WRITER" \
|
| 312 |
+
--path "$PROGRESS_FILE" \
|
| 313 |
+
--status completed \
|
| 314 |
+
--stage-key mask_gate \
|
| 315 |
+
--stage-label "3D withheld: re-prompt or review masks" \
|
| 316 |
+
--percent 100 \
|
| 317 |
+
--stage-end-percent 100 \
|
| 318 |
+
--stage-expected-seconds 0 \
|
| 319 |
+
--total-estimated-seconds "$TOTAL_WEIGHT" \
|
| 320 |
+
--message "3D preflight=$PREFLIGHT_DECISION; see 05_quality_checks/preflight.json"
|
| 321 |
+
echo "3D generation withheld. Review: $PREFLIGHT_DIR/preflight.json"
|
| 322 |
+
exit 0
|
| 323 |
+
fi
|
| 324 |
+
|
| 325 |
+
if [[ "$RUN_FAST_2D" == "1" ]]; then
|
| 326 |
+
start_stage "fast_2d" "Optional OpenCV 2D baseline" "$FAST_2D_WEIGHT"
|
| 327 |
+
completion_cmd=(
|
| 328 |
+
"$AMODAL3D_PYTHON" tools/accessibility_fast_2d_baseline.py
|
| 329 |
+
--image "$IMAGE"
|
| 330 |
+
--single-sample-id "$SAMPLE_ID"
|
| 331 |
+
--target-visible-mask "$MASK_OUTPUT/target_visible.png"
|
| 332 |
+
--target-amodal-mask "$MASK_OUTPUT/target_amodal.png"
|
| 333 |
+
--hidden-mask "$MASK_OUTPUT/hidden.png"
|
| 334 |
+
--obstacle-mask "$MASK_OUTPUT/obstacle.png"
|
| 335 |
+
--output-dir "$OUTPUT_DIR/03_fast_2d_baseline"
|
| 336 |
+
--mask-mode hidden
|
| 337 |
+
--opencv-mode pyramid
|
| 338 |
+
--overwrite
|
| 339 |
+
)
|
| 340 |
+
if [[ "$ALLOW_LARGE_2D" == "1" ]]; then
|
| 341 |
+
completion_cmd+=(--allow-large-mask)
|
| 342 |
+
fi
|
| 343 |
+
"${completion_cmd[@]}"
|
| 344 |
+
finish_stage "$FAST_2D_WEIGHT"
|
| 345 |
+
fi
|
| 346 |
+
|
| 347 |
+
if [[ "$RUN_GPU_2D" == "1" ]]; then
|
| 348 |
+
start_stage "gpu_2d" "GPU generative 2D completion and candidate selection" "$GPU_2D_WEIGHT"
|
| 349 |
+
"$AMODAL3D_PYTHON" tools/accessibility_2d_completion.py \
|
| 350 |
+
--image "$IMAGE" \
|
| 351 |
+
--mask "$MASK_OUTPUT/hidden.png" \
|
| 352 |
+
--obstacle-mask "$MASK_OUTPUT/obstacle_all_detected.png" \
|
| 353 |
+
--target-visible-mask "$MASK_OUTPUT/target_visible.png" \
|
| 354 |
+
--target-amodal-mask "$MASK_OUTPUT/target_amodal.png" \
|
| 355 |
+
--category "$CATEGORY" \
|
| 356 |
+
--output-dir "$OUTPUT_DIR/03_2d_completion" \
|
| 357 |
+
--model "$SD_MODEL" \
|
| 358 |
+
--device cuda
|
| 359 |
+
finish_stage "$GPU_2D_WEIGHT"
|
| 360 |
+
COMPLETION_STATUS="$("$AMODAL3D_PYTHON" -c 'import json,sys; print(json.load(open(sys.argv[1], encoding="utf-8")).get("status", "completed"))' "$OUTPUT_DIR/03_2d_completion/manifest.json")"
|
| 361 |
+
echo "2D completion quality status: $COMPLETION_STATUS"
|
| 362 |
+
case "$COMPLETION_STATUS" in
|
| 363 |
+
candidate_selected_for_review|skipped_empty_removal_mask)
|
| 364 |
+
;;
|
| 365 |
+
*)
|
| 366 |
+
echo "The 2D candidate needs review and will not condition either 3D backend."
|
| 367 |
+
echo "Accessibility3D will still use the original RGB plus the strict three-value mask."
|
| 368 |
+
;;
|
| 369 |
+
esac
|
| 370 |
+
fi
|
| 371 |
+
|
| 372 |
+
start_stage "geometry" "Depth and 3D geometry reconstruction" "$GEOMETRY_WEIGHT"
|
| 373 |
+
REFERENCE_IMAGE=""
|
| 374 |
+
if [[ "$RUN_GPU_2D" == "1" ]] \
|
| 375 |
+
&& [[ "${COMPLETION_STATUS:-}" =~ ^(candidate_selected_for_review|skipped_empty_removal_mask)$ ]]; then
|
| 376 |
+
REFERENCE_IMAGE="$OUTPUT_DIR/03_2d_completion/completed_rgb_selected.png"
|
| 377 |
+
fi
|
| 378 |
+
pipeline_cmd=(
|
| 379 |
+
"$AMODAL3D_PYTHON" -m accessibilityamodal.pipeline
|
| 380 |
+
--image "$IMAGE"
|
| 381 |
+
--sample-id "$SAMPLE_ID"
|
| 382 |
+
--preset "$CATEGORY"
|
| 383 |
+
--target-visible-mask "$MASK_OUTPUT/target_visible.png"
|
| 384 |
+
--target-amodal-mask "$MASK_OUTPUT/target_amodal.png"
|
| 385 |
+
--obstacle-mask "$MASK_OUTPUT/obstacle.png"
|
| 386 |
+
--output-dir "$OUTPUT_DIR/04_3d_completion"
|
| 387 |
+
)
|
| 388 |
+
if [[ -n "$REFERENCE_IMAGE" ]]; then
|
| 389 |
+
pipeline_cmd+=(
|
| 390 |
+
--reference-image "$REFERENCE_IMAGE"
|
| 391 |
+
--completed-rgb "$REFERENCE_IMAGE"
|
| 392 |
+
)
|
| 393 |
+
fi
|
| 394 |
+
if [[ "$RUN_DEPTH" == "1" ]]; then
|
| 395 |
+
pipeline_cmd+=(
|
| 396 |
+
--estimate-depth
|
| 397 |
+
--depth-engine depth_anything_v2
|
| 398 |
+
--depth-device cuda
|
| 399 |
+
--depth-anything-repo "$DEPTH_REPO"
|
| 400 |
+
--depth-checkpoint "$DEPTH_CHECKPOINT"
|
| 401 |
+
--depth-encoder vitl
|
| 402 |
+
--metric-depth
|
| 403 |
+
--max-depth 20
|
| 404 |
+
--depth-fallback error
|
| 405 |
+
)
|
| 406 |
+
fi
|
| 407 |
+
"${pipeline_cmd[@]}"
|
| 408 |
+
finish_stage "$GEOMETRY_WEIGHT"
|
| 409 |
+
|
| 410 |
+
start_stage "surface_variants" "Auxiliary surface and canonical 3D views" "$VARIANTS_WEIGHT"
|
| 411 |
+
variants_cmd=(
|
| 412 |
+
"$AMODAL3D_PYTHON" tools/accessibility_3d_variants.py
|
| 413 |
+
--geometry-dir "$OUTPUT_DIR/04_3d_completion/geometry"
|
| 414 |
+
--output-dir "$OUTPUT_DIR/04_3d_completion/geometry/presentation_models"
|
| 415 |
+
--sample-id "$SAMPLE_ID"
|
| 416 |
+
--category "$CATEGORY"
|
| 417 |
+
--frames 72
|
| 418 |
+
--duration-ms 140
|
| 419 |
+
--hidden-tint 0
|
| 420 |
+
--model-only
|
| 421 |
+
)
|
| 422 |
+
if [[ -n "$REFERENCE_IMAGE" ]]; then
|
| 423 |
+
variants_cmd+=(--texture-image "$REFERENCE_IMAGE")
|
| 424 |
+
fi
|
| 425 |
+
"${variants_cmd[@]}"
|
| 426 |
+
finish_stage "$VARIANTS_WEIGHT"
|
| 427 |
+
|
| 428 |
+
PRESENTATION_MESH="$OUTPUT_DIR/04_3d_completion/geometry/completed_mesh.ply"
|
| 429 |
+
if [[ -f "$OUTPUT_DIR/04_3d_completion/geometry/presentation_models/02_canonical_stairs/02_canonical_stairs.ply" ]]; then
|
| 430 |
+
PRESENTATION_MESH="$OUTPUT_DIR/04_3d_completion/geometry/presentation_models/02_canonical_stairs/02_canonical_stairs.ply"
|
| 431 |
+
elif [[ -f "$OUTPUT_DIR/04_3d_completion/geometry/presentation_models/02_continuous_support_surface/02_continuous_support_surface.ply" ]]; then
|
| 432 |
+
PRESENTATION_MESH="$OUTPUT_DIR/04_3d_completion/geometry/presentation_models/02_continuous_support_surface/02_continuous_support_surface.ply"
|
| 433 |
+
fi
|
| 434 |
+
|
| 435 |
+
start_stage "turntable" "Diagnostic category-geometry turntable" "$TURNTABLE_WEIGHT"
|
| 436 |
+
"$AMODAL3D_PYTHON" "$GPU_TURNTABLE_TOOL" \
|
| 437 |
+
--ply "$PRESENTATION_MESH" \
|
| 438 |
+
--output-dir "$OUTPUT_DIR/04_3d_completion/turntable" \
|
| 439 |
+
--sample-id "$SAMPLE_ID" \
|
| 440 |
+
--category "$CATEGORY" \
|
| 441 |
+
--frames 96 \
|
| 442 |
+
--fps 24 \
|
| 443 |
+
--shader defaultUnlit \
|
| 444 |
+
--background 0.82,0.82,0.80,1 \
|
| 445 |
+
--orbit-mode front_arc \
|
| 446 |
+
--orbit-span 140 \
|
| 447 |
+
--multiview-yaws=-60,-30,0,30,60 \
|
| 448 |
+
--require-vertex-colors
|
| 449 |
+
finish_stage "$TURNTABLE_WEIGHT"
|
| 450 |
+
|
| 451 |
+
if [[ "$RUN_AMODAL3D_3D" == "1" ]]; then
|
| 452 |
+
start_stage "amodal3d" "Accessibility3D CUDA Gaussian rotation and dense mesh" "$AMODAL3D_WEIGHT"
|
| 453 |
+
amodal3d_cmd=(
|
| 454 |
+
"$AMODAL3D_PYTHON" tools/accessibility_3d_completion.py
|
| 455 |
+
--image "$IMAGE"
|
| 456 |
+
--conditioning-rgb original
|
| 457 |
+
--mask "$OUTPUT_DIR/04_3d_completion/amodal3d_three_value_mask.png"
|
| 458 |
+
--output-dir "$ACCESSIBILITY3D_DIR"
|
| 459 |
+
--model "$AMODAL3D_MODEL"
|
| 460 |
+
)
|
| 461 |
+
# Match the known-good 9527 run: the learned backend receives the original
|
| 462 |
+
# display-oriented RGB and the strict reviewed three-value condition mask.
|
| 463 |
+
# The separate 2D inpaint must never become foggy texture conditioning here.
|
| 464 |
+
if [[ "$RUN_AMODAL3D_GLB" == "1" ]]; then
|
| 465 |
+
amodal3d_cmd+=(--export-glb)
|
| 466 |
+
fi
|
| 467 |
+
AMODAL3D_TORCH_HOME="$AMODAL3D_TORCH_HOME" "${amodal3d_cmd[@]}"
|
| 468 |
+
finish_stage "$AMODAL3D_WEIGHT"
|
| 469 |
+
|
| 470 |
+
POST_3D_DECISION="$("$AMODAL3D_PYTHON" "$VERIFICATION_TOOL" \
|
| 471 |
+
--run-dir "$OUTPUT_DIR" \
|
| 472 |
+
--sample-id "$SAMPLE_ID" \
|
| 473 |
+
--category "$CATEGORY" \
|
| 474 |
+
--output "$PREFLIGHT_DIR/verification.json" \
|
| 475 |
+
--print-decision)"
|
| 476 |
+
echo "Generated 3D verification decision: $POST_3D_DECISION"
|
| 477 |
+
|
| 478 |
+
if [[ "$POST_3D_DECISION" == "accept" ]]; then
|
| 479 |
+
echo "Local 3D candidate passed the reconstruction-quality gate."
|
| 480 |
+
echo "Paper/GitHub packaging is intentionally withheld pending explicit human release approval for license/privacy and calibrated-geometry review."
|
| 481 |
+
else
|
| 482 |
+
echo "Learned 3D debug artifacts are withheld because generated 3D verification=$POST_3D_DECISION"
|
| 483 |
+
fi
|
| 484 |
+
fi
|
| 485 |
+
|
| 486 |
+
# Normalize persistent upstream manifests before the review bundle records
|
| 487 |
+
# their byte hashes. The EXIT trap runs the same sanitizer again as a final
|
| 488 |
+
# privacy guard, but it is idempotent and must not invalidate review provenance.
|
| 489 |
+
sanitize_output_metadata
|
| 490 |
+
|
| 491 |
+
if [[ "$RUN_GPU_2D" == "1" ]]; then
|
| 492 |
+
PRIMARY_3D_ROLE="open_world_accessibility_surface"
|
| 493 |
+
PRIMARY_3D_GIF="$OUTPUT_DIR/04_3d_completion/turntable/turntable.gif"
|
| 494 |
+
PRIMARY_3D_MULTIVIEW="$OUTPUT_DIR/04_3d_completion/turntable/multiview.jpg"
|
| 495 |
+
PRIMARY_3D_MESH="$PRESENTATION_MESH"
|
| 496 |
+
if [[
|
| 497 |
+
"$RUN_AMODAL3D_3D" == "1" &&
|
| 498 |
+
"${POST_3D_DECISION:-}" == "accept" &&
|
| 499 |
+
-f "$ACCESSIBILITY3D_DIR/sample_gaussian.gif" &&
|
| 500 |
+
-f "$ACCESSIBILITY3D_DIR/multiview_contact_sheet.jpg" &&
|
| 501 |
+
-f "$ACCESSIBILITY3D_DIR/mesh.ply"
|
| 502 |
+
]]; then
|
| 503 |
+
PRIMARY_3D_ROLE="learned_amodal3d_gaussian"
|
| 504 |
+
PRIMARY_3D_GIF="$ACCESSIBILITY3D_DIR/sample_gaussian.gif"
|
| 505 |
+
PRIMARY_3D_MULTIVIEW="$ACCESSIBILITY3D_DIR/multiview_contact_sheet.jpg"
|
| 506 |
+
PRIMARY_3D_MESH="$ACCESSIBILITY3D_DIR/mesh.ply"
|
| 507 |
+
fi
|
| 508 |
+
review_cmd=(
|
| 509 |
+
"$AMODAL3D_PYTHON" "$REVIEW_BUNDLE_TOOL"
|
| 510 |
+
--source "$IMAGE"
|
| 511 |
+
--completion-2d "$OUTPUT_DIR/03_2d_completion/completed_rgb_selected.png"
|
| 512 |
+
--turntable-gif "$PRIMARY_3D_GIF"
|
| 513 |
+
--multiview "$PRIMARY_3D_MULTIVIEW"
|
| 514 |
+
--mesh "$PRIMARY_3D_MESH"
|
| 515 |
+
--primary-3d-role "$PRIMARY_3D_ROLE"
|
| 516 |
+
--completion-manifest "$OUTPUT_DIR/03_2d_completion/manifest.json"
|
| 517 |
+
--geometry-manifest "$OUTPUT_DIR/04_3d_completion/geometry/geometry_manifest.json"
|
| 518 |
+
--category "$CATEGORY"
|
| 519 |
+
--sample-id "$SAMPLE_ID"
|
| 520 |
+
--output-dir "$OUTPUT_DIR"
|
| 521 |
+
)
|
| 522 |
+
if [[ -f "$PREFLIGHT_DIR/verification.json" ]]; then
|
| 523 |
+
review_cmd+=(
|
| 524 |
+
--verification-manifest "$PREFLIGHT_DIR/verification.json"
|
| 525 |
+
)
|
| 526 |
+
fi
|
| 527 |
+
"${review_cmd[@]}"
|
| 528 |
+
fi
|
| 529 |
+
|
| 530 |
+
FINAL_STAGE_LABEL="All requested local outputs completed"
|
| 531 |
+
if [[ "$RUN_AMODAL3D_3D" == "1" ]]; then
|
| 532 |
+
FINAL_STAGE_LABEL="Local 3D outputs completed; public release withheld pending human approval"
|
| 533 |
+
if [[ "$POST_3D_DECISION" != "accept" ]]; then
|
| 534 |
+
FINAL_STAGE_LABEL="3D debug artifacts generated but withheld after verification"
|
| 535 |
+
fi
|
| 536 |
+
fi
|
| 537 |
+
"$AMODAL3D_PYTHON" "$PROGRESS_WRITER" \
|
| 538 |
+
--path "$PROGRESS_FILE" \
|
| 539 |
+
--status completed \
|
| 540 |
+
--stage-key completed \
|
| 541 |
+
--stage-label "$FINAL_STAGE_LABEL" \
|
| 542 |
+
--percent 100 \
|
| 543 |
+
--stage-end-percent 100 \
|
| 544 |
+
--stage-expected-seconds 0 \
|
| 545 |
+
--total-estimated-seconds "$TOTAL_WEIGHT"
|
| 546 |
+
|
| 547 |
+
echo "Completed. Review these first:"
|
| 548 |
+
if [[ "$RUN_GPU_2D" == "1" ]]; then
|
| 549 |
+
echo " $OUTPUT_DIR/00_quick_review/overview.jpg"
|
| 550 |
+
echo " $OUTPUT_DIR/00_quick_review/03_3d_turntable.gif"
|
| 551 |
+
fi
|
| 552 |
+
echo " $MASK_OUTPUT/mask_overlay.png"
|
| 553 |
+
if [[ "$RUN_GPU_2D" == "1" ]]; then
|
| 554 |
+
echo " $OUTPUT_DIR/03_2d_completion/completed_rgb_selected.png"
|
| 555 |
+
echo " $OUTPUT_DIR/03_2d_completion/candidate_comparison.jpg"
|
| 556 |
+
fi
|
| 557 |
+
if [[ "$RUN_FAST_2D" == "1" ]]; then
|
| 558 |
+
echo " $OUTPUT_DIR/03_fast_2d_baseline/$SAMPLE_ID/contact_sheet.jpg"
|
| 559 |
+
fi
|
| 560 |
+
echo " $OUTPUT_DIR/04_3d_completion/geometry/completed_region_overlay.png"
|
| 561 |
+
echo " $OUTPUT_DIR/04_3d_completion/geometry/completed_mesh.ply"
|
| 562 |
+
echo " $OUTPUT_DIR/04_3d_completion/geometry/presentation_models/manifest.json"
|
| 563 |
+
echo " diagnostic geometry mesh: $PRESENTATION_MESH"
|
| 564 |
+
if [[ "$RUN_AMODAL3D_3D" == "1" ]]; then
|
| 565 |
+
echo " $OUTPUT_DIR/05_quality_checks/verification.json"
|
| 566 |
+
echo " source-colored CUDA rotation: $ACCESSIBILITY3D_DIR/sample_gaussian.gif"
|
| 567 |
+
echo " CUDA mesh-normal diagnostic: $ACCESSIBILITY3D_DIR/sample_mesh.gif"
|
| 568 |
+
echo " primary multiview: $ACCESSIBILITY3D_DIR/multiview_contact_sheet.jpg"
|
| 569 |
+
echo " dense triangle mesh: $ACCESSIBILITY3D_DIR/mesh.ply"
|
| 570 |
+
if [[ "$RUN_AMODAL3D_GLB" == "1" ]]; then
|
| 571 |
+
echo " dense GLB mesh: $ACCESSIBILITY3D_DIR/mesh.glb"
|
| 572 |
+
fi
|
| 573 |
+
fi
|
tools/accessibility_2d_completion.py
ADDED
|
@@ -0,0 +1,604 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""GPU generative 2D completion for accessibility-scene occlusions.
|
| 3 |
+
|
| 4 |
+
The module is a project-owned orchestration layer. It uses a locally supplied
|
| 5 |
+
Diffusers inpainting checkpoint as a backend, preserves all pixels outside the
|
| 6 |
+
reviewed completion mask, ranks several candidates using staircase-oriented
|
| 7 |
+
image evidence, and publishes one deterministic selected result.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import argparse
|
| 13 |
+
import json
|
| 14 |
+
import math
|
| 15 |
+
import shutil
|
| 16 |
+
import sys
|
| 17 |
+
from datetime import datetime, timezone
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
from typing import Any
|
| 20 |
+
|
| 21 |
+
import cv2
|
| 22 |
+
import numpy as np
|
| 23 |
+
import torch
|
| 24 |
+
from PIL import Image, ImageDraw, ImageFilter, ImageOps
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
| 28 |
+
if str(PROJECT_ROOT) not in sys.path:
|
| 29 |
+
sys.path.insert(0, str(PROJECT_ROOT))
|
| 30 |
+
|
| 31 |
+
from accessibilityamodal.visual_completion import ( # noqa: E402
|
| 32 |
+
NEGATIVE_PROMPT,
|
| 33 |
+
PROMPTS,
|
| 34 |
+
apply_selection_clutter_penalty,
|
| 35 |
+
build_completion_envelope,
|
| 36 |
+
build_visual_removal_mask,
|
| 37 |
+
candidate_clutter_metrics,
|
| 38 |
+
candidate_quality,
|
| 39 |
+
derive_hidden_mask,
|
| 40 |
+
mask_statistics,
|
| 41 |
+
select_candidate,
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def resolve_path(value: str | Path) -> Path:
|
| 46 |
+
path = Path(value).expanduser()
|
| 47 |
+
return path.resolve() if path.is_absolute() else (PROJECT_ROOT / path).resolve()
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def load_rgb(path: Path) -> Image.Image:
|
| 51 |
+
"""Read RGB in the display orientation used to generate the proposal masks."""
|
| 52 |
+
return ImageOps.exif_transpose(Image.open(path)).convert("RGB")
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def load_binary_mask(path: Path, size: tuple[int, int]) -> Image.Image:
|
| 56 |
+
mask = ImageOps.exif_transpose(Image.open(path)).convert("L")
|
| 57 |
+
if mask.size != size:
|
| 58 |
+
raise ValueError(
|
| 59 |
+
f"Mask/RGB raster mismatch for {path}: mask={mask.size}, rgb={size}. "
|
| 60 |
+
"Refusing to resize because this can hide EXIF-orientation misalignment."
|
| 61 |
+
)
|
| 62 |
+
return Image.fromarray((np.asarray(mask) > 127).astype(np.uint8) * 255, mode="L")
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def portable_input_reference(path: Path) -> dict[str, str]:
|
| 66 |
+
"""Describe an input without embedding a machine-specific absolute path."""
|
| 67 |
+
|
| 68 |
+
try:
|
| 69 |
+
return {
|
| 70 |
+
"base": "project_root",
|
| 71 |
+
"path": path.resolve().relative_to(PROJECT_ROOT).as_posix(),
|
| 72 |
+
}
|
| 73 |
+
except ValueError:
|
| 74 |
+
return {"base": "external_input", "path": path.name}
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def load_completion_masks(
|
| 78 |
+
*,
|
| 79 |
+
size: tuple[int, int],
|
| 80 |
+
hidden_path: Path | None,
|
| 81 |
+
obstacle_path: Path | None,
|
| 82 |
+
target_amodal_path: Path | None,
|
| 83 |
+
target_visible_path: Path | None,
|
| 84 |
+
) -> tuple[Image.Image, Image.Image, dict[str, Any]]:
|
| 85 |
+
"""Load geometry hidden and build the separate visual removal mask."""
|
| 86 |
+
|
| 87 |
+
if bool(target_amodal_path) != bool(target_visible_path):
|
| 88 |
+
raise ValueError(
|
| 89 |
+
"--target-amodal-mask and --target-visible-mask must be provided together"
|
| 90 |
+
)
|
| 91 |
+
if hidden_path is None and target_amodal_path is None:
|
| 92 |
+
raise ValueError(
|
| 93 |
+
"Provide --mask, or provide both --target-amodal-mask and --target-visible-mask"
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
hidden_from_file = (
|
| 97 |
+
np.asarray(load_binary_mask(hidden_path, size)) > 127
|
| 98 |
+
if hidden_path is not None
|
| 99 |
+
else None
|
| 100 |
+
)
|
| 101 |
+
target_amodal = (
|
| 102 |
+
np.asarray(load_binary_mask(target_amodal_path, size)) > 127
|
| 103 |
+
if target_amodal_path is not None
|
| 104 |
+
else None
|
| 105 |
+
)
|
| 106 |
+
target_visible = (
|
| 107 |
+
np.asarray(load_binary_mask(target_visible_path, size)) > 127
|
| 108 |
+
if target_visible_path is not None
|
| 109 |
+
else None
|
| 110 |
+
)
|
| 111 |
+
hidden_from_targets = (
|
| 112 |
+
derive_hidden_mask(target_amodal, target_visible)
|
| 113 |
+
if target_amodal is not None and target_visible is not None
|
| 114 |
+
else None
|
| 115 |
+
)
|
| 116 |
+
if (
|
| 117 |
+
hidden_from_file is not None
|
| 118 |
+
and hidden_from_targets is not None
|
| 119 |
+
and not np.array_equal(hidden_from_file, hidden_from_targets)
|
| 120 |
+
):
|
| 121 |
+
mismatch = int(np.count_nonzero(hidden_from_file ^ hidden_from_targets))
|
| 122 |
+
raise ValueError(
|
| 123 |
+
"--mask disagrees with target_amodal AND NOT target_visible "
|
| 124 |
+
f"at {mismatch} pixels"
|
| 125 |
+
)
|
| 126 |
+
|
| 127 |
+
geometry_hidden = (
|
| 128 |
+
hidden_from_targets if hidden_from_targets is not None else hidden_from_file
|
| 129 |
+
)
|
| 130 |
+
assert geometry_hidden is not None
|
| 131 |
+
obstacle = (
|
| 132 |
+
np.asarray(load_binary_mask(obstacle_path, size)) > 127
|
| 133 |
+
if obstacle_path is not None
|
| 134 |
+
else None
|
| 135 |
+
)
|
| 136 |
+
visual_removal, policy_stats = build_visual_removal_mask(
|
| 137 |
+
geometry_hidden,
|
| 138 |
+
obstacle,
|
| 139 |
+
)
|
| 140 |
+
report: dict[str, Any] = {
|
| 141 |
+
"hidden_source": (
|
| 142 |
+
"target_amodal_minus_target_visible"
|
| 143 |
+
if hidden_from_targets is not None
|
| 144 |
+
else "legacy_hidden_mask"
|
| 145 |
+
),
|
| 146 |
+
"geometry_hidden": policy_stats["geometry_hidden"],
|
| 147 |
+
"obstacle_input": {
|
| 148 |
+
"provided": obstacle is not None,
|
| 149 |
+
**policy_stats["obstacle_input"],
|
| 150 |
+
},
|
| 151 |
+
"obstacle_retained": policy_stats["obstacle_retained"],
|
| 152 |
+
"visual_removal": policy_stats["visual_removal"],
|
| 153 |
+
"visual_removal_policy": policy_stats,
|
| 154 |
+
"target_amodal": (
|
| 155 |
+
{"provided": True, **mask_statistics(target_amodal)}
|
| 156 |
+
if target_amodal is not None
|
| 157 |
+
else {"provided": False}
|
| 158 |
+
),
|
| 159 |
+
"target_visible": (
|
| 160 |
+
{"provided": True, **mask_statistics(target_visible)}
|
| 161 |
+
if target_visible is not None
|
| 162 |
+
else {"provided": False}
|
| 163 |
+
),
|
| 164 |
+
}
|
| 165 |
+
return (
|
| 166 |
+
Image.fromarray(geometry_hidden.astype(np.uint8) * 255, mode="L"),
|
| 167 |
+
Image.fromarray(visual_removal.astype(np.uint8) * 255, mode="L"),
|
| 168 |
+
report,
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def dilate(mask: Image.Image, radius: int) -> Image.Image:
|
| 173 |
+
if radius <= 0:
|
| 174 |
+
return mask
|
| 175 |
+
array = np.asarray(mask) > 127
|
| 176 |
+
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (radius * 2 + 1, radius * 2 + 1))
|
| 177 |
+
return Image.fromarray(cv2.dilate(array.astype(np.uint8), kernel) * 255, mode="L")
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
def crop_around_mask(mask: Image.Image, padding: int) -> tuple[int, int, int, int]:
|
| 181 |
+
binary = np.asarray(mask) > 127
|
| 182 |
+
ys, xs = np.where(binary)
|
| 183 |
+
if not len(xs):
|
| 184 |
+
raise ValueError("The 2D completion mask is empty")
|
| 185 |
+
width, height = mask.size
|
| 186 |
+
x1, x2 = int(xs.min()), int(xs.max()) + 1
|
| 187 |
+
y1, y2 = int(ys.min()), int(ys.max()) + 1
|
| 188 |
+
x1, y1 = max(0, x1 - padding), max(0, y1 - padding)
|
| 189 |
+
x2, y2 = min(width, x2 + padding), min(height, y2 + padding)
|
| 190 |
+
|
| 191 |
+
# A near-square crop gives the diffusion model enough visible context on
|
| 192 |
+
# both sides of a narrow structural occluder.
|
| 193 |
+
side = max(x2 - x1, y2 - y1)
|
| 194 |
+
cx, cy = (x1 + x2) / 2.0, (y1 + y2) / 2.0
|
| 195 |
+
x1, y1 = int(round(cx - side / 2.0)), int(round(cy - side / 2.0))
|
| 196 |
+
x2, y2 = x1 + side, y1 + side
|
| 197 |
+
if x1 < 0:
|
| 198 |
+
x2, x1 = x2 - x1, 0
|
| 199 |
+
if y1 < 0:
|
| 200 |
+
y2, y1 = y2 - y1, 0
|
| 201 |
+
if x2 > width:
|
| 202 |
+
x1, x2 = max(0, x1 - (x2 - width)), width
|
| 203 |
+
if y2 > height:
|
| 204 |
+
y1, y2 = max(0, y1 - (y2 - height)), height
|
| 205 |
+
return x1, y1, x2, y2
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def model_size(size: tuple[int, int], maximum: int) -> tuple[int, int]:
|
| 209 |
+
width, height = size
|
| 210 |
+
scale = min(maximum / max(width, height), 1.0)
|
| 211 |
+
return (
|
| 212 |
+
max(64, int(round(width * scale / 8.0)) * 8),
|
| 213 |
+
max(64, int(round(height * scale / 8.0)) * 8),
|
| 214 |
+
)
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def composite_generated_crop(
|
| 218 |
+
source: Image.Image,
|
| 219 |
+
generated: Image.Image,
|
| 220 |
+
mask_crop: Image.Image,
|
| 221 |
+
box: tuple[int, int, int, int],
|
| 222 |
+
feather_radius: float,
|
| 223 |
+
preserve_crop: Image.Image | None = None,
|
| 224 |
+
) -> Image.Image:
|
| 225 |
+
x1, y1, x2, y2 = box
|
| 226 |
+
crop_size = (x2 - x1, y2 - y1)
|
| 227 |
+
generated = generated.convert("RGB").resize(crop_size, Image.Resampling.LANCZOS)
|
| 228 |
+
alpha = mask_crop.resize(crop_size, Image.Resampling.NEAREST)
|
| 229 |
+
if feather_radius > 0:
|
| 230 |
+
alpha = alpha.filter(ImageFilter.GaussianBlur(feather_radius))
|
| 231 |
+
if preserve_crop is not None:
|
| 232 |
+
preserve = (
|
| 233 |
+
np.asarray(
|
| 234 |
+
preserve_crop.resize(crop_size, Image.Resampling.NEAREST).convert("L")
|
| 235 |
+
)
|
| 236 |
+
> 127
|
| 237 |
+
)
|
| 238 |
+
alpha_array = np.asarray(alpha).copy()
|
| 239 |
+
alpha_array[preserve] = 0
|
| 240 |
+
alpha = Image.fromarray(alpha_array, mode="L")
|
| 241 |
+
result = source.copy()
|
| 242 |
+
result.paste(generated, (x1, y1), alpha)
|
| 243 |
+
return result
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def labelled_panel(image: Image.Image, label: str, size: tuple[int, int]) -> Image.Image:
|
| 247 |
+
body = ImageOps.contain(image.convert("RGB"), (size[0], size[1] - 34))
|
| 248 |
+
panel = Image.new("RGB", size, "white")
|
| 249 |
+
ImageDraw.Draw(panel).text((9, 10), label, fill=(20, 20, 20))
|
| 250 |
+
panel.paste(body, ((size[0] - body.width) // 2, 34 + (size[1] - 34 - body.height) // 2))
|
| 251 |
+
return panel
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def write_candidate_grid(
|
| 255 |
+
path: Path,
|
| 256 |
+
source: Image.Image,
|
| 257 |
+
mask: Image.Image,
|
| 258 |
+
candidates: list[dict[str, Any]],
|
| 259 |
+
) -> None:
|
| 260 |
+
overlay = source.copy()
|
| 261 |
+
overlay.paste(Image.new("RGB", source.size, (255, 45, 30)), mask=mask)
|
| 262 |
+
overlay = Image.blend(source, overlay, 0.52)
|
| 263 |
+
panel_size = (400, 560)
|
| 264 |
+
items = [labelled_panel(source, "source", panel_size), labelled_panel(overlay, "completion mask", panel_size)]
|
| 265 |
+
for row in candidates:
|
| 266 |
+
quality = row["quality"]
|
| 267 |
+
label = (
|
| 268 |
+
f"candidate {row['index']} | score {quality['score']:.3f} "
|
| 269 |
+
f"| gate {quality.get('gate_score', quality['score']):.3f}"
|
| 270 |
+
)
|
| 271 |
+
if "selection_score" in quality:
|
| 272 |
+
label += f" | select {quality['selection_score']:.3f}"
|
| 273 |
+
if quality.get("quality_flags"):
|
| 274 |
+
label += " | " + ",".join(quality["quality_flags"])
|
| 275 |
+
if row.get("selected"):
|
| 276 |
+
label += " | REVIEW PICK"
|
| 277 |
+
items.append(labelled_panel(Image.open(row["completed_rgb"]), label, panel_size))
|
| 278 |
+
columns = 3
|
| 279 |
+
rows = math.ceil(len(items) / columns)
|
| 280 |
+
sheet = Image.new("RGB", (columns * panel_size[0], rows * panel_size[1]), (240, 240, 240))
|
| 281 |
+
for index, item in enumerate(items):
|
| 282 |
+
sheet.paste(item, ((index % columns) * panel_size[0], (index // columns) * panel_size[1]))
|
| 283 |
+
sheet.save(path, quality=94, subsampling=0)
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def load_backend(model: Path, device: str):
|
| 287 |
+
from diffusers import StableDiffusionInpaintPipeline
|
| 288 |
+
|
| 289 |
+
dtype = torch.float16 if device.startswith("cuda") else torch.float32
|
| 290 |
+
pipeline = StableDiffusionInpaintPipeline.from_pretrained(
|
| 291 |
+
str(model),
|
| 292 |
+
torch_dtype=dtype,
|
| 293 |
+
variant="fp16" if dtype == torch.float16 else None,
|
| 294 |
+
safety_checker=None,
|
| 295 |
+
requires_safety_checker=False,
|
| 296 |
+
local_files_only=True,
|
| 297 |
+
)
|
| 298 |
+
pipeline = pipeline.to(device)
|
| 299 |
+
pipeline.set_progress_bar_config(desc="Accessibility GPU 2D", leave=False)
|
| 300 |
+
return pipeline
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
def build_parser() -> argparse.ArgumentParser:
|
| 304 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 305 |
+
parser.add_argument("--image", required=True)
|
| 306 |
+
parser.add_argument(
|
| 307 |
+
"--mask",
|
| 308 |
+
default=None,
|
| 309 |
+
help=(
|
| 310 |
+
"Legacy reviewed hidden mask. Optional when both target masks are supplied; "
|
| 311 |
+
"this geometry mask is never expanded to whole obstacle instances."
|
| 312 |
+
),
|
| 313 |
+
)
|
| 314 |
+
parser.add_argument(
|
| 315 |
+
"--obstacle-mask",
|
| 316 |
+
default=None,
|
| 317 |
+
help=(
|
| 318 |
+
"Detected obstacle mask. Visual removal keeps only complete connected "
|
| 319 |
+
"components intersecting hidden; nearby non-occluding people remain."
|
| 320 |
+
),
|
| 321 |
+
)
|
| 322 |
+
parser.add_argument("--target-amodal-mask", default=None)
|
| 323 |
+
parser.add_argument("--target-visible-mask", default=None)
|
| 324 |
+
parser.add_argument("--category", choices=tuple(PROMPTS), required=True)
|
| 325 |
+
parser.add_argument("--output-dir", required=True)
|
| 326 |
+
parser.add_argument("--model", default="weights/stable-diffusion-inpainting")
|
| 327 |
+
parser.add_argument("--prompt", default=None)
|
| 328 |
+
parser.add_argument("--negative-prompt", default=NEGATIVE_PROMPT)
|
| 329 |
+
parser.add_argument("--seed", type=int, default=9527)
|
| 330 |
+
parser.add_argument("--steps", type=int, default=45)
|
| 331 |
+
parser.add_argument("--guidance-scale", type=float, default=6.5)
|
| 332 |
+
parser.add_argument("--num-candidates", type=int, default=4)
|
| 333 |
+
parser.add_argument("--crop-padding", type=int, default=220)
|
| 334 |
+
parser.add_argument("--resolution", type=int, default=768)
|
| 335 |
+
parser.add_argument("--mask-dilate", type=int, default=8)
|
| 336 |
+
parser.add_argument("--feather-radius", type=float, default=3.0)
|
| 337 |
+
parser.add_argument("--device", default="cuda")
|
| 338 |
+
return parser
|
| 339 |
+
|
| 340 |
+
|
| 341 |
+
def main() -> int:
|
| 342 |
+
args = build_parser().parse_args()
|
| 343 |
+
if args.num_candidates < 1 or args.steps < 1:
|
| 344 |
+
raise ValueError("num-candidates and steps must be positive")
|
| 345 |
+
image_path = resolve_path(args.image)
|
| 346 |
+
mask_path = resolve_path(args.mask) if args.mask else None
|
| 347 |
+
obstacle_path = resolve_path(args.obstacle_mask) if args.obstacle_mask else None
|
| 348 |
+
target_amodal_path = (
|
| 349 |
+
resolve_path(args.target_amodal_mask) if args.target_amodal_mask else None
|
| 350 |
+
)
|
| 351 |
+
target_visible_path = (
|
| 352 |
+
resolve_path(args.target_visible_mask) if args.target_visible_mask else None
|
| 353 |
+
)
|
| 354 |
+
model_path = resolve_path(args.model)
|
| 355 |
+
output_dir = resolve_path(args.output_dir)
|
| 356 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 357 |
+
source = load_rgb(image_path)
|
| 358 |
+
geometry_hidden_mask, visual_removal_mask, mask_report = load_completion_masks(
|
| 359 |
+
size=source.size,
|
| 360 |
+
hidden_path=mask_path,
|
| 361 |
+
obstacle_path=obstacle_path,
|
| 362 |
+
target_amodal_path=target_amodal_path,
|
| 363 |
+
target_visible_path=target_visible_path,
|
| 364 |
+
)
|
| 365 |
+
if obstacle_path is not None:
|
| 366 |
+
obstacle_array = np.asarray(load_binary_mask(obstacle_path, source.size)) > 127
|
| 367 |
+
protected_non_occluding = obstacle_array & ~(
|
| 368 |
+
np.asarray(visual_removal_mask) > 127
|
| 369 |
+
)
|
| 370 |
+
else:
|
| 371 |
+
protected_non_occluding = np.zeros(
|
| 372 |
+
(source.height, source.width),
|
| 373 |
+
dtype=bool,
|
| 374 |
+
)
|
| 375 |
+
obstacle_array = None
|
| 376 |
+
completion_envelope, envelope_report = build_completion_envelope(
|
| 377 |
+
np.asarray(visual_removal_mask) > 127,
|
| 378 |
+
obstacle_array,
|
| 379 |
+
)
|
| 380 |
+
completion_envelope_mask = Image.fromarray(
|
| 381 |
+
completion_envelope.astype(np.uint8) * 255,
|
| 382 |
+
mode="L",
|
| 383 |
+
)
|
| 384 |
+
target_amodal_array = (
|
| 385 |
+
np.asarray(load_binary_mask(target_amodal_path, source.size)) > 127
|
| 386 |
+
if target_amodal_path is not None
|
| 387 |
+
else None
|
| 388 |
+
)
|
| 389 |
+
generation_mask = dilate(completion_envelope_mask, args.mask_dilate)
|
| 390 |
+
# Dilation and feathering must not nibble into a nearby person/object that
|
| 391 |
+
# the component policy intentionally excluded.
|
| 392 |
+
generation_array = (np.asarray(generation_mask) > 127) & ~protected_non_occluding
|
| 393 |
+
generation_mask = Image.fromarray(
|
| 394 |
+
generation_array.astype(np.uint8) * 255,
|
| 395 |
+
mode="L",
|
| 396 |
+
)
|
| 397 |
+
protected_non_occluding_mask = Image.fromarray(
|
| 398 |
+
protected_non_occluding.astype(np.uint8) * 255,
|
| 399 |
+
mode="L",
|
| 400 |
+
)
|
| 401 |
+
mask_report["generation_after_dilation"] = mask_statistics(
|
| 402 |
+
generation_array
|
| 403 |
+
)
|
| 404 |
+
mask_report["completion_envelope"] = envelope_report
|
| 405 |
+
mask_report["protected_non_occluding_obstacle"] = mask_statistics(
|
| 406 |
+
protected_non_occluding
|
| 407 |
+
)
|
| 408 |
+
mask_report["generation_mask_dilate_radius"] = args.mask_dilate
|
| 409 |
+
geometry_hidden_mask.save(output_dir / "geometry_hidden_mask.png")
|
| 410 |
+
# Preserve the legacy review filename for downstream readers.
|
| 411 |
+
geometry_hidden_mask.save(output_dir / "reviewed_hidden_mask.png")
|
| 412 |
+
visual_removal_mask.save(output_dir / "visual_removal_mask.png")
|
| 413 |
+
completion_envelope_mask.save(output_dir / "completion_envelope_mask.png")
|
| 414 |
+
generation_mask.save(output_dir / "generation_mask.png")
|
| 415 |
+
protected_non_occluding_mask.save(
|
| 416 |
+
output_dir / "protected_non_occluding_obstacle_mask.png"
|
| 417 |
+
)
|
| 418 |
+
|
| 419 |
+
input_references = {
|
| 420 |
+
"image": portable_input_reference(image_path),
|
| 421 |
+
"legacy_hidden": (
|
| 422 |
+
portable_input_reference(mask_path) if mask_path is not None else None
|
| 423 |
+
),
|
| 424 |
+
"obstacle": (
|
| 425 |
+
portable_input_reference(obstacle_path) if obstacle_path is not None else None
|
| 426 |
+
),
|
| 427 |
+
"target_amodal": (
|
| 428 |
+
portable_input_reference(target_amodal_path)
|
| 429 |
+
if target_amodal_path is not None
|
| 430 |
+
else None
|
| 431 |
+
),
|
| 432 |
+
"target_visible": (
|
| 433 |
+
portable_input_reference(target_visible_path)
|
| 434 |
+
if target_visible_path is not None
|
| 435 |
+
else None
|
| 436 |
+
),
|
| 437 |
+
}
|
| 438 |
+
if not np.any(np.asarray(generation_mask) > 127):
|
| 439 |
+
selected = output_dir / "completed_rgb_selected.png"
|
| 440 |
+
source.save(selected)
|
| 441 |
+
manifest = {
|
| 442 |
+
"schema_version": "accessibilityamodal_visual_completion_v1",
|
| 443 |
+
"status": "skipped_empty_removal_mask",
|
| 444 |
+
"human_review_required": True,
|
| 445 |
+
"automatic_passability_claim": False,
|
| 446 |
+
"inputs": input_references,
|
| 447 |
+
"mask_statistics": mask_report,
|
| 448 |
+
"files": {
|
| 449 |
+
"geometry_hidden_mask": "geometry_hidden_mask.png",
|
| 450 |
+
"visual_removal_mask": "visual_removal_mask.png",
|
| 451 |
+
"completion_envelope_mask": "completion_envelope_mask.png",
|
| 452 |
+
"generation_mask": "generation_mask.png",
|
| 453 |
+
"protected_non_occluding_obstacle_mask": (
|
| 454 |
+
"protected_non_occluding_obstacle_mask.png"
|
| 455 |
+
),
|
| 456 |
+
"selected_completed_rgb": selected.name,
|
| 457 |
+
},
|
| 458 |
+
"selected_completed_rgb": selected.name,
|
| 459 |
+
"warning": (
|
| 460 |
+
"Visual completion outputs are review candidates, not ground truth "
|
| 461 |
+
"or evidence that a route is passable."
|
| 462 |
+
),
|
| 463 |
+
}
|
| 464 |
+
(output_dir / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
|
| 465 |
+
return 0
|
| 466 |
+
|
| 467 |
+
box = crop_around_mask(generation_mask, args.crop_padding)
|
| 468 |
+
source_crop = source.crop(box)
|
| 469 |
+
mask_crop = generation_mask.crop(box)
|
| 470 |
+
preserve_crop = protected_non_occluding_mask.crop(box)
|
| 471 |
+
inference_size = model_size(source_crop.size, args.resolution)
|
| 472 |
+
model_image = source_crop.resize(inference_size, Image.Resampling.LANCZOS)
|
| 473 |
+
model_mask = mask_crop.resize(inference_size, Image.Resampling.NEAREST)
|
| 474 |
+
device = args.device if not args.device.startswith("cuda") or torch.cuda.is_available() else "cpu"
|
| 475 |
+
prompt = args.prompt or PROMPTS[args.category]
|
| 476 |
+
pipeline = load_backend(model_path, device)
|
| 477 |
+
|
| 478 |
+
candidates: list[dict[str, Any]] = []
|
| 479 |
+
for index in range(args.num_candidates):
|
| 480 |
+
seed = args.seed + index * 1009
|
| 481 |
+
generator = torch.Generator(device=device).manual_seed(seed)
|
| 482 |
+
generated = pipeline(
|
| 483 |
+
prompt=prompt,
|
| 484 |
+
negative_prompt=args.negative_prompt,
|
| 485 |
+
image=model_image,
|
| 486 |
+
mask_image=model_mask,
|
| 487 |
+
num_inference_steps=args.steps,
|
| 488 |
+
guidance_scale=args.guidance_scale,
|
| 489 |
+
generator=generator,
|
| 490 |
+
).images[0]
|
| 491 |
+
completed = composite_generated_crop(
|
| 492 |
+
source,
|
| 493 |
+
generated,
|
| 494 |
+
mask_crop,
|
| 495 |
+
box,
|
| 496 |
+
args.feather_radius,
|
| 497 |
+
preserve_crop=preserve_crop,
|
| 498 |
+
)
|
| 499 |
+
completed_path = output_dir / f"completed_rgb_candidate_{index:02d}.png"
|
| 500 |
+
crop_path = output_dir / f"generated_crop_candidate_{index:02d}.png"
|
| 501 |
+
completed.save(completed_path)
|
| 502 |
+
generated.save(crop_path)
|
| 503 |
+
quality = candidate_quality(
|
| 504 |
+
completed,
|
| 505 |
+
geometry_hidden_mask,
|
| 506 |
+
args.category,
|
| 507 |
+
)
|
| 508 |
+
quality["clutter_metrics"] = candidate_clutter_metrics(
|
| 509 |
+
completed,
|
| 510 |
+
source,
|
| 511 |
+
completion_envelope,
|
| 512 |
+
target_amodal_array,
|
| 513 |
+
)
|
| 514 |
+
candidates.append(
|
| 515 |
+
{
|
| 516 |
+
"index": index,
|
| 517 |
+
"seed": seed,
|
| 518 |
+
"completed_rgb": str(completed_path),
|
| 519 |
+
"generated_crop": str(crop_path),
|
| 520 |
+
"quality": quality,
|
| 521 |
+
}
|
| 522 |
+
)
|
| 523 |
+
|
| 524 |
+
apply_selection_clutter_penalty(candidates)
|
| 525 |
+
selected_row, status = select_candidate(candidates)
|
| 526 |
+
selected_row["selected"] = True
|
| 527 |
+
selected_path = output_dir / "completed_rgb_selected.png"
|
| 528 |
+
shutil.copy2(selected_row["completed_rgb"], selected_path)
|
| 529 |
+
source_crop.save(output_dir / "source_context_crop.png")
|
| 530 |
+
write_candidate_grid(output_dir / "candidate_comparison.jpg", source, generation_mask, candidates)
|
| 531 |
+
|
| 532 |
+
manifest_candidates = []
|
| 533 |
+
for row in candidates:
|
| 534 |
+
portable_row = dict(row)
|
| 535 |
+
portable_row["completed_rgb"] = Path(row["completed_rgb"]).name
|
| 536 |
+
portable_row["generated_crop"] = Path(row["generated_crop"]).name
|
| 537 |
+
manifest_candidates.append(portable_row)
|
| 538 |
+
manifest = {
|
| 539 |
+
"schema_version": "accessibilityamodal_visual_completion_v1",
|
| 540 |
+
"created_at_utc": datetime.now(timezone.utc).isoformat(),
|
| 541 |
+
"status": status,
|
| 542 |
+
"human_review_required": True,
|
| 543 |
+
"automatic_passability_claim": False,
|
| 544 |
+
"pipeline": "AccessibilityAmodal GPU 2D Visual Completion",
|
| 545 |
+
"backend": "Diffusers StableDiffusionInpaintPipeline",
|
| 546 |
+
"inputs": input_references,
|
| 547 |
+
"mask_statistics": mask_report,
|
| 548 |
+
"quality_evaluation_mask": "geometry_hidden_mask.png",
|
| 549 |
+
"category": args.category,
|
| 550 |
+
"model": portable_input_reference(model_path),
|
| 551 |
+
"prompt": prompt,
|
| 552 |
+
"negative_prompt": args.negative_prompt,
|
| 553 |
+
"device": device,
|
| 554 |
+
"steps": args.steps,
|
| 555 |
+
"guidance_scale": args.guidance_scale,
|
| 556 |
+
"crop_box": box,
|
| 557 |
+
"inference_size": inference_size,
|
| 558 |
+
"mask_dilate": args.mask_dilate,
|
| 559 |
+
"feather_radius": args.feather_radius,
|
| 560 |
+
"selected_index": selected_row["index"],
|
| 561 |
+
"selection_score_policy": (
|
| 562 |
+
"absolute_hidden_surface_quality_minus_up_to_0.15_"
|
| 563 |
+
"cohort_relative_outside_target_clutter_rank"
|
| 564 |
+
),
|
| 565 |
+
"selected_completed_rgb": selected_path.name,
|
| 566 |
+
"candidate_comparison": "candidate_comparison.jpg",
|
| 567 |
+
"files": {
|
| 568 |
+
"geometry_hidden_mask": "geometry_hidden_mask.png",
|
| 569 |
+
"visual_removal_mask": "visual_removal_mask.png",
|
| 570 |
+
"completion_envelope_mask": "completion_envelope_mask.png",
|
| 571 |
+
"generation_mask": "generation_mask.png",
|
| 572 |
+
"protected_non_occluding_obstacle_mask": (
|
| 573 |
+
"protected_non_occluding_obstacle_mask.png"
|
| 574 |
+
),
|
| 575 |
+
"source_context_crop": "source_context_crop.png",
|
| 576 |
+
"selected_completed_rgb": selected_path.name,
|
| 577 |
+
"candidate_comparison": "candidate_comparison.jpg",
|
| 578 |
+
},
|
| 579 |
+
"candidates": manifest_candidates,
|
| 580 |
+
"warning": (
|
| 581 |
+
"Generative 2D outputs are visual review candidates, not ground truth "
|
| 582 |
+
"or evidence that a route is passable."
|
| 583 |
+
),
|
| 584 |
+
}
|
| 585 |
+
(output_dir / "manifest.json").write_text(
|
| 586 |
+
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n",
|
| 587 |
+
encoding="utf-8",
|
| 588 |
+
)
|
| 589 |
+
if status == "withheld_needs_review":
|
| 590 |
+
print(
|
| 591 |
+
f"All candidates carry quality risks; retained candidate "
|
| 592 |
+
f"{selected_row['index']} for review and withheld automatic acceptance."
|
| 593 |
+
)
|
| 594 |
+
else:
|
| 595 |
+
print(
|
| 596 |
+
f"Selected visual candidate {selected_row['index']} by gate score "
|
| 597 |
+
f"{selected_row['quality']['gate_score']:.4f}; human review remains required."
|
| 598 |
+
)
|
| 599 |
+
print(f"Wrote GPU 2D review candidate to {selected_path}")
|
| 600 |
+
return 0
|
| 601 |
+
|
| 602 |
+
|
| 603 |
+
if __name__ == "__main__":
|
| 604 |
+
raise SystemExit(main())
|
tools/accessibility_3d_completion.py
ADDED
|
@@ -0,0 +1,1150 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""AccessibilityAmodal adapter for the licensed Amodal3R visual 3D backend.
|
| 3 |
+
|
| 4 |
+
This file contains project-owned input/output orchestration. The model package
|
| 5 |
+
imported as ``amodal3d`` remains third-party code under its upstream license.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import argparse
|
| 11 |
+
import hashlib
|
| 12 |
+
import importlib
|
| 13 |
+
import importlib.util
|
| 14 |
+
import json
|
| 15 |
+
import math
|
| 16 |
+
import os
|
| 17 |
+
import shutil
|
| 18 |
+
import sys
|
| 19 |
+
from datetime import datetime, timezone
|
| 20 |
+
from pathlib import Path
|
| 21 |
+
from typing import Any
|
| 22 |
+
|
| 23 |
+
import cv2
|
| 24 |
+
import imageio
|
| 25 |
+
import numpy as np
|
| 26 |
+
import trimesh
|
| 27 |
+
from PIL import Image, ImageOps
|
| 28 |
+
|
| 29 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
| 30 |
+
sys.path.insert(0, str(PROJECT_ROOT))
|
| 31 |
+
|
| 32 |
+
os.environ["ATTN_BACKEND"] = "xformers"
|
| 33 |
+
os.environ["SPARSE_ATTN_BACKEND"] = "xformers"
|
| 34 |
+
os.environ["XFORMERS_DISABLED"] = "1"
|
| 35 |
+
os.environ["SPCONV_ALGO"] = "native"
|
| 36 |
+
os.environ["TORCH_HOME"] = os.environ.get(
|
| 37 |
+
"AMODAL3D_TORCH_HOME", str(PROJECT_ROOT / "weights" / "torch")
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
VISIBLE_VALUE = 188
|
| 41 |
+
OCCLUDED_VALUE = 0
|
| 42 |
+
BACKGROUND_VALUE = 255
|
| 43 |
+
THREE_VALUE_MASK_VALUES = (OCCLUDED_VALUE, VISIBLE_VALUE, BACKGROUND_VALUE)
|
| 44 |
+
ACCEPTED_COMPLETION_STATUSES = frozenset({"candidate_selected_for_review"})
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def resolve_path(value: str | Path) -> Path:
|
| 48 |
+
path = Path(value).expanduser()
|
| 49 |
+
return path.resolve() if path.is_absolute() else (PROJECT_ROOT / path).resolve()
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def sha256_file(path: Path) -> str:
|
| 53 |
+
digest = hashlib.sha256()
|
| 54 |
+
with path.open("rb") as handle:
|
| 55 |
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
| 56 |
+
digest.update(chunk)
|
| 57 |
+
return digest.hexdigest()
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def portable_file_record(path: Path) -> dict[str, Any]:
|
| 61 |
+
"""Record an input without embedding a workstation-specific absolute path."""
|
| 62 |
+
|
| 63 |
+
resolved = path.resolve()
|
| 64 |
+
try:
|
| 65 |
+
base = "project_root"
|
| 66 |
+
relative = resolved.relative_to(PROJECT_ROOT).as_posix()
|
| 67 |
+
except ValueError:
|
| 68 |
+
base = "external_input"
|
| 69 |
+
relative = resolved.name
|
| 70 |
+
return {
|
| 71 |
+
"base": base,
|
| 72 |
+
"path": relative,
|
| 73 |
+
"sha256": sha256_file(resolved),
|
| 74 |
+
"bytes": resolved.stat().st_size,
|
| 75 |
+
}
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def _declared_selected_path_matches(
|
| 79 |
+
declared: str,
|
| 80 |
+
*,
|
| 81 |
+
completion_manifest: Path,
|
| 82 |
+
completed_image: Path,
|
| 83 |
+
) -> bool:
|
| 84 |
+
path = Path(declared).expanduser()
|
| 85 |
+
if path.is_absolute():
|
| 86 |
+
return path.resolve() == completed_image.resolve()
|
| 87 |
+
candidates = (
|
| 88 |
+
completion_manifest.parent / path,
|
| 89 |
+
completion_manifest.parent.parent / path,
|
| 90 |
+
)
|
| 91 |
+
return any(candidate.resolve() == completed_image.resolve() for candidate in candidates)
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def _resolve_completion_manifest_artifact(
|
| 95 |
+
declared: str,
|
| 96 |
+
*,
|
| 97 |
+
completion_manifest: Path,
|
| 98 |
+
) -> Path:
|
| 99 |
+
path = Path(declared).expanduser()
|
| 100 |
+
candidates = (
|
| 101 |
+
(path,) if path.is_absolute() else (
|
| 102 |
+
completion_manifest.parent / path,
|
| 103 |
+
completion_manifest.parent.parent / path,
|
| 104 |
+
)
|
| 105 |
+
)
|
| 106 |
+
existing = [candidate.resolve() for candidate in candidates if candidate.is_file()]
|
| 107 |
+
if not existing:
|
| 108 |
+
raise FileNotFoundError(
|
| 109 |
+
f"Completion manifest artifact does not exist: {declared}"
|
| 110 |
+
)
|
| 111 |
+
return existing[0]
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def validate_completed_rgb_texture_preservation(
|
| 115 |
+
*,
|
| 116 |
+
original: Path,
|
| 117 |
+
completed: Path,
|
| 118 |
+
completion_manifest: Path,
|
| 119 |
+
payload: dict[str, Any],
|
| 120 |
+
) -> dict[str, Any]:
|
| 121 |
+
"""Prove that a 2D completion preserves source RGB outside its edit mask."""
|
| 122 |
+
|
| 123 |
+
files = payload.get("files")
|
| 124 |
+
file_records = files if isinstance(files, dict) else {}
|
| 125 |
+
declared_mask = (
|
| 126 |
+
payload.get("appearance_generation_mask")
|
| 127 |
+
or file_records.get("generation_mask")
|
| 128 |
+
)
|
| 129 |
+
if not isinstance(declared_mask, str):
|
| 130 |
+
raise ValueError(
|
| 131 |
+
"Accepted completed RGB manifest must declare its exact generation "
|
| 132 |
+
"mask as appearance_generation_mask or files.generation_mask"
|
| 133 |
+
)
|
| 134 |
+
generation_mask = _resolve_completion_manifest_artifact(
|
| 135 |
+
declared_mask,
|
| 136 |
+
completion_manifest=completion_manifest,
|
| 137 |
+
)
|
| 138 |
+
with (
|
| 139 |
+
Image.open(original) as original_source,
|
| 140 |
+
Image.open(completed) as completed_source,
|
| 141 |
+
Image.open(generation_mask) as mask_source,
|
| 142 |
+
):
|
| 143 |
+
original_image = ImageOps.exif_transpose(original_source).convert("RGB")
|
| 144 |
+
completed_image = ImageOps.exif_transpose(completed_source).convert("RGB")
|
| 145 |
+
mask_image = ImageOps.exif_transpose(mask_source).convert("L")
|
| 146 |
+
if original_image.size != completed_image.size:
|
| 147 |
+
raise ValueError(
|
| 148 |
+
"Accepted completed RGB must use the same display raster as the original"
|
| 149 |
+
)
|
| 150 |
+
if mask_image.size != original_image.size:
|
| 151 |
+
raise ValueError(
|
| 152 |
+
"Completion generation mask must align exactly with original/completed RGB"
|
| 153 |
+
)
|
| 154 |
+
|
| 155 |
+
original_array = np.asarray(original_image, dtype=np.uint8)
|
| 156 |
+
completed_array = np.asarray(completed_image, dtype=np.uint8)
|
| 157 |
+
generation = np.asarray(mask_image, dtype=np.uint8) > 127
|
| 158 |
+
feather_radius = float(payload.get("feather_radius") or 0.0)
|
| 159 |
+
if not math.isfinite(feather_radius) or feather_radius < 0:
|
| 160 |
+
raise ValueError("Completion manifest feather_radius must be non-negative")
|
| 161 |
+
allowed = generation.astype(np.uint8)
|
| 162 |
+
feather_support_radius = int(math.ceil(3.0 * feather_radius))
|
| 163 |
+
if feather_support_radius > 0:
|
| 164 |
+
kernel = cv2.getStructuringElement(
|
| 165 |
+
cv2.MORPH_ELLIPSE,
|
| 166 |
+
(
|
| 167 |
+
feather_support_radius * 2 + 1,
|
| 168 |
+
feather_support_radius * 2 + 1,
|
| 169 |
+
),
|
| 170 |
+
)
|
| 171 |
+
allowed = cv2.dilate(allowed, kernel)
|
| 172 |
+
allowed = allowed > 0
|
| 173 |
+
changed = np.any(original_array != completed_array, axis=2)
|
| 174 |
+
changed_outside = changed & ~allowed
|
| 175 |
+
changed_inside = changed & allowed
|
| 176 |
+
changed_outside_count = int(changed_outside.sum())
|
| 177 |
+
if changed_outside_count:
|
| 178 |
+
raise ValueError(
|
| 179 |
+
"Completed RGB changes source texture outside its declared generation "
|
| 180 |
+
f"mask/feather support at {changed_outside_count} pixels"
|
| 181 |
+
)
|
| 182 |
+
changed_inside_count = int(changed_inside.sum())
|
| 183 |
+
if changed_inside_count == 0:
|
| 184 |
+
raise ValueError(
|
| 185 |
+
"Completed RGB does not change any pixel inside its declared edit region"
|
| 186 |
+
)
|
| 187 |
+
outside_count = int((~allowed).sum())
|
| 188 |
+
return {
|
| 189 |
+
"validated": True,
|
| 190 |
+
"generation_mask": portable_file_record(generation_mask),
|
| 191 |
+
"feather_radius": feather_radius,
|
| 192 |
+
"feather_support_radius_pixels": feather_support_radius,
|
| 193 |
+
"allowed_edit_pixel_count": int(allowed.sum()),
|
| 194 |
+
"changed_inside_allowed_region_pixel_count": changed_inside_count,
|
| 195 |
+
"changed_outside_allowed_region_pixel_count": 0,
|
| 196 |
+
"source_rgb_identity_outside_allowed_region": True,
|
| 197 |
+
"source_rgb_identity_outside_allowed_region_ratio": (
|
| 198 |
+
1.0 if outside_count else None
|
| 199 |
+
),
|
| 200 |
+
}
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
def select_backend_rgb(args: argparse.Namespace) -> tuple[Path, dict[str, Any]]:
|
| 204 |
+
"""Choose the explicitly requested RGB conditioning source.
|
| 205 |
+
|
| 206 |
+
The original image remains the canonical geometry/provenance source.
|
| 207 |
+
Completion arguments cannot replace it unless ``--conditioning-rgb
|
| 208 |
+
completed`` is explicit and the completion manifest passes the quality
|
| 209 |
+
gate.
|
| 210 |
+
"""
|
| 211 |
+
|
| 212 |
+
original = resolve_path(args.image)
|
| 213 |
+
if not original.is_file():
|
| 214 |
+
raise FileNotFoundError(original)
|
| 215 |
+
conditioning_rgb = str(getattr(args, "conditioning_rgb", "original"))
|
| 216 |
+
completed_value = getattr(args, "completed_image", None)
|
| 217 |
+
manifest_value = getattr(args, "completion_manifest", None)
|
| 218 |
+
completion_inputs_supplied = bool(completed_value or manifest_value)
|
| 219 |
+
|
| 220 |
+
if conditioning_rgb == "original":
|
| 221 |
+
original_record = portable_file_record(original)
|
| 222 |
+
return original, {
|
| 223 |
+
"conditioning_rgb_mode": "original",
|
| 224 |
+
"backend_rgb_role": "original_rgb_no_accepted_2d_completion",
|
| 225 |
+
"backend_rgb": original_record,
|
| 226 |
+
"original_rgb": original_record,
|
| 227 |
+
"selected_completion_rgb": None,
|
| 228 |
+
"completion_manifest": None,
|
| 229 |
+
"completion_status": None,
|
| 230 |
+
"quality_gate_accepted": False,
|
| 231 |
+
"completion_inputs_supplied": completion_inputs_supplied,
|
| 232 |
+
"completion_inputs_ignored": completion_inputs_supplied,
|
| 233 |
+
"original_vs_completed": {
|
| 234 |
+
"same_raster_size": None,
|
| 235 |
+
"same_sha256": None,
|
| 236 |
+
},
|
| 237 |
+
}
|
| 238 |
+
|
| 239 |
+
if conditioning_rgb != "completed":
|
| 240 |
+
raise ValueError(
|
| 241 |
+
"--conditioning-rgb must be either 'original' or 'completed'"
|
| 242 |
+
)
|
| 243 |
+
if not completed_value or not manifest_value:
|
| 244 |
+
raise ValueError(
|
| 245 |
+
"--conditioning-rgb completed requires both --completed-image and "
|
| 246 |
+
"--completion-manifest"
|
| 247 |
+
)
|
| 248 |
+
|
| 249 |
+
completed = resolve_path(completed_value)
|
| 250 |
+
completion_manifest = resolve_path(manifest_value)
|
| 251 |
+
if not completed.is_file():
|
| 252 |
+
raise FileNotFoundError(completed)
|
| 253 |
+
if not completion_manifest.is_file():
|
| 254 |
+
raise FileNotFoundError(completion_manifest)
|
| 255 |
+
payload = json.loads(completion_manifest.read_text(encoding="utf-8"))
|
| 256 |
+
if not isinstance(payload, dict):
|
| 257 |
+
raise ValueError("Completion manifest must contain a JSON object")
|
| 258 |
+
status = str(payload.get("status") or "")
|
| 259 |
+
if status not in ACCEPTED_COMPLETION_STATUSES:
|
| 260 |
+
raise ValueError(
|
| 261 |
+
"Refusing completed RGB for learned visual 3D because completion "
|
| 262 |
+
f"status is not accepted: {status or '<missing>'}"
|
| 263 |
+
)
|
| 264 |
+
files = payload.get("files")
|
| 265 |
+
declared_from_files = (
|
| 266 |
+
files.get("selected_completed_rgb") if isinstance(files, dict) else None
|
| 267 |
+
)
|
| 268 |
+
declared = payload.get("selected_completed_rgb") or declared_from_files
|
| 269 |
+
if not isinstance(declared, str) or not _declared_selected_path_matches(
|
| 270 |
+
declared,
|
| 271 |
+
completion_manifest=completion_manifest,
|
| 272 |
+
completed_image=completed,
|
| 273 |
+
):
|
| 274 |
+
raise ValueError(
|
| 275 |
+
"Completion manifest does not identify --completed-image as its "
|
| 276 |
+
"selected completion"
|
| 277 |
+
)
|
| 278 |
+
|
| 279 |
+
texture_preservation = validate_completed_rgb_texture_preservation(
|
| 280 |
+
original=original,
|
| 281 |
+
completed=completed,
|
| 282 |
+
completion_manifest=completion_manifest,
|
| 283 |
+
payload=payload,
|
| 284 |
+
)
|
| 285 |
+
original_record = portable_file_record(original)
|
| 286 |
+
completed_record = portable_file_record(completed)
|
| 287 |
+
return completed, {
|
| 288 |
+
"conditioning_rgb_mode": "completed",
|
| 289 |
+
"backend_rgb_role": (
|
| 290 |
+
"quality_gate_accepted_obstacle_removed_rgb_preserving_original_texture"
|
| 291 |
+
),
|
| 292 |
+
"backend_rgb": completed_record,
|
| 293 |
+
"original_rgb": original_record,
|
| 294 |
+
"selected_completion_rgb": completed_record,
|
| 295 |
+
"completion_manifest": portable_file_record(completion_manifest),
|
| 296 |
+
"completion_status": status,
|
| 297 |
+
"completion_manifest_selected_rgb": Path(declared).name,
|
| 298 |
+
"quality_gate_accepted": True,
|
| 299 |
+
"completion_inputs_supplied": True,
|
| 300 |
+
"completion_inputs_ignored": False,
|
| 301 |
+
"original_vs_completed": {
|
| 302 |
+
"same_raster_size": True,
|
| 303 |
+
"same_sha256": original_record["sha256"] == completed_record["sha256"],
|
| 304 |
+
},
|
| 305 |
+
"original_texture_preservation": texture_preservation,
|
| 306 |
+
}
|
| 307 |
+
|
| 308 |
+
|
| 309 |
+
def load_backend_runtime():
|
| 310 |
+
"""Import the licensed third-party runtime only for an actual GPU run."""
|
| 311 |
+
|
| 312 |
+
from amodal3d.pipelines import Amodal3RImageTo3DPipeline
|
| 313 |
+
from amodal3d.utils import render_utils
|
| 314 |
+
|
| 315 |
+
return Amodal3RImageTo3DPipeline, render_utils
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
def extract_glb(gs, mesh, mesh_simplify=0.95, texture_size=1024, export_path="output.glb"):
|
| 319 |
+
from amodal3d.utils import postprocessing_utils
|
| 320 |
+
|
| 321 |
+
glb = postprocessing_utils.to_glb(
|
| 322 |
+
gs,
|
| 323 |
+
mesh,
|
| 324 |
+
simplify=mesh_simplify,
|
| 325 |
+
texture_size=texture_size,
|
| 326 |
+
verbose=False,
|
| 327 |
+
)
|
| 328 |
+
glb.export(export_path)
|
| 329 |
+
return export_path
|
| 330 |
+
|
| 331 |
+
|
| 332 |
+
def save_mesh(mesh_result, filename):
|
| 333 |
+
vertices = (
|
| 334 |
+
mesh_result.vertices.cpu().numpy()
|
| 335 |
+
if hasattr(mesh_result.vertices, "cpu")
|
| 336 |
+
else mesh_result.vertices
|
| 337 |
+
)
|
| 338 |
+
faces = (
|
| 339 |
+
mesh_result.faces.cpu().numpy()
|
| 340 |
+
if hasattr(mesh_result.faces, "cpu")
|
| 341 |
+
else mesh_result.faces
|
| 342 |
+
)
|
| 343 |
+
mesh = trimesh.Trimesh(vertices=vertices, faces=faces, process=False)
|
| 344 |
+
if mesh_result.vertex_attrs is not None:
|
| 345 |
+
attrs = (
|
| 346 |
+
mesh_result.vertex_attrs.cpu().numpy()
|
| 347 |
+
if hasattr(mesh_result.vertex_attrs, "cpu")
|
| 348 |
+
else mesh_result.vertex_attrs
|
| 349 |
+
)
|
| 350 |
+
mesh.visual.vertex_colors = attrs
|
| 351 |
+
mesh.export(filename)
|
| 352 |
+
|
| 353 |
+
|
| 354 |
+
def parse_box(box):
|
| 355 |
+
values = [float(value) for value in box.split(",")]
|
| 356 |
+
if len(values) != 4:
|
| 357 |
+
raise argparse.ArgumentTypeError("box must be x1,y1,x2,y2")
|
| 358 |
+
if any(value < 0 or value > 1 for value in values):
|
| 359 |
+
raise argparse.ArgumentTypeError("box coordinates must be normalized to [0, 1]")
|
| 360 |
+
x1, y1, x2, y2 = values
|
| 361 |
+
if x2 <= x1 or y2 <= y1:
|
| 362 |
+
raise argparse.ArgumentTypeError("box must satisfy x2>x1 and y2>y1")
|
| 363 |
+
return values
|
| 364 |
+
|
| 365 |
+
|
| 366 |
+
def make_stair_scene_mask(image, occlusion_boxes=None, save_path=None):
|
| 367 |
+
width, height = image.size
|
| 368 |
+
mask = np.full((height, width), VISIBLE_VALUE, dtype=np.uint8)
|
| 369 |
+
mask[: int(height * 0.08), :] = BACKGROUND_VALUE
|
| 370 |
+
for x1, y1, x2, y2 in occlusion_boxes or []:
|
| 371 |
+
left = int(round(x1 * width))
|
| 372 |
+
top = int(round(y1 * height))
|
| 373 |
+
right = int(round(x2 * width))
|
| 374 |
+
bottom = int(round(y2 * height))
|
| 375 |
+
mask[top:bottom, left:right] = OCCLUDED_VALUE
|
| 376 |
+
result = Image.fromarray(mask, mode="L")
|
| 377 |
+
if save_path is not None:
|
| 378 |
+
result.save(save_path)
|
| 379 |
+
return result
|
| 380 |
+
|
| 381 |
+
|
| 382 |
+
def load_three_value_condition_mask(
|
| 383 |
+
path: str | Path,
|
| 384 |
+
*,
|
| 385 |
+
expected_size: tuple[int, int],
|
| 386 |
+
) -> tuple[Image.Image, Path, dict[str, Any]]:
|
| 387 |
+
"""Load and strictly validate an aligned Amodal3R condition PNG."""
|
| 388 |
+
|
| 389 |
+
resolved = resolve_path(path)
|
| 390 |
+
if not resolved.is_file():
|
| 391 |
+
raise FileNotFoundError(resolved)
|
| 392 |
+
with Image.open(resolved) as source:
|
| 393 |
+
image_format = source.format
|
| 394 |
+
condition = ImageOps.exif_transpose(source)
|
| 395 |
+
if image_format != "PNG":
|
| 396 |
+
raise ValueError(
|
| 397 |
+
f"Condition mask must be a PNG file; detected {image_format or 'unknown'}"
|
| 398 |
+
)
|
| 399 |
+
if condition.mode != "L":
|
| 400 |
+
raise ValueError(
|
| 401 |
+
"Condition mask PNG must be single-channel 8-bit grayscale "
|
| 402 |
+
f"(mode L); got mode {condition.mode}"
|
| 403 |
+
)
|
| 404 |
+
if condition.size != expected_size:
|
| 405 |
+
raise ValueError(
|
| 406 |
+
"Condition mask/RGB raster mismatch: "
|
| 407 |
+
f"mask={condition.size}, rgb={expected_size}. Refusing to resize."
|
| 408 |
+
)
|
| 409 |
+
values = np.asarray(condition, dtype=np.uint8).copy()
|
| 410 |
+
|
| 411 |
+
observed_values, observed_counts = np.unique(values, return_counts=True)
|
| 412 |
+
observed = {
|
| 413 |
+
int(value): int(count)
|
| 414 |
+
for value, count in zip(observed_values.tolist(), observed_counts.tolist())
|
| 415 |
+
}
|
| 416 |
+
invalid_values = sorted(set(observed) - set(THREE_VALUE_MASK_VALUES))
|
| 417 |
+
if invalid_values:
|
| 418 |
+
raise ValueError(
|
| 419 |
+
"Condition mask PNG contains invalid pixel values "
|
| 420 |
+
f"{invalid_values}; allowed exact values are "
|
| 421 |
+
f"{list(THREE_VALUE_MASK_VALUES)}"
|
| 422 |
+
)
|
| 423 |
+
visible_count = observed.get(VISIBLE_VALUE, 0)
|
| 424 |
+
if visible_count == 0:
|
| 425 |
+
raise ValueError(
|
| 426 |
+
"Condition mask PNG must contain at least one visible-target pixel "
|
| 427 |
+
f"with value {VISIBLE_VALUE}"
|
| 428 |
+
)
|
| 429 |
+
|
| 430 |
+
width, height = expected_size
|
| 431 |
+
statistics = {
|
| 432 |
+
"strict_three_value_validation": True,
|
| 433 |
+
"allowed_values": list(THREE_VALUE_MASK_VALUES),
|
| 434 |
+
"observed_values": sorted(observed),
|
| 435 |
+
"width": int(width),
|
| 436 |
+
"height": int(height),
|
| 437 |
+
"total_pixel_count": int(values.size),
|
| 438 |
+
"hidden_pixel_count": observed.get(OCCLUDED_VALUE, 0),
|
| 439 |
+
"visible_pixel_count": visible_count,
|
| 440 |
+
"background_pixel_count": observed.get(BACKGROUND_VALUE, 0),
|
| 441 |
+
"hidden_region_present": observed.get(OCCLUDED_VALUE, 0) > 0,
|
| 442 |
+
}
|
| 443 |
+
return Image.fromarray(values, mode="L"), resolved, statistics
|
| 444 |
+
|
| 445 |
+
|
| 446 |
+
def square_focus_crop(
|
| 447 |
+
image: Image.Image,
|
| 448 |
+
mask: Image.Image,
|
| 449 |
+
*,
|
| 450 |
+
padding_ratio: float,
|
| 451 |
+
) -> tuple[Image.Image, Image.Image, dict[str, Any]]:
|
| 452 |
+
"""Crop/pad aligned RGB and mask around the target without distortion."""
|
| 453 |
+
|
| 454 |
+
if not math.isfinite(padding_ratio) or padding_ratio < 0:
|
| 455 |
+
raise ValueError("--focus-crop-padding-ratio must be non-negative")
|
| 456 |
+
if image.size != mask.size:
|
| 457 |
+
raise ValueError("Focus crop requires aligned RGB and condition mask")
|
| 458 |
+
|
| 459 |
+
values = np.asarray(mask, dtype=np.uint8)
|
| 460 |
+
target_y, target_x = np.nonzero(values != BACKGROUND_VALUE)
|
| 461 |
+
if target_x.size == 0:
|
| 462 |
+
raise ValueError(
|
| 463 |
+
"Focus crop requires at least one non-background target pixel"
|
| 464 |
+
)
|
| 465 |
+
|
| 466 |
+
bbox_left = int(target_x.min())
|
| 467 |
+
bbox_top = int(target_y.min())
|
| 468 |
+
bbox_right = int(target_x.max()) + 1
|
| 469 |
+
bbox_bottom = int(target_y.max()) + 1
|
| 470 |
+
bbox_width = bbox_right - bbox_left
|
| 471 |
+
bbox_height = bbox_bottom - bbox_top
|
| 472 |
+
side = max(
|
| 473 |
+
1,
|
| 474 |
+
int(
|
| 475 |
+
math.ceil(
|
| 476 |
+
max(bbox_width, bbox_height)
|
| 477 |
+
* (1.0 + 2.0 * padding_ratio)
|
| 478 |
+
)
|
| 479 |
+
),
|
| 480 |
+
)
|
| 481 |
+
center_x = 0.5 * (bbox_left + bbox_right)
|
| 482 |
+
center_y = 0.5 * (bbox_top + bbox_bottom)
|
| 483 |
+
crop_left = int(math.floor(center_x - 0.5 * side))
|
| 484 |
+
crop_top = int(math.floor(center_y - 0.5 * side))
|
| 485 |
+
crop_right = crop_left + side
|
| 486 |
+
crop_bottom = crop_top + side
|
| 487 |
+
|
| 488 |
+
source_width, source_height = image.size
|
| 489 |
+
source_left = max(crop_left, 0)
|
| 490 |
+
source_top = max(crop_top, 0)
|
| 491 |
+
source_right = min(crop_right, source_width)
|
| 492 |
+
source_bottom = min(crop_bottom, source_height)
|
| 493 |
+
paste_left = source_left - crop_left
|
| 494 |
+
paste_top = source_top - crop_top
|
| 495 |
+
|
| 496 |
+
focused_image = Image.new("RGB", (side, side), (0, 0, 0))
|
| 497 |
+
focused_mask = Image.new("L", (side, side), BACKGROUND_VALUE)
|
| 498 |
+
source_box = (source_left, source_top, source_right, source_bottom)
|
| 499 |
+
focused_image.paste(image.crop(source_box), (paste_left, paste_top))
|
| 500 |
+
focused_mask.paste(mask.crop(source_box), (paste_left, paste_top))
|
| 501 |
+
|
| 502 |
+
focused_values = np.asarray(focused_mask, dtype=np.uint8)
|
| 503 |
+
observed_values, observed_counts = np.unique(
|
| 504 |
+
focused_values, return_counts=True
|
| 505 |
+
)
|
| 506 |
+
observed = {
|
| 507 |
+
int(value): int(count)
|
| 508 |
+
for value, count in zip(
|
| 509 |
+
observed_values.tolist(), observed_counts.tolist()
|
| 510 |
+
)
|
| 511 |
+
}
|
| 512 |
+
metadata = {
|
| 513 |
+
"applied": True,
|
| 514 |
+
"policy": "square_target_bbox_crop_with_background_padding",
|
| 515 |
+
"padding_ratio": float(padding_ratio),
|
| 516 |
+
"source_size": [int(source_width), int(source_height)],
|
| 517 |
+
"target_bbox_xyxy": [
|
| 518 |
+
bbox_left,
|
| 519 |
+
bbox_top,
|
| 520 |
+
bbox_right,
|
| 521 |
+
bbox_bottom,
|
| 522 |
+
],
|
| 523 |
+
"crop_box_xyxy": [crop_left, crop_top, crop_right, crop_bottom],
|
| 524 |
+
"source_intersection_xyxy": [
|
| 525 |
+
source_left,
|
| 526 |
+
source_top,
|
| 527 |
+
source_right,
|
| 528 |
+
source_bottom,
|
| 529 |
+
],
|
| 530 |
+
"output_size": [side, side],
|
| 531 |
+
"observed_values": sorted(observed),
|
| 532 |
+
"hidden_pixel_count": observed.get(OCCLUDED_VALUE, 0),
|
| 533 |
+
"visible_pixel_count": observed.get(VISIBLE_VALUE, 0),
|
| 534 |
+
"background_pixel_count": observed.get(BACKGROUND_VALUE, 0),
|
| 535 |
+
}
|
| 536 |
+
return focused_image, focused_mask, metadata
|
| 537 |
+
|
| 538 |
+
|
| 539 |
+
def load_inputs(args, output_dir):
|
| 540 |
+
backend_rgb_path, input_provenance = select_backend_rgb(args)
|
| 541 |
+
with Image.open(backend_rgb_path) as source:
|
| 542 |
+
image = ImageOps.exif_transpose(source).convert("RGB")
|
| 543 |
+
if args.mask:
|
| 544 |
+
mask_source = resolve_path(args.mask)
|
| 545 |
+
else:
|
| 546 |
+
mask_path = output_dir / "auto_stair_mask.png"
|
| 547 |
+
make_stair_scene_mask(image, args.occlusion_box, mask_path)
|
| 548 |
+
mask_source = mask_path.resolve()
|
| 549 |
+
mask, mask_source, mask_statistics = load_three_value_condition_mask(
|
| 550 |
+
mask_source,
|
| 551 |
+
expected_size=image.size,
|
| 552 |
+
)
|
| 553 |
+
focus_crop_padding_ratio = getattr(
|
| 554 |
+
args, "focus_crop_padding_ratio", None
|
| 555 |
+
)
|
| 556 |
+
if focus_crop_padding_ratio is not None:
|
| 557 |
+
source_mask_statistics = dict(mask_statistics)
|
| 558 |
+
image, mask, focus_crop = square_focus_crop(
|
| 559 |
+
image,
|
| 560 |
+
mask,
|
| 561 |
+
padding_ratio=float(focus_crop_padding_ratio),
|
| 562 |
+
)
|
| 563 |
+
mask_statistics = {
|
| 564 |
+
**mask_statistics,
|
| 565 |
+
"width": int(mask.width),
|
| 566 |
+
"height": int(mask.height),
|
| 567 |
+
"total_pixel_count": int(mask.width * mask.height),
|
| 568 |
+
"observed_values": focus_crop["observed_values"],
|
| 569 |
+
"hidden_pixel_count": focus_crop["hidden_pixel_count"],
|
| 570 |
+
"visible_pixel_count": focus_crop["visible_pixel_count"],
|
| 571 |
+
"background_pixel_count": focus_crop["background_pixel_count"],
|
| 572 |
+
"hidden_region_present": focus_crop["hidden_pixel_count"] > 0,
|
| 573 |
+
"focus_crop": focus_crop,
|
| 574 |
+
"pre_focus_crop_statistics": source_mask_statistics,
|
| 575 |
+
}
|
| 576 |
+
return (
|
| 577 |
+
image,
|
| 578 |
+
mask,
|
| 579 |
+
mask_source,
|
| 580 |
+
mask_statistics,
|
| 581 |
+
backend_rgb_path,
|
| 582 |
+
input_provenance,
|
| 583 |
+
)
|
| 584 |
+
|
| 585 |
+
|
| 586 |
+
def require_cuda(torch_module: Any | None = None) -> dict[str, Any]:
|
| 587 |
+
"""Fail before model loading unless the process has a usable CUDA allocation."""
|
| 588 |
+
|
| 589 |
+
if torch_module is None:
|
| 590 |
+
try:
|
| 591 |
+
import torch as torch_module
|
| 592 |
+
except ImportError as exc:
|
| 593 |
+
raise RuntimeError(
|
| 594 |
+
"Amodal3R visual 3D requires PyTorch with CUDA support"
|
| 595 |
+
) from exc
|
| 596 |
+
cuda = getattr(torch_module, "cuda", None)
|
| 597 |
+
if cuda is None or not cuda.is_available():
|
| 598 |
+
raise RuntimeError(
|
| 599 |
+
"Amodal3R visual 3D requires an available CUDA GPU for learned "
|
| 600 |
+
"generation and diff_gaussian_rasterization. Run this command inside "
|
| 601 |
+
"a Slurm GPU allocation."
|
| 602 |
+
)
|
| 603 |
+
device_count = getattr(cuda, "device_count", lambda: 1)()
|
| 604 |
+
torch_version = getattr(torch_module, "__version__", None)
|
| 605 |
+
version_namespace = getattr(torch_module, "version", None)
|
| 606 |
+
return {
|
| 607 |
+
"required": True,
|
| 608 |
+
"available": True,
|
| 609 |
+
"device_type": "cuda",
|
| 610 |
+
"device_count": int(device_count),
|
| 611 |
+
"torch_version": str(torch_version) if torch_version is not None else None,
|
| 612 |
+
"torch_cuda_build": (
|
| 613 |
+
str(getattr(version_namespace, "cuda"))
|
| 614 |
+
if version_namespace is not None
|
| 615 |
+
and getattr(version_namespace, "cuda", None) is not None
|
| 616 |
+
else None
|
| 617 |
+
),
|
| 618 |
+
}
|
| 619 |
+
|
| 620 |
+
|
| 621 |
+
def require_gpu_renderers(
|
| 622 |
+
*,
|
| 623 |
+
allow_gaussian_only: bool,
|
| 624 |
+
find_spec=importlib.util.find_spec,
|
| 625 |
+
import_module=importlib.import_module,
|
| 626 |
+
) -> dict[str, Any]:
|
| 627 |
+
"""Preflight the CUDA rasterizers before loading the Amodal3R weights."""
|
| 628 |
+
|
| 629 |
+
gaussian_available = find_spec("diff_gaussian_rasterization") is not None
|
| 630 |
+
if not gaussian_available:
|
| 631 |
+
raise RuntimeError(
|
| 632 |
+
"Amodal3R requires diff_gaussian_rasterization for the primary "
|
| 633 |
+
"CUDA rotating render."
|
| 634 |
+
)
|
| 635 |
+
try:
|
| 636 |
+
import_module("diff_gaussian_rasterization")
|
| 637 |
+
except Exception as exc:
|
| 638 |
+
raise RuntimeError(
|
| 639 |
+
"diff_gaussian_rasterization is installed but its CUDA extension "
|
| 640 |
+
"could not be imported."
|
| 641 |
+
) from exc
|
| 642 |
+
mesh_available = find_spec("nvdiffrast") is not None
|
| 643 |
+
if not mesh_available and not allow_gaussian_only:
|
| 644 |
+
raise RuntimeError(
|
| 645 |
+
"Full GPU rendering requires nvdiffrast for the dense FlexiCubes "
|
| 646 |
+
"mesh rotation. Install the project --nvdiffrast dependency, or use "
|
| 647 |
+
"--allow-gaussian-only only for an explicit debug run."
|
| 648 |
+
)
|
| 649 |
+
mesh_context_ready = False
|
| 650 |
+
if mesh_available:
|
| 651 |
+
try:
|
| 652 |
+
dr = import_module("nvdiffrast.torch")
|
| 653 |
+
context = dr.RasterizeCudaContext(device="cuda")
|
| 654 |
+
del context
|
| 655 |
+
mesh_context_ready = True
|
| 656 |
+
except Exception as exc:
|
| 657 |
+
raise RuntimeError(
|
| 658 |
+
"nvdiffrast is installed but its CUDA raster context could not "
|
| 659 |
+
"be created on the allocated GPU."
|
| 660 |
+
) from exc
|
| 661 |
+
return {
|
| 662 |
+
"gaussian": {
|
| 663 |
+
"package": "diff_gaussian_rasterization",
|
| 664 |
+
"available": gaussian_available,
|
| 665 |
+
"required": True,
|
| 666 |
+
"device": "cuda",
|
| 667 |
+
"runtime_import_succeeded": True,
|
| 668 |
+
},
|
| 669 |
+
"dense_mesh": {
|
| 670 |
+
"package": "nvdiffrast",
|
| 671 |
+
"available": mesh_available,
|
| 672 |
+
"required": not allow_gaussian_only,
|
| 673 |
+
"device": "cuda" if mesh_available else None,
|
| 674 |
+
"runtime_import_succeeded": mesh_available,
|
| 675 |
+
"cuda_context_preflight_succeeded": mesh_context_ready,
|
| 676 |
+
},
|
| 677 |
+
"cpu_render_fallback_allowed": False,
|
| 678 |
+
"gaussian_only_debug_mode": bool(allow_gaussian_only),
|
| 679 |
+
}
|
| 680 |
+
|
| 681 |
+
|
| 682 |
+
def write_gif_atomic(path: Path, frames: list[np.ndarray], *, fps: int) -> None:
|
| 683 |
+
temporary = path.with_name(f".{path.stem}.{os.getpid()}.tmp.gif")
|
| 684 |
+
try:
|
| 685 |
+
imageio.mimsave(
|
| 686 |
+
temporary,
|
| 687 |
+
frames,
|
| 688 |
+
duration=1000.0 / float(fps),
|
| 689 |
+
loop=0,
|
| 690 |
+
)
|
| 691 |
+
os.replace(temporary, path)
|
| 692 |
+
finally:
|
| 693 |
+
temporary.unlink(missing_ok=True)
|
| 694 |
+
|
| 695 |
+
|
| 696 |
+
def copy_file_atomic(source: Path, destination: Path) -> None:
|
| 697 |
+
"""Atomically materialize a byte-identical compatibility artifact."""
|
| 698 |
+
|
| 699 |
+
temporary = destination.with_name(
|
| 700 |
+
f".{destination.stem}.{os.getpid()}.tmp{destination.suffix}"
|
| 701 |
+
)
|
| 702 |
+
try:
|
| 703 |
+
shutil.copyfile(source, temporary)
|
| 704 |
+
os.replace(temporary, destination)
|
| 705 |
+
finally:
|
| 706 |
+
temporary.unlink(missing_ok=True)
|
| 707 |
+
|
| 708 |
+
|
| 709 |
+
def inspect_gif(path: Path) -> dict[str, Any]:
|
| 710 |
+
with Image.open(path) as image:
|
| 711 |
+
return {
|
| 712 |
+
"path": path.name,
|
| 713 |
+
"frames": int(getattr(image, "n_frames", 1)),
|
| 714 |
+
"width": int(image.width),
|
| 715 |
+
"height": int(image.height),
|
| 716 |
+
}
|
| 717 |
+
|
| 718 |
+
|
| 719 |
+
def validate_render_artifacts(
|
| 720 |
+
*,
|
| 721 |
+
output_dir: Path,
|
| 722 |
+
video_frames: int,
|
| 723 |
+
nviews: int,
|
| 724 |
+
dense_mesh_rendered: bool,
|
| 725 |
+
mesh_face_count: int,
|
| 726 |
+
) -> dict[str, Any]:
|
| 727 |
+
"""Validate the full-GPU publication contract before writing manifest.json."""
|
| 728 |
+
|
| 729 |
+
gaussian = inspect_gif(output_dir / "sample_gaussian.gif")
|
| 730 |
+
combined = inspect_gif(output_dir / "sample_multi.gif")
|
| 731 |
+
if gaussian["frames"] != video_frames or combined["frames"] != video_frames:
|
| 732 |
+
raise RuntimeError(
|
| 733 |
+
"GPU GIF frame-count mismatch: "
|
| 734 |
+
f"gaussian={gaussian['frames']}, combined={combined['frames']}, "
|
| 735 |
+
f"expected={video_frames}"
|
| 736 |
+
)
|
| 737 |
+
# The 9527 reference contract uses sample_multi.gif as a compatibility
|
| 738 |
+
# alias for the source-colored Gaussian render. Dense-mesh normal maps are
|
| 739 |
+
# diagnostics and must never widen or recolor the primary presentation.
|
| 740 |
+
expected_combined_width = gaussian["width"]
|
| 741 |
+
if (
|
| 742 |
+
combined["width"] != expected_combined_width
|
| 743 |
+
or combined["height"] != gaussian["height"]
|
| 744 |
+
):
|
| 745 |
+
raise RuntimeError(
|
| 746 |
+
"Reference-compatible GIF dimensions do not match the Gaussian render"
|
| 747 |
+
)
|
| 748 |
+
if sha256_file(output_dir / "sample_gaussian.gif") != sha256_file(
|
| 749 |
+
output_dir / "sample_multi.gif"
|
| 750 |
+
):
|
| 751 |
+
raise RuntimeError(
|
| 752 |
+
"sample_multi.gif must be a byte-identical compatibility alias of "
|
| 753 |
+
"the source-colored sample_gaussian.gif"
|
| 754 |
+
)
|
| 755 |
+
gaussian_views = [
|
| 756 |
+
output_dir / f"{index:03d}_gs.png" for index in range(nviews)
|
| 757 |
+
]
|
| 758 |
+
if not all(path.is_file() and path.stat().st_size > 0 for path in gaussian_views):
|
| 759 |
+
raise RuntimeError("GPU Gaussian multiview output is incomplete")
|
| 760 |
+
|
| 761 |
+
mesh_gif = None
|
| 762 |
+
mesh_views: list[Path] = []
|
| 763 |
+
if dense_mesh_rendered:
|
| 764 |
+
mesh_gif = inspect_gif(output_dir / "sample_mesh.gif")
|
| 765 |
+
if (
|
| 766 |
+
mesh_gif["frames"] != video_frames
|
| 767 |
+
or mesh_gif["width"] != gaussian["width"]
|
| 768 |
+
or mesh_gif["height"] != gaussian["height"]
|
| 769 |
+
):
|
| 770 |
+
raise RuntimeError("GPU dense-mesh GIF does not align with Gaussian GIF")
|
| 771 |
+
mesh_views = [
|
| 772 |
+
output_dir / f"{index:03d}_mesh.png" for index in range(nviews)
|
| 773 |
+
]
|
| 774 |
+
if not all(path.is_file() and path.stat().st_size > 0 for path in mesh_views):
|
| 775 |
+
raise RuntimeError("GPU dense-mesh multiview output is incomplete")
|
| 776 |
+
if mesh_face_count <= 0:
|
| 777 |
+
raise RuntimeError("FlexiCubes output contains no triangle faces")
|
| 778 |
+
contact_sheet = output_dir / "multiview_contact_sheet.jpg"
|
| 779 |
+
if not contact_sheet.is_file() or contact_sheet.stat().st_size <= 0:
|
| 780 |
+
raise RuntimeError("GPU multiview contact sheet is missing")
|
| 781 |
+
with Image.open(contact_sheet) as sheet:
|
| 782 |
+
expected_sheet_size = (
|
| 783 |
+
gaussian["width"] * 2,
|
| 784 |
+
gaussian["height"] * ((nviews + 1) // 2),
|
| 785 |
+
)
|
| 786 |
+
if sheet.size != expected_sheet_size:
|
| 787 |
+
raise RuntimeError(
|
| 788 |
+
"Primary contact sheet must use only source-colored Gaussian "
|
| 789 |
+
f"views in a 2-column layout: got={sheet.size}, "
|
| 790 |
+
f"expected={expected_sheet_size}"
|
| 791 |
+
)
|
| 792 |
+
return {
|
| 793 |
+
"validated": True,
|
| 794 |
+
"gaussian_gif": gaussian,
|
| 795 |
+
"mesh_gif": mesh_gif,
|
| 796 |
+
"combined_gif": combined,
|
| 797 |
+
"combined_gif_is_byte_identical_gaussian_alias": True,
|
| 798 |
+
"contact_sheet_layout": "gaussian_color_only_2_columns",
|
| 799 |
+
"gaussian_view_count": len(gaussian_views),
|
| 800 |
+
"mesh_view_count": len(mesh_views),
|
| 801 |
+
"dense_mesh_rendered_on_gpu": dense_mesh_rendered,
|
| 802 |
+
"mesh_face_count": int(mesh_face_count),
|
| 803 |
+
"cpu_render_fallback_used": False,
|
| 804 |
+
}
|
| 805 |
+
|
| 806 |
+
|
| 807 |
+
def build_output_manifest(
|
| 808 |
+
*,
|
| 809 |
+
args: argparse.Namespace,
|
| 810 |
+
input_provenance: dict[str, Any],
|
| 811 |
+
mask_source: str | Path,
|
| 812 |
+
mask_statistics: dict[str, Any],
|
| 813 |
+
cuda_runtime: dict[str, Any],
|
| 814 |
+
gpu_renderer_runtime: dict[str, Any],
|
| 815 |
+
render_validation: dict[str, Any],
|
| 816 |
+
mesh_gif: str | None,
|
| 817 |
+
nvdiffrast_available: bool,
|
| 818 |
+
mesh_renderer_mode: str,
|
| 819 |
+
glb_export_mode: str | None,
|
| 820 |
+
) -> dict[str, Any]:
|
| 821 |
+
"""Build the portable visual-candidate manifest and evidence policy."""
|
| 822 |
+
|
| 823 |
+
return {
|
| 824 |
+
"schema_version": "accessibilityamodal_visual_3d_candidate_v1",
|
| 825 |
+
"created_at_utc": datetime.now(timezone.utc).isoformat(),
|
| 826 |
+
"input_provenance": input_provenance,
|
| 827 |
+
"three_value_mask": portable_file_record(resolve_path(mask_source)),
|
| 828 |
+
"three_value_mask_statistics": dict(mask_statistics),
|
| 829 |
+
"third_party_backend_provenance": {
|
| 830 |
+
"name": "Amodal3R",
|
| 831 |
+
"python_package": "amodal3d",
|
| 832 |
+
"pipeline_class": "Amodal3RImageTo3DPipeline",
|
| 833 |
+
"model_identifier_or_path": args.model,
|
| 834 |
+
"upstream_identity_preserved": True,
|
| 835 |
+
},
|
| 836 |
+
"representation_contract": {
|
| 837 |
+
"primary_representation": "Amodal3R Gaussian",
|
| 838 |
+
"primary_render_backend": "CUDA diff_gaussian_rasterization",
|
| 839 |
+
"primary_render_device": "cuda",
|
| 840 |
+
"primary_output": "sample_gaussian.gif",
|
| 841 |
+
"reference_compatible_output": "sample_multi.gif",
|
| 842 |
+
"reference_compatible_output_matches_primary": True,
|
| 843 |
+
"primary_appearance": (
|
| 844 |
+
"quality_gated_obstacle_removed_color_preserving_original_texture"
|
| 845 |
+
if input_provenance["conditioning_rgb_mode"] == "completed"
|
| 846 |
+
else "source_rgb_conditioned_color"
|
| 847 |
+
),
|
| 848 |
+
"appearance_conditioning_mode": input_provenance[
|
| 849 |
+
"conditioning_rgb_mode"
|
| 850 |
+
],
|
| 851 |
+
"primary_contact_sheet": "multiview_contact_sheet.jpg",
|
| 852 |
+
"primary_contact_sheet_content": "gaussian_color_only",
|
| 853 |
+
"paired_dense_geometry": "FlexiCubes triangle mesh",
|
| 854 |
+
"paired_dense_geometry_output": "mesh.ply",
|
| 855 |
+
"paired_dense_geometry_render": (
|
| 856 |
+
"sample_mesh.gif" if nvdiffrast_available else None
|
| 857 |
+
),
|
| 858 |
+
"paired_dense_geometry_render_role": (
|
| 859 |
+
"diagnostic_normal_map_only_not_source_rgb_texture"
|
| 860 |
+
if nvdiffrast_available
|
| 861 |
+
else None
|
| 862 |
+
),
|
| 863 |
+
"interactive_textured_surface": (
|
| 864 |
+
"mesh.glb" if args.export_glb else None
|
| 865 |
+
),
|
| 866 |
+
"interactive_texture_source": (
|
| 867 |
+
"GPU Gaussian appearance baked to UV/PBR base-color texture"
|
| 868 |
+
if args.export_glb
|
| 869 |
+
else None
|
| 870 |
+
),
|
| 871 |
+
"vggt_is_primary": False,
|
| 872 |
+
"discrete_point_cloud_is_primary": False,
|
| 873 |
+
},
|
| 874 |
+
"cuda_runtime": dict(cuda_runtime),
|
| 875 |
+
"gpu_renderer_runtime": dict(gpu_renderer_runtime),
|
| 876 |
+
"render_validation": dict(render_validation),
|
| 877 |
+
"seed": args.seed,
|
| 878 |
+
"nviews": args.nviews,
|
| 879 |
+
"video_frames": args.video_frames,
|
| 880 |
+
"output_kind": (
|
| 881 |
+
"AccessibilityAmodal learned visual candidate via the licensed "
|
| 882 |
+
"third-party Amodal3R backend"
|
| 883 |
+
),
|
| 884 |
+
"outputs": {
|
| 885 |
+
"combined_gif": "sample_multi.gif",
|
| 886 |
+
"gaussian_gif": "sample_gaussian.gif",
|
| 887 |
+
"mesh_gif": mesh_gif,
|
| 888 |
+
"mesh_normal_diagnostic_gif": mesh_gif,
|
| 889 |
+
"contact_sheet": "multiview_contact_sheet.jpg",
|
| 890 |
+
"mesh": "mesh.ply",
|
| 891 |
+
"glb": "mesh.glb" if args.export_glb else None,
|
| 892 |
+
"conditioning_rgb_model_input": "conditioning_rgb_model_input.png",
|
| 893 |
+
"condition_mask_model_input": "condition_mask_model_input.png",
|
| 894 |
+
},
|
| 895 |
+
"nvdiffrast_available": nvdiffrast_available,
|
| 896 |
+
"mesh_renderer_mode": mesh_renderer_mode,
|
| 897 |
+
"glb_export_mode": glb_export_mode,
|
| 898 |
+
"metric_geometry": False,
|
| 899 |
+
"passability_evidence": False,
|
| 900 |
+
"automatic_passability_claim": False,
|
| 901 |
+
"human_review_required": True,
|
| 902 |
+
"evidence_policy": {
|
| 903 |
+
"role": "learned_visual_candidate_only",
|
| 904 |
+
"metric_geometry": False,
|
| 905 |
+
"passability_evidence": False,
|
| 906 |
+
"automatic_passability_claim": False,
|
| 907 |
+
"warning": (
|
| 908 |
+
"This learned visual 3D candidate is not calibrated geometry and "
|
| 909 |
+
"must not be used to decide whether a person can pass."
|
| 910 |
+
),
|
| 911 |
+
},
|
| 912 |
+
}
|
| 913 |
+
|
| 914 |
+
|
| 915 |
+
def run(args):
|
| 916 |
+
output_dir = Path(args.output_dir)
|
| 917 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 918 |
+
# manifest.json is the success marker. Remove stale primary markers before a
|
| 919 |
+
# rerun so a failed GPU mesh pass cannot be mistaken for a complete result.
|
| 920 |
+
for filename in (
|
| 921 |
+
"manifest.json",
|
| 922 |
+
"sample_multi.gif",
|
| 923 |
+
"sample_mesh.gif",
|
| 924 |
+
"mesh.glb",
|
| 925 |
+
):
|
| 926 |
+
(output_dir / filename).unlink(missing_ok=True)
|
| 927 |
+
(
|
| 928 |
+
image,
|
| 929 |
+
mask,
|
| 930 |
+
mask_source,
|
| 931 |
+
mask_statistics,
|
| 932 |
+
backend_rgb_path,
|
| 933 |
+
input_provenance,
|
| 934 |
+
) = load_inputs(args, output_dir)
|
| 935 |
+
image.save(output_dir / "conditioning_rgb_model_input.png")
|
| 936 |
+
mask.save(output_dir / "condition_mask_model_input.png")
|
| 937 |
+
cuda_runtime = require_cuda()
|
| 938 |
+
gpu_renderer_runtime = require_gpu_renderers(
|
| 939 |
+
allow_gaussian_only=args.allow_gaussian_only,
|
| 940 |
+
)
|
| 941 |
+
nvdiffrast_available = bool(
|
| 942 |
+
gpu_renderer_runtime["dense_mesh"]["available"]
|
| 943 |
+
)
|
| 944 |
+
print(f"Original image: {args.image}")
|
| 945 |
+
print(f"Learned visual backend RGB: {backend_rgb_path}")
|
| 946 |
+
print(f"Backend RGB role: {input_provenance['backend_rgb_role']}")
|
| 947 |
+
if input_provenance["completion_inputs_ignored"]:
|
| 948 |
+
print(
|
| 949 |
+
"Completion RGB arguments were supplied but ignored because "
|
| 950 |
+
"--conditioning-rgb defaults to original."
|
| 951 |
+
)
|
| 952 |
+
print(f"Mask: {mask_source}")
|
| 953 |
+
print(f"Output dir: {output_dir}")
|
| 954 |
+
|
| 955 |
+
pipeline_class, render_utils = load_backend_runtime()
|
| 956 |
+
pipeline = pipeline_class.from_pretrained(args.model)
|
| 957 |
+
pipeline.cuda()
|
| 958 |
+
outputs = pipeline.run_multi_image(
|
| 959 |
+
[image],
|
| 960 |
+
[mask],
|
| 961 |
+
seed=args.seed,
|
| 962 |
+
sparse_structure_sampler_params={"steps": args.ss_steps, "cfg_strength": args.ss_cfg},
|
| 963 |
+
slat_sampler_params={"steps": args.slat_steps, "cfg_strength": args.slat_cfg},
|
| 964 |
+
erode_kernel_size=args.erode_kernel_size,
|
| 965 |
+
)
|
| 966 |
+
|
| 967 |
+
video_gs = render_utils.render_video(
|
| 968 |
+
outputs["gaussian"][0],
|
| 969 |
+
bg_color=(1, 1, 1),
|
| 970 |
+
num_frames=args.video_frames,
|
| 971 |
+
)["color"]
|
| 972 |
+
write_gif_atomic(output_dir / "sample_gaussian.gif", video_gs, fps=24)
|
| 973 |
+
gaussian = outputs["gaussian"][0]
|
| 974 |
+
multi_view_gs, _, _ = render_utils.render_multiview(
|
| 975 |
+
gaussian, nviews=args.nviews, bg_color=(1, 1, 1)
|
| 976 |
+
)
|
| 977 |
+
mesh = outputs["mesh"][0]
|
| 978 |
+
for index, output in enumerate(multi_view_gs["color"]):
|
| 979 |
+
output = cv2.cvtColor(output, cv2.COLOR_RGB2BGR)
|
| 980 |
+
cv2.imwrite(str(output_dir / f"{index:03d}_gs.png"), output)
|
| 981 |
+
mesh_path = output_dir / "mesh.ply"
|
| 982 |
+
save_mesh(mesh, mesh_path)
|
| 983 |
+
mesh_face_count = int(mesh.faces.shape[0])
|
| 984 |
+
|
| 985 |
+
mesh_gif: str | None = None
|
| 986 |
+
mesh_renderer_mode = "skipped: explicit Gaussian-only debug mode"
|
| 987 |
+
if nvdiffrast_available:
|
| 988 |
+
video_mesh = render_utils.render_video(
|
| 989 |
+
mesh,
|
| 990 |
+
bg_color=(1, 1, 1),
|
| 991 |
+
num_frames=args.video_frames,
|
| 992 |
+
)["normal"]
|
| 993 |
+
write_gif_atomic(output_dir / "sample_mesh.gif", video_mesh, fps=24)
|
| 994 |
+
copy_file_atomic(
|
| 995 |
+
output_dir / "sample_gaussian.gif",
|
| 996 |
+
output_dir / "sample_multi.gif",
|
| 997 |
+
)
|
| 998 |
+
multi_view_mesh, _, _ = render_utils.render_multiview(
|
| 999 |
+
mesh, nviews=args.nviews, bg_color=(1, 1, 1)
|
| 1000 |
+
)
|
| 1001 |
+
for index, output in enumerate(multi_view_mesh["normal"]):
|
| 1002 |
+
output = cv2.cvtColor(output, cv2.COLOR_RGB2BGR)
|
| 1003 |
+
cv2.imwrite(str(output_dir / f"{index:03d}_mesh.png"), output)
|
| 1004 |
+
previews = list(multi_view_gs["color"])
|
| 1005 |
+
mesh_gif = "sample_mesh.gif"
|
| 1006 |
+
mesh_renderer_mode = "nvdiffrast normal-map diagnostic rendering"
|
| 1007 |
+
else:
|
| 1008 |
+
previews = list(multi_view_gs["color"])
|
| 1009 |
+
copy_file_atomic(
|
| 1010 |
+
output_dir / "sample_gaussian.gif",
|
| 1011 |
+
output_dir / "sample_multi.gif",
|
| 1012 |
+
)
|
| 1013 |
+
|
| 1014 |
+
rows = []
|
| 1015 |
+
for start in range(0, len(previews), 2):
|
| 1016 |
+
row = previews[start : start + 2]
|
| 1017 |
+
if len(row) == 1:
|
| 1018 |
+
row.append(np.full_like(row[0], 255))
|
| 1019 |
+
rows.append(np.concatenate(row, axis=1))
|
| 1020 |
+
contact_sheet = np.concatenate(rows, axis=0)
|
| 1021 |
+
contact_sheet_path = output_dir / "multiview_contact_sheet.jpg"
|
| 1022 |
+
temporary_contact_sheet = contact_sheet_path.with_name(
|
| 1023 |
+
f".{contact_sheet_path.stem}.{os.getpid()}.tmp.jpg"
|
| 1024 |
+
)
|
| 1025 |
+
try:
|
| 1026 |
+
Image.fromarray(contact_sheet).save(temporary_contact_sheet, quality=92)
|
| 1027 |
+
os.replace(temporary_contact_sheet, contact_sheet_path)
|
| 1028 |
+
finally:
|
| 1029 |
+
temporary_contact_sheet.unlink(missing_ok=True)
|
| 1030 |
+
glb_export_mode = None
|
| 1031 |
+
if args.export_glb:
|
| 1032 |
+
if nvdiffrast_available:
|
| 1033 |
+
extract_glb(
|
| 1034 |
+
outputs["gaussian"][0],
|
| 1035 |
+
outputs["mesh"][0],
|
| 1036 |
+
mesh_simplify=args.mesh_simplify,
|
| 1037 |
+
texture_size=args.texture_size,
|
| 1038 |
+
export_path=str(output_dir / "mesh.glb"),
|
| 1039 |
+
)
|
| 1040 |
+
glb_export_mode = "Amodal3R textured GLB"
|
| 1041 |
+
else:
|
| 1042 |
+
raise RuntimeError(
|
| 1043 |
+
"--export-glb requires nvdiffrast; CPU GLB fallback is disabled "
|
| 1044 |
+
"by the full-GPU rendering contract."
|
| 1045 |
+
)
|
| 1046 |
+
render_validation = validate_render_artifacts(
|
| 1047 |
+
output_dir=output_dir,
|
| 1048 |
+
video_frames=args.video_frames,
|
| 1049 |
+
nviews=args.nviews,
|
| 1050 |
+
dense_mesh_rendered=nvdiffrast_available,
|
| 1051 |
+
mesh_face_count=mesh_face_count,
|
| 1052 |
+
)
|
| 1053 |
+
manifest = build_output_manifest(
|
| 1054 |
+
args=args,
|
| 1055 |
+
input_provenance=input_provenance,
|
| 1056 |
+
mask_source=mask_source,
|
| 1057 |
+
mask_statistics=mask_statistics,
|
| 1058 |
+
cuda_runtime=cuda_runtime,
|
| 1059 |
+
gpu_renderer_runtime=gpu_renderer_runtime,
|
| 1060 |
+
render_validation=render_validation,
|
| 1061 |
+
mesh_gif=mesh_gif,
|
| 1062 |
+
nvdiffrast_available=nvdiffrast_available,
|
| 1063 |
+
mesh_renderer_mode=mesh_renderer_mode,
|
| 1064 |
+
glb_export_mode=glb_export_mode,
|
| 1065 |
+
)
|
| 1066 |
+
(output_dir / "manifest.json").write_text(
|
| 1067 |
+
json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8"
|
| 1068 |
+
)
|
| 1069 |
+
print("Done.")
|
| 1070 |
+
|
| 1071 |
+
|
| 1072 |
+
def build_parser():
|
| 1073 |
+
parser = argparse.ArgumentParser(
|
| 1074 |
+
description="Run the AccessibilityAmodal visual-3D backend adapter."
|
| 1075 |
+
)
|
| 1076 |
+
parser.add_argument(
|
| 1077 |
+
"--image",
|
| 1078 |
+
required=True,
|
| 1079 |
+
help=(
|
| 1080 |
+
"Canonical original RGB. It remains provenance/geometry source even "
|
| 1081 |
+
"when an accepted selected 2D completion drives this visual backend."
|
| 1082 |
+
),
|
| 1083 |
+
)
|
| 1084 |
+
parser.add_argument(
|
| 1085 |
+
"--conditioning-rgb",
|
| 1086 |
+
choices=("original", "completed"),
|
| 1087 |
+
default="original",
|
| 1088 |
+
help=(
|
| 1089 |
+
"RGB used to condition Amodal3R. Defaults to the canonical original. "
|
| 1090 |
+
"Use 'completed' explicitly to opt in to a quality-gated completion."
|
| 1091 |
+
),
|
| 1092 |
+
)
|
| 1093 |
+
parser.add_argument(
|
| 1094 |
+
"--completed-image",
|
| 1095 |
+
default=None,
|
| 1096 |
+
help=(
|
| 1097 |
+
"Selected 2D completion used only with --conditioning-rgb completed. "
|
| 1098 |
+
"Requires an accepted --completion-manifest."
|
| 1099 |
+
),
|
| 1100 |
+
)
|
| 1101 |
+
parser.add_argument(
|
| 1102 |
+
"--completion-manifest",
|
| 1103 |
+
default=None,
|
| 1104 |
+
help=(
|
| 1105 |
+
"2D manifest proving --completed-image is the selected candidate. "
|
| 1106 |
+
"Required only with --conditioning-rgb completed."
|
| 1107 |
+
),
|
| 1108 |
+
)
|
| 1109 |
+
parser.add_argument(
|
| 1110 |
+
"--mask",
|
| 1111 |
+
default=None,
|
| 1112 |
+
help="Optional three-value mask: white background, gray visible, black occluded.",
|
| 1113 |
+
)
|
| 1114 |
+
parser.add_argument(
|
| 1115 |
+
"--focus-crop-padding-ratio",
|
| 1116 |
+
type=float,
|
| 1117 |
+
default=None,
|
| 1118 |
+
help=(
|
| 1119 |
+
"Optionally crop/pad RGB and mask to a square around the modeled "
|
| 1120 |
+
"target before Amodal3R's 518x518 resize. This preserves source "
|
| 1121 |
+
"aspect ratio and makes small target surfaces more prominent."
|
| 1122 |
+
),
|
| 1123 |
+
)
|
| 1124 |
+
parser.add_argument("--output-dir", default="./output/accessibilityamodal/visual_candidate")
|
| 1125 |
+
parser.add_argument("--model", default="Sm0kyWu/Amodal3R")
|
| 1126 |
+
parser.add_argument("--occlusion-box", action="append", type=parse_box, default=[])
|
| 1127 |
+
parser.add_argument("--seed", type=int, default=1)
|
| 1128 |
+
parser.add_argument("--ss-steps", type=int, default=12)
|
| 1129 |
+
parser.add_argument("--ss-cfg", type=float, default=7.5)
|
| 1130 |
+
parser.add_argument("--slat-steps", type=int, default=12)
|
| 1131 |
+
parser.add_argument("--slat-cfg", type=float, default=3.0)
|
| 1132 |
+
parser.add_argument("--erode-kernel-size", type=int, default=3)
|
| 1133 |
+
parser.add_argument("--nviews", type=int, default=8)
|
| 1134 |
+
parser.add_argument("--video-frames", type=int, default=120)
|
| 1135 |
+
parser.add_argument(
|
| 1136 |
+
"--allow-gaussian-only",
|
| 1137 |
+
action="store_true",
|
| 1138 |
+
help=(
|
| 1139 |
+
"Explicit debug escape hatch when nvdiffrast is unavailable. "
|
| 1140 |
+
"The default requires both CUDA Gaussian and CUDA dense-mesh renders."
|
| 1141 |
+
),
|
| 1142 |
+
)
|
| 1143 |
+
parser.add_argument("--export-glb", action="store_true")
|
| 1144 |
+
parser.add_argument("--mesh-simplify", type=float, default=0.5)
|
| 1145 |
+
parser.add_argument("--texture-size", type=int, default=1024)
|
| 1146 |
+
return parser
|
| 1147 |
+
|
| 1148 |
+
|
| 1149 |
+
if __name__ == "__main__":
|
| 1150 |
+
run(build_parser().parse_args())
|
tools/accessibility_3d_variants.py
ADDED
|
@@ -0,0 +1,858 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Build face-rendered, solidified, and canonical 3D views for one result."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import hashlib
|
| 8 |
+
import json
|
| 9 |
+
import os
|
| 10 |
+
import sys
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from typing import Any
|
| 13 |
+
|
| 14 |
+
import cv2
|
| 15 |
+
import numpy as np
|
| 16 |
+
import trimesh
|
| 17 |
+
from PIL import Image, ImageDraw
|
| 18 |
+
|
| 19 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
| 20 |
+
sys.path.insert(0, str(PROJECT_ROOT))
|
| 21 |
+
|
| 22 |
+
from tools.build_accessibility_solid_mesh_showcase import ( # noqa: E402
|
| 23 |
+
CONTINUOUS_SURFACE_CATEGORIES,
|
| 24 |
+
build_canonical_staircase_prism_mesh,
|
| 25 |
+
build_continuous_stair_mesh,
|
| 26 |
+
build_solid_mesh,
|
| 27 |
+
clean_target_mask,
|
| 28 |
+
continuous_depth_for_category,
|
| 29 |
+
fit_plane,
|
| 30 |
+
plane_depth_for_pixels,
|
| 31 |
+
prepare_texture,
|
| 32 |
+
read_json,
|
| 33 |
+
read_mask,
|
| 34 |
+
render_mesh_frame,
|
| 35 |
+
trusted_stair_edges_from_manifest,
|
| 36 |
+
write_ply,
|
| 37 |
+
write_turntable_gif,
|
| 38 |
+
)
|
| 39 |
+
from accessibilityamodal.reconstruct import camera_intrinsics, pixels_to_points # noqa: E402
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def continuous_presentation_mask(
|
| 43 |
+
target_mask: np.ndarray,
|
| 44 |
+
*,
|
| 45 |
+
maximum_hull_expansion_ratio: float = 1.8,
|
| 46 |
+
) -> tuple[np.ndarray, dict[str, Any]]:
|
| 47 |
+
"""Return a hole-free, category-constrained surface presentation mask.
|
| 48 |
+
|
| 49 |
+
The semantic/review masks remain untouched on disk. This mask is used only
|
| 50 |
+
for the nonmetric solid visualization so that segmentation notches do not
|
| 51 |
+
become vertical person-shaped walls. A modest convex hull is accepted for
|
| 52 |
+
perspective walkway footprints; strongly concave routes keep their outer
|
| 53 |
+
contour and only close internal holes.
|
| 54 |
+
"""
|
| 55 |
+
|
| 56 |
+
target = np.asarray(target_mask, dtype=bool)
|
| 57 |
+
if target.ndim != 2:
|
| 58 |
+
raise ValueError("target_mask must be a 2D mask")
|
| 59 |
+
if maximum_hull_expansion_ratio < 1:
|
| 60 |
+
raise ValueError("maximum_hull_expansion_ratio must be at least 1")
|
| 61 |
+
pixels = int(target.sum())
|
| 62 |
+
if pixels == 0:
|
| 63 |
+
return target.copy(), {
|
| 64 |
+
"policy": "unchanged_empty_target",
|
| 65 |
+
"source_pixel_count": 0,
|
| 66 |
+
"presentation_pixel_count": 0,
|
| 67 |
+
"hull_expansion_ratio": None,
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
external = np.zeros_like(target, dtype=np.uint8)
|
| 71 |
+
contours, _ = cv2.findContours(
|
| 72 |
+
target.astype(np.uint8),
|
| 73 |
+
cv2.RETR_EXTERNAL,
|
| 74 |
+
cv2.CHAIN_APPROX_SIMPLE,
|
| 75 |
+
)
|
| 76 |
+
cv2.drawContours(external, contours, -1, 1, cv2.FILLED)
|
| 77 |
+
external_mask = external.astype(bool)
|
| 78 |
+
ys, xs = np.where(external_mask)
|
| 79 |
+
points = np.column_stack([xs, ys]).astype(np.int32)
|
| 80 |
+
hull = np.zeros_like(external, dtype=np.uint8)
|
| 81 |
+
if points.shape[0] >= 3:
|
| 82 |
+
cv2.fillConvexPoly(hull, cv2.convexHull(points), 1)
|
| 83 |
+
else:
|
| 84 |
+
hull[external_mask] = 1
|
| 85 |
+
hull_mask = hull.astype(bool)
|
| 86 |
+
hull_ratio = int(hull_mask.sum()) / max(int(external_mask.sum()), 1)
|
| 87 |
+
use_hull = hull_ratio <= maximum_hull_expansion_ratio
|
| 88 |
+
presentation = hull_mask if use_hull else external_mask
|
| 89 |
+
return presentation, {
|
| 90 |
+
"policy": (
|
| 91 |
+
"bounded_convex_support_presentation"
|
| 92 |
+
if use_hull
|
| 93 |
+
else "external_contour_hole_fill_due_to_ambiguous_concavity"
|
| 94 |
+
),
|
| 95 |
+
"source_pixel_count": pixels,
|
| 96 |
+
"external_contour_pixel_count": int(external_mask.sum()),
|
| 97 |
+
"presentation_pixel_count": int(presentation.sum()),
|
| 98 |
+
"hull_expansion_ratio": round(hull_ratio, 8),
|
| 99 |
+
"maximum_hull_expansion_ratio": float(maximum_hull_expansion_ratio),
|
| 100 |
+
"semantic_mask_unchanged": True,
|
| 101 |
+
"metric_geometry": False,
|
| 102 |
+
"human_review_required": True,
|
| 103 |
+
}
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def idealized_continuous_presentation_depth(
|
| 107 |
+
depth: np.ndarray,
|
| 108 |
+
presentation_mask: np.ndarray,
|
| 109 |
+
visible_mask: np.ndarray | None,
|
| 110 |
+
) -> tuple[np.ndarray, dict[str, Any]]:
|
| 111 |
+
"""Fit one bounded support plane for a clean, nonmetric solid preview."""
|
| 112 |
+
|
| 113 |
+
source = np.asarray(depth, dtype=np.float32)
|
| 114 |
+
target = np.asarray(presentation_mask, dtype=bool)
|
| 115 |
+
visible = target if visible_mask is None else (
|
| 116 |
+
np.asarray(visible_mask, dtype=bool) & target
|
| 117 |
+
)
|
| 118 |
+
h, w = source.shape
|
| 119 |
+
fx, fy, cx, cy = camera_intrinsics(w, h, None, None, None, None)
|
| 120 |
+
points = pixels_to_points(source, fx, fy, cx, cy)
|
| 121 |
+
fit_mask = visible & np.isfinite(source) & (source > 0)
|
| 122 |
+
plane = fit_plane(points[fit_mask])
|
| 123 |
+
if plane is None:
|
| 124 |
+
fallback, metadata = continuous_depth_for_category(
|
| 125 |
+
source,
|
| 126 |
+
target,
|
| 127 |
+
visible_mask,
|
| 128 |
+
"walkway",
|
| 129 |
+
)
|
| 130 |
+
return fallback, {
|
| 131 |
+
**metadata,
|
| 132 |
+
"presentation_depth_policy": "fallback_continuous_depth_plane_fit_failed",
|
| 133 |
+
"human_review_required": True,
|
| 134 |
+
}
|
| 135 |
+
|
| 136 |
+
observed = source[fit_mask]
|
| 137 |
+
low, high = np.percentile(observed, [1, 99])
|
| 138 |
+
lower_bound = max(1e-4, 0.50 * float(low))
|
| 139 |
+
upper_bound = max(lower_bound + 1e-4, 1.50 * float(high))
|
| 140 |
+
plane_depth = plane_depth_for_pixels(
|
| 141 |
+
source.shape,
|
| 142 |
+
plane,
|
| 143 |
+
fx,
|
| 144 |
+
fy,
|
| 145 |
+
cx,
|
| 146 |
+
cy,
|
| 147 |
+
)
|
| 148 |
+
invalid = target & (~np.isfinite(plane_depth) | (plane_depth <= 0))
|
| 149 |
+
bounded_plane = np.clip(plane_depth, lower_bound, upper_bound)
|
| 150 |
+
result = source.copy()
|
| 151 |
+
result[target & ~invalid] = bounded_plane[target & ~invalid]
|
| 152 |
+
if invalid.any():
|
| 153 |
+
result[invalid] = float(np.median(observed))
|
| 154 |
+
clamped = target & (
|
| 155 |
+
(plane_depth < lower_bound)
|
| 156 |
+
| (plane_depth > upper_bound)
|
| 157 |
+
)
|
| 158 |
+
return result, {
|
| 159 |
+
"surface_model": "bounded_robust_single_plane_presentation",
|
| 160 |
+
"fit_points": int(fit_mask.sum()),
|
| 161 |
+
"plane_normal": [float(value) for value in plane[0]],
|
| 162 |
+
"plane_d": float(plane[1]),
|
| 163 |
+
"presentation_pixels": int(target.sum()),
|
| 164 |
+
"invalid_plane_pixels_replaced": int(invalid.sum()),
|
| 165 |
+
"bounded_plane_pixels": int(clamped.sum()),
|
| 166 |
+
"depth_bounds": [lower_bound, upper_bound],
|
| 167 |
+
"presentation_depth_policy": (
|
| 168 |
+
"visible_target_plane_extended_over_nonmetric_presentation_mask"
|
| 169 |
+
),
|
| 170 |
+
"metric_geometry": False,
|
| 171 |
+
"safe_passage_claim": False,
|
| 172 |
+
"human_review_required": True,
|
| 173 |
+
}
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def load_ply(path: Path) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
| 177 |
+
mesh = trimesh.load(path, process=False)
|
| 178 |
+
if not isinstance(mesh, trimesh.Trimesh):
|
| 179 |
+
raise ValueError(f"Expected one triangle mesh in {path}")
|
| 180 |
+
vertices = np.asarray(mesh.vertices, dtype=np.float32)
|
| 181 |
+
faces = np.asarray(mesh.faces, dtype=np.int64)
|
| 182 |
+
vertex_colors = np.asarray(mesh.visual.vertex_colors)
|
| 183 |
+
if vertex_colors.ndim == 2 and vertex_colors.shape[0] == len(vertices):
|
| 184 |
+
colors = vertex_colors[:, :3].astype(np.uint8)
|
| 185 |
+
else:
|
| 186 |
+
colors = np.full((len(vertices), 3), (190, 190, 185), dtype=np.uint8)
|
| 187 |
+
return vertices, colors, faces
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def export_glb(
|
| 191 |
+
path: Path,
|
| 192 |
+
vertices: np.ndarray,
|
| 193 |
+
colors: np.ndarray,
|
| 194 |
+
faces: np.ndarray,
|
| 195 |
+
) -> dict[str, Any]:
|
| 196 |
+
rgba = np.concatenate(
|
| 197 |
+
[colors.astype(np.uint8), np.full((len(colors), 1), 255, dtype=np.uint8)],
|
| 198 |
+
axis=1,
|
| 199 |
+
)
|
| 200 |
+
mesh = trimesh.Trimesh(
|
| 201 |
+
vertices=vertices,
|
| 202 |
+
faces=faces,
|
| 203 |
+
vertex_colors=rgba,
|
| 204 |
+
process=False,
|
| 205 |
+
)
|
| 206 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 207 |
+
mesh.export(path)
|
| 208 |
+
return {
|
| 209 |
+
"watertight": bool(mesh.is_watertight),
|
| 210 |
+
"euler_number": int(mesh.euler_number),
|
| 211 |
+
"bounds": np.asarray(mesh.bounds).round(6).tolist(),
|
| 212 |
+
}
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def save_variant(
|
| 216 |
+
output_dir: Path,
|
| 217 |
+
name: str,
|
| 218 |
+
title: str,
|
| 219 |
+
vertices: np.ndarray,
|
| 220 |
+
colors: np.ndarray,
|
| 221 |
+
faces: np.ndarray,
|
| 222 |
+
*,
|
| 223 |
+
sample_id: str,
|
| 224 |
+
category: str,
|
| 225 |
+
frames: int,
|
| 226 |
+
duration_ms: int,
|
| 227 |
+
size: tuple[int, int],
|
| 228 |
+
pitch: float,
|
| 229 |
+
source_ply: Path | None = None,
|
| 230 |
+
render_outputs: bool = True,
|
| 231 |
+
) -> dict[str, Any]:
|
| 232 |
+
variant_dir = output_dir / name
|
| 233 |
+
variant_dir.mkdir(parents=True, exist_ok=True)
|
| 234 |
+
ply_path = source_ply or variant_dir / f"{name}.ply"
|
| 235 |
+
if source_ply is None:
|
| 236 |
+
write_ply(ply_path, vertices, colors, faces)
|
| 237 |
+
gif_path = variant_dir / f"{name}_slow.gif"
|
| 238 |
+
preview_path = variant_dir / f"{name}_preview.jpg"
|
| 239 |
+
glb_path = variant_dir / f"{name}.glb"
|
| 240 |
+
if render_outputs:
|
| 241 |
+
write_turntable_gif(
|
| 242 |
+
gif_path,
|
| 243 |
+
vertices,
|
| 244 |
+
colors,
|
| 245 |
+
faces,
|
| 246 |
+
sample_id=sample_id,
|
| 247 |
+
category=f"{category} | {title}",
|
| 248 |
+
frames=frames,
|
| 249 |
+
size=size,
|
| 250 |
+
pitch=pitch,
|
| 251 |
+
duration_ms=duration_ms,
|
| 252 |
+
)
|
| 253 |
+
render_mesh_frame(
|
| 254 |
+
vertices,
|
| 255 |
+
colors,
|
| 256 |
+
faces,
|
| 257 |
+
size=size,
|
| 258 |
+
yaw=35.0,
|
| 259 |
+
pitch=pitch,
|
| 260 |
+
title=f"{sample_id} | {title}",
|
| 261 |
+
).save(preview_path, quality=94)
|
| 262 |
+
glb_info = export_glb(glb_path, vertices, colors, faces)
|
| 263 |
+
return {
|
| 264 |
+
"name": name,
|
| 265 |
+
"title": title,
|
| 266 |
+
"vertices": int(len(vertices)),
|
| 267 |
+
"faces": int(len(faces)),
|
| 268 |
+
"ply": str(ply_path),
|
| 269 |
+
"glb": str(glb_path),
|
| 270 |
+
"gif": str(gif_path) if render_outputs else None,
|
| 271 |
+
"preview": str(preview_path) if render_outputs else None,
|
| 272 |
+
**glb_info,
|
| 273 |
+
}
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
def build_contact_sheet(path: Path, rows: list[dict[str, Any]]) -> None:
|
| 277 |
+
previews = [Image.open(row["preview"]).convert("RGB") for row in rows]
|
| 278 |
+
width = max(image.width for image in previews)
|
| 279 |
+
label_height = 42
|
| 280 |
+
height = sum(image.height + label_height for image in previews)
|
| 281 |
+
sheet = Image.new("RGB", (width, height), "white")
|
| 282 |
+
draw = ImageDraw.Draw(sheet)
|
| 283 |
+
top = 0
|
| 284 |
+
for image, row in zip(previews, rows):
|
| 285 |
+
sheet.paste(image, (0, top + label_height))
|
| 286 |
+
draw.text(
|
| 287 |
+
(10, top + 10),
|
| 288 |
+
f"{row['title']} | {row['vertices']} vertices | {row['faces']} faces",
|
| 289 |
+
fill=(20, 20, 20),
|
| 290 |
+
)
|
| 291 |
+
top += image.height + label_height
|
| 292 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 293 |
+
sheet.save(path, quality=94)
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
def portable_path(path: str | Path, anchor: Path) -> str:
|
| 297 |
+
return Path(os.path.relpath(Path(path).resolve(), anchor.resolve())).as_posix()
|
| 298 |
+
|
| 299 |
+
|
| 300 |
+
def portable_variant_row(row: dict[str, Any], anchor: Path) -> dict[str, Any]:
|
| 301 |
+
portable = dict(row)
|
| 302 |
+
for key in ("ply", "glb", "gif", "preview"):
|
| 303 |
+
value = portable.get(key)
|
| 304 |
+
if value:
|
| 305 |
+
portable[key] = portable_path(value, anchor)
|
| 306 |
+
return portable
|
| 307 |
+
|
| 308 |
+
|
| 309 |
+
def sha256_file(path: Path) -> str:
|
| 310 |
+
digest = hashlib.sha256()
|
| 311 |
+
with path.open("rb") as handle:
|
| 312 |
+
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
| 313 |
+
digest.update(block)
|
| 314 |
+
return digest.hexdigest()
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
def portable_file_record(path: Path, anchor: Path) -> dict[str, Any]:
|
| 318 |
+
resolved = path.expanduser().resolve()
|
| 319 |
+
return {
|
| 320 |
+
"path": portable_path(resolved, anchor),
|
| 321 |
+
"sha256": sha256_file(resolved),
|
| 322 |
+
"bytes": int(resolved.stat().st_size),
|
| 323 |
+
}
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
def resolve_declared_path(value: Any, base_dir: Path) -> Path | None:
|
| 327 |
+
if not isinstance(value, str) or not value:
|
| 328 |
+
return None
|
| 329 |
+
path = Path(value).expanduser()
|
| 330 |
+
return path if path.is_absolute() else base_dir / path
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
def _portable_model_identity(value: Any) -> dict[str, Any] | None:
|
| 334 |
+
if not isinstance(value, dict):
|
| 335 |
+
return None
|
| 336 |
+
allowed = {
|
| 337 |
+
"model_id",
|
| 338 |
+
"family",
|
| 339 |
+
"encoder",
|
| 340 |
+
"metric_training_dataset",
|
| 341 |
+
"checkpoint_filename",
|
| 342 |
+
"checkpoint_sha256",
|
| 343 |
+
"selection_policy",
|
| 344 |
+
"license",
|
| 345 |
+
"third_party",
|
| 346 |
+
"revision",
|
| 347 |
+
"revision_review_required_before_public_release",
|
| 348 |
+
}
|
| 349 |
+
identity = {
|
| 350 |
+
key: item
|
| 351 |
+
for key, item in value.items()
|
| 352 |
+
if key in allowed and isinstance(item, (str, int, float, bool, type(None)))
|
| 353 |
+
}
|
| 354 |
+
return identity or None
|
| 355 |
+
|
| 356 |
+
|
| 357 |
+
def load_depth_context(
|
| 358 |
+
geometry_dir: Path,
|
| 359 |
+
) -> tuple[dict[str, Any], Path | None]:
|
| 360 |
+
"""Read the depth sidecar without promoting monocular scale to truth."""
|
| 361 |
+
|
| 362 |
+
candidates = (
|
| 363 |
+
geometry_dir.parent / "depth_manifest.json",
|
| 364 |
+
geometry_dir / "depth_manifest.json",
|
| 365 |
+
)
|
| 366 |
+
manifest_path = next((path for path in candidates if path.is_file()), None)
|
| 367 |
+
payload: dict[str, Any] = {}
|
| 368 |
+
if manifest_path is not None:
|
| 369 |
+
value = read_json(manifest_path)
|
| 370 |
+
if not isinstance(value, dict):
|
| 371 |
+
raise ValueError(f"Depth manifest must contain one JSON object: {manifest_path}")
|
| 372 |
+
payload = value
|
| 373 |
+
|
| 374 |
+
raw_model = payload.get("model")
|
| 375 |
+
model = raw_model if isinstance(raw_model, str) else None
|
| 376 |
+
if model and Path(model).is_absolute():
|
| 377 |
+
model = Path(model).name
|
| 378 |
+
checkpoint = payload.get("checkpoint")
|
| 379 |
+
checkpoint_path = resolve_declared_path(
|
| 380 |
+
checkpoint,
|
| 381 |
+
manifest_path.parent if manifest_path is not None else geometry_dir,
|
| 382 |
+
)
|
| 383 |
+
checkpoint_name = (
|
| 384 |
+
Path(checkpoint).name
|
| 385 |
+
if isinstance(checkpoint, str) and checkpoint
|
| 386 |
+
else None
|
| 387 |
+
)
|
| 388 |
+
checkpoint_sha256 = (
|
| 389 |
+
sha256_file(checkpoint_path)
|
| 390 |
+
if checkpoint_path is not None and checkpoint_path.is_file()
|
| 391 |
+
else None
|
| 392 |
+
)
|
| 393 |
+
model_identity = _portable_model_identity(payload.get("model_identity"))
|
| 394 |
+
if model_identity is None and model:
|
| 395 |
+
# Older depth manifests predate the structured identity object. Keep
|
| 396 |
+
# their declared model identifier instead of silently dropping it.
|
| 397 |
+
model_identity = {"model_id": model}
|
| 398 |
+
if (
|
| 399 |
+
model_identity is None
|
| 400 |
+
and payload.get("engine") == "depth_anything_v2"
|
| 401 |
+
and checkpoint_name == "depth_anything_v2_metric_hypersim_vitl.pth"
|
| 402 |
+
):
|
| 403 |
+
model_identity = {
|
| 404 |
+
"model_id": "Depth-Anything-V2-Metric-Hypersim-Large",
|
| 405 |
+
"family": "Depth Anything V2",
|
| 406 |
+
"encoder": payload.get("encoder") or "vitl",
|
| 407 |
+
"metric_training_dataset": "Hypersim",
|
| 408 |
+
"checkpoint_filename": checkpoint_name,
|
| 409 |
+
"checkpoint_sha256": checkpoint_sha256,
|
| 410 |
+
"selection_policy": (
|
| 411 |
+
"best_available_local_single_image_metric_depth_model"
|
| 412 |
+
),
|
| 413 |
+
}
|
| 414 |
+
metric_depth_value = payload.get("metric_depth")
|
| 415 |
+
if metric_depth_value not in (None, True, False):
|
| 416 |
+
raise ValueError(
|
| 417 |
+
"depth_manifest.metric_depth must be a JSON boolean or null: "
|
| 418 |
+
f"{manifest_path}"
|
| 419 |
+
)
|
| 420 |
+
metric_style = metric_depth_value is True
|
| 421 |
+
return {
|
| 422 |
+
"manifest_available": manifest_path is not None,
|
| 423 |
+
"engine": payload.get("engine"),
|
| 424 |
+
"model": model,
|
| 425 |
+
"model_identity": model_identity,
|
| 426 |
+
"checkpoint_filename": checkpoint_name,
|
| 427 |
+
"checkpoint_sha256": checkpoint_sha256,
|
| 428 |
+
"encoder": payload.get("encoder"),
|
| 429 |
+
"metric_depth_manifest_value": metric_depth_value,
|
| 430 |
+
"metric_style_depth": metric_style,
|
| 431 |
+
"maximum_model_depth": payload.get("max_depth") if metric_style else None,
|
| 432 |
+
"calibrated_metric_truth": False,
|
| 433 |
+
"scale_policy": (
|
| 434 |
+
"metric_style_monocular_output_not_calibrated_truth"
|
| 435 |
+
if metric_style
|
| 436 |
+
else "relative_or_unknown_depth_scale"
|
| 437 |
+
),
|
| 438 |
+
}, manifest_path
|
| 439 |
+
|
| 440 |
+
|
| 441 |
+
def build_argument_parser() -> argparse.ArgumentParser:
|
| 442 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 443 |
+
parser.add_argument("--geometry-dir", required=True)
|
| 444 |
+
parser.add_argument("--output-dir", required=True)
|
| 445 |
+
parser.add_argument("--sample-id", default="single_mesh")
|
| 446 |
+
parser.add_argument("--category", default="stairs")
|
| 447 |
+
parser.add_argument("--frames", type=int, default=72)
|
| 448 |
+
parser.add_argument("--duration-ms", type=int, default=140)
|
| 449 |
+
parser.add_argument("--width", type=int, default=640)
|
| 450 |
+
parser.add_argument("--height", type=int, default=480)
|
| 451 |
+
parser.add_argument("--pitch", type=float, default=-25.0)
|
| 452 |
+
parser.add_argument("--stride", type=int, default=6)
|
| 453 |
+
parser.add_argument("--thickness-ratio", type=float, default=0.12)
|
| 454 |
+
parser.add_argument("--stair-cross-samples", type=int, default=48)
|
| 455 |
+
parser.add_argument("--stair-step-count", type=int, default=0)
|
| 456 |
+
parser.add_argument(
|
| 457 |
+
"--texture-image",
|
| 458 |
+
default=None,
|
| 459 |
+
help=(
|
| 460 |
+
"Optional frozen RGB texture. When supplied it is required and takes "
|
| 461 |
+
"strict precedence over every discovered completion/fallback image."
|
| 462 |
+
),
|
| 463 |
+
)
|
| 464 |
+
parser.add_argument(
|
| 465 |
+
"--hidden-tint",
|
| 466 |
+
type=float,
|
| 467 |
+
default=0.0,
|
| 468 |
+
help="Hidden-region review tint in [0,1]. Default 0 preserves source texture.",
|
| 469 |
+
)
|
| 470 |
+
parser.add_argument(
|
| 471 |
+
"--include-depth-solid",
|
| 472 |
+
action="store_true",
|
| 473 |
+
help=(
|
| 474 |
+
"Also render the experimental depth-extruded solid. It is disabled by "
|
| 475 |
+
"default because long side-wall triangles are not publication quality."
|
| 476 |
+
),
|
| 477 |
+
)
|
| 478 |
+
parser.add_argument(
|
| 479 |
+
"--model-only",
|
| 480 |
+
action="store_true",
|
| 481 |
+
help=(
|
| 482 |
+
"Build PLY/GLB presentation models without duplicate CPU GIFs; "
|
| 483 |
+
"use the dedicated GPU turntable renderer for final review media."
|
| 484 |
+
),
|
| 485 |
+
)
|
| 486 |
+
return parser
|
| 487 |
+
|
| 488 |
+
|
| 489 |
+
def main() -> int:
|
| 490 |
+
parser = build_argument_parser()
|
| 491 |
+
args = parser.parse_args()
|
| 492 |
+
|
| 493 |
+
geometry_dir = Path(args.geometry_dir).expanduser().resolve()
|
| 494 |
+
output_dir = Path(args.output_dir).expanduser().resolve()
|
| 495 |
+
texture_image = (
|
| 496 |
+
Path(args.texture_image).expanduser().resolve()
|
| 497 |
+
if args.texture_image
|
| 498 |
+
else None
|
| 499 |
+
)
|
| 500 |
+
required = [
|
| 501 |
+
geometry_dir / "completed_mesh.ply",
|
| 502 |
+
geometry_dir / "completed_depth.npy",
|
| 503 |
+
geometry_dir / "amodal_target_mask.png",
|
| 504 |
+
]
|
| 505 |
+
if texture_image is not None:
|
| 506 |
+
required.append(texture_image)
|
| 507 |
+
missing = [str(path) for path in required if not path.is_file()]
|
| 508 |
+
if missing:
|
| 509 |
+
parser.error("missing geometry input(s): " + ", ".join(missing))
|
| 510 |
+
if args.frames < 12 or args.duration_ms < 20:
|
| 511 |
+
parser.error("use at least 12 frames and 20 ms per frame")
|
| 512 |
+
if not 0.0 <= args.hidden_tint <= 1.0:
|
| 513 |
+
parser.error("--hidden-tint must be in [0,1]")
|
| 514 |
+
|
| 515 |
+
depth = np.load(geometry_dir / "completed_depth.npy").astype(np.float32)
|
| 516 |
+
shape = depth.shape
|
| 517 |
+
target_mask = clean_target_mask(
|
| 518 |
+
read_mask(geometry_dir / "amodal_target_mask.png", shape),
|
| 519 |
+
close_kernel=5,
|
| 520 |
+
keep_largest=True,
|
| 521 |
+
)
|
| 522 |
+
visible_path = geometry_dir / "target_visible_mask.png"
|
| 523 |
+
visible_mask = read_mask(visible_path, shape) if visible_path.is_file() else None
|
| 524 |
+
colors, texture_source = prepare_texture(
|
| 525 |
+
geometry_dir.parent,
|
| 526 |
+
geometry_dir,
|
| 527 |
+
shape,
|
| 528 |
+
hidden_tint=args.hidden_tint,
|
| 529 |
+
texture_override=texture_image,
|
| 530 |
+
)
|
| 531 |
+
selected_texture_path = Path(texture_source.split(" + ", 1)[0]).resolve()
|
| 532 |
+
if (
|
| 533 |
+
texture_image is not None
|
| 534 |
+
and selected_texture_path != texture_image
|
| 535 |
+
):
|
| 536 |
+
raise RuntimeError(
|
| 537 |
+
"Explicit --texture-image lost strict priority during texture selection"
|
| 538 |
+
)
|
| 539 |
+
geometry_manifest_path = geometry_dir / "geometry_manifest.json"
|
| 540 |
+
geometry_manifest = (
|
| 541 |
+
read_json(geometry_manifest_path)
|
| 542 |
+
if geometry_manifest_path.is_file()
|
| 543 |
+
else {}
|
| 544 |
+
)
|
| 545 |
+
depth_context, depth_manifest_path = load_depth_context(geometry_dir)
|
| 546 |
+
metric_style_depth = bool(depth_context["metric_style_depth"])
|
| 547 |
+
depth_manifest = (
|
| 548 |
+
read_json(depth_manifest_path)
|
| 549 |
+
if depth_manifest_path is not None
|
| 550 |
+
else {}
|
| 551 |
+
)
|
| 552 |
+
manifest_depth_output = resolve_declared_path(
|
| 553 |
+
depth_manifest.get("output_depth"),
|
| 554 |
+
depth_manifest_path.parent if depth_manifest_path is not None else geometry_dir,
|
| 555 |
+
)
|
| 556 |
+
geometry_depth_path = resolve_declared_path(
|
| 557 |
+
geometry_manifest.get("depth_source"),
|
| 558 |
+
geometry_manifest_path.parent,
|
| 559 |
+
)
|
| 560 |
+
depth_context["geometry_depth_source_matches_manifest"] = (
|
| 561 |
+
geometry_depth_path.resolve() == manifest_depth_output.resolve()
|
| 562 |
+
if geometry_depth_path is not None and manifest_depth_output is not None
|
| 563 |
+
else None
|
| 564 |
+
)
|
| 565 |
+
depth_context["manifest_output_depth_filename"] = (
|
| 566 |
+
manifest_depth_output.name
|
| 567 |
+
if manifest_depth_output is not None
|
| 568 |
+
else None
|
| 569 |
+
)
|
| 570 |
+
depth_context["geometry_depth_source_filename"] = (
|
| 571 |
+
geometry_depth_path.name
|
| 572 |
+
if geometry_depth_path is not None
|
| 573 |
+
else None
|
| 574 |
+
)
|
| 575 |
+
stair_edges, stair_edge_provenance = (
|
| 576 |
+
trusted_stair_edges_from_manifest(geometry_manifest)
|
| 577 |
+
)
|
| 578 |
+
stair_slope = float(geometry_manifest.get("stair_edge_slope", 0.0))
|
| 579 |
+
size = (args.width, args.height)
|
| 580 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 581 |
+
|
| 582 |
+
variants: list[dict[str, Any]] = []
|
| 583 |
+
source_mesh_path = geometry_dir / "completed_mesh.ply"
|
| 584 |
+
source_vertices, source_colors, source_faces = load_ply(source_mesh_path)
|
| 585 |
+
variants.append(
|
| 586 |
+
save_variant(
|
| 587 |
+
output_dir,
|
| 588 |
+
"01_surface_faces",
|
| 589 |
+
"Original continuous triangle faces",
|
| 590 |
+
source_vertices,
|
| 591 |
+
source_colors,
|
| 592 |
+
source_faces,
|
| 593 |
+
sample_id=args.sample_id,
|
| 594 |
+
category=args.category,
|
| 595 |
+
frames=args.frames,
|
| 596 |
+
duration_ms=args.duration_ms,
|
| 597 |
+
size=size,
|
| 598 |
+
pitch=args.pitch,
|
| 599 |
+
source_ply=source_mesh_path,
|
| 600 |
+
render_outputs=not args.model_only,
|
| 601 |
+
)
|
| 602 |
+
)
|
| 603 |
+
|
| 604 |
+
if args.include_depth_solid:
|
| 605 |
+
if args.category == "stairs" and stair_edges:
|
| 606 |
+
solid_vertices, solid_colors, solid_faces, solid_metadata = build_continuous_stair_mesh(
|
| 607 |
+
depth,
|
| 608 |
+
colors,
|
| 609 |
+
target_mask,
|
| 610 |
+
stride=args.stride,
|
| 611 |
+
thickness=None,
|
| 612 |
+
thickness_ratio=args.thickness_ratio,
|
| 613 |
+
support_mode="stair_sidewall",
|
| 614 |
+
wedge_base_drop_ratio=0.12,
|
| 615 |
+
stair_edges_y=stair_edges,
|
| 616 |
+
stair_edge_slope=stair_slope,
|
| 617 |
+
cross_samples=args.stair_cross_samples,
|
| 618 |
+
)
|
| 619 |
+
solid_title = "Experimental depth-extruded stairs"
|
| 620 |
+
else:
|
| 621 |
+
solid_vertices, solid_colors, solid_faces, solid_metadata = build_solid_mesh(
|
| 622 |
+
depth,
|
| 623 |
+
colors,
|
| 624 |
+
target_mask,
|
| 625 |
+
stride=args.stride,
|
| 626 |
+
max_depth_jump=None,
|
| 627 |
+
thickness=None,
|
| 628 |
+
thickness_ratio=args.thickness_ratio,
|
| 629 |
+
support_mode="boundary",
|
| 630 |
+
wedge_base_drop_ratio=0.12,
|
| 631 |
+
category=args.category,
|
| 632 |
+
stair_edges_y=stair_edges,
|
| 633 |
+
)
|
| 634 |
+
solid_title = "Experimental depth-extruded surface"
|
| 635 |
+
solid_row = save_variant(
|
| 636 |
+
output_dir,
|
| 637 |
+
"debug_depth_solid",
|
| 638 |
+
solid_title,
|
| 639 |
+
solid_vertices,
|
| 640 |
+
solid_colors,
|
| 641 |
+
solid_faces,
|
| 642 |
+
sample_id=args.sample_id,
|
| 643 |
+
category=args.category,
|
| 644 |
+
frames=args.frames,
|
| 645 |
+
duration_ms=args.duration_ms,
|
| 646 |
+
size=size,
|
| 647 |
+
pitch=args.pitch,
|
| 648 |
+
render_outputs=not args.model_only,
|
| 649 |
+
)
|
| 650 |
+
solid_row["construction"] = solid_metadata
|
| 651 |
+
solid_row["publication_ready"] = False
|
| 652 |
+
variants.append(solid_row)
|
| 653 |
+
|
| 654 |
+
if args.category == "stairs":
|
| 655 |
+
(
|
| 656 |
+
canonical_vertices,
|
| 657 |
+
canonical_colors,
|
| 658 |
+
canonical_faces,
|
| 659 |
+
canonical_metadata,
|
| 660 |
+
) = build_canonical_staircase_prism_mesh(
|
| 661 |
+
depth,
|
| 662 |
+
colors,
|
| 663 |
+
target_mask,
|
| 664 |
+
stride=args.stride,
|
| 665 |
+
thickness=None,
|
| 666 |
+
thickness_ratio=args.thickness_ratio,
|
| 667 |
+
stair_edges_y=stair_edges,
|
| 668 |
+
stair_edge_slope=stair_slope,
|
| 669 |
+
step_count=args.stair_step_count,
|
| 670 |
+
metric_style_depth=metric_style_depth,
|
| 671 |
+
)
|
| 672 |
+
canonical_row = save_variant(
|
| 673 |
+
output_dir,
|
| 674 |
+
"02_canonical_stairs",
|
| 675 |
+
"Depth-scaled textured open-world staircase",
|
| 676 |
+
canonical_vertices,
|
| 677 |
+
canonical_colors,
|
| 678 |
+
canonical_faces,
|
| 679 |
+
sample_id=args.sample_id,
|
| 680 |
+
category=args.category,
|
| 681 |
+
frames=args.frames,
|
| 682 |
+
duration_ms=args.duration_ms,
|
| 683 |
+
size=size,
|
| 684 |
+
pitch=args.pitch,
|
| 685 |
+
render_outputs=not args.model_only,
|
| 686 |
+
)
|
| 687 |
+
canonical_row["construction"] = canonical_metadata
|
| 688 |
+
canonical_row["publication_ready"] = True
|
| 689 |
+
variants.append(canonical_row)
|
| 690 |
+
elif args.category in CONTINUOUS_SURFACE_CATEGORIES:
|
| 691 |
+
presentation_mask, presentation_mask_metadata = (
|
| 692 |
+
continuous_presentation_mask(target_mask)
|
| 693 |
+
)
|
| 694 |
+
continuous_depth, surface_metadata = idealized_continuous_presentation_depth(
|
| 695 |
+
depth,
|
| 696 |
+
presentation_mask,
|
| 697 |
+
visible_mask,
|
| 698 |
+
)
|
| 699 |
+
presentation_depth_bounds = surface_metadata.get("depth_bounds", [0.0, 1.0])
|
| 700 |
+
presentation_depth_jump_limit = max(
|
| 701 |
+
1.0,
|
| 702 |
+
float(presentation_depth_bounds[1])
|
| 703 |
+
- float(presentation_depth_bounds[0])
|
| 704 |
+
+ 1e-3,
|
| 705 |
+
)
|
| 706 |
+
(
|
| 707 |
+
support_vertices,
|
| 708 |
+
support_colors,
|
| 709 |
+
support_faces,
|
| 710 |
+
support_metadata,
|
| 711 |
+
) = build_solid_mesh(
|
| 712 |
+
continuous_depth,
|
| 713 |
+
colors,
|
| 714 |
+
presentation_mask,
|
| 715 |
+
stride=args.stride,
|
| 716 |
+
max_depth_jump=presentation_depth_jump_limit,
|
| 717 |
+
thickness=None,
|
| 718 |
+
thickness_ratio=args.thickness_ratio,
|
| 719 |
+
support_mode="vertical_slab",
|
| 720 |
+
wedge_base_drop_ratio=0.12,
|
| 721 |
+
category=args.category,
|
| 722 |
+
stair_edges_y=None,
|
| 723 |
+
)
|
| 724 |
+
support_row = save_variant(
|
| 725 |
+
output_dir,
|
| 726 |
+
"02_continuous_support_surface",
|
| 727 |
+
"Category-constrained continuous support surface",
|
| 728 |
+
support_vertices,
|
| 729 |
+
support_colors,
|
| 730 |
+
support_faces,
|
| 731 |
+
sample_id=args.sample_id,
|
| 732 |
+
category=args.category,
|
| 733 |
+
frames=args.frames,
|
| 734 |
+
duration_ms=args.duration_ms,
|
| 735 |
+
size=size,
|
| 736 |
+
pitch=args.pitch,
|
| 737 |
+
render_outputs=not args.model_only,
|
| 738 |
+
)
|
| 739 |
+
support_row["construction"] = {
|
| 740 |
+
**support_metadata,
|
| 741 |
+
**surface_metadata,
|
| 742 |
+
"presentation_mask": presentation_mask_metadata,
|
| 743 |
+
"role": "nonmetric_category_constrained_geometry_hypothesis",
|
| 744 |
+
"safe_passage_claim": False,
|
| 745 |
+
"human_review_required": True,
|
| 746 |
+
}
|
| 747 |
+
support_row["publication_ready"] = True
|
| 748 |
+
variants.append(support_row)
|
| 749 |
+
|
| 750 |
+
contact_sheet = output_dir / "3d_variants_contact_sheet.jpg"
|
| 751 |
+
if not args.model_only:
|
| 752 |
+
build_contact_sheet(contact_sheet, variants)
|
| 753 |
+
texture_parts = texture_source.split(" + ")
|
| 754 |
+
portable_texture_source = " + ".join(
|
| 755 |
+
[portable_path(texture_parts[0], output_dir), *texture_parts[1:]]
|
| 756 |
+
)
|
| 757 |
+
selected_texture_record = portable_file_record(
|
| 758 |
+
selected_texture_path,
|
| 759 |
+
output_dir,
|
| 760 |
+
)
|
| 761 |
+
explicit_texture_record = (
|
| 762 |
+
portable_file_record(texture_image, output_dir)
|
| 763 |
+
if texture_image is not None
|
| 764 |
+
else None
|
| 765 |
+
)
|
| 766 |
+
if depth_manifest_path is not None:
|
| 767 |
+
depth_context["manifest"] = portable_path(depth_manifest_path, output_dir)
|
| 768 |
+
else:
|
| 769 |
+
depth_context["manifest"] = None
|
| 770 |
+
portable_variants = [
|
| 771 |
+
portable_variant_row(row, output_dir)
|
| 772 |
+
for row in variants
|
| 773 |
+
]
|
| 774 |
+
canonical_construction = next(
|
| 775 |
+
(
|
| 776 |
+
row.get("construction", {})
|
| 777 |
+
for row in variants
|
| 778 |
+
if row.get("name") == "02_canonical_stairs"
|
| 779 |
+
),
|
| 780 |
+
{},
|
| 781 |
+
)
|
| 782 |
+
manifest = {
|
| 783 |
+
"schema_version": "accessibilityamodal_3d_presentation_variants_v2",
|
| 784 |
+
"sample_id": args.sample_id,
|
| 785 |
+
"category": args.category,
|
| 786 |
+
"geometry_dir": portable_path(geometry_dir, output_dir),
|
| 787 |
+
"texture_source": portable_texture_source,
|
| 788 |
+
"texture": {
|
| 789 |
+
"selection_policy": (
|
| 790 |
+
"explicit_frozen_texture_strict_priority"
|
| 791 |
+
if texture_image is not None
|
| 792 |
+
else "pipeline_texture_discovery"
|
| 793 |
+
),
|
| 794 |
+
"explicit_frozen_texture": explicit_texture_record,
|
| 795 |
+
"selected_texture": selected_texture_record,
|
| 796 |
+
"selected_source_description": portable_texture_source,
|
| 797 |
+
"hidden_tint": float(args.hidden_tint),
|
| 798 |
+
"source_file_modified": False,
|
| 799 |
+
"used_for_vertex_colors": True,
|
| 800 |
+
},
|
| 801 |
+
"geometry": {
|
| 802 |
+
"source_geometry_manifest": (
|
| 803 |
+
portable_path(geometry_manifest_path, output_dir)
|
| 804 |
+
if geometry_manifest_path.is_file()
|
| 805 |
+
else None
|
| 806 |
+
),
|
| 807 |
+
"geometry_mode": geometry_manifest.get("geometry_mode"),
|
| 808 |
+
"depth": depth_context,
|
| 809 |
+
"metric_style_depth": metric_style_depth,
|
| 810 |
+
"calibrated_metric_truth": False,
|
| 811 |
+
"stair_edge_rows": stair_edges,
|
| 812 |
+
"stair_edge_provenance": stair_edge_provenance,
|
| 813 |
+
"stair_edge_source": (
|
| 814 |
+
geometry_manifest.get("stair_edge_source")
|
| 815 |
+
or geometry_manifest.get("edge_source")
|
| 816 |
+
),
|
| 817 |
+
"stair_step_count": canonical_construction.get("stair_step_count"),
|
| 818 |
+
"stair_step_count_source": canonical_construction.get(
|
| 819 |
+
"stair_step_count_source"
|
| 820 |
+
),
|
| 821 |
+
"role": "open_world_accessibility_surface_hypothesis",
|
| 822 |
+
},
|
| 823 |
+
"depth_model_identity": depth_context.get("model_identity"),
|
| 824 |
+
"metric_style_depth": metric_style_depth,
|
| 825 |
+
"frames": args.frames,
|
| 826 |
+
"duration_ms": args.duration_ms,
|
| 827 |
+
"seconds_per_rotation": round(args.frames * args.duration_ms / 1000.0, 3),
|
| 828 |
+
"contact_sheet": (
|
| 829 |
+
portable_path(contact_sheet, output_dir)
|
| 830 |
+
if not args.model_only
|
| 831 |
+
else None
|
| 832 |
+
),
|
| 833 |
+
"model_only": bool(args.model_only),
|
| 834 |
+
"variants": portable_variants,
|
| 835 |
+
"publication_variants": [
|
| 836 |
+
row["name"] for row in variants if row.get("publication_ready", True)
|
| 837 |
+
],
|
| 838 |
+
"experimental_depth_solid_included": args.include_depth_solid,
|
| 839 |
+
"depth_is_metric_ground_truth": False,
|
| 840 |
+
"solid_models_are_visual_geometry_hypotheses": True,
|
| 841 |
+
"open_world_semantics": {
|
| 842 |
+
"scene_role": "local_accessibility_surface_patch",
|
| 843 |
+
"freestanding_object_model": False,
|
| 844 |
+
"context_landings": canonical_construction.get("context_landings"),
|
| 845 |
+
"automatic_passability_claim": False,
|
| 846 |
+
"human_review_required": True,
|
| 847 |
+
},
|
| 848 |
+
"path_policy": "All filesystem paths are relative to manifest.json.",
|
| 849 |
+
}
|
| 850 |
+
(output_dir / "manifest.json").write_text(
|
| 851 |
+
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
| 852 |
+
)
|
| 853 |
+
print(json.dumps(manifest, ensure_ascii=False, indent=2))
|
| 854 |
+
return 0
|
| 855 |
+
|
| 856 |
+
|
| 857 |
+
if __name__ == "__main__":
|
| 858 |
+
raise SystemExit(main())
|
tools/accessibility_amodal_mask.py
ADDED
|
@@ -0,0 +1,265 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Predict review-only hidden/amodal masks for one accessibility image.
|
| 3 |
+
|
| 4 |
+
The tiny adapter consumes RGB, a visible-target proposal, an obstacle proposal,
|
| 5 |
+
and a coarse stairs/non-stairs category plane. Predictions are constrained to
|
| 6 |
+
the obstacle support and written as candidates, never as ground truth.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import argparse
|
| 12 |
+
import json
|
| 13 |
+
import sys
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
from typing import Any
|
| 16 |
+
|
| 17 |
+
import cv2
|
| 18 |
+
import numpy as np
|
| 19 |
+
import torch
|
| 20 |
+
from PIL import Image, ImageOps
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
TOOLS_DIR = Path(__file__).resolve().parent
|
| 24 |
+
if str(TOOLS_DIR) not in sys.path:
|
| 25 |
+
sys.path.insert(0, str(TOOLS_DIR))
|
| 26 |
+
|
| 27 |
+
import train_accessibility_amodal_adapter as trainer # noqa: E402
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
CATEGORIES = ("curb_cut", "ramp", "stairs", "tactile_paving", "walkway")
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def load_rgb(path: Path) -> Image.Image:
|
| 34 |
+
return ImageOps.exif_transpose(Image.open(path)).convert("RGB")
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def load_mask(path: Path, size: tuple[int, int]) -> np.ndarray:
|
| 38 |
+
image = ImageOps.exif_transpose(Image.open(path)).convert("L")
|
| 39 |
+
if image.size != size:
|
| 40 |
+
raise ValueError(
|
| 41 |
+
f"Mask/RGB raster mismatch for {path}: mask={image.size}, rgb={size}. "
|
| 42 |
+
"Refusing to resize because this can hide EXIF-orientation misalignment."
|
| 43 |
+
)
|
| 44 |
+
return np.asarray(image) > 127
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def save_mask(path: Path, mask: np.ndarray) -> None:
|
| 48 |
+
Image.fromarray(mask.astype(np.uint8) * 255, mode="L").save(path)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def retain_hidden_components(obstacle: np.ndarray, hidden: np.ndarray) -> np.ndarray:
|
| 52 |
+
if not hidden.any():
|
| 53 |
+
return np.zeros_like(obstacle)
|
| 54 |
+
_, labels = cv2.connectedComponents(obstacle.astype(np.uint8), connectivity=8)
|
| 55 |
+
keep = np.unique(labels[obstacle & hidden])
|
| 56 |
+
keep = keep[keep != 0]
|
| 57 |
+
return np.isin(labels, keep)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def checkpoint_category_names(checkpoint: dict[str, Any], model_input_channels: int) -> tuple[str, ...]:
|
| 61 |
+
"""Recover the category-plane order used to train a checkpoint.
|
| 62 |
+
|
| 63 |
+
The original adapter used only ``stairs`` and ``walkway`` category planes
|
| 64 |
+
(7 total input channels). The current adapter uses one plane for each
|
| 65 |
+
canonical category (10 total). Keeping this explicit makes old reviewed
|
| 66 |
+
runs reproducible while preventing a silent channel-order mismatch.
|
| 67 |
+
"""
|
| 68 |
+
category_count = model_input_channels - 5 # RGB + visible + obstacle
|
| 69 |
+
if category_count < 0:
|
| 70 |
+
raise ValueError(
|
| 71 |
+
f"Checkpoint expects {model_input_channels} inputs; at least 5 are required."
|
| 72 |
+
)
|
| 73 |
+
recorded = checkpoint.get("input_channels", [])
|
| 74 |
+
if isinstance(recorded, list):
|
| 75 |
+
names = tuple(name for name in recorded if name in CATEGORIES)
|
| 76 |
+
if len(names) == category_count:
|
| 77 |
+
return names
|
| 78 |
+
if category_count == len(CATEGORIES):
|
| 79 |
+
return CATEGORIES
|
| 80 |
+
raise ValueError(
|
| 81 |
+
"Checkpoint category-plane metadata is incompatible with its model input shape: "
|
| 82 |
+
f"expected {category_count} category planes, recorded={recorded!r}."
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def category_planes(category: str, names: tuple[str, ...], size: int) -> np.ndarray:
|
| 87 |
+
return np.stack(
|
| 88 |
+
[np.full((size, size), category == name, dtype=np.float32) for name in names],
|
| 89 |
+
axis=0,
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def predict_probability(
|
| 94 |
+
model: torch.nn.Module,
|
| 95 |
+
device: torch.device,
|
| 96 |
+
image: Image.Image,
|
| 97 |
+
visible: np.ndarray,
|
| 98 |
+
obstacle: np.ndarray,
|
| 99 |
+
category: str,
|
| 100 |
+
category_names: tuple[str, ...],
|
| 101 |
+
image_size: int,
|
| 102 |
+
) -> np.ndarray:
|
| 103 |
+
rgb = np.asarray(
|
| 104 |
+
image.resize((image_size, image_size), Image.Resampling.BILINEAR),
|
| 105 |
+
dtype=np.float32,
|
| 106 |
+
) / 255.0
|
| 107 |
+
visible_small = np.asarray(
|
| 108 |
+
Image.fromarray(visible.astype(np.uint8) * 255).resize(
|
| 109 |
+
(image_size, image_size), Image.Resampling.NEAREST
|
| 110 |
+
)
|
| 111 |
+
) > 127
|
| 112 |
+
obstacle_small = np.asarray(
|
| 113 |
+
Image.fromarray(obstacle.astype(np.uint8) * 255).resize(
|
| 114 |
+
(image_size, image_size), Image.Resampling.NEAREST
|
| 115 |
+
)
|
| 116 |
+
) > 127
|
| 117 |
+
inputs = np.concatenate(
|
| 118 |
+
[
|
| 119 |
+
rgb.transpose(2, 0, 1),
|
| 120 |
+
visible_small[None].astype(np.float32),
|
| 121 |
+
obstacle_small[None].astype(np.float32),
|
| 122 |
+
category_planes(category, category_names, image_size),
|
| 123 |
+
],
|
| 124 |
+
axis=0,
|
| 125 |
+
)
|
| 126 |
+
with torch.inference_mode():
|
| 127 |
+
logits = model(torch.from_numpy(inputs[None]).to(device)).sigmoid()[0, 0]
|
| 128 |
+
probability = np.asarray(
|
| 129 |
+
Image.fromarray(logits.detach().cpu().numpy().astype(np.float32), mode="F").resize(
|
| 130 |
+
image.size, Image.Resampling.BILINEAR
|
| 131 |
+
)
|
| 132 |
+
).copy()
|
| 133 |
+
probability *= (obstacle & ~visible).astype(np.float32)
|
| 134 |
+
return probability
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def overlay(
|
| 138 |
+
image: Image.Image,
|
| 139 |
+
visible: np.ndarray,
|
| 140 |
+
hidden: np.ndarray,
|
| 141 |
+
obstacle: np.ndarray,
|
| 142 |
+
) -> Image.Image:
|
| 143 |
+
result = np.asarray(image, dtype=np.float32).copy()
|
| 144 |
+
for mask, color, alpha in (
|
| 145 |
+
(visible, np.asarray((30, 210, 70), dtype=np.float32), 0.42),
|
| 146 |
+
(hidden, np.asarray((40, 100, 245), dtype=np.float32), 0.62),
|
| 147 |
+
(obstacle, np.asarray((230, 45, 45), dtype=np.float32), 0.52),
|
| 148 |
+
):
|
| 149 |
+
result[mask] = result[mask] * (1.0 - alpha) + color * alpha
|
| 150 |
+
return Image.fromarray(np.clip(result, 0, 255).astype(np.uint8), mode="RGB")
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def build_parser() -> argparse.ArgumentParser:
|
| 154 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 155 |
+
parser.add_argument("--image", type=Path, required=True)
|
| 156 |
+
parser.add_argument("--target-visible-mask", type=Path, required=True)
|
| 157 |
+
parser.add_argument("--obstacle-candidate-mask", type=Path, required=True)
|
| 158 |
+
parser.add_argument("--category", choices=CATEGORIES, required=True)
|
| 159 |
+
parser.add_argument("--checkpoint", type=Path, required=True)
|
| 160 |
+
parser.add_argument("--output-dir", type=Path, required=True)
|
| 161 |
+
parser.add_argument("--device", default="cuda")
|
| 162 |
+
parser.add_argument("--image-size", type=int, default=256)
|
| 163 |
+
parser.add_argument(
|
| 164 |
+
"--threshold",
|
| 165 |
+
type=float,
|
| 166 |
+
default=None,
|
| 167 |
+
help="Defaults to validation_threshold stored in the checkpoint.",
|
| 168 |
+
)
|
| 169 |
+
return parser
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def main() -> int:
|
| 173 |
+
args = build_parser().parse_args()
|
| 174 |
+
image_path = args.image.expanduser().resolve()
|
| 175 |
+
checkpoint_path = args.checkpoint.expanduser().resolve()
|
| 176 |
+
output_dir = args.output_dir.expanduser().resolve()
|
| 177 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 178 |
+
|
| 179 |
+
image = load_rgb(image_path)
|
| 180 |
+
visible = load_mask(args.target_visible_mask.expanduser().resolve(), image.size)
|
| 181 |
+
obstacle_candidate = load_mask(
|
| 182 |
+
args.obstacle_candidate_mask.expanduser().resolve(), image.size
|
| 183 |
+
)
|
| 184 |
+
visible &= ~obstacle_candidate
|
| 185 |
+
|
| 186 |
+
device = torch.device(args.device)
|
| 187 |
+
if device.type == "cuda" and not torch.cuda.is_available():
|
| 188 |
+
raise RuntimeError("CUDA requested but unavailable; run through Slurm or use --device cpu")
|
| 189 |
+
checkpoint: dict[str, Any] = torch.load(
|
| 190 |
+
checkpoint_path, map_location=device, weights_only=False
|
| 191 |
+
)
|
| 192 |
+
model_state = checkpoint.get("model_state")
|
| 193 |
+
if not isinstance(model_state, dict) or "enc1.block.0.weight" not in model_state:
|
| 194 |
+
raise ValueError("Checkpoint does not contain a TinyAmodalUNet model_state.")
|
| 195 |
+
model_input_channels = int(model_state["enc1.block.0.weight"].shape[1])
|
| 196 |
+
category_names = checkpoint_category_names(checkpoint, model_input_channels)
|
| 197 |
+
threshold = float(
|
| 198 |
+
args.threshold
|
| 199 |
+
if args.threshold is not None
|
| 200 |
+
else checkpoint.get("validation_threshold", 0.6)
|
| 201 |
+
)
|
| 202 |
+
model = trainer.TinyAmodalUNet(in_channels=model_input_channels).to(device)
|
| 203 |
+
model.load_state_dict(model_state)
|
| 204 |
+
model.eval()
|
| 205 |
+
|
| 206 |
+
probability = predict_probability(
|
| 207 |
+
model,
|
| 208 |
+
device,
|
| 209 |
+
image,
|
| 210 |
+
visible,
|
| 211 |
+
obstacle_candidate,
|
| 212 |
+
args.category,
|
| 213 |
+
category_names,
|
| 214 |
+
args.image_size,
|
| 215 |
+
)
|
| 216 |
+
hidden = probability >= threshold
|
| 217 |
+
obstacle = retain_hidden_components(obstacle_candidate & ~visible, hidden)
|
| 218 |
+
hidden &= obstacle
|
| 219 |
+
amodal = visible | hidden
|
| 220 |
+
|
| 221 |
+
if np.any(visible & obstacle):
|
| 222 |
+
raise AssertionError("target_visible overlaps obstacle")
|
| 223 |
+
if np.any(hidden & ~obstacle):
|
| 224 |
+
raise AssertionError("hidden lies outside obstacle")
|
| 225 |
+
if np.any(hidden != (amodal & ~visible)):
|
| 226 |
+
raise AssertionError("hidden formula failed")
|
| 227 |
+
|
| 228 |
+
save_mask(output_dir / "target_visible.png", visible)
|
| 229 |
+
save_mask(output_dir / "obstacle_all_detected.png", obstacle_candidate)
|
| 230 |
+
save_mask(output_dir / "obstacle.png", obstacle)
|
| 231 |
+
save_mask(output_dir / "hidden.png", hidden)
|
| 232 |
+
save_mask(output_dir / "target_amodal.png", amodal)
|
| 233 |
+
Image.fromarray(
|
| 234 |
+
np.clip(probability * 255.0, 0, 255).astype(np.uint8), mode="L"
|
| 235 |
+
).save(output_dir / "hidden_probability.png")
|
| 236 |
+
overlay(image, visible, hidden, obstacle).save(output_dir / "mask_overlay.png")
|
| 237 |
+
|
| 238 |
+
metadata = {
|
| 239 |
+
"image": str(image_path),
|
| 240 |
+
"raster_orientation_policy": "RGB and every input mask are decoded with PIL ImageOps.exif_transpose and must match exactly.",
|
| 241 |
+
"display_raster_size": {"width": image.width, "height": image.height},
|
| 242 |
+
"category": args.category,
|
| 243 |
+
"checkpoint": str(checkpoint_path),
|
| 244 |
+
"model_input_channels": model_input_channels,
|
| 245 |
+
"category_plane_names": list(category_names),
|
| 246 |
+
"threshold": threshold,
|
| 247 |
+
"target_visible_pixels": int(visible.sum()),
|
| 248 |
+
"obstacle_candidate_pixels": int(obstacle_candidate.sum()),
|
| 249 |
+
"obstacle_pixels": int(obstacle.sum()),
|
| 250 |
+
"hidden_pixels": int(hidden.sum()),
|
| 251 |
+
"target_amodal_pixels": int(amodal.sum()),
|
| 252 |
+
"automatic_masks_are_ground_truth": False,
|
| 253 |
+
"review_status": "single_image_candidate_requires_human_review",
|
| 254 |
+
"mask_invariants_passed": True,
|
| 255 |
+
}
|
| 256 |
+
(output_dir / "metadata.json").write_text(
|
| 257 |
+
json.dumps(metadata, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
| 258 |
+
encoding="utf-8",
|
| 259 |
+
)
|
| 260 |
+
print(json.dumps(metadata, ensure_ascii=False, indent=2, sort_keys=True))
|
| 261 |
+
return 0
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
if __name__ == "__main__":
|
| 265 |
+
raise SystemExit(main())
|
tools/accessibility_fast_2d_baseline.py
ADDED
|
@@ -0,0 +1,581 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Run the optional fast CPU 2D baseline on accessibility samples.
|
| 3 |
+
|
| 4 |
+
The script is intentionally non-destructive: it only reads dataset samples and
|
| 5 |
+
writes a mirrored result tree under --output-dir.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import argparse
|
| 11 |
+
import json
|
| 12 |
+
import random
|
| 13 |
+
from dataclasses import dataclass
|
| 14 |
+
from datetime import datetime, timezone
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
from typing import Any
|
| 17 |
+
|
| 18 |
+
import cv2
|
| 19 |
+
import numpy as np
|
| 20 |
+
from PIL import Image, ImageDraw, ImageOps
|
| 21 |
+
|
| 22 |
+
try:
|
| 23 |
+
from tools.accessibility_dataset_layout import iter_sample_dirs, resolve_sample_dir
|
| 24 |
+
except ModuleNotFoundError: # Direct execution from tools/
|
| 25 |
+
from accessibility_dataset_layout import iter_sample_dirs, resolve_sample_dir
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
| 29 |
+
IMAGE_CANDIDATES = ("image.jpg", "image.jpeg", "image.png")
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
@dataclass(frozen=True)
|
| 33 |
+
class Sample:
|
| 34 |
+
sample_id: str
|
| 35 |
+
sample_dir: Path
|
| 36 |
+
image_path: Path
|
| 37 |
+
metadata: dict[str, Any]
|
| 38 |
+
target_visible_path: Path | None = None
|
| 39 |
+
target_amodal_path: Path | None = None
|
| 40 |
+
hidden_path: Path | None = None
|
| 41 |
+
obstacle_path: Path | None = None
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def resolve_path(path: str | Path) -> Path:
|
| 45 |
+
candidate = Path(path)
|
| 46 |
+
if candidate.is_absolute():
|
| 47 |
+
return candidate
|
| 48 |
+
return PROJECT_ROOT / candidate
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def read_json(path: Path) -> dict[str, Any]:
|
| 52 |
+
return json.loads(path.read_text(encoding="utf-8"))
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def read_rgb(path: Path) -> np.ndarray:
|
| 56 |
+
return np.array(ImageOps.exif_transpose(Image.open(path)).convert("RGB"))
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def read_mask(path: Path | None, shape: tuple[int, int]) -> np.ndarray | None:
|
| 60 |
+
if path is None or not path.is_file():
|
| 61 |
+
return None
|
| 62 |
+
mask = np.array(ImageOps.exif_transpose(Image.open(path)).convert("L")) > 127
|
| 63 |
+
h, w = shape
|
| 64 |
+
if mask.shape != (h, w):
|
| 65 |
+
raise ValueError(
|
| 66 |
+
f"Mask/RGB raster mismatch for {path}: mask={mask.shape}, rgb={(h, w)}. "
|
| 67 |
+
"Refusing to resize because this can hide EXIF-orientation misalignment."
|
| 68 |
+
)
|
| 69 |
+
return mask.astype(bool)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def save_mask(path: Path, mask: np.ndarray) -> None:
|
| 73 |
+
Image.fromarray(mask.astype(np.uint8) * 255).save(path)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def save_rgb(path: Path, image: np.ndarray) -> None:
|
| 77 |
+
Image.fromarray(np.clip(image, 0, 255).astype(np.uint8), mode="RGB").save(path)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def maybe_path(sample_dir: Path, name: str) -> Path | None:
|
| 81 |
+
path = sample_dir / name
|
| 82 |
+
return path if path.is_file() else None
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def find_image_path(sample_dir: Path, metadata: dict[str, Any], dataset_root: Path) -> Path:
|
| 86 |
+
image_file = metadata.get("image_file")
|
| 87 |
+
if isinstance(image_file, str):
|
| 88 |
+
candidates = [
|
| 89 |
+
sample_dir / Path(image_file).name,
|
| 90 |
+
dataset_root / image_file,
|
| 91 |
+
PROJECT_ROOT / image_file,
|
| 92 |
+
]
|
| 93 |
+
for candidate in candidates:
|
| 94 |
+
if candidate.is_file():
|
| 95 |
+
return candidate
|
| 96 |
+
for name in IMAGE_CANDIDATES:
|
| 97 |
+
candidate = sample_dir / name
|
| 98 |
+
if candidate.is_file():
|
| 99 |
+
return candidate
|
| 100 |
+
raise FileNotFoundError(f"No image file found in {sample_dir}")
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def sample_from_dir(sample_dir: Path, dataset_root: Path) -> Sample:
|
| 104 |
+
metadata_path = sample_dir / "metadata.json"
|
| 105 |
+
metadata = read_json(metadata_path) if metadata_path.is_file() else {}
|
| 106 |
+
sample_id = str(metadata.get("sample_id") or sample_dir.name)
|
| 107 |
+
return Sample(
|
| 108 |
+
sample_id=sample_id,
|
| 109 |
+
sample_dir=sample_dir,
|
| 110 |
+
image_path=find_image_path(sample_dir, metadata, dataset_root),
|
| 111 |
+
metadata=metadata,
|
| 112 |
+
target_visible_path=maybe_path(sample_dir, "target_visible.png"),
|
| 113 |
+
target_amodal_path=maybe_path(sample_dir, "target_amodal.png"),
|
| 114 |
+
hidden_path=maybe_path(sample_dir, "hidden.png"),
|
| 115 |
+
obstacle_path=maybe_path(sample_dir, "obstacle.png"),
|
| 116 |
+
)
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def discover_samples(
|
| 120 |
+
dataset_root: Path,
|
| 121 |
+
sample_ids: list[str],
|
| 122 |
+
categories: set[str],
|
| 123 |
+
splits: set[str],
|
| 124 |
+
) -> list[Sample]:
|
| 125 |
+
sample_root = dataset_root / "samples" if (dataset_root / "samples").is_dir() else dataset_root
|
| 126 |
+
if sample_ids:
|
| 127 |
+
dirs = [resolve_sample_dir(sample_root, sample_id) for sample_id in sample_ids]
|
| 128 |
+
else:
|
| 129 |
+
dirs = list(iter_sample_dirs(sample_root))
|
| 130 |
+
|
| 131 |
+
samples: list[Sample] = []
|
| 132 |
+
for sample_dir in dirs:
|
| 133 |
+
if not sample_dir.is_dir():
|
| 134 |
+
raise FileNotFoundError(f"Missing sample directory: {sample_dir}")
|
| 135 |
+
sample = sample_from_dir(sample_dir, dataset_root)
|
| 136 |
+
category = str(sample.metadata.get("category") or sample.metadata.get("taxonomy_category") or "")
|
| 137 |
+
split = str(sample.metadata.get("split") or sample.metadata.get("strict_gt_split") or "")
|
| 138 |
+
if categories and category not in categories:
|
| 139 |
+
continue
|
| 140 |
+
if splits and split not in splits:
|
| 141 |
+
continue
|
| 142 |
+
samples.append(sample)
|
| 143 |
+
return samples
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def hidden_pixel_count(sample: Sample) -> int:
|
| 147 |
+
value = sample.metadata.get("hidden_pixels")
|
| 148 |
+
if isinstance(value, int):
|
| 149 |
+
return value
|
| 150 |
+
if sample.hidden_path and sample.hidden_path.is_file():
|
| 151 |
+
return int((np.array(Image.open(sample.hidden_path).convert("L")) > 127).sum())
|
| 152 |
+
return 0
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def choose_samples(samples: list[Sample], args: argparse.Namespace) -> list[Sample]:
|
| 156 |
+
eligible = [sample for sample in samples if hidden_pixel_count(sample) >= args.min_hidden_pixels]
|
| 157 |
+
if args.sample_policy == "largest-hidden":
|
| 158 |
+
eligible.sort(key=lambda sample: (-hidden_pixel_count(sample), sample.sample_id))
|
| 159 |
+
elif args.sample_policy == "random":
|
| 160 |
+
rng = random.Random(args.seed)
|
| 161 |
+
rng.shuffle(eligible)
|
| 162 |
+
else:
|
| 163 |
+
eligible.sort(key=lambda sample: sample.sample_id)
|
| 164 |
+
|
| 165 |
+
if args.all:
|
| 166 |
+
return eligible
|
| 167 |
+
return eligible[: args.limit]
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def ellipse_kernel(radius: int) -> np.ndarray | None:
|
| 171 |
+
if radius <= 0:
|
| 172 |
+
return None
|
| 173 |
+
size = radius * 2 + 1
|
| 174 |
+
return cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (size, size))
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def dilate_mask(mask: np.ndarray, radius: int) -> np.ndarray:
|
| 178 |
+
kernel = ellipse_kernel(radius)
|
| 179 |
+
if kernel is None or not mask.any():
|
| 180 |
+
return mask.astype(bool)
|
| 181 |
+
return cv2.dilate(mask.astype(np.uint8), kernel, iterations=1).astype(bool)
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def close_mask(mask: np.ndarray, radius: int) -> np.ndarray:
|
| 185 |
+
kernel = ellipse_kernel(radius)
|
| 186 |
+
if kernel is None or not mask.any():
|
| 187 |
+
return mask.astype(bool)
|
| 188 |
+
return cv2.morphologyEx(mask.astype(np.uint8), cv2.MORPH_CLOSE, kernel).astype(bool)
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def derive_hidden_mask(
|
| 192 |
+
hidden: np.ndarray | None,
|
| 193 |
+
target_visible: np.ndarray | None,
|
| 194 |
+
target_amodal: np.ndarray | None,
|
| 195 |
+
) -> tuple[np.ndarray, str]:
|
| 196 |
+
if hidden is not None and hidden.any():
|
| 197 |
+
derived = hidden.astype(bool).copy()
|
| 198 |
+
source = "hidden mask"
|
| 199 |
+
if target_visible is not None and target_amodal is not None:
|
| 200 |
+
derived |= target_amodal & ~target_visible
|
| 201 |
+
source += " union target_amodal-minus-target_visible"
|
| 202 |
+
return derived, source
|
| 203 |
+
if target_visible is not None and target_amodal is not None:
|
| 204 |
+
return (target_amodal & ~target_visible).astype(bool), "target_amodal-minus-target_visible"
|
| 205 |
+
if target_amodal is not None:
|
| 206 |
+
return target_amodal.astype(bool), "target_amodal fallback"
|
| 207 |
+
raise ValueError("Need hidden.png or target_amodal.png + target_visible.png to derive completion mask")
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def build_inpaint_mask(
|
| 211 |
+
hidden: np.ndarray,
|
| 212 |
+
target_visible: np.ndarray | None,
|
| 213 |
+
target_amodal: np.ndarray | None,
|
| 214 |
+
obstacle: np.ndarray | None,
|
| 215 |
+
mode: str,
|
| 216 |
+
target_band_dilate: int,
|
| 217 |
+
final_dilate: int,
|
| 218 |
+
close_radius: int,
|
| 219 |
+
) -> tuple[np.ndarray, dict[str, Any]]:
|
| 220 |
+
mask = hidden.astype(bool).copy()
|
| 221 |
+
metadata: dict[str, Any] = {
|
| 222 |
+
"mode": mode,
|
| 223 |
+
"hidden_pixels": int(hidden.sum()),
|
| 224 |
+
"added_obstacle_pixels": 0,
|
| 225 |
+
}
|
| 226 |
+
|
| 227 |
+
if mode == "hidden":
|
| 228 |
+
pass
|
| 229 |
+
elif mode == "hidden_dilated":
|
| 230 |
+
mask = dilate_mask(mask, target_band_dilate)
|
| 231 |
+
elif mode == "target_occluder":
|
| 232 |
+
if obstacle is not None:
|
| 233 |
+
obstacle_on_target = obstacle & dilate_mask(hidden, target_band_dilate)
|
| 234 |
+
metadata["added_obstacle_pixels"] = int((obstacle_on_target & ~mask).sum())
|
| 235 |
+
mask |= obstacle_on_target
|
| 236 |
+
elif mode == "obstacle":
|
| 237 |
+
if obstacle is not None:
|
| 238 |
+
metadata["added_obstacle_pixels"] = int((obstacle & ~mask).sum())
|
| 239 |
+
mask |= obstacle
|
| 240 |
+
else:
|
| 241 |
+
raise ValueError(f"Unsupported mask mode: {mode}")
|
| 242 |
+
|
| 243 |
+
mask = close_mask(mask, close_radius)
|
| 244 |
+
mask = dilate_mask(mask, final_dilate)
|
| 245 |
+
metadata["final_pixels"] = int(mask.sum())
|
| 246 |
+
return mask.astype(bool), metadata
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def inpaint_opencv(rgb: np.ndarray, mask: np.ndarray, radius: float, method: str) -> np.ndarray:
|
| 250 |
+
if not mask.any():
|
| 251 |
+
return rgb.copy()
|
| 252 |
+
flag = cv2.INPAINT_TELEA if method == "telea" else cv2.INPAINT_NS
|
| 253 |
+
bgr = cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR)
|
| 254 |
+
completed = cv2.inpaint(bgr, mask.astype(np.uint8) * 255, radius, flag)
|
| 255 |
+
return cv2.cvtColor(completed, cv2.COLOR_BGR2RGB)
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
def inpaint_opencv_pyramid(
|
| 259 |
+
rgb: np.ndarray,
|
| 260 |
+
mask: np.ndarray,
|
| 261 |
+
radius: float,
|
| 262 |
+
method: str,
|
| 263 |
+
levels: int,
|
| 264 |
+
seam_radius: int,
|
| 265 |
+
) -> np.ndarray:
|
| 266 |
+
if not mask.any() or levels <= 1:
|
| 267 |
+
return inpaint_opencv(rgb, mask, radius, method)
|
| 268 |
+
|
| 269 |
+
h, w = rgb.shape[:2]
|
| 270 |
+
scale = 1.0 / float(2 ** (levels - 1))
|
| 271 |
+
low_w = max(32, int(round(w * scale)))
|
| 272 |
+
low_h = max(32, int(round(h * scale)))
|
| 273 |
+
low_rgb = cv2.resize(rgb, (low_w, low_h), interpolation=cv2.INTER_AREA)
|
| 274 |
+
low_mask = cv2.resize(mask.astype(np.uint8), (low_w, low_h), interpolation=cv2.INTER_NEAREST) > 0
|
| 275 |
+
low_completed = inpaint_opencv(low_rgb, low_mask, radius, method)
|
| 276 |
+
up_completed = cv2.resize(low_completed, (w, h), interpolation=cv2.INTER_CUBIC)
|
| 277 |
+
|
| 278 |
+
composite = rgb.copy()
|
| 279 |
+
composite[mask] = up_completed[mask]
|
| 280 |
+
|
| 281 |
+
seam = dilate_mask(mask, seam_radius)
|
| 282 |
+
inner = cv2.erode(
|
| 283 |
+
mask.astype(np.uint8),
|
| 284 |
+
ellipse_kernel(max(1, seam_radius // 2)),
|
| 285 |
+
iterations=1,
|
| 286 |
+
).astype(bool)
|
| 287 |
+
seam = seam & ~inner
|
| 288 |
+
if seam.any():
|
| 289 |
+
composite = inpaint_opencv(composite, seam, max(1.0, radius * 0.5), method)
|
| 290 |
+
return composite
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
def run_inpaint(rgb: np.ndarray, mask: np.ndarray, args: argparse.Namespace) -> np.ndarray:
|
| 294 |
+
if args.opencv_mode == "pyramid":
|
| 295 |
+
return inpaint_opencv_pyramid(
|
| 296 |
+
rgb,
|
| 297 |
+
mask,
|
| 298 |
+
args.inpaint_radius,
|
| 299 |
+
args.method,
|
| 300 |
+
args.pyramid_levels,
|
| 301 |
+
args.seam_radius,
|
| 302 |
+
)
|
| 303 |
+
return inpaint_opencv(rgb, mask, args.inpaint_radius, args.method)
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
def blend_masks(rgb: np.ndarray, masks: list[tuple[np.ndarray, tuple[int, int, int], float]]) -> np.ndarray:
|
| 307 |
+
out = rgb.astype(np.float32).copy()
|
| 308 |
+
for mask, color, alpha in masks:
|
| 309 |
+
if mask is not None and mask.any():
|
| 310 |
+
out[mask] = out[mask] * (1.0 - alpha) + np.array(color, dtype=np.float32) * alpha
|
| 311 |
+
return np.clip(out, 0, 255).astype(np.uint8)
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
def write_target_rgba(path: Path, completed: np.ndarray, target_amodal: np.ndarray | None, inpaint_mask: np.ndarray) -> None:
|
| 315 |
+
alpha = target_amodal if target_amodal is not None and target_amodal.any() else inpaint_mask
|
| 316 |
+
rgba = np.dstack([completed, alpha.astype(np.uint8) * 255])
|
| 317 |
+
Image.fromarray(rgba, mode="RGBA").save(path)
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
def checkerboard(size: tuple[int, int], cell: int = 16) -> Image.Image:
|
| 321 |
+
width, height = size
|
| 322 |
+
yy, xx = np.indices((height, width))
|
| 323 |
+
pattern = ((xx // cell + yy // cell) % 2).astype(np.uint8)
|
| 324 |
+
values = np.where(pattern[..., None] == 0, 228, 188).astype(np.uint8)
|
| 325 |
+
image = np.repeat(values, 3, axis=2)
|
| 326 |
+
return Image.fromarray(image, mode="RGB")
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
def rgba_on_checker(rgba_path: Path, size: tuple[int, int]) -> Image.Image:
|
| 330 |
+
image = Image.open(rgba_path).convert("RGBA")
|
| 331 |
+
canvas = checkerboard(image.size)
|
| 332 |
+
canvas.paste(image, (0, 0), image)
|
| 333 |
+
return ImageOps.contain(canvas, size)
|
| 334 |
+
|
| 335 |
+
|
| 336 |
+
def label_panel(image: Image.Image, label: str, size: tuple[int, int]) -> Image.Image:
|
| 337 |
+
body = ImageOps.contain(image.convert("RGB"), size)
|
| 338 |
+
panel = Image.new("RGB", (size[0], size[1] + 28), "white")
|
| 339 |
+
draw = ImageDraw.Draw(panel)
|
| 340 |
+
draw.text((8, 8), label, fill=(0, 0, 0))
|
| 341 |
+
panel.paste(body, ((size[0] - body.width) // 2, 28 + (size[1] - body.height) // 2))
|
| 342 |
+
return panel
|
| 343 |
+
|
| 344 |
+
|
| 345 |
+
def write_contact_sheet(
|
| 346 |
+
path: Path,
|
| 347 |
+
original: np.ndarray,
|
| 348 |
+
overlay: np.ndarray,
|
| 349 |
+
completed: np.ndarray,
|
| 350 |
+
rgba_path: Path,
|
| 351 |
+
panel_width: int,
|
| 352 |
+
) -> None:
|
| 353 |
+
h, w = original.shape[:2]
|
| 354 |
+
panel_height = max(160, int(panel_width * h / max(w, 1)))
|
| 355 |
+
size = (panel_width, panel_height)
|
| 356 |
+
panels = [
|
| 357 |
+
label_panel(Image.fromarray(original), "source image", size),
|
| 358 |
+
label_panel(Image.fromarray(overlay), "mask guide", size),
|
| 359 |
+
label_panel(Image.fromarray(completed), "completed RGB", size),
|
| 360 |
+
label_panel(rgba_on_checker(rgba_path, size), "amodal target RGBA", size),
|
| 361 |
+
]
|
| 362 |
+
sheet = Image.new("RGB", (sum(panel.width for panel in panels), max(panel.height for panel in panels)), "white")
|
| 363 |
+
x = 0
|
| 364 |
+
for panel in panels:
|
| 365 |
+
sheet.paste(panel, (x, 0))
|
| 366 |
+
x += panel.width
|
| 367 |
+
sheet.save(path, quality=92)
|
| 368 |
+
|
| 369 |
+
|
| 370 |
+
def path_for_manifest(path: Path) -> str:
|
| 371 |
+
try:
|
| 372 |
+
return str(path.relative_to(PROJECT_ROOT))
|
| 373 |
+
except ValueError:
|
| 374 |
+
return str(path)
|
| 375 |
+
|
| 376 |
+
|
| 377 |
+
def process_sample(sample: Sample, output_dir: Path, args: argparse.Namespace) -> dict[str, Any]:
|
| 378 |
+
sample_out = output_dir / sample.sample_id
|
| 379 |
+
if sample_out.exists() and not args.overwrite:
|
| 380 |
+
return {
|
| 381 |
+
"sample_id": sample.sample_id,
|
| 382 |
+
"status": "skipped_existing_output",
|
| 383 |
+
"output_dir": path_for_manifest(sample_out),
|
| 384 |
+
}
|
| 385 |
+
sample_out.mkdir(parents=True, exist_ok=True)
|
| 386 |
+
|
| 387 |
+
rgb = read_rgb(sample.image_path)
|
| 388 |
+
shape = rgb.shape[:2]
|
| 389 |
+
target_visible = read_mask(sample.target_visible_path, shape)
|
| 390 |
+
target_amodal = read_mask(sample.target_amodal_path, shape)
|
| 391 |
+
hidden_input = read_mask(sample.hidden_path, shape)
|
| 392 |
+
obstacle = read_mask(sample.obstacle_path, shape)
|
| 393 |
+
hidden, hidden_source = derive_hidden_mask(hidden_input, target_visible, target_amodal)
|
| 394 |
+
inpaint_mask, mask_meta = build_inpaint_mask(
|
| 395 |
+
hidden=hidden,
|
| 396 |
+
target_visible=target_visible,
|
| 397 |
+
target_amodal=target_amodal,
|
| 398 |
+
obstacle=obstacle,
|
| 399 |
+
mode=args.mask_mode,
|
| 400 |
+
target_band_dilate=args.target_band_dilate,
|
| 401 |
+
final_dilate=args.mask_dilate,
|
| 402 |
+
close_radius=args.mask_close,
|
| 403 |
+
)
|
| 404 |
+
|
| 405 |
+
mask_ratio = float(inpaint_mask.sum()) / float(inpaint_mask.size)
|
| 406 |
+
row: dict[str, Any] = {
|
| 407 |
+
"sample_id": sample.sample_id,
|
| 408 |
+
"category": sample.metadata.get("category") or sample.metadata.get("taxonomy_category"),
|
| 409 |
+
"split": sample.metadata.get("split") or sample.metadata.get("strict_gt_split"),
|
| 410 |
+
"source_image": path_for_manifest(sample.image_path),
|
| 411 |
+
"source_sample_dir": path_for_manifest(sample.sample_dir),
|
| 412 |
+
"hidden_source": hidden_source,
|
| 413 |
+
"mask": {
|
| 414 |
+
**mask_meta,
|
| 415 |
+
"mask_ratio": mask_ratio,
|
| 416 |
+
"max_mask_area_ratio": args.max_mask_area_ratio,
|
| 417 |
+
},
|
| 418 |
+
"backend": {
|
| 419 |
+
"name": "opencv",
|
| 420 |
+
"method": args.method,
|
| 421 |
+
"opencv_mode": args.opencv_mode,
|
| 422 |
+
"inpaint_radius": args.inpaint_radius,
|
| 423 |
+
"pyramid_levels": args.pyramid_levels,
|
| 424 |
+
"seam_radius": args.seam_radius,
|
| 425 |
+
},
|
| 426 |
+
}
|
| 427 |
+
if not inpaint_mask.any():
|
| 428 |
+
row["status"] = "skipped_empty_mask"
|
| 429 |
+
(sample_out / "sample_manifest.json").write_text(json.dumps(row, indent=2, ensure_ascii=False), encoding="utf-8")
|
| 430 |
+
return row
|
| 431 |
+
if mask_ratio > args.max_mask_area_ratio and not args.allow_large_mask:
|
| 432 |
+
save_mask(sample_out / "inpaint_mask.png", inpaint_mask)
|
| 433 |
+
row["status"] = "skipped_large_mask"
|
| 434 |
+
(sample_out / "sample_manifest.json").write_text(json.dumps(row, indent=2, ensure_ascii=False), encoding="utf-8")
|
| 435 |
+
return row
|
| 436 |
+
|
| 437 |
+
completed = run_inpaint(rgb, inpaint_mask, args)
|
| 438 |
+
mask_overlay = blend_masks(
|
| 439 |
+
rgb,
|
| 440 |
+
[
|
| 441 |
+
(target_amodal if target_amodal is not None else np.zeros(shape, dtype=bool), (0, 150, 255), 0.28),
|
| 442 |
+
(target_visible if target_visible is not None else np.zeros(shape, dtype=bool), (0, 220, 80), 0.42),
|
| 443 |
+
(inpaint_mask, (255, 48, 48), 0.65),
|
| 444 |
+
],
|
| 445 |
+
)
|
| 446 |
+
completion_delta = np.abs(completed.astype(np.int16) - rgb.astype(np.int16)).max(axis=2) > 8
|
| 447 |
+
|
| 448 |
+
outputs = {
|
| 449 |
+
"completed_rgb": sample_out / "completed_rgb.png",
|
| 450 |
+
"amodal_target_rgba": sample_out / "amodal_target_rgba.png",
|
| 451 |
+
"inpaint_mask": sample_out / "inpaint_mask.png",
|
| 452 |
+
"hidden_mask": sample_out / "hidden_mask.png",
|
| 453 |
+
"mask_overlay": sample_out / "mask_overlay.jpg",
|
| 454 |
+
"completion_delta": sample_out / "completion_delta.png",
|
| 455 |
+
"contact_sheet": sample_out / "contact_sheet.jpg",
|
| 456 |
+
"manifest": sample_out / "sample_manifest.json",
|
| 457 |
+
}
|
| 458 |
+
save_rgb(outputs["completed_rgb"], completed)
|
| 459 |
+
write_target_rgba(outputs["amodal_target_rgba"], completed, target_amodal, inpaint_mask)
|
| 460 |
+
save_mask(outputs["inpaint_mask"], inpaint_mask)
|
| 461 |
+
save_mask(outputs["hidden_mask"], hidden)
|
| 462 |
+
save_rgb(outputs["mask_overlay"], mask_overlay)
|
| 463 |
+
save_mask(outputs["completion_delta"], completion_delta)
|
| 464 |
+
write_contact_sheet(
|
| 465 |
+
outputs["contact_sheet"],
|
| 466 |
+
rgb,
|
| 467 |
+
mask_overlay,
|
| 468 |
+
completed,
|
| 469 |
+
outputs["amodal_target_rgba"],
|
| 470 |
+
args.panel_width,
|
| 471 |
+
)
|
| 472 |
+
|
| 473 |
+
row["status"] = "completed"
|
| 474 |
+
row["outputs"] = {key: path_for_manifest(value) for key, value in outputs.items() if key != "manifest"}
|
| 475 |
+
outputs["manifest"].write_text(json.dumps(row, indent=2, ensure_ascii=False), encoding="utf-8")
|
| 476 |
+
return row
|
| 477 |
+
|
| 478 |
+
|
| 479 |
+
def sample_from_single_image(args: argparse.Namespace) -> Sample:
|
| 480 |
+
image_path = resolve_path(args.image)
|
| 481 |
+
if not image_path.is_file():
|
| 482 |
+
raise FileNotFoundError(image_path)
|
| 483 |
+
sample_id = args.single_sample_id or image_path.stem
|
| 484 |
+
return Sample(
|
| 485 |
+
sample_id=sample_id,
|
| 486 |
+
sample_dir=image_path.parent,
|
| 487 |
+
image_path=image_path,
|
| 488 |
+
metadata={"sample_id": sample_id, "source": "single_image_cli"},
|
| 489 |
+
target_visible_path=resolve_path(args.target_visible_mask) if args.target_visible_mask else None,
|
| 490 |
+
target_amodal_path=resolve_path(args.target_amodal_mask) if args.target_amodal_mask else None,
|
| 491 |
+
hidden_path=resolve_path(args.hidden_mask) if args.hidden_mask else None,
|
| 492 |
+
obstacle_path=resolve_path(args.obstacle_mask) if args.obstacle_mask else None,
|
| 493 |
+
)
|
| 494 |
+
|
| 495 |
+
|
| 496 |
+
def write_run_manifest(output_dir: Path, rows: list[dict[str, Any]], args: argparse.Namespace) -> None:
|
| 497 |
+
manifest = {
|
| 498 |
+
"created_at_utc": datetime.now(timezone.utc).isoformat(),
|
| 499 |
+
"tool": "tools/accessibility_fast_2d_baseline.py",
|
| 500 |
+
"non_destructive_policy": "Read source samples only; write all derived RGB/RGBA outputs under output_dir.",
|
| 501 |
+
"output_dir": path_for_manifest(output_dir),
|
| 502 |
+
"args": {
|
| 503 |
+
"dataset_root": args.dataset_root,
|
| 504 |
+
"sample_id": args.sample_id,
|
| 505 |
+
"image": args.image,
|
| 506 |
+
"limit": args.limit,
|
| 507 |
+
"all": args.all,
|
| 508 |
+
"sample_policy": args.sample_policy,
|
| 509 |
+
"mask_mode": args.mask_mode,
|
| 510 |
+
"method": args.method,
|
| 511 |
+
},
|
| 512 |
+
"references": {
|
| 513 |
+
"saraao_amodal": "https://github.com/saraao/amodal",
|
| 514 |
+
"pix2gestalt": "https://github.com/cvlab-columbia/pix2gestalt",
|
| 515 |
+
"amodal_completion_in_the_wild": "https://github.com/Championchess/Amodal-Completion-in-the-Wild",
|
| 516 |
+
"local_amodal": "external backend; path supplied by user",
|
| 517 |
+
"local_pix2gestalt": "external backend; path supplied by user",
|
| 518 |
+
"local_amodal_wild": "external backend; path supplied by user",
|
| 519 |
+
},
|
| 520 |
+
"samples": rows,
|
| 521 |
+
}
|
| 522 |
+
(output_dir / "run_manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8")
|
| 523 |
+
|
| 524 |
+
|
| 525 |
+
def build_parser() -> argparse.ArgumentParser:
|
| 526 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 527 |
+
parser.add_argument("--dataset-root", default="output/Accessibility", help="Dataset root containing samples/<sample_id>/ directories.")
|
| 528 |
+
parser.add_argument("--sample-id", action="append", default=[], help="Specific sample id to process. Repeat for multiple samples.")
|
| 529 |
+
parser.add_argument("--category", action="append", default=[], help="Optional category filter, e.g. stairs or ramp.")
|
| 530 |
+
parser.add_argument("--split", action="append", default=[], help="Optional split filter.")
|
| 531 |
+
parser.add_argument("--limit", type=int, default=8, help="Maximum samples when --all is not set.")
|
| 532 |
+
parser.add_argument("--all", action="store_true", help="Process all eligible samples.")
|
| 533 |
+
parser.add_argument("--sample-policy", choices=["largest-hidden", "random", "sorted"], default="largest-hidden")
|
| 534 |
+
parser.add_argument("--seed", type=int, default=13)
|
| 535 |
+
parser.add_argument("--min-hidden-pixels", type=int, default=32)
|
| 536 |
+
|
| 537 |
+
parser.add_argument("--image", default=None, help="Single-image mode input image.")
|
| 538 |
+
parser.add_argument("--single-sample-id", default=None)
|
| 539 |
+
parser.add_argument("--target-visible-mask", default=None)
|
| 540 |
+
parser.add_argument("--target-amodal-mask", default=None)
|
| 541 |
+
parser.add_argument("--hidden-mask", default=None)
|
| 542 |
+
parser.add_argument("--obstacle-mask", default=None)
|
| 543 |
+
|
| 544 |
+
parser.add_argument("--output-dir", default="output/amodal2d_color_completion")
|
| 545 |
+
parser.add_argument("--overwrite", action="store_true", help="Overwrite only files inside --output-dir.")
|
| 546 |
+
parser.add_argument("--mask-mode", choices=["hidden", "hidden_dilated", "target_occluder", "obstacle"], default="hidden")
|
| 547 |
+
parser.add_argument("--target-band-dilate", type=int, default=16, help="Pixels to dilate hidden completion support before intersecting obstacle mask.")
|
| 548 |
+
parser.add_argument("--mask-dilate", type=int, default=3, help="Final dilation radius for the inpaint mask.")
|
| 549 |
+
parser.add_argument("--mask-close", type=int, default=3, help="Closing radius for small holes in the inpaint mask.")
|
| 550 |
+
parser.add_argument("--max-mask-area-ratio", type=float, default=0.18, help="Skip unexpectedly huge masks unless --allow-large-mask is set.")
|
| 551 |
+
parser.add_argument("--allow-large-mask", action="store_true")
|
| 552 |
+
parser.add_argument("--opencv-mode", choices=["single", "pyramid"], default="single")
|
| 553 |
+
parser.add_argument("--pyramid-levels", type=int, default=3, help="Coarse-to-full resolution levels used with --opencv-mode pyramid.")
|
| 554 |
+
parser.add_argument("--seam-radius", type=int, default=8, help="Boundary refinement radius used with --opencv-mode pyramid.")
|
| 555 |
+
parser.add_argument("--method", choices=["telea", "ns"], default="telea", help="OpenCV inpainting method.")
|
| 556 |
+
parser.add_argument("--inpaint-radius", type=float, default=5.0)
|
| 557 |
+
parser.add_argument("--panel-width", type=int, default=420)
|
| 558 |
+
return parser
|
| 559 |
+
|
| 560 |
+
|
| 561 |
+
def main() -> int:
|
| 562 |
+
args = build_parser().parse_args()
|
| 563 |
+
output_dir = resolve_path(args.output_dir)
|
| 564 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 565 |
+
|
| 566 |
+
if args.image:
|
| 567 |
+
samples = [sample_from_single_image(args)]
|
| 568 |
+
else:
|
| 569 |
+
dataset_root = resolve_path(args.dataset_root)
|
| 570 |
+
samples = discover_samples(dataset_root, args.sample_id, set(args.category), set(args.split))
|
| 571 |
+
samples = choose_samples(samples, args)
|
| 572 |
+
|
| 573 |
+
rows = [process_sample(sample, output_dir, args) for sample in samples]
|
| 574 |
+
write_run_manifest(output_dir, rows, args)
|
| 575 |
+
completed = sum(1 for row in rows if row.get("status") == "completed")
|
| 576 |
+
print(f"Wrote {completed}/{len(rows)} completed samples to {output_dir}")
|
| 577 |
+
return 0
|
| 578 |
+
|
| 579 |
+
|
| 580 |
+
if __name__ == "__main__":
|
| 581 |
+
raise SystemExit(main())
|
tools/accessibility_mask_proposals.py
ADDED
|
@@ -0,0 +1,1332 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Run prompt-consensus accessibility masks on a small reviewed pilot set.
|
| 3 |
+
|
| 4 |
+
Supported backends:
|
| 5 |
+
sam3 Meta SAM 3 text-prompt concept segmentation (preferred)
|
| 6 |
+
grounded_sam GroundingDINO boxes refined by SAM ViT-H (local control)
|
| 7 |
+
|
| 8 |
+
The tool saves per-prompt masks, a high-recall union, a majority-vote core, an
|
| 9 |
+
uncertainty mask, both raw/high-recall and filtered obstacle masks, overlays,
|
| 10 |
+
and machine-readable quality scores. Automatic output is a review proposal,
|
| 11 |
+
never ground truth.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import argparse
|
| 17 |
+
import html
|
| 18 |
+
import json
|
| 19 |
+
import math
|
| 20 |
+
import os
|
| 21 |
+
import re
|
| 22 |
+
import sys
|
| 23 |
+
import traceback
|
| 24 |
+
from collections import Counter
|
| 25 |
+
from dataclasses import dataclass
|
| 26 |
+
from pathlib import Path
|
| 27 |
+
from typing import Any
|
| 28 |
+
|
| 29 |
+
import cv2
|
| 30 |
+
import numpy as np
|
| 31 |
+
from PIL import Image, ImageOps
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def read_jsonl(path: Path) -> list[dict[str, Any]]:
|
| 35 |
+
rows: list[dict[str, Any]] = []
|
| 36 |
+
with path.open("r", encoding="utf-8") as handle:
|
| 37 |
+
for line in handle:
|
| 38 |
+
if line.strip():
|
| 39 |
+
rows.append(json.loads(line))
|
| 40 |
+
return rows
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
|
| 44 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 45 |
+
tmp_path = path.with_suffix(path.suffix + ".tmp")
|
| 46 |
+
with tmp_path.open("w", encoding="utf-8") as handle:
|
| 47 |
+
for row in rows:
|
| 48 |
+
handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n")
|
| 49 |
+
tmp_path.replace(path)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def json_dump(path: Path, value: Any) -> None:
|
| 53 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 54 |
+
tmp_path = path.with_suffix(path.suffix + ".tmp")
|
| 55 |
+
with tmp_path.open("w", encoding="utf-8") as handle:
|
| 56 |
+
json.dump(value, handle, ensure_ascii=False, indent=2, sort_keys=True)
|
| 57 |
+
tmp_path.replace(path)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
STANDARD_SAMPLE_OUTPUTS = (
|
| 61 |
+
"target_visible_candidate.png",
|
| 62 |
+
"target_visible_core.png",
|
| 63 |
+
"target_uncertain.png",
|
| 64 |
+
"obstacle_unfiltered_evidence.png",
|
| 65 |
+
"obstacle_all_detected.png",
|
| 66 |
+
"obstacle.png",
|
| 67 |
+
"obstacle_red.png",
|
| 68 |
+
"overlay.png",
|
| 69 |
+
"quality.json",
|
| 70 |
+
)
|
| 71 |
+
OBSTACLE_EVIDENCE_POLICY_VERSION = 1
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def load_resume_quality(sample_dir: Path, sample_id: str, backend_name: str) -> dict[str, Any] | None:
|
| 75 |
+
quality_path = sample_dir / "quality.json"
|
| 76 |
+
if not quality_path.is_file():
|
| 77 |
+
return None
|
| 78 |
+
if any(not (sample_dir / filename).is_file() for filename in STANDARD_SAMPLE_OUTPUTS):
|
| 79 |
+
return None
|
| 80 |
+
with quality_path.open("r", encoding="utf-8") as handle:
|
| 81 |
+
quality = json.load(handle)
|
| 82 |
+
if quality.get("sample_id") != sample_id:
|
| 83 |
+
return None
|
| 84 |
+
if quality.get("backend") != backend_name:
|
| 85 |
+
return None
|
| 86 |
+
# Old output roots did not distinguish raw prompt evidence from the
|
| 87 |
+
# high-recall-but-sanity-filtered evidence used by later hidden completion.
|
| 88 |
+
# Force an explicit rerun rather than silently reusing that weaker audit
|
| 89 |
+
# trail after this schema change.
|
| 90 |
+
if quality.get("obstacle_evidence_policy_version") != OBSTACLE_EVIDENCE_POLICY_VERSION:
|
| 91 |
+
return None
|
| 92 |
+
if quality.get("review_status") == "error" or quality.get("error"):
|
| 93 |
+
return None
|
| 94 |
+
return quality
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def safe_name(text: str) -> str:
|
| 98 |
+
return re.sub(r"[^a-z0-9]+", "_", text.lower()).strip("_")
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def prompt_list(value: Any) -> list[str]:
|
| 102 |
+
"""Normalize a prompt list while accepting the legacy JSON shape."""
|
| 103 |
+
if isinstance(value, str):
|
| 104 |
+
raw = [value]
|
| 105 |
+
elif isinstance(value, (list, tuple)):
|
| 106 |
+
raw = value
|
| 107 |
+
else:
|
| 108 |
+
raw = []
|
| 109 |
+
return list(dict.fromkeys(str(prompt).strip() for prompt in raw if str(prompt).strip()))
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def obstacle_prompt_resolution(
|
| 113 |
+
config: dict[str, Any], category: str, backend_name: str | None = None
|
| 114 |
+
) -> dict[str, Any]:
|
| 115 |
+
"""Resolve backend-aware obstacle prompts without breaking legacy configs.
|
| 116 |
+
|
| 117 |
+
When a backend-specific list is present it replaces the corresponding
|
| 118 |
+
legacy global/category list. The other level still falls back separately,
|
| 119 |
+
so a config may specialize only global prompts or only one category. With
|
| 120 |
+
``backend_name=None`` the output is exactly the historical legacy union.
|
| 121 |
+
"""
|
| 122 |
+
global_prompts = prompt_list(config.get("obstacle_prompts"))
|
| 123 |
+
category_map = config.get("category_obstacle_prompts")
|
| 124 |
+
category_prompts = prompt_list(
|
| 125 |
+
category_map.get(category) if isinstance(category_map, dict) else None
|
| 126 |
+
)
|
| 127 |
+
global_source = "legacy.obstacle_prompts"
|
| 128 |
+
category_source = "legacy.category_obstacle_prompts"
|
| 129 |
+
if backend_name:
|
| 130 |
+
by_backend = config.get("obstacle_prompts_by_backend")
|
| 131 |
+
if isinstance(by_backend, dict) and backend_name in by_backend:
|
| 132 |
+
global_prompts = prompt_list(by_backend.get(backend_name))
|
| 133 |
+
global_source = f"obstacle_prompts_by_backend.{backend_name}"
|
| 134 |
+
category_by_backend = config.get("category_obstacle_prompts_by_backend")
|
| 135 |
+
backend_categories = (
|
| 136 |
+
category_by_backend.get(backend_name)
|
| 137 |
+
if isinstance(category_by_backend, dict)
|
| 138 |
+
else None
|
| 139 |
+
)
|
| 140 |
+
if isinstance(backend_categories, dict) and category in backend_categories:
|
| 141 |
+
category_prompts = prompt_list(backend_categories.get(category))
|
| 142 |
+
category_source = (
|
| 143 |
+
f"category_obstacle_prompts_by_backend.{backend_name}.{category}"
|
| 144 |
+
)
|
| 145 |
+
prompts = list(dict.fromkeys([*global_prompts, *category_prompts]))
|
| 146 |
+
return {
|
| 147 |
+
"backend": backend_name,
|
| 148 |
+
"category": category,
|
| 149 |
+
"global_source": global_source,
|
| 150 |
+
"category_source": category_source,
|
| 151 |
+
"prompts": prompts,
|
| 152 |
+
}
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def obstacle_prompts_for_category(
|
| 156 |
+
config: dict[str, Any], category: str, backend_name: str | None = None
|
| 157 |
+
) -> list[str]:
|
| 158 |
+
"""Return backend-aware obstacle prompts, with legacy fallback support."""
|
| 159 |
+
return list(obstacle_prompt_resolution(config, category, backend_name)["prompts"])
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def as_mapping(value: Any) -> dict[str, Any]:
|
| 163 |
+
return dict(value) if isinstance(value, dict) else {}
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def merge_mappings(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
|
| 167 |
+
"""Small recursive merge for backend-specific evidence policy overrides."""
|
| 168 |
+
result = dict(base)
|
| 169 |
+
for key, value in override.items():
|
| 170 |
+
if isinstance(value, dict) and isinstance(result.get(key), dict):
|
| 171 |
+
result[key] = merge_mappings(dict(result[key]), value)
|
| 172 |
+
else:
|
| 173 |
+
result[key] = value
|
| 174 |
+
return result
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def merge_prompt_config(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
|
| 178 |
+
"""Merge a small, output-run-specific prompt overlay safely.
|
| 179 |
+
|
| 180 |
+
Nested objects are merged, while prompt lists are appended in stable
|
| 181 |
+
de-duplicated order. This lets a finite remediation run add a narrowly
|
| 182 |
+
justified prompt without copying or silently editing the frozen base
|
| 183 |
+
configuration used by a control run. Scalar policy values remain an
|
| 184 |
+
explicit override, so their provenance can be locked by the Slurm wrapper.
|
| 185 |
+
"""
|
| 186 |
+
result = dict(base)
|
| 187 |
+
for key, value in override.items():
|
| 188 |
+
current = result.get(key)
|
| 189 |
+
if isinstance(current, dict) and isinstance(value, dict):
|
| 190 |
+
result[key] = merge_prompt_config(dict(current), value)
|
| 191 |
+
elif isinstance(current, list) and isinstance(value, list):
|
| 192 |
+
result[key] = list(dict.fromkeys([*current, *value]))
|
| 193 |
+
else:
|
| 194 |
+
result[key] = value
|
| 195 |
+
return result
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def float_or_default(value: Any, default: float) -> float:
|
| 199 |
+
try:
|
| 200 |
+
return float(value)
|
| 201 |
+
except (TypeError, ValueError):
|
| 202 |
+
return default
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
def normalized_float_mapping(value: Any) -> dict[str, float]:
|
| 206 |
+
mapping = as_mapping(value)
|
| 207 |
+
return {
|
| 208 |
+
str(key).strip().lower(): float_or_default(item, 0.0)
|
| 209 |
+
for key, item in mapping.items()
|
| 210 |
+
if str(key).strip()
|
| 211 |
+
}
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def normalized_override_mapping(value: Any) -> dict[str, dict[str, Any]]:
|
| 215 |
+
mapping = as_mapping(value)
|
| 216 |
+
return {
|
| 217 |
+
str(key).strip().lower(): as_mapping(item)
|
| 218 |
+
for key, item in mapping.items()
|
| 219 |
+
if str(key).strip() and isinstance(item, dict)
|
| 220 |
+
}
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def resolve_obstacle_evidence_policy(
|
| 224 |
+
config: dict[str, Any], backend_name: str
|
| 225 |
+
) -> dict[str, Any]:
|
| 226 |
+
"""Compile a serializable trusted-obstacle-evidence policy.
|
| 227 |
+
|
| 228 |
+
New configs may use ``obstacle_evidence_policy`` and optional
|
| 229 |
+
``obstacle_evidence_policy_by_backend.<backend>``. The aliases
|
| 230 |
+
``obstacle_all_detected_policy`` / ``..._by_backend`` are accepted for
|
| 231 |
+
early experiment configs. In a legacy config, the existing
|
| 232 |
+
``obstacle_maximum_area_ratio`` map becomes the per-prompt area cap,
|
| 233 |
+
confidence filtering remains disabled, and a conservative route-like
|
| 234 |
+
component guard protects downstream hidden completion from broad scene
|
| 235 |
+
surfaces.
|
| 236 |
+
"""
|
| 237 |
+
base = as_mapping(
|
| 238 |
+
config.get("obstacle_evidence_policy", config.get("obstacle_all_detected_policy", {}))
|
| 239 |
+
)
|
| 240 |
+
by_backend = as_mapping(
|
| 241 |
+
config.get(
|
| 242 |
+
"obstacle_evidence_policy_by_backend",
|
| 243 |
+
config.get("obstacle_all_detected_policy_by_backend", {}),
|
| 244 |
+
)
|
| 245 |
+
)
|
| 246 |
+
backend_override = as_mapping(by_backend.get(backend_name))
|
| 247 |
+
raw = merge_mappings(base, backend_override)
|
| 248 |
+
legacy_maximums = normalized_float_mapping(config.get("obstacle_maximum_area_ratio"))
|
| 249 |
+
maximums = dict(legacy_maximums)
|
| 250 |
+
maximums.update(
|
| 251 |
+
normalized_float_mapping(
|
| 252 |
+
raw.get("maximum_area_ratio", raw.get("maximum_area_ratio_by_prompt", {}))
|
| 253 |
+
)
|
| 254 |
+
)
|
| 255 |
+
prompt_overrides = normalized_override_mapping(
|
| 256 |
+
raw.get("prompt_overrides", raw.get("prompts", {}))
|
| 257 |
+
)
|
| 258 |
+
confidence_by_prompt = normalized_float_mapping(
|
| 259 |
+
raw.get("minimum_confidence_by_prompt", raw.get("minimum_prompt_confidence_by_prompt", {}))
|
| 260 |
+
)
|
| 261 |
+
route_like_raw = as_mapping(
|
| 262 |
+
raw.get("route_like", raw.get("route_like_component_policy", {}))
|
| 263 |
+
)
|
| 264 |
+
# The default is intentionally conservative: broad horizontal/ground-like
|
| 265 |
+
# components are rejected from trusted evidence, but remain visible in
|
| 266 |
+
# obstacle_unfiltered_evidence.png for human recovery.
|
| 267 |
+
route_like = {
|
| 268 |
+
"enabled": bool(route_like_raw.get("enabled", True)),
|
| 269 |
+
"minimum_bbox_width_ratio": float_or_default(
|
| 270 |
+
route_like_raw.get("minimum_bbox_width_ratio"), 0.80
|
| 271 |
+
),
|
| 272 |
+
"minimum_bbox_height_ratio": float_or_default(
|
| 273 |
+
route_like_raw.get("minimum_bbox_height_ratio"), 0.12
|
| 274 |
+
),
|
| 275 |
+
"minimum_component_area_ratio": float_or_default(
|
| 276 |
+
route_like_raw.get("minimum_component_area_ratio"), 0.03
|
| 277 |
+
),
|
| 278 |
+
}
|
| 279 |
+
default_maximum = float_or_default(
|
| 280 |
+
raw.get("default_maximum_area_ratio", maximums.get("default", 0.15)),
|
| 281 |
+
0.15,
|
| 282 |
+
)
|
| 283 |
+
return {
|
| 284 |
+
"version": OBSTACLE_EVIDENCE_POLICY_VERSION,
|
| 285 |
+
"backend": backend_name,
|
| 286 |
+
"source": {
|
| 287 |
+
"global": "obstacle_evidence_policy" if base else "legacy_defaults",
|
| 288 |
+
"backend_override_present": bool(backend_override),
|
| 289 |
+
"legacy_obstacle_maximum_area_ratio_used": bool(legacy_maximums),
|
| 290 |
+
},
|
| 291 |
+
"minimum_component_area_pixels": max(
|
| 292 |
+
1, int(float_or_default(raw.get("minimum_component_area_pixels"), 16.0))
|
| 293 |
+
),
|
| 294 |
+
"minimum_component_area_ratio": max(
|
| 295 |
+
0.0, float_or_default(raw.get("minimum_component_area_ratio"), 0.00002)
|
| 296 |
+
),
|
| 297 |
+
"minimum_prompt_confidence": min(
|
| 298 |
+
1.0,
|
| 299 |
+
max(
|
| 300 |
+
0.0,
|
| 301 |
+
float_or_default(
|
| 302 |
+
raw.get(
|
| 303 |
+
"minimum_prompt_confidence",
|
| 304 |
+
raw.get("minimum_confidence", 0.0),
|
| 305 |
+
),
|
| 306 |
+
0.0,
|
| 307 |
+
),
|
| 308 |
+
),
|
| 309 |
+
),
|
| 310 |
+
"minimum_confidence_by_prompt": confidence_by_prompt,
|
| 311 |
+
"default_maximum_area_ratio": min(1.0, max(0.0, default_maximum)),
|
| 312 |
+
"maximum_area_ratio_by_prompt": maximums,
|
| 313 |
+
"full_frame_area_ratio": min(
|
| 314 |
+
1.0, max(0.0, float_or_default(raw.get("full_frame_area_ratio"), 0.90))
|
| 315 |
+
),
|
| 316 |
+
"prompt_overrides": prompt_overrides,
|
| 317 |
+
"route_like": route_like,
|
| 318 |
+
}
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
def evidence_minimum_component_area(policy: dict[str, Any], image_area: int) -> int:
|
| 322 |
+
return max(
|
| 323 |
+
int(policy["minimum_component_area_pixels"]),
|
| 324 |
+
int(round(image_area * float(policy["minimum_component_area_ratio"]))),
|
| 325 |
+
)
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
def evidence_prompt_limits(prompt: str, policy: dict[str, Any]) -> tuple[float, float]:
|
| 329 |
+
key = prompt.strip().lower()
|
| 330 |
+
override = as_mapping(policy["prompt_overrides"].get(key))
|
| 331 |
+
confidence = float_or_default(
|
| 332 |
+
override.get(
|
| 333 |
+
"minimum_confidence",
|
| 334 |
+
policy["minimum_confidence_by_prompt"].get(key, policy["minimum_prompt_confidence"]),
|
| 335 |
+
),
|
| 336 |
+
float(policy["minimum_prompt_confidence"]),
|
| 337 |
+
)
|
| 338 |
+
maximum_area = float_or_default(
|
| 339 |
+
override.get(
|
| 340 |
+
"maximum_area_ratio",
|
| 341 |
+
policy["maximum_area_ratio_by_prompt"].get(
|
| 342 |
+
key, policy["default_maximum_area_ratio"]
|
| 343 |
+
),
|
| 344 |
+
),
|
| 345 |
+
float(policy["default_maximum_area_ratio"]),
|
| 346 |
+
)
|
| 347 |
+
return min(1.0, max(0.0, confidence)), min(1.0, max(0.0, maximum_area))
|
| 348 |
+
|
| 349 |
+
|
| 350 |
+
def save_mask(path: Path, mask: np.ndarray) -> None:
|
| 351 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 352 |
+
Image.fromarray(mask.astype(np.uint8) * 255, mode="L").save(path)
|
| 353 |
+
|
| 354 |
+
|
| 355 |
+
def save_red_mask(path: Path, mask: np.ndarray) -> None:
|
| 356 |
+
"""Save an RGB review visualization while preserving obstacle.png as binary."""
|
| 357 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 358 |
+
color = np.zeros((*mask.shape, 3), dtype=np.uint8)
|
| 359 |
+
color[mask] = (230, 45, 45)
|
| 360 |
+
Image.fromarray(color, mode="RGB").save(path)
|
| 361 |
+
|
| 362 |
+
|
| 363 |
+
def remove_small_components(mask: np.ndarray, minimum_area: int) -> np.ndarray:
|
| 364 |
+
count, labels, stats, _ = cv2.connectedComponentsWithStats(mask.astype(np.uint8), 8)
|
| 365 |
+
output = np.zeros_like(mask, dtype=bool)
|
| 366 |
+
for index in range(1, count):
|
| 367 |
+
if int(stats[index, cv2.CC_STAT_AREA]) >= minimum_area:
|
| 368 |
+
output |= labels == index
|
| 369 |
+
return output
|
| 370 |
+
|
| 371 |
+
|
| 372 |
+
def retain_core_connected_union(union: np.ndarray, core: np.ndarray, minimum_area: int) -> np.ndarray:
|
| 373 |
+
union = remove_small_components(union, minimum_area)
|
| 374 |
+
count, labels, stats, _ = cv2.connectedComponentsWithStats(union.astype(np.uint8), 8)
|
| 375 |
+
output = np.zeros_like(union, dtype=bool)
|
| 376 |
+
for index in range(1, count):
|
| 377 |
+
component = labels == index
|
| 378 |
+
if int(stats[index, cv2.CC_STAT_AREA]) >= minimum_area and np.any(component & core):
|
| 379 |
+
output |= component
|
| 380 |
+
return output
|
| 381 |
+
|
| 382 |
+
|
| 383 |
+
def retain_nearby_obstacles(
|
| 384 |
+
obstacle: np.ndarray,
|
| 385 |
+
target: np.ndarray,
|
| 386 |
+
minimum_area: int,
|
| 387 |
+
maximum_area_ratio: float = 0.15,
|
| 388 |
+
) -> np.ndarray:
|
| 389 |
+
"""Keep compact obstacle components spatially adjacent to the target surface."""
|
| 390 |
+
distance = max(9, int(round(min(target.shape) * 0.035)) | 1)
|
| 391 |
+
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (distance, distance))
|
| 392 |
+
nearby = cv2.dilate(target.astype(np.uint8), kernel).astype(bool)
|
| 393 |
+
count, labels, stats, _ = cv2.connectedComponentsWithStats(obstacle.astype(np.uint8), 8)
|
| 394 |
+
output = np.zeros_like(obstacle, dtype=bool)
|
| 395 |
+
image_area = obstacle.size
|
| 396 |
+
for index in range(1, count):
|
| 397 |
+
area = int(stats[index, cv2.CC_STAT_AREA])
|
| 398 |
+
component = labels == index
|
| 399 |
+
if (
|
| 400 |
+
area >= minimum_area
|
| 401 |
+
and area / image_area <= maximum_area_ratio
|
| 402 |
+
and np.any(component & nearby)
|
| 403 |
+
):
|
| 404 |
+
output |= component
|
| 405 |
+
return output
|
| 406 |
+
|
| 407 |
+
|
| 408 |
+
def retain_compact_obstacles(
|
| 409 |
+
obstacle: np.ndarray,
|
| 410 |
+
minimum_area: int,
|
| 411 |
+
maximum_area_ratio: float = 0.12,
|
| 412 |
+
) -> np.ndarray:
|
| 413 |
+
"""Keep compact scene obstacles without requiring target adjacency."""
|
| 414 |
+
count, labels, stats, _ = cv2.connectedComponentsWithStats(obstacle.astype(np.uint8), 8)
|
| 415 |
+
output = np.zeros_like(obstacle, dtype=bool)
|
| 416 |
+
image_area = obstacle.size
|
| 417 |
+
for index in range(1, count):
|
| 418 |
+
area = int(stats[index, cv2.CC_STAT_AREA])
|
| 419 |
+
if minimum_area <= area <= image_area * maximum_area_ratio:
|
| 420 |
+
output |= labels == index
|
| 421 |
+
return output
|
| 422 |
+
|
| 423 |
+
|
| 424 |
+
def route_like_component_count(mask: np.ndarray, route_policy: dict[str, Any]) -> int:
|
| 425 |
+
"""Count broad ground/route-like components that must not seed hidden masks."""
|
| 426 |
+
if not bool(route_policy.get("enabled", False)) or not np.any(mask):
|
| 427 |
+
return 0
|
| 428 |
+
count, _, stats, _ = cv2.connectedComponentsWithStats(mask.astype(np.uint8), 8)
|
| 429 |
+
height, width = mask.shape
|
| 430 |
+
route_like = 0
|
| 431 |
+
for index in range(1, count):
|
| 432 |
+
area = int(stats[index, cv2.CC_STAT_AREA])
|
| 433 |
+
width_ratio = float(stats[index, cv2.CC_STAT_WIDTH] / max(width, 1))
|
| 434 |
+
height_ratio = float(stats[index, cv2.CC_STAT_HEIGHT] / max(height, 1))
|
| 435 |
+
area_ratio = float(area / max(mask.size, 1))
|
| 436 |
+
if (
|
| 437 |
+
width_ratio >= float(route_policy["minimum_bbox_width_ratio"])
|
| 438 |
+
and height_ratio >= float(route_policy["minimum_bbox_height_ratio"])
|
| 439 |
+
and area_ratio >= float(route_policy["minimum_component_area_ratio"])
|
| 440 |
+
):
|
| 441 |
+
route_like += 1
|
| 442 |
+
return route_like
|
| 443 |
+
|
| 444 |
+
|
| 445 |
+
def trusted_obstacle_prompt_evidence(
|
| 446 |
+
*,
|
| 447 |
+
prompt: str,
|
| 448 |
+
group: str,
|
| 449 |
+
prediction: PromptPrediction,
|
| 450 |
+
minimum_component_area: int,
|
| 451 |
+
policy: dict[str, Any],
|
| 452 |
+
) -> tuple[np.ndarray, np.ndarray, dict[str, Any]]:
|
| 453 |
+
"""Return all cleaned evidence and the subset safe to seed hidden completion.
|
| 454 |
+
|
| 455 |
+
The first mask is intentionally high recall: it only removes microscopic
|
| 456 |
+
components. The second is accepted only if its prompt confidence and
|
| 457 |
+
image-area sanity checks pass and it is not route-like. Rejecting at the
|
| 458 |
+
prompt level makes the audit rationale easy to understand and guarantees
|
| 459 |
+
that a broad false positive cannot leak into ``obstacle_all_detected``.
|
| 460 |
+
"""
|
| 461 |
+
cleaned = remove_small_components(prediction.mask, minimum_component_area)
|
| 462 |
+
minimum_confidence, maximum_area_ratio = evidence_prompt_limits(prompt, policy)
|
| 463 |
+
area_ratio = float(cleaned.mean())
|
| 464 |
+
reasons: list[str] = []
|
| 465 |
+
if not np.any(cleaned):
|
| 466 |
+
reasons.append("empty_after_small_component_removal")
|
| 467 |
+
if float(prediction.confidence) < minimum_confidence:
|
| 468 |
+
reasons.append("confidence_below_minimum")
|
| 469 |
+
if area_ratio > maximum_area_ratio:
|
| 470 |
+
reasons.append("area_ratio_exceeds_prompt_maximum")
|
| 471 |
+
if area_ratio > float(policy["full_frame_area_ratio"]):
|
| 472 |
+
reasons.append("area_ratio_exceeds_full_frame_guard")
|
| 473 |
+
route_like_count = route_like_component_count(cleaned, policy["route_like"])
|
| 474 |
+
if route_like_count:
|
| 475 |
+
reasons.append("route_like_component_detected")
|
| 476 |
+
accepted = not reasons
|
| 477 |
+
trusted = cleaned if accepted else np.zeros_like(cleaned)
|
| 478 |
+
decision = {
|
| 479 |
+
"prompt": prompt,
|
| 480 |
+
"group": group,
|
| 481 |
+
"confidence": round(float(prediction.confidence), 6),
|
| 482 |
+
"instance_count": int(prediction.instance_count),
|
| 483 |
+
"minimum_confidence": round(minimum_confidence, 6),
|
| 484 |
+
"maximum_area_ratio": round(maximum_area_ratio, 6),
|
| 485 |
+
"cleaned_pixels": int(cleaned.sum()),
|
| 486 |
+
"cleaned_area_ratio": round(area_ratio, 6),
|
| 487 |
+
"route_like_component_count": route_like_count,
|
| 488 |
+
"decision": "accepted" if accepted else "rejected",
|
| 489 |
+
"reasons": reasons,
|
| 490 |
+
"trusted_pixels": int(trusted.sum()),
|
| 491 |
+
}
|
| 492 |
+
return cleaned, trusted, decision
|
| 493 |
+
|
| 494 |
+
|
| 495 |
+
def area_plausibility(area_ratio: float, minimum: float, maximum: float) -> float:
|
| 496 |
+
if area_ratio <= 0 or area_ratio < minimum / 2 or area_ratio > min(1.0, maximum * 1.2):
|
| 497 |
+
return 0.0
|
| 498 |
+
if minimum <= area_ratio <= maximum:
|
| 499 |
+
return 1.0
|
| 500 |
+
if area_ratio < minimum:
|
| 501 |
+
return max(0.0, area_ratio / minimum)
|
| 502 |
+
return max(0.0, 1.0 - (area_ratio - maximum) / max(1.0 - maximum, 1e-6))
|
| 503 |
+
|
| 504 |
+
|
| 505 |
+
@dataclass
|
| 506 |
+
class PromptPrediction:
|
| 507 |
+
mask: np.ndarray
|
| 508 |
+
confidence: float
|
| 509 |
+
instance_count: int
|
| 510 |
+
boxes: list[list[float]]
|
| 511 |
+
|
| 512 |
+
|
| 513 |
+
class Sam3Backend:
|
| 514 |
+
name = "sam3"
|
| 515 |
+
|
| 516 |
+
def __init__(self, args: argparse.Namespace):
|
| 517 |
+
repo = Path(args.sam3_repo).resolve()
|
| 518 |
+
checkpoint = Path(args.sam3_checkpoint).resolve()
|
| 519 |
+
if not repo.is_dir():
|
| 520 |
+
raise FileNotFoundError(f"SAM 3 repository not found: {repo}")
|
| 521 |
+
if not checkpoint.is_file():
|
| 522 |
+
raise FileNotFoundError(
|
| 523 |
+
f"SAM 3 checkpoint not found: {checkpoint}. Accept the model terms and download sam3.pt first."
|
| 524 |
+
)
|
| 525 |
+
sys.path.insert(0, str(repo))
|
| 526 |
+
import torch
|
| 527 |
+
from sam3.model.sam3_image_processor import Sam3Processor
|
| 528 |
+
from sam3.model_builder import build_sam3_image_model
|
| 529 |
+
|
| 530 |
+
self.torch = torch
|
| 531 |
+
self.device = args.device
|
| 532 |
+
self.autocast_device = str(args.device).split(":", 1)[0]
|
| 533 |
+
self.autocast_dtype = torch.bfloat16
|
| 534 |
+
model = build_sam3_image_model(
|
| 535 |
+
device=args.device,
|
| 536 |
+
checkpoint_path=str(checkpoint),
|
| 537 |
+
load_from_HF=False,
|
| 538 |
+
compile=args.compile,
|
| 539 |
+
)
|
| 540 |
+
self.processor = Sam3Processor(
|
| 541 |
+
model,
|
| 542 |
+
device=args.device,
|
| 543 |
+
confidence_threshold=args.confidence_threshold,
|
| 544 |
+
)
|
| 545 |
+
self.state: dict[str, Any] | None = None
|
| 546 |
+
|
| 547 |
+
def begin_image(self, image: Image.Image) -> None:
|
| 548 |
+
with self.torch.autocast(
|
| 549 |
+
device_type=self.autocast_device,
|
| 550 |
+
dtype=self.autocast_dtype,
|
| 551 |
+
enabled=self.autocast_device == "cuda",
|
| 552 |
+
):
|
| 553 |
+
self.state = self.processor.set_image(image)
|
| 554 |
+
|
| 555 |
+
def predict(self, prompt: str) -> PromptPrediction:
|
| 556 |
+
if self.state is None:
|
| 557 |
+
raise RuntimeError("begin_image must be called before predict")
|
| 558 |
+
with self.torch.autocast(
|
| 559 |
+
device_type=self.autocast_device,
|
| 560 |
+
dtype=self.autocast_dtype,
|
| 561 |
+
enabled=self.autocast_device == "cuda",
|
| 562 |
+
):
|
| 563 |
+
output = self.processor.set_text_prompt(prompt, self.state)
|
| 564 |
+
masks_tensor = output["masks"]
|
| 565 |
+
scores_tensor = output["scores"]
|
| 566 |
+
boxes_tensor = output["boxes"]
|
| 567 |
+
if masks_tensor.numel() == 0:
|
| 568 |
+
shape = (int(output["original_height"]), int(output["original_width"]))
|
| 569 |
+
return PromptPrediction(np.zeros(shape, dtype=bool), 0.0, 0, [])
|
| 570 |
+
masks = masks_tensor.detach().cpu().numpy().astype(bool)
|
| 571 |
+
while masks.ndim > 3 and masks.shape[1] == 1:
|
| 572 |
+
masks = masks[:, 0]
|
| 573 |
+
union = np.any(masks, axis=0)
|
| 574 |
+
scores = scores_tensor.detach().float().cpu().numpy()
|
| 575 |
+
boxes = boxes_tensor.detach().float().cpu().numpy().tolist()
|
| 576 |
+
return PromptPrediction(union, float(scores.max()), int(len(scores)), boxes)
|
| 577 |
+
|
| 578 |
+
|
| 579 |
+
class GroundedSamBackend:
|
| 580 |
+
name = "grounded_sam"
|
| 581 |
+
|
| 582 |
+
def __init__(self, args: argparse.Namespace):
|
| 583 |
+
repo = Path(args.grounded_sam_repo).resolve()
|
| 584 |
+
config = Path(args.grounding_dino_config).resolve()
|
| 585 |
+
dino_checkpoint = Path(args.grounding_dino_checkpoint).resolve()
|
| 586 |
+
sam_checkpoint = Path(args.sam_checkpoint).resolve()
|
| 587 |
+
for path in (repo, config, dino_checkpoint, sam_checkpoint):
|
| 588 |
+
if not path.exists():
|
| 589 |
+
raise FileNotFoundError(f"Required Grounded-SAM path not found: {path}")
|
| 590 |
+
sys.path[:0] = [str(repo), str(repo / "GroundingDINO"), str(repo / "segment_anything")]
|
| 591 |
+
import torch
|
| 592 |
+
import GroundingDINO.groundingdino.datasets.transforms as T
|
| 593 |
+
from GroundingDINO.groundingdino.models import build_model
|
| 594 |
+
from GroundingDINO.groundingdino.util.slconfig import SLConfig
|
| 595 |
+
from GroundingDINO.groundingdino.util.utils import clean_state_dict
|
| 596 |
+
from segment_anything import SamPredictor, sam_model_registry
|
| 597 |
+
|
| 598 |
+
model_args = SLConfig.fromfile(str(config))
|
| 599 |
+
model_args.device = args.device
|
| 600 |
+
if args.bert_path:
|
| 601 |
+
model_args.bert_base_uncased_path = str(Path(args.bert_path).resolve())
|
| 602 |
+
self.torch = torch
|
| 603 |
+
self.transforms = T.Compose(
|
| 604 |
+
[
|
| 605 |
+
T.RandomResize([800], max_size=1333),
|
| 606 |
+
T.ToTensor(),
|
| 607 |
+
T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
|
| 608 |
+
]
|
| 609 |
+
)
|
| 610 |
+
self.device = args.device
|
| 611 |
+
self.box_threshold = args.box_threshold
|
| 612 |
+
self.model = build_model(model_args)
|
| 613 |
+
checkpoint = torch.load(str(dino_checkpoint), map_location="cpu", weights_only=False)
|
| 614 |
+
self.model.load_state_dict(clean_state_dict(checkpoint["model"]), strict=False)
|
| 615 |
+
self.model.to(args.device).eval()
|
| 616 |
+
sam = sam_model_registry[args.sam_model_type](checkpoint=str(sam_checkpoint)).to(args.device)
|
| 617 |
+
self.predictor = SamPredictor(sam)
|
| 618 |
+
self.image_pil: Image.Image | None = None
|
| 619 |
+
self.image_tensor = None
|
| 620 |
+
self.image_rgb: np.ndarray | None = None
|
| 621 |
+
|
| 622 |
+
def begin_image(self, image: Image.Image) -> None:
|
| 623 |
+
self.image_pil = image
|
| 624 |
+
self.image_rgb = np.asarray(image)
|
| 625 |
+
self.image_tensor, _ = self.transforms(image, None)
|
| 626 |
+
self.predictor.set_image(self.image_rgb)
|
| 627 |
+
|
| 628 |
+
def predict(self, prompt: str) -> PromptPrediction:
|
| 629 |
+
if self.image_pil is None or self.image_tensor is None or self.image_rgb is None:
|
| 630 |
+
raise RuntimeError("begin_image must be called before predict")
|
| 631 |
+
caption = prompt.lower().strip()
|
| 632 |
+
if not caption.endswith("."):
|
| 633 |
+
caption += "."
|
| 634 |
+
with self.torch.inference_mode():
|
| 635 |
+
outputs = self.model(self.image_tensor[None].to(self.device), captions=[caption])
|
| 636 |
+
logits = outputs["pred_logits"].detach().cpu().sigmoid()[0]
|
| 637 |
+
boxes = outputs["pred_boxes"].detach().cpu()[0]
|
| 638 |
+
confidence = logits.max(dim=1).values
|
| 639 |
+
keep = confidence > self.box_threshold
|
| 640 |
+
boxes = boxes[keep]
|
| 641 |
+
confidence = confidence[keep]
|
| 642 |
+
height, width = self.image_rgb.shape[:2]
|
| 643 |
+
if boxes.shape[0] == 0:
|
| 644 |
+
return PromptPrediction(np.zeros((height, width), dtype=bool), 0.0, 0, [])
|
| 645 |
+
scale = self.torch.tensor([width, height, width, height], dtype=boxes.dtype)
|
| 646 |
+
boxes = boxes * scale
|
| 647 |
+
boxes[:, :2] -= boxes[:, 2:] / 2
|
| 648 |
+
boxes[:, 2:] += boxes[:, :2]
|
| 649 |
+
transformed = self.predictor.transform.apply_boxes_torch(boxes, (height, width)).to(self.device)
|
| 650 |
+
with self.torch.inference_mode():
|
| 651 |
+
masks, sam_scores, _ = self.predictor.predict_torch(
|
| 652 |
+
point_coords=None,
|
| 653 |
+
point_labels=None,
|
| 654 |
+
boxes=transformed,
|
| 655 |
+
multimask_output=False,
|
| 656 |
+
)
|
| 657 |
+
masks_np = masks[:, 0].detach().cpu().numpy().astype(bool)
|
| 658 |
+
combined_confidence = float(
|
| 659 |
+
math.sqrt(float(confidence.max()) * float(sam_scores.detach().cpu().max()))
|
| 660 |
+
)
|
| 661 |
+
return PromptPrediction(
|
| 662 |
+
np.any(masks_np, axis=0),
|
| 663 |
+
combined_confidence,
|
| 664 |
+
int(len(masks_np)),
|
| 665 |
+
boxes.tolist(),
|
| 666 |
+
)
|
| 667 |
+
|
| 668 |
+
|
| 669 |
+
def build_backend(args: argparse.Namespace):
|
| 670 |
+
if args.backend == "sam3":
|
| 671 |
+
return Sam3Backend(args)
|
| 672 |
+
if args.backend == "grounded_sam":
|
| 673 |
+
return GroundedSamBackend(args)
|
| 674 |
+
raise ValueError(f"Unsupported backend: {args.backend}")
|
| 675 |
+
|
| 676 |
+
|
| 677 |
+
def prompt_consensus(
|
| 678 |
+
predictions: list[PromptPrediction], minimum_area: int
|
| 679 |
+
) -> tuple[np.ndarray, np.ndarray, np.ndarray, dict[str, Any]]:
|
| 680 |
+
if not predictions:
|
| 681 |
+
raise ValueError("At least one prompt prediction is required")
|
| 682 |
+
masks = np.stack([prediction.mask for prediction in predictions])
|
| 683 |
+
votes = masks.sum(axis=0)
|
| 684 |
+
union = votes > 0
|
| 685 |
+
majority = max(2, math.ceil(len(predictions) / 2)) if len(predictions) > 1 else 1
|
| 686 |
+
core = votes >= majority
|
| 687 |
+
fallback = False
|
| 688 |
+
if not np.any(core) and np.any(union):
|
| 689 |
+
fallback = True
|
| 690 |
+
best_index = int(np.argmax([prediction.confidence for prediction in predictions]))
|
| 691 |
+
core = predictions[best_index].mask.copy()
|
| 692 |
+
candidate = retain_core_connected_union(union, core, minimum_area)
|
| 693 |
+
core = remove_small_components(core & candidate, minimum_area)
|
| 694 |
+
uncertain = candidate & ~core
|
| 695 |
+
union_area = int(candidate.sum())
|
| 696 |
+
agreement = float(core.sum() / union_area) if union_area else 0.0
|
| 697 |
+
confidences = [prediction.confidence for prediction in predictions]
|
| 698 |
+
metadata = {
|
| 699 |
+
"majority_vote": majority,
|
| 700 |
+
"fallback_to_best_prompt": fallback,
|
| 701 |
+
"agreement": round(agreement, 6),
|
| 702 |
+
"mean_prompt_confidence": round(float(np.mean(confidences)), 6),
|
| 703 |
+
"maximum_prompt_confidence": round(float(np.max(confidences)), 6),
|
| 704 |
+
"detected_prompt_count": sum(value > 0 for value in confidences),
|
| 705 |
+
}
|
| 706 |
+
return candidate, core, uncertain, metadata
|
| 707 |
+
|
| 708 |
+
|
| 709 |
+
def overlay_masks(
|
| 710 |
+
image: np.ndarray, target: np.ndarray, obstacle: np.ndarray, uncertain: np.ndarray
|
| 711 |
+
) -> np.ndarray:
|
| 712 |
+
overlay = image.astype(np.float32).copy()
|
| 713 |
+
colors = [
|
| 714 |
+
(target, np.asarray([30, 210, 70], dtype=np.float32), 0.38),
|
| 715 |
+
(uncertain, np.asarray([250, 210, 30], dtype=np.float32), 0.55),
|
| 716 |
+
# Draw obstacles last so target/uncertainty colors never hide red occluders.
|
| 717 |
+
(obstacle, np.asarray([230, 45, 45], dtype=np.float32), 0.58),
|
| 718 |
+
]
|
| 719 |
+
for mask, color, alpha in colors:
|
| 720 |
+
overlay[mask] = overlay[mask] * (1.0 - alpha) + color * alpha
|
| 721 |
+
return np.clip(overlay, 0, 255).astype(np.uint8)
|
| 722 |
+
|
| 723 |
+
|
| 724 |
+
def score_category(
|
| 725 |
+
mask: np.ndarray,
|
| 726 |
+
consensus: dict[str, Any],
|
| 727 |
+
thresholds: dict[str, float],
|
| 728 |
+
category: str,
|
| 729 |
+
) -> tuple[float, dict[str, float]]:
|
| 730 |
+
area_ratio = float(mask.mean())
|
| 731 |
+
plausibility = area_plausibility(
|
| 732 |
+
area_ratio,
|
| 733 |
+
thresholds["minimum_target_area_ratio"],
|
| 734 |
+
thresholds["maximum_target_area_ratio"],
|
| 735 |
+
)
|
| 736 |
+
bottom_contact = float(mask[int(mask.shape[0] * 0.8) :, :].mean() > 0.005)
|
| 737 |
+
confidence = float(consensus["mean_prompt_confidence"])
|
| 738 |
+
agreement = float(consensus["agreement"])
|
| 739 |
+
evidence = 0.55 * confidence + 0.35 * agreement + 0.10 * bottom_contact
|
| 740 |
+
score = evidence * (0.20 + 0.80 * plausibility)
|
| 741 |
+
if category == "walkway":
|
| 742 |
+
score *= 0.94 # prevent the generic class from dominating specific structures
|
| 743 |
+
signals = {
|
| 744 |
+
"area_ratio": round(area_ratio, 6),
|
| 745 |
+
"area_plausibility": round(plausibility, 6),
|
| 746 |
+
"bottom_contact": bottom_contact,
|
| 747 |
+
"score": round(score, 6),
|
| 748 |
+
}
|
| 749 |
+
return score, signals
|
| 750 |
+
|
| 751 |
+
|
| 752 |
+
def predict_prompt_set(
|
| 753 |
+
backend,
|
| 754 |
+
prompts: list[str],
|
| 755 |
+
output_dir: Path,
|
| 756 |
+
minimum_component_area: int,
|
| 757 |
+
save_prompt_masks: bool = True,
|
| 758 |
+
) -> tuple[np.ndarray, np.ndarray, np.ndarray, dict[str, Any]]:
|
| 759 |
+
predictions = []
|
| 760 |
+
prompt_metadata = []
|
| 761 |
+
for prompt in prompts:
|
| 762 |
+
prediction = backend.predict(prompt)
|
| 763 |
+
predictions.append(prediction)
|
| 764 |
+
if save_prompt_masks:
|
| 765 |
+
save_mask(output_dir / f"{safe_name(prompt)}.png", prediction.mask)
|
| 766 |
+
prompt_metadata.append(
|
| 767 |
+
{
|
| 768 |
+
"prompt": prompt,
|
| 769 |
+
"confidence": round(prediction.confidence, 6),
|
| 770 |
+
"instance_count": prediction.instance_count,
|
| 771 |
+
"boxes_xyxy": prediction.boxes,
|
| 772 |
+
}
|
| 773 |
+
)
|
| 774 |
+
candidate, core, uncertain, consensus = prompt_consensus(
|
| 775 |
+
predictions, minimum_component_area
|
| 776 |
+
)
|
| 777 |
+
consensus["prompts"] = prompt_metadata
|
| 778 |
+
return candidate, core, uncertain, consensus
|
| 779 |
+
|
| 780 |
+
|
| 781 |
+
def process_sample(
|
| 782 |
+
row: dict[str, Any],
|
| 783 |
+
backend,
|
| 784 |
+
config: dict[str, Any],
|
| 785 |
+
output_root: Path,
|
| 786 |
+
args: argparse.Namespace,
|
| 787 |
+
) -> dict[str, Any]:
|
| 788 |
+
sample_id = row["sample_id"]
|
| 789 |
+
sample_dir = output_root / "samples" / sample_id
|
| 790 |
+
quality_path = sample_dir / "quality.json"
|
| 791 |
+
if args.resume:
|
| 792 |
+
quality = load_resume_quality(sample_dir, sample_id, backend.name)
|
| 793 |
+
if quality is not None:
|
| 794 |
+
return quality
|
| 795 |
+
|
| 796 |
+
# Phone photos commonly store the camera pixels in landscape orientation
|
| 797 |
+
# and rely on EXIF to display them upright. Normalize that orientation
|
| 798 |
+
# before segmentation so every downstream mask uses the displayed frame.
|
| 799 |
+
image = ImageOps.exif_transpose(Image.open(row["image_path"])).convert("RGB")
|
| 800 |
+
rgb = np.asarray(image)
|
| 801 |
+
image_area = rgb.shape[0] * rgb.shape[1]
|
| 802 |
+
minimum_component_area = max(args.minimum_component_area, int(image_area * 0.0002))
|
| 803 |
+
backend.begin_image(image)
|
| 804 |
+
|
| 805 |
+
requested_categories = args.categories or list(config["categories"])
|
| 806 |
+
fixed_category = row.get("category")
|
| 807 |
+
if fixed_category:
|
| 808 |
+
requested_categories = [fixed_category]
|
| 809 |
+
if args.verify_manifest_category and not row.get("category_reviewed", False):
|
| 810 |
+
requested_categories.extend(
|
| 811 |
+
config.get("category_confusions", {}).get(fixed_category, [])
|
| 812 |
+
)
|
| 813 |
+
requested_categories = list(dict.fromkeys(requested_categories))
|
| 814 |
+
|
| 815 |
+
category_results: dict[str, dict[str, Any]] = {}
|
| 816 |
+
masks: dict[str, tuple[np.ndarray, np.ndarray, np.ndarray]] = {}
|
| 817 |
+
thresholds = config["quality_thresholds"]
|
| 818 |
+
save_full_artifacts = args.artifact_level == "full"
|
| 819 |
+
for category in requested_categories:
|
| 820 |
+
if category not in config["categories"]:
|
| 821 |
+
raise ValueError(f"Unknown category in pilot manifest: {category}")
|
| 822 |
+
prompt_dir = sample_dir / "prompts" / category
|
| 823 |
+
candidate, core, uncertain, consensus = predict_prompt_set(
|
| 824 |
+
backend,
|
| 825 |
+
config["categories"][category]["prompts"],
|
| 826 |
+
prompt_dir,
|
| 827 |
+
minimum_component_area,
|
| 828 |
+
save_prompt_masks=save_full_artifacts,
|
| 829 |
+
)
|
| 830 |
+
# --- per-category area cap (prevents whole-image over-segmentation) ---
|
| 831 |
+
cat_max_area = config["categories"][category].get(
|
| 832 |
+
"max_area_ratio",
|
| 833 |
+
config.get("tactile_paving_max_area_ratio", None),
|
| 834 |
+
)
|
| 835 |
+
candidate_area_ratio = float(candidate.mean()) if candidate.any() else 0.0
|
| 836 |
+
if cat_max_area is not None and candidate_area_ratio > float(cat_max_area):
|
| 837 |
+
consensus["area_cap_rejected"] = True
|
| 838 |
+
consensus["area_cap_ratio"] = round(candidate_area_ratio, 6)
|
| 839 |
+
consensus["area_cap_limit"] = float(cat_max_area)
|
| 840 |
+
candidate = np.zeros_like(candidate)
|
| 841 |
+
core = np.zeros_like(core)
|
| 842 |
+
uncertain = np.zeros_like(uncertain)
|
| 843 |
+
else:
|
| 844 |
+
consensus["area_cap_rejected"] = False
|
| 845 |
+
# --- end area cap ---
|
| 846 |
+
score, signals = score_category(candidate, consensus, thresholds, category)
|
| 847 |
+
if save_full_artifacts:
|
| 848 |
+
save_mask(sample_dir / "categories" / f"{category}_candidate.png", candidate)
|
| 849 |
+
save_mask(sample_dir / "categories" / f"{category}_core.png", core)
|
| 850 |
+
save_mask(sample_dir / "categories" / f"{category}_uncertain.png", uncertain)
|
| 851 |
+
masks[category] = (candidate, core, uncertain)
|
| 852 |
+
category_results[category] = {"consensus": consensus, **signals}
|
| 853 |
+
|
| 854 |
+
best_scored_category = max(
|
| 855 |
+
category_results, key=lambda key: category_results[key]["score"]
|
| 856 |
+
)
|
| 857 |
+
# A mask score is useful for flagging semantic disagreement, but it is not
|
| 858 |
+
# reliable enough to silently overwrite a manifest category. Human review
|
| 859 |
+
# overrides remain authoritative.
|
| 860 |
+
selected_category = fixed_category or best_scored_category
|
| 861 |
+
category_disagreement = bool(
|
| 862 |
+
fixed_category and best_scored_category != fixed_category
|
| 863 |
+
)
|
| 864 |
+
target, target_core, target_uncertain = masks[selected_category]
|
| 865 |
+
|
| 866 |
+
prompt_resolution = obstacle_prompt_resolution(
|
| 867 |
+
config, selected_category, backend.name
|
| 868 |
+
)
|
| 869 |
+
obstacle_predictions: list[tuple[str, PromptPrediction]] = []
|
| 870 |
+
obstacle_metadata = []
|
| 871 |
+
for prompt in prompt_resolution["prompts"]:
|
| 872 |
+
prediction = backend.predict(prompt)
|
| 873 |
+
obstacle_predictions.append((prompt, prediction))
|
| 874 |
+
if save_full_artifacts:
|
| 875 |
+
save_mask(
|
| 876 |
+
sample_dir / "prompts" / "obstacles" / f"{safe_name(prompt)}.png",
|
| 877 |
+
prediction.mask,
|
| 878 |
+
)
|
| 879 |
+
obstacle_metadata.append(
|
| 880 |
+
{
|
| 881 |
+
"prompt": prompt,
|
| 882 |
+
"group": "dynamic_or_compact",
|
| 883 |
+
"confidence": round(prediction.confidence, 6),
|
| 884 |
+
"instance_count": prediction.instance_count,
|
| 885 |
+
"boxes_xyxy": prediction.boxes,
|
| 886 |
+
}
|
| 887 |
+
)
|
| 888 |
+
maximum_ratios = config.get("obstacle_maximum_area_ratio", {})
|
| 889 |
+
filtered_dynamic_obstacles = []
|
| 890 |
+
for prompt, prediction in obstacle_predictions:
|
| 891 |
+
cleaned = remove_small_components(prediction.mask, minimum_component_area)
|
| 892 |
+
maximum_area_ratio = float(
|
| 893 |
+
maximum_ratios.get(prompt, maximum_ratios.get("default", 0.15))
|
| 894 |
+
)
|
| 895 |
+
filtered_dynamic_obstacles.append(
|
| 896 |
+
retain_nearby_obstacles(
|
| 897 |
+
cleaned,
|
| 898 |
+
target,
|
| 899 |
+
minimum_component_area,
|
| 900 |
+
maximum_area_ratio=maximum_area_ratio,
|
| 901 |
+
)
|
| 902 |
+
)
|
| 903 |
+
dynamic_obstacle = (
|
| 904 |
+
np.any(np.stack(filtered_dynamic_obstacles), axis=0)
|
| 905 |
+
if filtered_dynamic_obstacles
|
| 906 |
+
else np.zeros_like(target)
|
| 907 |
+
)
|
| 908 |
+
|
| 909 |
+
barrier_predictions: list[tuple[str, PromptPrediction]] = []
|
| 910 |
+
barrier_backends = config.get("barrier_obstacle_backends", ["sam3"])
|
| 911 |
+
barrier_prompts = (
|
| 912 |
+
config.get("barrier_obstacle_prompts", [])
|
| 913 |
+
if backend.name in barrier_backends
|
| 914 |
+
else []
|
| 915 |
+
)
|
| 916 |
+
for prompt in barrier_prompts:
|
| 917 |
+
prediction = backend.predict(prompt)
|
| 918 |
+
barrier_predictions.append((prompt, prediction))
|
| 919 |
+
if save_full_artifacts:
|
| 920 |
+
save_mask(
|
| 921 |
+
sample_dir / "prompts" / "obstacles" / f"{safe_name(prompt)}.png",
|
| 922 |
+
prediction.mask,
|
| 923 |
+
)
|
| 924 |
+
obstacle_metadata.append(
|
| 925 |
+
{
|
| 926 |
+
"prompt": prompt,
|
| 927 |
+
"group": "barrier_consensus",
|
| 928 |
+
"confidence": round(prediction.confidence, 6),
|
| 929 |
+
"instance_count": prediction.instance_count,
|
| 930 |
+
"boxes_xyxy": prediction.boxes,
|
| 931 |
+
}
|
| 932 |
+
)
|
| 933 |
+
if barrier_predictions:
|
| 934 |
+
barrier_votes = np.stack(
|
| 935 |
+
[prediction.mask for _, prediction in barrier_predictions]
|
| 936 |
+
).sum(axis=0)
|
| 937 |
+
required_votes = max(2, math.ceil(len(barrier_predictions) / 2))
|
| 938 |
+
barrier_obstacle = barrier_votes >= required_votes
|
| 939 |
+
barrier_obstacle = retain_compact_obstacles(
|
| 940 |
+
barrier_obstacle, minimum_component_area
|
| 941 |
+
)
|
| 942 |
+
else:
|
| 943 |
+
barrier_obstacle = np.zeros_like(target)
|
| 944 |
+
|
| 945 |
+
# Keep two explicit evidence layers. The unfiltered union is an audit
|
| 946 |
+
# artifact: every prompt after only microscopic-component removal. The
|
| 947 |
+
# trusted union applies per-prompt confidence, prompt-area, full-frame,
|
| 948 |
+
# and broad-route sanity checks before it can seed a later hidden proposal.
|
| 949 |
+
# ``obstacle.png`` below intentionally keeps its original nearby policy.
|
| 950 |
+
evidence_policy = resolve_obstacle_evidence_policy(config, backend.name)
|
| 951 |
+
evidence_minimum_area = evidence_minimum_component_area(evidence_policy, image_area)
|
| 952 |
+
unfiltered_evidence_masks: list[np.ndarray] = []
|
| 953 |
+
trusted_evidence_masks: list[np.ndarray] = []
|
| 954 |
+
obstacle_evidence_decisions: list[dict[str, Any]] = []
|
| 955 |
+
for group, predictions in (
|
| 956 |
+
("dynamic_or_compact", obstacle_predictions),
|
| 957 |
+
("barrier_consensus", barrier_predictions),
|
| 958 |
+
):
|
| 959 |
+
for prompt, prediction in predictions:
|
| 960 |
+
unfiltered, trusted, decision = trusted_obstacle_prompt_evidence(
|
| 961 |
+
prompt=prompt,
|
| 962 |
+
group=group,
|
| 963 |
+
prediction=prediction,
|
| 964 |
+
minimum_component_area=evidence_minimum_area,
|
| 965 |
+
policy=evidence_policy,
|
| 966 |
+
)
|
| 967 |
+
unfiltered_evidence_masks.append(unfiltered)
|
| 968 |
+
trusted_evidence_masks.append(trusted)
|
| 969 |
+
obstacle_evidence_decisions.append(decision)
|
| 970 |
+
obstacle_unfiltered_evidence = (
|
| 971 |
+
np.any(np.stack(unfiltered_evidence_masks), axis=0)
|
| 972 |
+
if unfiltered_evidence_masks
|
| 973 |
+
else np.zeros_like(target)
|
| 974 |
+
)
|
| 975 |
+
obstacle_all_detected = (
|
| 976 |
+
np.any(np.stack(trusted_evidence_masks), axis=0)
|
| 977 |
+
if trusted_evidence_masks
|
| 978 |
+
else np.zeros_like(target)
|
| 979 |
+
)
|
| 980 |
+
obstacle = dynamic_obstacle | barrier_obstacle
|
| 981 |
+
target = target & ~obstacle
|
| 982 |
+
target_core = target_core & target
|
| 983 |
+
target_uncertain = target_uncertain & target
|
| 984 |
+
|
| 985 |
+
kernel_size = max(5, int(round(min(rgb.shape[:2]) * 0.012)) | 1)
|
| 986 |
+
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (kernel_size, kernel_size))
|
| 987 |
+
target_dilated = cv2.dilate(target.astype(np.uint8), kernel).astype(bool)
|
| 988 |
+
obstacle_dilated = cv2.dilate(obstacle.astype(np.uint8), kernel).astype(bool)
|
| 989 |
+
adjacency = target_dilated & obstacle_dilated
|
| 990 |
+
adjacency_ratio = float(adjacency.sum() / max(target.sum(), 1))
|
| 991 |
+
|
| 992 |
+
save_mask(sample_dir / "target_visible_candidate.png", target)
|
| 993 |
+
save_mask(sample_dir / "target_visible_core.png", target_core)
|
| 994 |
+
save_mask(sample_dir / "target_uncertain.png", target_uncertain)
|
| 995 |
+
save_mask(sample_dir / "obstacle_unfiltered_evidence.png", obstacle_unfiltered_evidence)
|
| 996 |
+
save_mask(sample_dir / "obstacle_all_detected.png", obstacle_all_detected)
|
| 997 |
+
save_mask(sample_dir / "obstacle.png", obstacle)
|
| 998 |
+
save_red_mask(sample_dir / "obstacle_red.png", obstacle)
|
| 999 |
+
Image.fromarray(overlay_masks(rgb, target_core, obstacle, target_uncertain)).save(
|
| 1000 |
+
sample_dir / "overlay.png", quality=92
|
| 1001 |
+
)
|
| 1002 |
+
|
| 1003 |
+
selected = category_results[selected_category]
|
| 1004 |
+
score = float(selected["score"])
|
| 1005 |
+
target_area_ratio = float(target.mean())
|
| 1006 |
+
uncertainty_ratio = float(target_uncertain.sum() / max(target.sum(), 1))
|
| 1007 |
+
obstacle_area_ratio = float(obstacle.mean())
|
| 1008 |
+
area_valid = (
|
| 1009 |
+
thresholds["minimum_target_area_ratio"]
|
| 1010 |
+
<= target_area_ratio
|
| 1011 |
+
<= thresholds["maximum_target_area_ratio"]
|
| 1012 |
+
)
|
| 1013 |
+
if not area_valid:
|
| 1014 |
+
review_status = "reject_or_reprompt"
|
| 1015 |
+
elif (
|
| 1016 |
+
score >= thresholds["auto_accept_score"]
|
| 1017 |
+
and not selected["consensus"]["fallback_to_best_prompt"]
|
| 1018 |
+
and uncertainty_ratio <= 0.35
|
| 1019 |
+
and obstacle_area_ratio <= 0.20
|
| 1020 |
+
):
|
| 1021 |
+
review_status = "candidate_accept_after_visual_review"
|
| 1022 |
+
elif score >= thresholds["manual_review_score"]:
|
| 1023 |
+
review_status = "manual_review"
|
| 1024 |
+
else:
|
| 1025 |
+
review_status = "reject_or_reprompt"
|
| 1026 |
+
if (
|
| 1027 |
+
category_disagreement
|
| 1028 |
+
and not row.get("category_reviewed", False)
|
| 1029 |
+
and review_status == "candidate_accept_after_visual_review"
|
| 1030 |
+
):
|
| 1031 |
+
review_status = "manual_review"
|
| 1032 |
+
quality = {
|
| 1033 |
+
"sample_id": sample_id,
|
| 1034 |
+
"image_path": row["image_path"],
|
| 1035 |
+
"backend": backend.name,
|
| 1036 |
+
"selected_category": selected_category,
|
| 1037 |
+
"category_verification": {
|
| 1038 |
+
"manifest_category": fixed_category,
|
| 1039 |
+
"best_scored_category": best_scored_category,
|
| 1040 |
+
"category_disagreement": category_disagreement,
|
| 1041 |
+
"category_reviewed": bool(row.get("category_reviewed", False)),
|
| 1042 |
+
},
|
| 1043 |
+
"quality_score": round(score, 6),
|
| 1044 |
+
"review_status": review_status,
|
| 1045 |
+
"target_area_ratio": round(target_area_ratio, 6),
|
| 1046 |
+
"target_core_ratio": round(float(target_core.mean()), 6),
|
| 1047 |
+
"uncertainty_ratio_within_target": round(
|
| 1048 |
+
uncertainty_ratio, 6
|
| 1049 |
+
),
|
| 1050 |
+
"obstacle_area_ratio": round(obstacle_area_ratio, 6),
|
| 1051 |
+
"obstacle_unfiltered_evidence_pixels": int(obstacle_unfiltered_evidence.sum()),
|
| 1052 |
+
"obstacle_unfiltered_evidence_area_ratio": round(
|
| 1053 |
+
float(obstacle_unfiltered_evidence.mean()), 6
|
| 1054 |
+
),
|
| 1055 |
+
"obstacle_all_detected_pixels": int(obstacle_all_detected.sum()),
|
| 1056 |
+
"obstacle_all_detected_area_ratio": round(float(obstacle_all_detected.mean()), 6),
|
| 1057 |
+
"obstacle_all_detected_policy": "sanitized_high_recall_prompt_union_v2",
|
| 1058 |
+
"obstacle_evidence_policy_version": OBSTACLE_EVIDENCE_POLICY_VERSION,
|
| 1059 |
+
"obstacle_evidence_minimum_component_area_pixels": evidence_minimum_area,
|
| 1060 |
+
"obstacle_evidence_artifact_semantics": {
|
| 1061 |
+
"obstacle_unfiltered_evidence.png": (
|
| 1062 |
+
"union of every obstacle/barrier prompt after only small-component removal; "
|
| 1063 |
+
"audit evidence only, never direct hidden-completion support"
|
| 1064 |
+
),
|
| 1065 |
+
"obstacle_all_detected.png": (
|
| 1066 |
+
"high-recall prompt union after confidence, per-prompt area, full-frame, "
|
| 1067 |
+
"and route-like sanity filters; candidate evidence for later hidden completion"
|
| 1068 |
+
),
|
| 1069 |
+
"obstacle.png": (
|
| 1070 |
+
"unchanged nearby/compact obstacle proposal used for visible-mask cleanup; "
|
| 1071 |
+
"not the high-recall evidence layer"
|
| 1072 |
+
),
|
| 1073 |
+
},
|
| 1074 |
+
"obstacle_prompt_resolution": prompt_resolution,
|
| 1075 |
+
"obstacle_evidence_policy": evidence_policy,
|
| 1076 |
+
"obstacle_evidence_prompt_decisions": obstacle_evidence_decisions,
|
| 1077 |
+
"obstacle_evidence_decision_counts": dict(
|
| 1078 |
+
sorted(Counter(item["decision"] for item in obstacle_evidence_decisions).items())
|
| 1079 |
+
),
|
| 1080 |
+
"barrier_obstacle_area_ratio": round(float(barrier_obstacle.mean()), 6),
|
| 1081 |
+
"target_obstacle_adjacency_ratio": round(adjacency_ratio, 6),
|
| 1082 |
+
"category_results": category_results,
|
| 1083 |
+
"obstacle_prompts": obstacle_metadata,
|
| 1084 |
+
"automatic_mask_is_ground_truth": False,
|
| 1085 |
+
"artifact_level": args.artifact_level,
|
| 1086 |
+
}
|
| 1087 |
+
json_dump(quality_path, quality)
|
| 1088 |
+
return quality
|
| 1089 |
+
|
| 1090 |
+
|
| 1091 |
+
def build_html_report(output_dir: Path, results: list[dict[str, Any]]) -> None:
|
| 1092 |
+
rows = []
|
| 1093 |
+
for result in sorted(results, key=lambda item: item.get("quality_score", -1), reverse=True):
|
| 1094 |
+
sample_id = result["sample_id"]
|
| 1095 |
+
overlay = f"samples/{sample_id}/overlay.png"
|
| 1096 |
+
error = result.get("error")
|
| 1097 |
+
details = html.escape(error) if error else (
|
| 1098 |
+
f'{html.escape(result["selected_category"])} | '
|
| 1099 |
+
f'score={result["quality_score"]:.3f} | '
|
| 1100 |
+
f'{html.escape(result["review_status"])}'
|
| 1101 |
+
)
|
| 1102 |
+
rows.append(
|
| 1103 |
+
f'<article><img src="{html.escape(overlay)}" loading="lazy">'
|
| 1104 |
+
f'<div><strong>{html.escape(sample_id)}</strong><br>{details}</div></article>'
|
| 1105 |
+
)
|
| 1106 |
+
document = """<!doctype html><meta charset="utf-8"><title>Accessibility mask pilot</title>
|
| 1107 |
+
<style>body{font-family:sans-serif;margin:20px}main{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:16px}article{border:1px solid #bbb;padding:8px}img{width:100%;height:auto}div{margin-top:6px;font-size:14px}</style>
|
| 1108 |
+
<h1>Accessibility mask pilot</h1><p>Green: consensus core; yellow: uncertain target; red: obstacle. All masks require visual review.</p><main>"""
|
| 1109 |
+
document += "\n".join(rows) + "</main>"
|
| 1110 |
+
(output_dir / "report.html").write_text(document, encoding="utf-8")
|
| 1111 |
+
|
| 1112 |
+
|
| 1113 |
+
def build_parser() -> argparse.ArgumentParser:
|
| 1114 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 1115 |
+
parser.add_argument("--pilot-manifest", default="output/accessibility_clip_screen/pilot_manifest.jsonl")
|
| 1116 |
+
parser.add_argument("--image", default=None, help="Single-image mode input image; bypasses --pilot-manifest.")
|
| 1117 |
+
parser.add_argument("--single-sample-id", default=None, help="Stable output ID used with --image.")
|
| 1118 |
+
parser.add_argument(
|
| 1119 |
+
"--single-category",
|
| 1120 |
+
choices=["curb_cut", "ramp", "stairs", "tactile_paving", "walkway"],
|
| 1121 |
+
default=None,
|
| 1122 |
+
help="Known category for --image. Omit to score all configured categories.",
|
| 1123 |
+
)
|
| 1124 |
+
parser.add_argument("--prompt-config", default="configs/accessibility_mask_prompts.json")
|
| 1125 |
+
parser.add_argument(
|
| 1126 |
+
"--prompt-config-override",
|
| 1127 |
+
default=None,
|
| 1128 |
+
help=(
|
| 1129 |
+
"Optional JSON overlay merged into --prompt-config for this output-only run. "
|
| 1130 |
+
"Nested prompt lists are appended and de-duplicated; the base config is never edited."
|
| 1131 |
+
),
|
| 1132 |
+
)
|
| 1133 |
+
parser.add_argument("--output-dir", default="output/accessibility_mask_proposals/sam3")
|
| 1134 |
+
parser.add_argument("--backend", choices=["sam3", "grounded_sam"], default="sam3")
|
| 1135 |
+
parser.add_argument("--device", default="cuda")
|
| 1136 |
+
parser.add_argument("--categories", nargs="*", default=None)
|
| 1137 |
+
parser.add_argument("--limit", type=int, default=0)
|
| 1138 |
+
parser.add_argument("--minimum-component-area", type=int, default=128)
|
| 1139 |
+
parser.add_argument("--confidence-threshold", type=float, default=0.38)
|
| 1140 |
+
parser.add_argument("--compile", action="store_true")
|
| 1141 |
+
parser.add_argument("--resume", action="store_true")
|
| 1142 |
+
parser.add_argument(
|
| 1143 |
+
"--artifact-level",
|
| 1144 |
+
choices=["full", "standard"],
|
| 1145 |
+
default="full",
|
| 1146 |
+
help="standard omits regenerable per-prompt/category PNGs for large batch runs.",
|
| 1147 |
+
)
|
| 1148 |
+
parser.add_argument(
|
| 1149 |
+
"--verify-manifest-category",
|
| 1150 |
+
action="store_true",
|
| 1151 |
+
help="Compare configured confusable categories unless category_reviewed=true.",
|
| 1152 |
+
)
|
| 1153 |
+
parser.add_argument(
|
| 1154 |
+
"--allow-test-split",
|
| 1155 |
+
action="store_true",
|
| 1156 |
+
help=(
|
| 1157 |
+
"Explicitly allow frozen test RGB rows for a pre-registered front-end "
|
| 1158 |
+
"inference run. Default behavior remains validation-only."
|
| 1159 |
+
),
|
| 1160 |
+
)
|
| 1161 |
+
parser.add_argument("--sam3-repo", default="repos/sam3")
|
| 1162 |
+
parser.add_argument("--sam3-checkpoint", default="weights/sam3/sam3.pt")
|
| 1163 |
+
parser.add_argument(
|
| 1164 |
+
"--grounded-sam-repo",
|
| 1165 |
+
default="repos/grounded-sam-compat",
|
| 1166 |
+
)
|
| 1167 |
+
parser.add_argument(
|
| 1168 |
+
"--grounding-dino-config",
|
| 1169 |
+
default="repos/grounded-sam-compat/GroundingDINO/groundingdino/config/GroundingDINO_SwinT_OGC.py",
|
| 1170 |
+
)
|
| 1171 |
+
parser.add_argument(
|
| 1172 |
+
"--grounding-dino-checkpoint",
|
| 1173 |
+
default="weights/grounded-sam/groundingdino_swint_ogc.pth",
|
| 1174 |
+
)
|
| 1175 |
+
parser.add_argument(
|
| 1176 |
+
"--sam-checkpoint",
|
| 1177 |
+
default="weights/grounded-sam/sam_vit_h_4b8939.pth",
|
| 1178 |
+
)
|
| 1179 |
+
parser.add_argument("--sam-model-type", choices=["vit_h", "vit_l", "vit_b"], default="vit_h")
|
| 1180 |
+
parser.add_argument("--bert-path", default=None)
|
| 1181 |
+
parser.add_argument("--box-threshold", type=float, default=0.28)
|
| 1182 |
+
return parser
|
| 1183 |
+
|
| 1184 |
+
|
| 1185 |
+
def main() -> int:
|
| 1186 |
+
args = build_parser().parse_args()
|
| 1187 |
+
config_path = Path(args.prompt_config).resolve()
|
| 1188 |
+
config_override_path = (
|
| 1189 |
+
Path(args.prompt_config_override).resolve()
|
| 1190 |
+
if args.prompt_config_override
|
| 1191 |
+
else None
|
| 1192 |
+
)
|
| 1193 |
+
output_dir = Path(args.output_dir).resolve()
|
| 1194 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 1195 |
+
with config_path.open("r", encoding="utf-8") as handle:
|
| 1196 |
+
config = json.load(handle)
|
| 1197 |
+
if not isinstance(config, dict):
|
| 1198 |
+
raise ValueError(f"prompt config must be a JSON object: {config_path}")
|
| 1199 |
+
if config_override_path is not None:
|
| 1200 |
+
with config_override_path.open("r", encoding="utf-8") as handle:
|
| 1201 |
+
override = json.load(handle)
|
| 1202 |
+
if not isinstance(override, dict):
|
| 1203 |
+
raise ValueError(f"prompt config override must be a JSON object: {config_override_path}")
|
| 1204 |
+
config = merge_prompt_config(config, override)
|
| 1205 |
+
if args.image:
|
| 1206 |
+
image_path = Path(args.image).expanduser().resolve()
|
| 1207 |
+
if not image_path.is_file():
|
| 1208 |
+
raise FileNotFoundError(image_path)
|
| 1209 |
+
sample_id = args.single_sample_id or safe_name(image_path.stem) or "single_image"
|
| 1210 |
+
rows = [
|
| 1211 |
+
{
|
| 1212 |
+
"sample_id": sample_id,
|
| 1213 |
+
"image_path": str(image_path),
|
| 1214 |
+
"category": args.single_category,
|
| 1215 |
+
"category_reviewed": bool(args.single_category),
|
| 1216 |
+
}
|
| 1217 |
+
]
|
| 1218 |
+
else:
|
| 1219 |
+
pilot_path = Path(args.pilot_manifest).resolve()
|
| 1220 |
+
rows = read_jsonl(pilot_path)
|
| 1221 |
+
if args.limit > 0:
|
| 1222 |
+
rows = rows[: args.limit]
|
| 1223 |
+
allowed_splits = {None, "validation"}
|
| 1224 |
+
if args.allow_test_split:
|
| 1225 |
+
allowed_splits.add("test")
|
| 1226 |
+
invalid_splits = sorted(
|
| 1227 |
+
{str(row.get("benchmark_split")) for row in rows if row.get("benchmark_split") not in allowed_splits}
|
| 1228 |
+
)
|
| 1229 |
+
if invalid_splits:
|
| 1230 |
+
raise ValueError(
|
| 1231 |
+
"Proposal manifest contains unsupported benchmark split(s): "
|
| 1232 |
+
f"{invalid_splits}. Pass --allow-test-split only for an explicitly "
|
| 1233 |
+
"registered test front-end inference run."
|
| 1234 |
+
)
|
| 1235 |
+
test_accessed = any(row.get("benchmark_split") == "test" for row in rows)
|
| 1236 |
+
backend = None
|
| 1237 |
+
initialization_error = None
|
| 1238 |
+
try:
|
| 1239 |
+
backend = build_backend(args)
|
| 1240 |
+
if backend.torch.cuda.is_available():
|
| 1241 |
+
backend.torch.cuda.reset_peak_memory_stats()
|
| 1242 |
+
except Exception as exc:
|
| 1243 |
+
traceback.print_exc()
|
| 1244 |
+
initialization_error = f"{type(exc).__name__}: {exc}"
|
| 1245 |
+
results = []
|
| 1246 |
+
if initialization_error is not None:
|
| 1247 |
+
for row in rows:
|
| 1248 |
+
results.append(
|
| 1249 |
+
{
|
| 1250 |
+
"sample_id": row["sample_id"],
|
| 1251 |
+
"image_path": row["image_path"],
|
| 1252 |
+
"backend": args.backend,
|
| 1253 |
+
"quality_score": -1.0,
|
| 1254 |
+
"review_status": "error",
|
| 1255 |
+
"error": f"model_initialization_failed: {initialization_error}",
|
| 1256 |
+
}
|
| 1257 |
+
)
|
| 1258 |
+
write_jsonl(output_dir / "results.jsonl", results)
|
| 1259 |
+
else:
|
| 1260 |
+
assert backend is not None
|
| 1261 |
+
for index, row in enumerate(rows, start=1):
|
| 1262 |
+
print(f"[{index}/{len(rows)}] {row['sample_id']}", flush=True)
|
| 1263 |
+
try:
|
| 1264 |
+
result = process_sample(row, backend, config, output_dir, args)
|
| 1265 |
+
except Exception as exc:
|
| 1266 |
+
traceback.print_exc()
|
| 1267 |
+
result = {
|
| 1268 |
+
"sample_id": row["sample_id"],
|
| 1269 |
+
"image_path": row["image_path"],
|
| 1270 |
+
"backend": args.backend,
|
| 1271 |
+
"quality_score": -1.0,
|
| 1272 |
+
"review_status": "error",
|
| 1273 |
+
"error": f"{type(exc).__name__}: {exc}",
|
| 1274 |
+
}
|
| 1275 |
+
results.append(result)
|
| 1276 |
+
write_jsonl(output_dir / "results.jsonl", results)
|
| 1277 |
+
build_html_report(output_dir, results)
|
| 1278 |
+
failed = sum(item.get("review_status") == "error" for item in results)
|
| 1279 |
+
peak_bytes = (
|
| 1280 |
+
int(backend.torch.cuda.max_memory_allocated())
|
| 1281 |
+
if backend is not None and backend.torch.cuda.is_available()
|
| 1282 |
+
else 0
|
| 1283 |
+
)
|
| 1284 |
+
summary = {
|
| 1285 |
+
"backend": args.backend,
|
| 1286 |
+
"sample_count": len(results),
|
| 1287 |
+
"completed": len(results) - failed,
|
| 1288 |
+
"failed": failed,
|
| 1289 |
+
"failure_rate": failed / len(results) if results else 1.0,
|
| 1290 |
+
"status_counts": dict(
|
| 1291 |
+
sorted(
|
| 1292 |
+
{
|
| 1293 |
+
status: sum(item.get("review_status") == status for item in results)
|
| 1294 |
+
for status in {item.get("review_status") for item in results}
|
| 1295 |
+
}.items()
|
| 1296 |
+
)
|
| 1297 |
+
),
|
| 1298 |
+
"mean_quality_score": round(
|
| 1299 |
+
float(np.mean([item["quality_score"] for item in results if item["quality_score"] >= 0])),
|
| 1300 |
+
6,
|
| 1301 |
+
)
|
| 1302 |
+
if any(item["quality_score"] >= 0 for item in results)
|
| 1303 |
+
else None,
|
| 1304 |
+
"automatic_masks_are_ground_truth": False,
|
| 1305 |
+
"prompt_config": str(config_path),
|
| 1306 |
+
"prompt_config_override": str(config_override_path) if config_override_path else None,
|
| 1307 |
+
"model_initialization_error": initialization_error,
|
| 1308 |
+
"peak_gpu_memory_bytes": peak_bytes,
|
| 1309 |
+
"peak_gpu_memory_gib": peak_bytes / (1024**3),
|
| 1310 |
+
"environment": {
|
| 1311 |
+
"python": sys.version,
|
| 1312 |
+
"executable": sys.executable,
|
| 1313 |
+
"torch": getattr(getattr(backend, "torch", None), "__version__", None),
|
| 1314 |
+
"cuda_runtime": getattr(getattr(backend, "torch", None), "version", None).cuda
|
| 1315 |
+
if backend is not None
|
| 1316 |
+
else None,
|
| 1317 |
+
"gpu_name": backend.torch.cuda.get_device_name(0)
|
| 1318 |
+
if backend is not None and backend.torch.cuda.is_available()
|
| 1319 |
+
else None,
|
| 1320 |
+
"slurm_job_id": os.environ.get("SLURM_JOB_ID"),
|
| 1321 |
+
},
|
| 1322 |
+
"resolved_arguments": vars(args),
|
| 1323 |
+
"test_accessed": test_accessed,
|
| 1324 |
+
"test_access_explicitly_allowed": bool(args.allow_test_split),
|
| 1325 |
+
}
|
| 1326 |
+
json_dump(output_dir / "summary.json", summary)
|
| 1327 |
+
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
| 1328 |
+
return 0 if all(item.get("review_status") != "error" for item in results) else 2
|
| 1329 |
+
|
| 1330 |
+
|
| 1331 |
+
if __name__ == "__main__":
|
| 1332 |
+
raise SystemExit(main())
|
tools/audit_accessibility_3d_candidate.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Write an auditable preflight and multiview verification report for one run."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
import sys
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
import numpy as np
|
| 12 |
+
from PIL import Image
|
| 13 |
+
|
| 14 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
| 15 |
+
sys.path.insert(0, str(PROJECT_ROOT))
|
| 16 |
+
|
| 17 |
+
from accessibilityamodal.verification import build_verification # noqa: E402
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def read_json(path: Path) -> dict:
|
| 21 |
+
return json.loads(path.read_text(encoding="utf-8"))
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def read_mask(path: Path) -> np.ndarray:
|
| 25 |
+
mask = np.asarray(Image.open(path).convert("L")) > 127
|
| 26 |
+
return mask
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def find_sam_quality(run_dir: Path, sample_id: str) -> dict | None:
|
| 30 |
+
for root in (run_dir / "01_sam3", run_dir / "01_sam3_retry"):
|
| 31 |
+
path = root / "samples" / sample_id / "quality.json"
|
| 32 |
+
if path.is_file():
|
| 33 |
+
return read_json(path)
|
| 34 |
+
return None
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def find_reviewed_visible_metadata(run_dir: Path) -> dict | None:
|
| 38 |
+
path = run_dir / "01_reviewed_visible" / "metadata.json"
|
| 39 |
+
return read_json(path) if path.is_file() else None
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def main() -> int:
|
| 43 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 44 |
+
parser.add_argument("--run-dir", required=True)
|
| 45 |
+
parser.add_argument("--sample-id", required=True)
|
| 46 |
+
parser.add_argument("--category", required=True, choices=["curb_cut", "ramp", "stairs", "tactile_paving", "walkway"])
|
| 47 |
+
parser.add_argument("--output", default=None)
|
| 48 |
+
parser.add_argument("--preflight-only", action="store_true")
|
| 49 |
+
parser.add_argument("--print-decision", action="store_true")
|
| 50 |
+
args = parser.parse_args()
|
| 51 |
+
|
| 52 |
+
run_dir = Path(args.run_dir).expanduser().resolve()
|
| 53 |
+
masks_dir = run_dir / "02_masks"
|
| 54 |
+
visible = read_mask(masks_dir / "target_visible.png")
|
| 55 |
+
amodal = read_mask(masks_dir / "target_amodal.png")
|
| 56 |
+
hidden = read_mask(masks_dir / "hidden.png")
|
| 57 |
+
obstacle = read_mask(masks_dir / "obstacle.png")
|
| 58 |
+
geometry_path = run_dir / "04_3d_completion" / "geometry" / "geometry_manifest.json"
|
| 59 |
+
geometry = None if args.preflight_only or not geometry_path.is_file() else read_json(geometry_path)
|
| 60 |
+
learned_dir = None
|
| 61 |
+
if not args.preflight_only:
|
| 62 |
+
learned_candidates = (
|
| 63 |
+
run_dir / "accessibility3d",
|
| 64 |
+
run_dir / "04_3d_completion" / "visual_candidate",
|
| 65 |
+
)
|
| 66 |
+
learned_dir = next(
|
| 67 |
+
(candidate for candidate in learned_candidates if candidate.is_dir()),
|
| 68 |
+
learned_candidates[0],
|
| 69 |
+
)
|
| 70 |
+
report = build_verification(
|
| 71 |
+
sample_id=args.sample_id,
|
| 72 |
+
category=args.category,
|
| 73 |
+
visible_mask=visible,
|
| 74 |
+
amodal_mask=amodal,
|
| 75 |
+
hidden_mask=hidden,
|
| 76 |
+
obstacle_mask=obstacle,
|
| 77 |
+
sam3_quality=find_sam_quality(run_dir, args.sample_id),
|
| 78 |
+
reviewed_visible_metadata=find_reviewed_visible_metadata(run_dir),
|
| 79 |
+
geometry_manifest=geometry,
|
| 80 |
+
learned_dir=learned_dir,
|
| 81 |
+
)
|
| 82 |
+
output = (
|
| 83 |
+
Path(args.output).expanduser().resolve()
|
| 84 |
+
if args.output
|
| 85 |
+
else run_dir / "05_quality_checks" / "verification.json"
|
| 86 |
+
)
|
| 87 |
+
output.parent.mkdir(parents=True, exist_ok=True)
|
| 88 |
+
output.write_text(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
| 89 |
+
if args.print_decision:
|
| 90 |
+
print(report["decision"])
|
| 91 |
+
else:
|
| 92 |
+
print(json.dumps({"output": str(output), "decision": report["decision"], "gate": report["gate"]}, ensure_ascii=False))
|
| 93 |
+
return 0
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
if __name__ == "__main__":
|
| 97 |
+
raise SystemExit(main())
|
tools/build_accessibility_review_bundle.py
ADDED
|
@@ -0,0 +1,746 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Build one orderly 2D/3D accessibility-completion quick-review directory."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import hashlib
|
| 8 |
+
import json
|
| 9 |
+
import math
|
| 10 |
+
import os
|
| 11 |
+
import shutil
|
| 12 |
+
from datetime import datetime, timezone
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
from typing import Any, Iterable, Mapping, Sequence
|
| 15 |
+
|
| 16 |
+
from PIL import Image, ImageDraw, ImageOps
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
PROFILES = ("walking", "blind_low_vision", "wheelchair_wheeled", "cyclist")
|
| 20 |
+
PRIMARY_3D_ROLES = (
|
| 21 |
+
"category_geometry",
|
| 22 |
+
"open_world_accessibility_surface",
|
| 23 |
+
"learned_amodal3d_gaussian",
|
| 24 |
+
)
|
| 25 |
+
CORE_OUTPUT_NAMES = {
|
| 26 |
+
"original": "01_original.jpg",
|
| 27 |
+
"completion_2d": "02_2d_completion.png",
|
| 28 |
+
"geometry_turntable": "03_3d_turntable.gif",
|
| 29 |
+
"geometry_multiview": "04_3d_multiview.jpg",
|
| 30 |
+
"geometry_mesh": "05_mesh.ply",
|
| 31 |
+
"overview": "overview.jpg",
|
| 32 |
+
"accessibility_review": "accessibility_review.json",
|
| 33 |
+
}
|
| 34 |
+
VISUAL_OUTPUT_NAMES = {
|
| 35 |
+
"visual_turntable": "05_visual_3d_turntable.gif",
|
| 36 |
+
"visual_multiview": "06_visual_3d_multiview.jpg",
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
METRIC_ALIASES = {
|
| 40 |
+
"width": (
|
| 41 |
+
"clear_width_m",
|
| 42 |
+
"minimum_clear_width_m",
|
| 43 |
+
"min_clear_width_m",
|
| 44 |
+
"path_width_m",
|
| 45 |
+
"walkable_width_m",
|
| 46 |
+
"width_m",
|
| 47 |
+
),
|
| 48 |
+
"slope": (
|
| 49 |
+
"slope_degrees",
|
| 50 |
+
"slope_percent",
|
| 51 |
+
"longitudinal_slope_percent",
|
| 52 |
+
"cross_slope_percent",
|
| 53 |
+
"slope_ratio",
|
| 54 |
+
"grade_percent",
|
| 55 |
+
),
|
| 56 |
+
"clearance": (
|
| 57 |
+
"clearance_m",
|
| 58 |
+
"minimum_clearance_m",
|
| 59 |
+
"min_clearance_m",
|
| 60 |
+
"vertical_clearance_m",
|
| 61 |
+
"obstacle_clearance_m",
|
| 62 |
+
),
|
| 63 |
+
}
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def build_argument_parser() -> argparse.ArgumentParser:
|
| 67 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 68 |
+
parser.add_argument("--source", "--original", dest="source", required=True)
|
| 69 |
+
parser.add_argument(
|
| 70 |
+
"--completion-2d",
|
| 71 |
+
"--selected-2d",
|
| 72 |
+
"--2d-selected",
|
| 73 |
+
dest="completion_2d",
|
| 74 |
+
required=True,
|
| 75 |
+
)
|
| 76 |
+
parser.add_argument(
|
| 77 |
+
"--turntable-gif",
|
| 78 |
+
"--geometry-turntable",
|
| 79 |
+
dest="turntable_gif",
|
| 80 |
+
required=True,
|
| 81 |
+
)
|
| 82 |
+
parser.add_argument(
|
| 83 |
+
"--multiview",
|
| 84 |
+
"--geometry-multiview",
|
| 85 |
+
dest="multiview",
|
| 86 |
+
required=True,
|
| 87 |
+
)
|
| 88 |
+
parser.add_argument("--mesh", "--geometry-mesh", dest="mesh", required=True)
|
| 89 |
+
parser.add_argument(
|
| 90 |
+
"--primary-3d-role",
|
| 91 |
+
choices=PRIMARY_3D_ROLES,
|
| 92 |
+
default="category_geometry",
|
| 93 |
+
help=(
|
| 94 |
+
"Declares whether the core rotating review files are diagnostic "
|
| 95 |
+
"category geometry, an open-world accessibility surface, or the "
|
| 96 |
+
"Accessibility3D CUDA Gaussian/dense-mesh result."
|
| 97 |
+
),
|
| 98 |
+
)
|
| 99 |
+
parser.add_argument("--completion-manifest", required=True)
|
| 100 |
+
parser.add_argument(
|
| 101 |
+
"--geometry-manifest",
|
| 102 |
+
default=None,
|
| 103 |
+
help="Optional geometry/depth manifest containing metric-evidence declarations.",
|
| 104 |
+
)
|
| 105 |
+
parser.add_argument(
|
| 106 |
+
"--verification-manifest",
|
| 107 |
+
default=None,
|
| 108 |
+
help=(
|
| 109 |
+
"Optional generated-3D verification report. Its exact bytes are bound "
|
| 110 |
+
"into the quick-review manifest so the selected primary role remains auditable."
|
| 111 |
+
),
|
| 112 |
+
)
|
| 113 |
+
parser.add_argument(
|
| 114 |
+
"--visual-turntable",
|
| 115 |
+
default=None,
|
| 116 |
+
help="Optional learned visual 3D candidate; never used as passability evidence.",
|
| 117 |
+
)
|
| 118 |
+
parser.add_argument(
|
| 119 |
+
"--visual-multiview",
|
| 120 |
+
default=None,
|
| 121 |
+
help="Optional learned visual 3D candidate; never used as passability evidence.",
|
| 122 |
+
)
|
| 123 |
+
parser.add_argument("--category", required=True)
|
| 124 |
+
parser.add_argument("--sample-id", required=True)
|
| 125 |
+
parser.add_argument(
|
| 126 |
+
"--output-dir",
|
| 127 |
+
required=True,
|
| 128 |
+
help="Sample directory; files are written below its 00_quick_review child.",
|
| 129 |
+
)
|
| 130 |
+
return parser
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def sha256_file(path: Path) -> str:
|
| 134 |
+
digest = hashlib.sha256()
|
| 135 |
+
with path.open("rb") as handle:
|
| 136 |
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
| 137 |
+
digest.update(chunk)
|
| 138 |
+
return digest.hexdigest()
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def relative_path(path: Path, anchor: Path) -> str:
|
| 142 |
+
return Path(os.path.relpath(path.resolve(), anchor.resolve())).as_posix()
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def file_record(path: Path, anchor: Path) -> dict[str, Any]:
|
| 146 |
+
return {
|
| 147 |
+
"path": relative_path(path, anchor),
|
| 148 |
+
"sha256": sha256_file(path),
|
| 149 |
+
"bytes": path.stat().st_size,
|
| 150 |
+
}
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def read_json_object(path: Path) -> dict[str, Any]:
|
| 154 |
+
value = json.loads(path.read_text(encoding="utf-8"))
|
| 155 |
+
if not isinstance(value, dict):
|
| 156 |
+
raise ValueError(f"Expected a JSON object: {path}")
|
| 157 |
+
return value
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def write_json(path: Path, payload: Mapping[str, Any]) -> None:
|
| 161 |
+
path.write_text(
|
| 162 |
+
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
| 163 |
+
encoding="utf-8",
|
| 164 |
+
)
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def _walk_json(value: Any, prefix: tuple[str, ...] = ()) -> Iterable[tuple[tuple[str, ...], Any]]:
|
| 168 |
+
if isinstance(value, dict):
|
| 169 |
+
for key, child in value.items():
|
| 170 |
+
key_path = (*prefix, str(key))
|
| 171 |
+
yield key_path, child
|
| 172 |
+
yield from _walk_json(child, key_path)
|
| 173 |
+
elif isinstance(value, list):
|
| 174 |
+
for index, child in enumerate(value):
|
| 175 |
+
yield from _walk_json(child, (*prefix, str(index)))
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def _number(value: Any) -> float | None:
|
| 179 |
+
if isinstance(value, bool):
|
| 180 |
+
return None
|
| 181 |
+
if isinstance(value, (int, float)) and math.isfinite(float(value)):
|
| 182 |
+
return float(value)
|
| 183 |
+
if isinstance(value, dict):
|
| 184 |
+
return _number(value.get("value"))
|
| 185 |
+
return None
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def _measurement_unit(field: str) -> str:
|
| 189 |
+
if field.endswith("_m"):
|
| 190 |
+
return "m"
|
| 191 |
+
if field.endswith("_degrees"):
|
| 192 |
+
return "degrees"
|
| 193 |
+
if field.endswith("_percent") or field == "grade_percent":
|
| 194 |
+
return "percent"
|
| 195 |
+
if field.endswith("_ratio"):
|
| 196 |
+
return "ratio"
|
| 197 |
+
return "declared_metric"
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def _find_measurement(
|
| 201 |
+
manifests: Sequence[tuple[str, Mapping[str, Any]]],
|
| 202 |
+
aliases: Sequence[str],
|
| 203 |
+
) -> dict[str, Any] | None:
|
| 204 |
+
alias_set = set(aliases)
|
| 205 |
+
for source_name, payload in manifests:
|
| 206 |
+
for key_path, value in _walk_json(payload):
|
| 207 |
+
field = key_path[-1]
|
| 208 |
+
if field not in alias_set:
|
| 209 |
+
continue
|
| 210 |
+
numeric_value = _number(value)
|
| 211 |
+
if numeric_value is None:
|
| 212 |
+
continue
|
| 213 |
+
unit = _measurement_unit(field)
|
| 214 |
+
if isinstance(value, dict) and isinstance(value.get("unit"), str):
|
| 215 |
+
unit = str(value["unit"])
|
| 216 |
+
return {
|
| 217 |
+
"value": numeric_value,
|
| 218 |
+
"unit": unit,
|
| 219 |
+
"source_manifest": source_name,
|
| 220 |
+
"source_field": ".".join(key_path),
|
| 221 |
+
}
|
| 222 |
+
return None
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
def _metric_truth_declarations(
|
| 226 |
+
manifests: Sequence[tuple[str, Mapping[str, Any]]],
|
| 227 |
+
) -> list[dict[str, Any]]:
|
| 228 |
+
truth_fields = {
|
| 229 |
+
"depth_is_metric_truth",
|
| 230 |
+
"depth_is_metric",
|
| 231 |
+
"geometry_is_metric",
|
| 232 |
+
"metric_geometry",
|
| 233 |
+
"metric_calibrated",
|
| 234 |
+
}
|
| 235 |
+
declarations: list[dict[str, Any]] = []
|
| 236 |
+
for source_name, payload in manifests:
|
| 237 |
+
for key_path, value in _walk_json(payload):
|
| 238 |
+
if key_path[-1] in truth_fields and isinstance(value, bool):
|
| 239 |
+
declarations.append(
|
| 240 |
+
{
|
| 241 |
+
"source_manifest": source_name,
|
| 242 |
+
"source_field": ".".join(key_path),
|
| 243 |
+
"value": value,
|
| 244 |
+
}
|
| 245 |
+
)
|
| 246 |
+
return declarations
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def extract_metric_evidence(
|
| 250 |
+
completion_manifest: Mapping[str, Any],
|
| 251 |
+
geometry_manifest: Mapping[str, Any] | None = None,
|
| 252 |
+
) -> dict[str, Any]:
|
| 253 |
+
"""Extract only explicitly named physical measurements.
|
| 254 |
+
|
| 255 |
+
A width-like pixel count is deliberately not treated as metric evidence.
|
| 256 |
+
Measurements are trusted for passability review only when a supplied
|
| 257 |
+
manifest explicitly declares calibrated/metric geometry.
|
| 258 |
+
"""
|
| 259 |
+
manifests: list[tuple[str, Mapping[str, Any]]] = [
|
| 260 |
+
("completion_manifest", completion_manifest)
|
| 261 |
+
]
|
| 262 |
+
if geometry_manifest is not None:
|
| 263 |
+
manifests.append(("geometry_manifest", geometry_manifest))
|
| 264 |
+
declarations = _metric_truth_declarations(manifests)
|
| 265 |
+
trusted_metric_geometry = any(item["value"] is True for item in declarations)
|
| 266 |
+
measurements = {
|
| 267 |
+
name: _find_measurement(manifests, aliases)
|
| 268 |
+
for name, aliases in METRIC_ALIASES.items()
|
| 269 |
+
}
|
| 270 |
+
missing = [name for name, measurement in measurements.items() if measurement is None]
|
| 271 |
+
complete = trusted_metric_geometry and not missing
|
| 272 |
+
return {
|
| 273 |
+
"trusted_metric_geometry": trusted_metric_geometry,
|
| 274 |
+
"metric_truth_declarations": declarations,
|
| 275 |
+
"measurements": measurements,
|
| 276 |
+
"missing_measurements": missing,
|
| 277 |
+
"complete_width_slope_clearance": complete,
|
| 278 |
+
"automatic_passability_allowed": False,
|
| 279 |
+
"reason": (
|
| 280 |
+
"Metric width, slope, and clearance are present, but thresholds and field "
|
| 281 |
+
"validation still require human review."
|
| 282 |
+
if complete
|
| 283 |
+
else "Trusted metric width, slope, and clearance are incomplete."
|
| 284 |
+
),
|
| 285 |
+
}
|
| 286 |
+
|
| 287 |
+
|
| 288 |
+
def is_stairs_category(category: str) -> bool:
|
| 289 |
+
normalized = category.strip().lower().replace("-", "_").replace(" ", "_")
|
| 290 |
+
return normalized in {"stairs", "stair", "staircase", "steps"} or normalized.startswith(
|
| 291 |
+
"stairs_"
|
| 292 |
+
)
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
def build_population_assessments(
|
| 296 |
+
category: str,
|
| 297 |
+
metric_evidence: Mapping[str, Any],
|
| 298 |
+
) -> dict[str, dict[str, Any]]:
|
| 299 |
+
"""Return conservative per-population decisions without a safety claim."""
|
| 300 |
+
assessments: dict[str, dict[str, Any]] = {}
|
| 301 |
+
complete_metrics = bool(metric_evidence.get("complete_width_slope_clearance", False))
|
| 302 |
+
for profile in PROFILES:
|
| 303 |
+
if is_stairs_category(category) and profile == "wheelchair_wheeled":
|
| 304 |
+
assessments[profile] = {
|
| 305 |
+
"status": "blocked",
|
| 306 |
+
"decision": "block",
|
| 307 |
+
"can_pass": False,
|
| 308 |
+
"review_action": "stop_and_replan",
|
| 309 |
+
"reason": "Stair geometry blocks a wheeled route; use an alternate reviewed route.",
|
| 310 |
+
"basis": "category_rule_stairs_wheelchair",
|
| 311 |
+
"human_review_required": True,
|
| 312 |
+
}
|
| 313 |
+
continue
|
| 314 |
+
assessments[profile] = {
|
| 315 |
+
"status": "unknown",
|
| 316 |
+
"decision": "unknown",
|
| 317 |
+
"can_pass": None,
|
| 318 |
+
"review_action": "manual_review",
|
| 319 |
+
"reason": (
|
| 320 |
+
"Metric evidence exists, but no jurisdiction-specific thresholds or "
|
| 321 |
+
"field validation were supplied."
|
| 322 |
+
if complete_metrics
|
| 323 |
+
else "Trusted metric width, slope, and clearance are incomplete."
|
| 324 |
+
),
|
| 325 |
+
"basis": (
|
| 326 |
+
"metric_evidence_requires_threshold_review"
|
| 327 |
+
if complete_metrics
|
| 328 |
+
else "insufficient_metric_width_slope_clearance"
|
| 329 |
+
),
|
| 330 |
+
"human_review_required": True,
|
| 331 |
+
}
|
| 332 |
+
return assessments
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
def _provenance_summary(payload: Mapping[str, Any]) -> dict[str, Any]:
|
| 336 |
+
"""Keep truthful source identifiers without copying absolute paths."""
|
| 337 |
+
summary: dict[str, Any] = {}
|
| 338 |
+
scalar_fields = (
|
| 339 |
+
"pipeline",
|
| 340 |
+
"backend",
|
| 341 |
+
"output_kind",
|
| 342 |
+
"completion_method",
|
| 343 |
+
"warning",
|
| 344 |
+
"device",
|
| 345 |
+
"category",
|
| 346 |
+
"sample_id",
|
| 347 |
+
)
|
| 348 |
+
for field in scalar_fields:
|
| 349 |
+
value = payload.get(field)
|
| 350 |
+
if isinstance(value, (str, int, float, bool)) or value is None:
|
| 351 |
+
summary[field] = value
|
| 352 |
+
model = payload.get("model")
|
| 353 |
+
if isinstance(model, str) and model.strip():
|
| 354 |
+
# A backend/model identifier is useful provenance. If it is a local
|
| 355 |
+
# filesystem path, retain the final identifier rather than publishing an
|
| 356 |
+
# absolute host path.
|
| 357 |
+
summary["model_identifier"] = Path(model).name if ("/" in model or "\\" in model) else model
|
| 358 |
+
return summary
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
def _save_image(source: Path, destination: Path, image_format: str) -> None:
|
| 362 |
+
image = ImageOps.exif_transpose(Image.open(source)).convert("RGB")
|
| 363 |
+
if image_format == "JPEG":
|
| 364 |
+
image.save(destination, format=image_format, quality=95, subsampling=0)
|
| 365 |
+
else:
|
| 366 |
+
image.save(destination, format=image_format)
|
| 367 |
+
|
| 368 |
+
|
| 369 |
+
def copy_review_assets(
|
| 370 |
+
*,
|
| 371 |
+
source: Path,
|
| 372 |
+
completion_2d: Path,
|
| 373 |
+
turntable_gif: Path,
|
| 374 |
+
multiview: Path,
|
| 375 |
+
mesh: Path,
|
| 376 |
+
review_dir: Path,
|
| 377 |
+
visual_turntable: Path | None = None,
|
| 378 |
+
visual_multiview: Path | None = None,
|
| 379 |
+
) -> dict[str, Path]:
|
| 380 |
+
review_dir.mkdir(parents=True, exist_ok=True)
|
| 381 |
+
outputs = {
|
| 382 |
+
role: review_dir / filename for role, filename in CORE_OUTPUT_NAMES.items()
|
| 383 |
+
}
|
| 384 |
+
_save_image(source, outputs["original"], "JPEG")
|
| 385 |
+
_save_image(completion_2d, outputs["completion_2d"], "PNG")
|
| 386 |
+
shutil.copy2(turntable_gif, outputs["geometry_turntable"])
|
| 387 |
+
_save_image(multiview, outputs["geometry_multiview"], "JPEG")
|
| 388 |
+
shutil.copy2(mesh, outputs["geometry_mesh"])
|
| 389 |
+
|
| 390 |
+
if visual_turntable is not None:
|
| 391 |
+
destination = review_dir / VISUAL_OUTPUT_NAMES["visual_turntable"]
|
| 392 |
+
shutil.copy2(visual_turntable, destination)
|
| 393 |
+
outputs["visual_turntable"] = destination
|
| 394 |
+
if visual_multiview is not None:
|
| 395 |
+
destination = review_dir / VISUAL_OUTPUT_NAMES["visual_multiview"]
|
| 396 |
+
_save_image(visual_multiview, destination, "JPEG")
|
| 397 |
+
outputs["visual_multiview"] = destination
|
| 398 |
+
return outputs
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
def _panel(path: Path, label: str, size: tuple[int, int]) -> Image.Image:
|
| 402 |
+
image = ImageOps.exif_transpose(Image.open(path)).convert("RGB")
|
| 403 |
+
header_height = 42
|
| 404 |
+
body = ImageOps.contain(image, (size[0], size[1] - header_height))
|
| 405 |
+
panel = Image.new("RGB", size, "white")
|
| 406 |
+
draw = ImageDraw.Draw(panel)
|
| 407 |
+
draw.rectangle((0, 0, size[0], header_height), fill=(241, 241, 238))
|
| 408 |
+
draw.text((13, 14), label, fill=(18, 18, 18))
|
| 409 |
+
panel.paste(
|
| 410 |
+
body,
|
| 411 |
+
(
|
| 412 |
+
(size[0] - body.width) // 2,
|
| 413 |
+
header_height + (size[1] - header_height - body.height) // 2,
|
| 414 |
+
),
|
| 415 |
+
)
|
| 416 |
+
return panel
|
| 417 |
+
|
| 418 |
+
|
| 419 |
+
def build_overview(
|
| 420 |
+
*,
|
| 421 |
+
original: Path,
|
| 422 |
+
completion_2d: Path,
|
| 423 |
+
geometry_multiview: Path,
|
| 424 |
+
destination: Path,
|
| 425 |
+
category: str,
|
| 426 |
+
primary_3d_role: str = "category_geometry",
|
| 427 |
+
) -> None:
|
| 428 |
+
if primary_3d_role == "learned_amodal3d_gaussian":
|
| 429 |
+
review_label = "03-05 Accessibility3D CUDA 3D review"
|
| 430 |
+
elif primary_3d_role == "open_world_accessibility_surface":
|
| 431 |
+
review_label = "03-05 Open-world accessibility surface review"
|
| 432 |
+
else:
|
| 433 |
+
review_label = "03-05 Diagnostic geometry review"
|
| 434 |
+
panel_size = (480, 500)
|
| 435 |
+
panels = [
|
| 436 |
+
_panel(original, "01 Original image", panel_size),
|
| 437 |
+
_panel(completion_2d, "02 2D completion candidate", panel_size),
|
| 438 |
+
_panel(
|
| 439 |
+
geometry_multiview,
|
| 440 |
+
review_label,
|
| 441 |
+
panel_size,
|
| 442 |
+
),
|
| 443 |
+
]
|
| 444 |
+
footer_height = 64
|
| 445 |
+
canvas = Image.new(
|
| 446 |
+
"RGB",
|
| 447 |
+
(panel_size[0] * len(panels), panel_size[1] + footer_height),
|
| 448 |
+
(231, 231, 228),
|
| 449 |
+
)
|
| 450 |
+
for index, panel in enumerate(panels):
|
| 451 |
+
canvas.paste(panel, (index * panel_size[0], 0))
|
| 452 |
+
draw = ImageDraw.Draw(canvas)
|
| 453 |
+
draw.text(
|
| 454 |
+
(14, panel_size[1] + 12),
|
| 455 |
+
(
|
| 456 |
+
f"Category: {category}. Review artifact only: do not infer safe passage "
|
| 457 |
+
"without metric width, slope, clearance, and human validation."
|
| 458 |
+
),
|
| 459 |
+
fill=(90, 35, 28),
|
| 460 |
+
)
|
| 461 |
+
draw.text(
|
| 462 |
+
(14, panel_size[1] + 36),
|
| 463 |
+
(
|
| 464 |
+
"The 2D result is generative; the Accessibility3D rotation is a nonmetric "
|
| 465 |
+
"visual reconstruction, not a navigation certification."
|
| 466 |
+
if primary_3d_role == "learned_amodal3d_gaussian"
|
| 467 |
+
else "The 2D result is generative; the rotating geometry is not a navigation certification."
|
| 468 |
+
),
|
| 469 |
+
fill=(70, 70, 70),
|
| 470 |
+
)
|
| 471 |
+
canvas.save(destination, quality=94, subsampling=0)
|
| 472 |
+
|
| 473 |
+
|
| 474 |
+
def build_accessibility_review(
|
| 475 |
+
*,
|
| 476 |
+
sample_id: str,
|
| 477 |
+
category: str,
|
| 478 |
+
completion_manifest: Mapping[str, Any],
|
| 479 |
+
metric_evidence: Mapping[str, Any],
|
| 480 |
+
visual_candidate_present: bool,
|
| 481 |
+
primary_3d_role: str = "category_geometry",
|
| 482 |
+
) -> dict[str, Any]:
|
| 483 |
+
assessments = build_population_assessments(category, metric_evidence)
|
| 484 |
+
if primary_3d_role == "learned_amodal3d_gaussian":
|
| 485 |
+
render_backend = "Accessibility3D CUDA Gaussian rasterization"
|
| 486 |
+
mesh_representation = "dense FlexiCubes triangle faces"
|
| 487 |
+
elif primary_3d_role == "open_world_accessibility_surface":
|
| 488 |
+
render_backend = "category-constrained open-world surface renderer"
|
| 489 |
+
mesh_representation = "open-world accessibility triangle surface patch"
|
| 490 |
+
else:
|
| 491 |
+
render_backend = "category-constrained geometry renderer"
|
| 492 |
+
mesh_representation = "category-constrained triangle mesh"
|
| 493 |
+
primary_review_name = (
|
| 494 |
+
"Accessibility3D learned nonmetric reconstruction"
|
| 495 |
+
if primary_3d_role == "learned_amodal3d_gaussian"
|
| 496 |
+
else "category-constrained diagnostic 3D geometry"
|
| 497 |
+
)
|
| 498 |
+
return {
|
| 499 |
+
"schema_version": "accessibilityamodal_review_v1",
|
| 500 |
+
"created_at_utc": datetime.now(timezone.utc).isoformat(),
|
| 501 |
+
"sample_id": sample_id,
|
| 502 |
+
"category": category,
|
| 503 |
+
"overall_status": "manual_review_required",
|
| 504 |
+
"safe_passage_claim": False,
|
| 505 |
+
"population_assessments": assessments,
|
| 506 |
+
"metric_evidence": dict(metric_evidence),
|
| 507 |
+
"primary_3d_evidence": {
|
| 508 |
+
"turntable": CORE_OUTPUT_NAMES["geometry_turntable"],
|
| 509 |
+
"multiview": CORE_OUTPUT_NAMES["geometry_multiview"],
|
| 510 |
+
"mesh": CORE_OUTPUT_NAMES["geometry_mesh"],
|
| 511 |
+
"role": primary_3d_role,
|
| 512 |
+
"render_backend": render_backend,
|
| 513 |
+
"mesh_representation": mesh_representation,
|
| 514 |
+
"is_metric_geometry": False,
|
| 515 |
+
"is_navigation_certification": False,
|
| 516 |
+
},
|
| 517 |
+
"completion_2d": {
|
| 518 |
+
"path": CORE_OUTPUT_NAMES["completion_2d"],
|
| 519 |
+
"role": "generative_visual_hypothesis",
|
| 520 |
+
"is_ground_truth": False,
|
| 521 |
+
},
|
| 522 |
+
"visual_3d_candidate": {
|
| 523 |
+
"present": visual_candidate_present,
|
| 524 |
+
"role": "learned_visual_candidate_only",
|
| 525 |
+
"metric_evidence": False,
|
| 526 |
+
"passability_evidence": False,
|
| 527 |
+
"warning": (
|
| 528 |
+
"The learned visual candidate must not be used to infer dimensions or passage."
|
| 529 |
+
if visual_candidate_present
|
| 530 |
+
else None
|
| 531 |
+
),
|
| 532 |
+
},
|
| 533 |
+
"completion_provenance": _provenance_summary(completion_manifest),
|
| 534 |
+
"required_human_checks": [
|
| 535 |
+
"Verify that the hidden ground/support surface is completed continuously, without fog, haze, or ghost obstacles.",
|
| 536 |
+
f"Verify that the 2D completion agrees with the selected {primary_review_name}.",
|
| 537 |
+
"Measure and validate route width, slope, and clearance before any passage decision.",
|
| 538 |
+
],
|
| 539 |
+
"limitations": [
|
| 540 |
+
"No population is declared safely passable by this automatic bundle.",
|
| 541 |
+
"A blocked stairs/wheelchair rule is a route-level constraint, not a complete site assessment.",
|
| 542 |
+
],
|
| 543 |
+
}
|
| 544 |
+
|
| 545 |
+
|
| 546 |
+
def build_bundle(
|
| 547 |
+
*,
|
| 548 |
+
source: Path,
|
| 549 |
+
completion_2d: Path,
|
| 550 |
+
turntable_gif: Path,
|
| 551 |
+
multiview: Path,
|
| 552 |
+
mesh: Path,
|
| 553 |
+
completion_manifest_path: Path,
|
| 554 |
+
category: str,
|
| 555 |
+
sample_id: str,
|
| 556 |
+
output_dir: Path,
|
| 557 |
+
geometry_manifest_path: Path | None = None,
|
| 558 |
+
verification_manifest_path: Path | None = None,
|
| 559 |
+
visual_turntable: Path | None = None,
|
| 560 |
+
visual_multiview: Path | None = None,
|
| 561 |
+
primary_3d_role: str = "category_geometry",
|
| 562 |
+
) -> Path:
|
| 563 |
+
if primary_3d_role not in PRIMARY_3D_ROLES:
|
| 564 |
+
raise ValueError(
|
| 565 |
+
f"Unsupported primary_3d_role={primary_3d_role!r}; "
|
| 566 |
+
f"expected one of {PRIMARY_3D_ROLES}"
|
| 567 |
+
)
|
| 568 |
+
input_paths = {
|
| 569 |
+
"source": source,
|
| 570 |
+
"completion_2d": completion_2d,
|
| 571 |
+
"geometry_turntable": turntable_gif,
|
| 572 |
+
"geometry_multiview": multiview,
|
| 573 |
+
"geometry_mesh": mesh,
|
| 574 |
+
"completion_manifest": completion_manifest_path,
|
| 575 |
+
}
|
| 576 |
+
if geometry_manifest_path is not None:
|
| 577 |
+
input_paths["geometry_manifest"] = geometry_manifest_path
|
| 578 |
+
if verification_manifest_path is not None:
|
| 579 |
+
input_paths["verification_manifest"] = verification_manifest_path
|
| 580 |
+
if visual_turntable is not None:
|
| 581 |
+
input_paths["visual_turntable"] = visual_turntable
|
| 582 |
+
if visual_multiview is not None:
|
| 583 |
+
input_paths["visual_multiview"] = visual_multiview
|
| 584 |
+
for role, path in input_paths.items():
|
| 585 |
+
if not path.is_file():
|
| 586 |
+
raise FileNotFoundError(f"Missing {role}: {path}")
|
| 587 |
+
if (visual_turntable is None) != (visual_multiview is None):
|
| 588 |
+
raise ValueError(
|
| 589 |
+
"--visual-turntable and --visual-multiview must be supplied together"
|
| 590 |
+
)
|
| 591 |
+
|
| 592 |
+
review_dir = (
|
| 593 |
+
output_dir if output_dir.name == "00_quick_review" else output_dir / "00_quick_review"
|
| 594 |
+
)
|
| 595 |
+
completion_manifest = read_json_object(completion_manifest_path)
|
| 596 |
+
geometry_manifest = (
|
| 597 |
+
read_json_object(geometry_manifest_path)
|
| 598 |
+
if geometry_manifest_path is not None
|
| 599 |
+
else None
|
| 600 |
+
)
|
| 601 |
+
verification_manifest = (
|
| 602 |
+
read_json_object(verification_manifest_path)
|
| 603 |
+
if verification_manifest_path is not None
|
| 604 |
+
else None
|
| 605 |
+
)
|
| 606 |
+
metric_evidence = extract_metric_evidence(completion_manifest, geometry_manifest)
|
| 607 |
+
outputs = copy_review_assets(
|
| 608 |
+
source=source,
|
| 609 |
+
completion_2d=completion_2d,
|
| 610 |
+
turntable_gif=turntable_gif,
|
| 611 |
+
multiview=multiview,
|
| 612 |
+
mesh=mesh,
|
| 613 |
+
review_dir=review_dir,
|
| 614 |
+
visual_turntable=visual_turntable,
|
| 615 |
+
visual_multiview=visual_multiview,
|
| 616 |
+
)
|
| 617 |
+
build_overview(
|
| 618 |
+
original=outputs["original"],
|
| 619 |
+
completion_2d=outputs["completion_2d"],
|
| 620 |
+
geometry_multiview=outputs["geometry_multiview"],
|
| 621 |
+
destination=outputs["overview"],
|
| 622 |
+
category=category,
|
| 623 |
+
primary_3d_role=primary_3d_role,
|
| 624 |
+
)
|
| 625 |
+
review = build_accessibility_review(
|
| 626 |
+
sample_id=sample_id,
|
| 627 |
+
category=category,
|
| 628 |
+
completion_manifest=completion_manifest,
|
| 629 |
+
metric_evidence=metric_evidence,
|
| 630 |
+
visual_candidate_present=(
|
| 631 |
+
visual_turntable is not None
|
| 632 |
+
or primary_3d_role == "learned_amodal3d_gaussian"
|
| 633 |
+
),
|
| 634 |
+
primary_3d_role=primary_3d_role,
|
| 635 |
+
)
|
| 636 |
+
write_json(outputs["accessibility_review"], review)
|
| 637 |
+
|
| 638 |
+
output_records = {
|
| 639 |
+
role: file_record(path, review_dir)
|
| 640 |
+
for role, path in outputs.items()
|
| 641 |
+
if path.is_file()
|
| 642 |
+
}
|
| 643 |
+
input_records = {
|
| 644 |
+
role: file_record(path, review_dir) for role, path in input_paths.items()
|
| 645 |
+
}
|
| 646 |
+
manifest = {
|
| 647 |
+
"schema_version": "accessibilityamodal_quick_review_manifest_v1",
|
| 648 |
+
"created_at_utc": datetime.now(timezone.utc).isoformat(),
|
| 649 |
+
"sample_id": sample_id,
|
| 650 |
+
"category": category,
|
| 651 |
+
"purpose": "fast_human_review_of_original_2d_and_3d_completion",
|
| 652 |
+
"primary_3d_role": primary_3d_role,
|
| 653 |
+
"inputs": input_records,
|
| 654 |
+
"files": output_records,
|
| 655 |
+
"review_summary": {
|
| 656 |
+
"overall_status": review["overall_status"],
|
| 657 |
+
"safe_passage_claim": False,
|
| 658 |
+
"wheelchair_wheeled": review["population_assessments"][
|
| 659 |
+
"wheelchair_wheeled"
|
| 660 |
+
]["status"],
|
| 661 |
+
},
|
| 662 |
+
"visual_candidate_policy": {
|
| 663 |
+
"present": (
|
| 664 |
+
visual_turntable is not None
|
| 665 |
+
or primary_3d_role == "learned_amodal3d_gaussian"
|
| 666 |
+
),
|
| 667 |
+
"role": "learned_visual_candidate",
|
| 668 |
+
"is_metric_evidence": False,
|
| 669 |
+
"is_passability_evidence": False,
|
| 670 |
+
},
|
| 671 |
+
"provenance": {
|
| 672 |
+
"completion_manifest": _provenance_summary(completion_manifest),
|
| 673 |
+
"geometry_manifest_supplied": geometry_manifest is not None,
|
| 674 |
+
"verification_manifest_supplied": verification_manifest is not None,
|
| 675 |
+
"verification_decision": (
|
| 676 |
+
verification_manifest.get("decision")
|
| 677 |
+
if verification_manifest is not None
|
| 678 |
+
else None
|
| 679 |
+
),
|
| 680 |
+
},
|
| 681 |
+
"path_policy": "All filesystem paths in this manifest are relative to manifest.json.",
|
| 682 |
+
"integrity_note": "manifest.json omits its own hash to avoid recursive self-hashing.",
|
| 683 |
+
}
|
| 684 |
+
write_json(review_dir / "manifest.json", manifest)
|
| 685 |
+
return review_dir
|
| 686 |
+
|
| 687 |
+
|
| 688 |
+
def main(argv: Sequence[str] | None = None) -> int:
|
| 689 |
+
parser = build_argument_parser()
|
| 690 |
+
args = parser.parse_args(argv)
|
| 691 |
+
category = str(args.category).strip().lower()
|
| 692 |
+
sample_id = str(args.sample_id).strip()
|
| 693 |
+
if not category:
|
| 694 |
+
parser.error("--category must not be empty")
|
| 695 |
+
if not sample_id:
|
| 696 |
+
parser.error("--sample-id must not be empty")
|
| 697 |
+
|
| 698 |
+
review_dir = build_bundle(
|
| 699 |
+
source=Path(args.source).expanduser().resolve(),
|
| 700 |
+
completion_2d=Path(args.completion_2d).expanduser().resolve(),
|
| 701 |
+
turntable_gif=Path(args.turntable_gif).expanduser().resolve(),
|
| 702 |
+
multiview=Path(args.multiview).expanduser().resolve(),
|
| 703 |
+
mesh=Path(args.mesh).expanduser().resolve(),
|
| 704 |
+
completion_manifest_path=Path(args.completion_manifest).expanduser().resolve(),
|
| 705 |
+
geometry_manifest_path=(
|
| 706 |
+
Path(args.geometry_manifest).expanduser().resolve()
|
| 707 |
+
if args.geometry_manifest
|
| 708 |
+
else None
|
| 709 |
+
),
|
| 710 |
+
verification_manifest_path=(
|
| 711 |
+
Path(args.verification_manifest).expanduser().resolve()
|
| 712 |
+
if args.verification_manifest
|
| 713 |
+
else None
|
| 714 |
+
),
|
| 715 |
+
visual_turntable=(
|
| 716 |
+
Path(args.visual_turntable).expanduser().resolve()
|
| 717 |
+
if args.visual_turntable
|
| 718 |
+
else None
|
| 719 |
+
),
|
| 720 |
+
visual_multiview=(
|
| 721 |
+
Path(args.visual_multiview).expanduser().resolve()
|
| 722 |
+
if args.visual_multiview
|
| 723 |
+
else None
|
| 724 |
+
),
|
| 725 |
+
primary_3d_role=args.primary_3d_role,
|
| 726 |
+
category=category,
|
| 727 |
+
sample_id=sample_id,
|
| 728 |
+
output_dir=Path(args.output_dir).expanduser().resolve(),
|
| 729 |
+
)
|
| 730 |
+
print(
|
| 731 |
+
json.dumps(
|
| 732 |
+
{
|
| 733 |
+
"status": "built",
|
| 734 |
+
"sample_id": sample_id,
|
| 735 |
+
"review_dir": str(review_dir),
|
| 736 |
+
"safe_passage_claim": False,
|
| 737 |
+
},
|
| 738 |
+
ensure_ascii=False,
|
| 739 |
+
sort_keys=True,
|
| 740 |
+
)
|
| 741 |
+
)
|
| 742 |
+
return 0
|
| 743 |
+
|
| 744 |
+
|
| 745 |
+
if __name__ == "__main__":
|
| 746 |
+
raise SystemExit(main())
|
tools/build_accessibility_solid_mesh_showcase.py
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tools/render_accessibility_turntable.py
ADDED
|
@@ -0,0 +1,1316 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Render one accessibility geometry mesh into a compact turntable review set.
|
| 3 |
+
|
| 4 |
+
Open3D is imported only after the render environment has been configured. This
|
| 5 |
+
keeps argument/manifest helpers usable on CPU-only review machines and makes a
|
| 6 |
+
software fallback an explicit, auditable user choice.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import argparse
|
| 12 |
+
import json
|
| 13 |
+
import math
|
| 14 |
+
import os
|
| 15 |
+
import re
|
| 16 |
+
from datetime import datetime, timezone
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
from typing import Any, Mapping, MutableMapping, Sequence
|
| 19 |
+
|
| 20 |
+
import imageio.v2 as imageio
|
| 21 |
+
import numpy as np
|
| 22 |
+
from PIL import Image, ImageDraw, ImageOps
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
REQUESTED_RENDERER = "Open3D/EGL"
|
| 26 |
+
RENDERER_IMPLEMENTATION = "open3d.visualization.rendering.OffscreenRenderer"
|
| 27 |
+
PRODUCTION_RENDER_DEVICE = "physical_nvidia_gpu"
|
| 28 |
+
DOMINANT_COMPONENT_MIN_FACE_FRACTION = 0.90
|
| 29 |
+
GL_VENDOR = 0x1F00
|
| 30 |
+
GL_RENDERER = 0x1F01
|
| 31 |
+
GL_VERSION = 0x1F02
|
| 32 |
+
GL_STRING_NAMES = {
|
| 33 |
+
GL_VENDOR: "GL_VENDOR",
|
| 34 |
+
GL_RENDERER: "GL_RENDERER",
|
| 35 |
+
GL_VERSION: "GL_VERSION",
|
| 36 |
+
}
|
| 37 |
+
SOFTWARE_OPENGL_MARKERS = (
|
| 38 |
+
"llvmpipe",
|
| 39 |
+
"softpipe",
|
| 40 |
+
"swrast",
|
| 41 |
+
"software",
|
| 42 |
+
"swiftshader",
|
| 43 |
+
"lavapipe",
|
| 44 |
+
)
|
| 45 |
+
NVIDIA_EGL_VENDOR_JSON = Path(
|
| 46 |
+
"/usr/share/glvnd/egl_vendor.d/10_nvidia.json"
|
| 47 |
+
)
|
| 48 |
+
OUTPUT_NAMES = {
|
| 49 |
+
"video": "turntable.mp4",
|
| 50 |
+
"gif": "turntable.gif",
|
| 51 |
+
"poster": "turntable_poster.png",
|
| 52 |
+
"multiview": "multiview.jpg",
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def parse_color(value: str) -> tuple[float, float, float, float]:
|
| 57 |
+
"""Parse an RGB/RGBA command-line color in the inclusive [0, 1] range."""
|
| 58 |
+
values = [
|
| 59 |
+
float(item.strip())
|
| 60 |
+
for item in value.replace(":", ",").replace(";", ",").split(",")
|
| 61 |
+
if item.strip()
|
| 62 |
+
]
|
| 63 |
+
if len(values) == 3:
|
| 64 |
+
values.append(1.0)
|
| 65 |
+
if len(values) != 4:
|
| 66 |
+
raise argparse.ArgumentTypeError("expected three or four comma-separated numbers")
|
| 67 |
+
if any(not math.isfinite(item) or item < 0.0 or item > 1.0 for item in values):
|
| 68 |
+
raise argparse.ArgumentTypeError("background components must be between 0 and 1")
|
| 69 |
+
return tuple(values) # type: ignore[return-value]
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def parse_yaws(value: str) -> tuple[float, ...]:
|
| 73 |
+
"""Parse the yaw angles used in the multiview contact sheet."""
|
| 74 |
+
try:
|
| 75 |
+
values = tuple(float(item.strip()) for item in value.split(",") if item.strip())
|
| 76 |
+
except ValueError as error:
|
| 77 |
+
raise argparse.ArgumentTypeError("multiview yaws must be comma-separated numbers") from error
|
| 78 |
+
if not values:
|
| 79 |
+
raise argparse.ArgumentTypeError("at least one multiview yaw is required")
|
| 80 |
+
if len(values) > 12:
|
| 81 |
+
raise argparse.ArgumentTypeError("at most 12 multiview yaws are supported")
|
| 82 |
+
if any(not math.isfinite(item) for item in values):
|
| 83 |
+
raise argparse.ArgumentTypeError("multiview yaws must be finite")
|
| 84 |
+
return values
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def build_argument_parser() -> argparse.ArgumentParser:
|
| 88 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 89 |
+
parser.add_argument(
|
| 90 |
+
"--mesh",
|
| 91 |
+
"--ply",
|
| 92 |
+
dest="mesh",
|
| 93 |
+
required=True,
|
| 94 |
+
help="Input triangle mesh. --ply is retained as a convenient alias.",
|
| 95 |
+
)
|
| 96 |
+
parser.add_argument("--output-dir", required=True)
|
| 97 |
+
parser.add_argument("--sample-id", default=None)
|
| 98 |
+
parser.add_argument("--category", default="unknown")
|
| 99 |
+
parser.add_argument("--width", type=int, default=960)
|
| 100 |
+
parser.add_argument("--height", type=int, default=720)
|
| 101 |
+
parser.add_argument("--frames", type=int, default=96)
|
| 102 |
+
parser.add_argument("--fps", type=int, default=24)
|
| 103 |
+
parser.add_argument("--fov", type=float, default=32.0)
|
| 104 |
+
parser.add_argument("--elevation", type=float, default=18.0)
|
| 105 |
+
parser.add_argument("--distance-scale", type=float, default=1.25)
|
| 106 |
+
parser.add_argument("--shader", default="defaultUnlit")
|
| 107 |
+
parser.add_argument(
|
| 108 |
+
"--vertex-colors-srgb",
|
| 109 |
+
action=argparse.BooleanOptionalAction,
|
| 110 |
+
default=True,
|
| 111 |
+
help=(
|
| 112 |
+
"Treat byte RGB vertex colors sampled from photographs as sRGB. "
|
| 113 |
+
"Enabled by default to avoid linear-color whitening."
|
| 114 |
+
),
|
| 115 |
+
)
|
| 116 |
+
parser.add_argument("--background", type=parse_color, default=parse_color("0.82,0.82,0.80,1"))
|
| 117 |
+
parser.add_argument("--smooth-iterations", type=int, default=0)
|
| 118 |
+
parser.add_argument(
|
| 119 |
+
"--orbit-mode",
|
| 120 |
+
choices=("full_360", "front_arc"),
|
| 121 |
+
default="front_arc",
|
| 122 |
+
help=(
|
| 123 |
+
"full_360 is a conventional object turntable. front_arc keeps an "
|
| 124 |
+
"open-world scene in camera-facing review views and avoids treating "
|
| 125 |
+
"its unobserved back side as reconstructed evidence."
|
| 126 |
+
),
|
| 127 |
+
)
|
| 128 |
+
parser.add_argument(
|
| 129 |
+
"--orbit-span",
|
| 130 |
+
type=float,
|
| 131 |
+
default=140.0,
|
| 132 |
+
help="Total yaw span in degrees for --orbit-mode front_arc.",
|
| 133 |
+
)
|
| 134 |
+
parser.add_argument(
|
| 135 |
+
"--multiview-yaws",
|
| 136 |
+
type=parse_yaws,
|
| 137 |
+
default=parse_yaws("-60,-30,0,30,60"),
|
| 138 |
+
help=(
|
| 139 |
+
"Comma-separated camera-facing yaw angles. In front_arc mode every "
|
| 140 |
+
"angle must stay within half of --orbit-span."
|
| 141 |
+
),
|
| 142 |
+
)
|
| 143 |
+
parser.add_argument(
|
| 144 |
+
"--require-vertex-colors",
|
| 145 |
+
action="store_true",
|
| 146 |
+
help=(
|
| 147 |
+
"Reject meshes without a complete finite RGB value for every rendered "
|
| 148 |
+
"vertex instead of silently producing an untextured white surface."
|
| 149 |
+
),
|
| 150 |
+
)
|
| 151 |
+
parser.add_argument(
|
| 152 |
+
"--cpu-fallback",
|
| 153 |
+
action="store_true",
|
| 154 |
+
# Retained only for low-level renderer diagnostics and unit tests. It is
|
| 155 |
+
# intentionally absent from normal CLI help and every production
|
| 156 |
+
# orchestrator; production rendering is physical-NVIDIA-GPU only.
|
| 157 |
+
help=argparse.SUPPRESS,
|
| 158 |
+
)
|
| 159 |
+
return parser
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def validate_arguments(args: argparse.Namespace) -> None:
|
| 163 |
+
if args.width < 64 or args.height < 64:
|
| 164 |
+
raise ValueError("--width and --height must each be at least 64")
|
| 165 |
+
if args.frames < 2:
|
| 166 |
+
raise ValueError("--frames must be at least 2")
|
| 167 |
+
if args.fps < 1:
|
| 168 |
+
raise ValueError("--fps must be at least 1")
|
| 169 |
+
if not math.isfinite(args.fov) or not (5.0 <= args.fov < 170.0):
|
| 170 |
+
raise ValueError("--fov must satisfy 5 <= fov < 170")
|
| 171 |
+
if not math.isfinite(args.elevation):
|
| 172 |
+
raise ValueError("--elevation must be finite")
|
| 173 |
+
if not math.isfinite(args.distance_scale) or args.distance_scale <= 0:
|
| 174 |
+
raise ValueError("--distance-scale must be positive")
|
| 175 |
+
if args.smooth_iterations < 0:
|
| 176 |
+
raise ValueError("--smooth-iterations must be non-negative")
|
| 177 |
+
if not (20.0 <= args.orbit_span <= 240.0):
|
| 178 |
+
raise ValueError("--orbit-span must satisfy 20 <= span <= 240")
|
| 179 |
+
if args.orbit_mode == "front_arc":
|
| 180 |
+
half_span = 0.5 * args.orbit_span
|
| 181 |
+
if any(abs(yaw) > half_span + 1e-6 for yaw in args.multiview_yaws):
|
| 182 |
+
raise ValueError(
|
| 183 |
+
"front_arc multiview yaws must stay within half of --orbit-span"
|
| 184 |
+
)
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
def _visibility_state(value: str | None) -> str:
|
| 188 |
+
if value is None:
|
| 189 |
+
return "unspecified"
|
| 190 |
+
if not value.strip() or value.strip().lower() in {"-1", "none", "void", "nodevfiles"}:
|
| 191 |
+
return "hidden_or_disabled"
|
| 192 |
+
return "configured_visible_set"
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def inspect_render_environment(
|
| 196 |
+
environment: Mapping[str, str] | None = None,
|
| 197 |
+
*,
|
| 198 |
+
cpu_fallback: bool = False,
|
| 199 |
+
) -> dict[str, Any]:
|
| 200 |
+
"""Return environment evidence without claiming that Open3D used a GPU.
|
| 201 |
+
|
| 202 |
+
CUDA visibility and EGL configuration are useful diagnostics, but neither
|
| 203 |
+
proves which physical renderer Filament selected. The returned status is
|
| 204 |
+
intentionally conservative.
|
| 205 |
+
"""
|
| 206 |
+
env = os.environ if environment is None else environment
|
| 207 |
+
cuda_visible = env.get("CUDA_VISIBLE_DEVICES")
|
| 208 |
+
egl_platform = env.get("EGL_PLATFORM")
|
| 209 |
+
software_value = env.get("LIBGL_ALWAYS_SOFTWARE")
|
| 210 |
+
software_requested = cpu_fallback or software_value == "1"
|
| 211 |
+
return {
|
| 212 |
+
"slurm": {
|
| 213 |
+
"job_id": env.get("SLURM_JOB_ID"),
|
| 214 |
+
"job_gpus": env.get("SLURM_JOB_GPUS"),
|
| 215 |
+
"gpu_allocation_present": bool(env.get("SLURM_JOB_ID")),
|
| 216 |
+
},
|
| 217 |
+
"egl": {
|
| 218 |
+
"platform": egl_platform,
|
| 219 |
+
"platform_configured": egl_platform is not None,
|
| 220 |
+
"visibility": "configured" if egl_platform is not None else "unspecified",
|
| 221 |
+
"device_id": env.get("EGL_DEVICE_ID"),
|
| 222 |
+
"gpu_use": "not_proven",
|
| 223 |
+
},
|
| 224 |
+
"cuda": {
|
| 225 |
+
"visible_devices": cuda_visible,
|
| 226 |
+
"visibility": _visibility_state(cuda_visible),
|
| 227 |
+
"used_by_renderer": "not_proven",
|
| 228 |
+
},
|
| 229 |
+
"software_rendering": {
|
| 230 |
+
"requested": software_requested,
|
| 231 |
+
"libgl_always_software": software_value,
|
| 232 |
+
"mesa_loader_driver_override": env.get("MESA_LOADER_DRIVER_OVERRIDE"),
|
| 233 |
+
"gallium_driver": env.get("GALLIUM_DRIVER"),
|
| 234 |
+
},
|
| 235 |
+
"hardware_acceleration": {
|
| 236 |
+
"status": (
|
| 237 |
+
"software_fallback_requested_backend_unverified"
|
| 238 |
+
if software_requested
|
| 239 |
+
else "hardware_backend_unverified"
|
| 240 |
+
),
|
| 241 |
+
"reason": (
|
| 242 |
+
"Open3D OffscreenRenderer does not expose reliable physical-device "
|
| 243 |
+
"identity through this render path; environment visibility alone is "
|
| 244 |
+
"not hardware proof."
|
| 245 |
+
),
|
| 246 |
+
},
|
| 247 |
+
}
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
def require_slurm_gpu_environment(
|
| 251 |
+
environment: Mapping[str, str] | None = None,
|
| 252 |
+
) -> dict[str, Any]:
|
| 253 |
+
"""Fail closed unless rendering runs inside a visible Slurm GPU allocation."""
|
| 254 |
+
|
| 255 |
+
env = os.environ if environment is None else environment
|
| 256 |
+
job_id = str(env.get("SLURM_JOB_ID") or "").strip()
|
| 257 |
+
cuda_visible = str(env.get("CUDA_VISIBLE_DEVICES") or "").strip()
|
| 258 |
+
if not job_id:
|
| 259 |
+
raise RuntimeError(
|
| 260 |
+
"GPU rendering is required by default and must run inside a Slurm "
|
| 261 |
+
"allocation. Use sbatch/srun, or explicitly pass --cpu-fallback "
|
| 262 |
+
"for a diagnostic software render."
|
| 263 |
+
)
|
| 264 |
+
if not cuda_visible or cuda_visible.lower() in {"-1", "none", "void", "nodevfiles"}:
|
| 265 |
+
raise RuntimeError(
|
| 266 |
+
"The Slurm render job has no visible CUDA device; refusing an "
|
| 267 |
+
"implicit CPU/software fallback."
|
| 268 |
+
)
|
| 269 |
+
return {
|
| 270 |
+
"policy": "slurm_gpu_required_unless_cpu_fallback_explicit",
|
| 271 |
+
"slurm_job_id": job_id,
|
| 272 |
+
"cuda_visible_devices": cuda_visible,
|
| 273 |
+
"gpu_allocation_contract_satisfied": True,
|
| 274 |
+
}
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
def configure_render_environment(
|
| 278 |
+
cpu_fallback: bool,
|
| 279 |
+
environment: MutableMapping[str, str] | None = None,
|
| 280 |
+
) -> dict[str, Any]:
|
| 281 |
+
"""Configure EGL before importing Open3D and return auditable evidence."""
|
| 282 |
+
env = os.environ if environment is None else environment
|
| 283 |
+
original = {
|
| 284 |
+
key: env.get(key)
|
| 285 |
+
for key in (
|
| 286 |
+
"EGL_PLATFORM",
|
| 287 |
+
"CUDA_VISIBLE_DEVICES",
|
| 288 |
+
"__EGL_VENDOR_LIBRARY_FILENAMES",
|
| 289 |
+
"LIBGL_ALWAYS_SOFTWARE",
|
| 290 |
+
"MESA_LOADER_DRIVER_OVERRIDE",
|
| 291 |
+
"GALLIUM_DRIVER",
|
| 292 |
+
)
|
| 293 |
+
}
|
| 294 |
+
env.setdefault("EGL_PLATFORM", "surfaceless")
|
| 295 |
+
gpu_contract = None
|
| 296 |
+
if cpu_fallback:
|
| 297 |
+
env["LIBGL_ALWAYS_SOFTWARE"] = "1"
|
| 298 |
+
env.setdefault("MESA_LOADER_DRIVER_OVERRIDE", "llvmpipe")
|
| 299 |
+
env.setdefault("GALLIUM_DRIVER", "llvmpipe")
|
| 300 |
+
else:
|
| 301 |
+
gpu_contract = require_slurm_gpu_environment(env)
|
| 302 |
+
if NVIDIA_EGL_VENDOR_JSON.is_file():
|
| 303 |
+
env.setdefault(
|
| 304 |
+
"__EGL_VENDOR_LIBRARY_FILENAMES",
|
| 305 |
+
str(NVIDIA_EGL_VENDOR_JSON),
|
| 306 |
+
)
|
| 307 |
+
evidence = inspect_render_environment(env, cpu_fallback=cpu_fallback)
|
| 308 |
+
evidence["configuration"] = {
|
| 309 |
+
"original": original,
|
| 310 |
+
"egl_platform_defaulted_by_tool": original["EGL_PLATFORM"] is None,
|
| 311 |
+
"nvidia_egl_vendor_defaulted_by_tool": (
|
| 312 |
+
not cpu_fallback
|
| 313 |
+
and original["__EGL_VENDOR_LIBRARY_FILENAMES"] is None
|
| 314 |
+
and env.get("__EGL_VENDOR_LIBRARY_FILENAMES")
|
| 315 |
+
== str(NVIDIA_EGL_VENDOR_JSON)
|
| 316 |
+
),
|
| 317 |
+
"cpu_fallback_explicit": cpu_fallback,
|
| 318 |
+
"gpu_contract": gpu_contract,
|
| 319 |
+
}
|
| 320 |
+
return evidence
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
def _load_gl_get_string() -> tuple[Any, str]:
|
| 324 |
+
"""Resolve ``glGetString`` without importing an OpenGL Python package.
|
| 325 |
+
|
| 326 |
+
Open3D wheels do not expose Filament's selected renderer through their
|
| 327 |
+
Python API. Once ``OffscreenRenderer`` has created its EGL context, the GL
|
| 328 |
+
dispatch function is sufficient to query the active context directly.
|
| 329 |
+
"""
|
| 330 |
+
|
| 331 |
+
import ctypes
|
| 332 |
+
import ctypes.util
|
| 333 |
+
|
| 334 |
+
attempts: list[str] = []
|
| 335 |
+
candidates: list[tuple[str, str | None]] = [("process", None)]
|
| 336 |
+
for library_name in ("GL", "OpenGL", "GLESv2"):
|
| 337 |
+
library_path = ctypes.util.find_library(library_name)
|
| 338 |
+
if library_path:
|
| 339 |
+
candidates.append((library_name, library_path))
|
| 340 |
+
|
| 341 |
+
for label, library_path in candidates:
|
| 342 |
+
try:
|
| 343 |
+
library = ctypes.CDLL(library_path)
|
| 344 |
+
gl_get_string = getattr(library, "glGetString")
|
| 345 |
+
except (AttributeError, OSError) as error:
|
| 346 |
+
attempts.append(f"{label}: {error}")
|
| 347 |
+
continue
|
| 348 |
+
gl_get_string.argtypes = [ctypes.c_uint]
|
| 349 |
+
gl_get_string.restype = ctypes.c_char_p
|
| 350 |
+
return gl_get_string, f"ctypes:{label}"
|
| 351 |
+
|
| 352 |
+
# Some EGL/GLES deployments expose GL entry points only through
|
| 353 |
+
# eglGetProcAddress. Keep this as a final standards-based resolution path.
|
| 354 |
+
egl_path = ctypes.util.find_library("EGL")
|
| 355 |
+
if egl_path:
|
| 356 |
+
try:
|
| 357 |
+
egl = ctypes.CDLL(egl_path)
|
| 358 |
+
egl_get_proc_address = egl.eglGetProcAddress
|
| 359 |
+
egl_get_proc_address.argtypes = [ctypes.c_char_p]
|
| 360 |
+
egl_get_proc_address.restype = ctypes.c_void_p
|
| 361 |
+
address = egl_get_proc_address(b"glGetString")
|
| 362 |
+
if address:
|
| 363 |
+
prototype = ctypes.CFUNCTYPE(ctypes.c_char_p, ctypes.c_uint)
|
| 364 |
+
return prototype(address), "ctypes:EGL.eglGetProcAddress"
|
| 365 |
+
attempts.append("EGL.eglGetProcAddress: returned NULL")
|
| 366 |
+
except (AttributeError, OSError) as error:
|
| 367 |
+
attempts.append(f"EGL: {error}")
|
| 368 |
+
|
| 369 |
+
detail = "; ".join(attempts) or "no OpenGL libraries were discoverable"
|
| 370 |
+
raise RuntimeError(f"Could not resolve glGetString ({detail})")
|
| 371 |
+
|
| 372 |
+
|
| 373 |
+
def _decode_gl_string(value: Any) -> str | None:
|
| 374 |
+
"""Convert a ``glGetString`` result into a non-empty Python string."""
|
| 375 |
+
|
| 376 |
+
if value is None:
|
| 377 |
+
return None
|
| 378 |
+
if isinstance(value, bytes):
|
| 379 |
+
decoded = value.decode("utf-8", errors="replace")
|
| 380 |
+
elif isinstance(value, str):
|
| 381 |
+
decoded = value
|
| 382 |
+
else:
|
| 383 |
+
decoded = str(value)
|
| 384 |
+
decoded = decoded.strip()
|
| 385 |
+
return decoded or None
|
| 386 |
+
|
| 387 |
+
|
| 388 |
+
def query_current_opengl_context(
|
| 389 |
+
gl_get_string: Any | None = None,
|
| 390 |
+
*,
|
| 391 |
+
query_source: str | None = None,
|
| 392 |
+
) -> dict[str, Any]:
|
| 393 |
+
"""Query the OpenGL strings for the context current on this thread.
|
| 394 |
+
|
| 395 |
+
The optional callable is an injection seam for CPU-only unit tests. In
|
| 396 |
+
production this function is called immediately after constructing Open3D's
|
| 397 |
+
``OffscreenRenderer``.
|
| 398 |
+
"""
|
| 399 |
+
|
| 400 |
+
evidence: dict[str, Any] = {
|
| 401 |
+
"query_api": "glGetString",
|
| 402 |
+
"query_source": query_source,
|
| 403 |
+
"query_succeeded": False,
|
| 404 |
+
"GL_VENDOR": None,
|
| 405 |
+
"GL_RENDERER": None,
|
| 406 |
+
"GL_VERSION": None,
|
| 407 |
+
"software_renderer_detected": False,
|
| 408 |
+
"software_markers": [],
|
| 409 |
+
"verification_status": "active_context_unverified",
|
| 410 |
+
"error": None,
|
| 411 |
+
}
|
| 412 |
+
try:
|
| 413 |
+
if gl_get_string is None:
|
| 414 |
+
gl_get_string, resolved_source = _load_gl_get_string()
|
| 415 |
+
evidence["query_source"] = resolved_source
|
| 416 |
+
elif evidence["query_source"] is None:
|
| 417 |
+
evidence["query_source"] = "injected_gl_get_string"
|
| 418 |
+
|
| 419 |
+
errors: list[str] = []
|
| 420 |
+
for enum_value, field_name in GL_STRING_NAMES.items():
|
| 421 |
+
try:
|
| 422 |
+
evidence[field_name] = _decode_gl_string(gl_get_string(enum_value))
|
| 423 |
+
except Exception as error: # ctypes failures must become fail-closed evidence.
|
| 424 |
+
errors.append(f"{field_name}: {type(error).__name__}: {error}")
|
| 425 |
+
missing = [name for name in GL_STRING_NAMES.values() if not evidence[name]]
|
| 426 |
+
if missing:
|
| 427 |
+
errors.append("NULL or empty values: " + ", ".join(missing))
|
| 428 |
+
evidence["query_succeeded"] = not errors
|
| 429 |
+
if errors:
|
| 430 |
+
evidence["error"] = "; ".join(errors)
|
| 431 |
+
except Exception as error:
|
| 432 |
+
evidence["error"] = f"{type(error).__name__}: {error}"
|
| 433 |
+
|
| 434 |
+
combined = " ".join(
|
| 435 |
+
str(evidence[name] or "").lower() for name in GL_STRING_NAMES.values()
|
| 436 |
+
)
|
| 437 |
+
markers = [marker for marker in SOFTWARE_OPENGL_MARKERS if marker in combined]
|
| 438 |
+
evidence["software_markers"] = markers
|
| 439 |
+
evidence["software_renderer_detected"] = bool(markers)
|
| 440 |
+
if evidence["query_succeeded"]:
|
| 441 |
+
evidence["verification_status"] = (
|
| 442 |
+
"active_software_context_verified"
|
| 443 |
+
if markers
|
| 444 |
+
else "active_non_software_context_verified"
|
| 445 |
+
)
|
| 446 |
+
return evidence
|
| 447 |
+
|
| 448 |
+
|
| 449 |
+
def inspect_process_graphics_backend(
|
| 450 |
+
*,
|
| 451 |
+
maps_text: str | None = None,
|
| 452 |
+
fd_targets: Sequence[str] | None = None,
|
| 453 |
+
environment: Mapping[str, str] | None = None,
|
| 454 |
+
) -> dict[str, Any]:
|
| 455 |
+
"""Inspect process-wide EGL libraries and open GPU device handles.
|
| 456 |
+
|
| 457 |
+
Filament owns its OpenGL context on an internal render thread, so
|
| 458 |
+
``glGetString`` from the Python thread may legitimately return NULL. In
|
| 459 |
+
that case a process-wide NVIDIA EGL binding plus an open physical
|
| 460 |
+
``/dev/nvidiaN`` handle is stronger evidence than CUDA visibility alone.
|
| 461 |
+
Injection seams keep this policy fully unit-testable without a GPU.
|
| 462 |
+
"""
|
| 463 |
+
|
| 464 |
+
env = os.environ if environment is None else environment
|
| 465 |
+
errors: list[str] = []
|
| 466 |
+
if maps_text is None:
|
| 467 |
+
try:
|
| 468 |
+
maps_text = Path("/proc/self/maps").read_text(
|
| 469 |
+
encoding="utf-8",
|
| 470 |
+
errors="replace",
|
| 471 |
+
)
|
| 472 |
+
except OSError as error:
|
| 473 |
+
maps_text = ""
|
| 474 |
+
errors.append(f"maps: {type(error).__name__}: {error}")
|
| 475 |
+
if fd_targets is None:
|
| 476 |
+
discovered_targets: list[str] = []
|
| 477 |
+
try:
|
| 478 |
+
for entry in Path("/proc/self/fd").iterdir():
|
| 479 |
+
try:
|
| 480 |
+
discovered_targets.append(os.readlink(entry))
|
| 481 |
+
except OSError:
|
| 482 |
+
continue
|
| 483 |
+
except OSError as error:
|
| 484 |
+
errors.append(f"fd: {type(error).__name__}: {error}")
|
| 485 |
+
fd_targets = discovered_targets
|
| 486 |
+
|
| 487 |
+
mapped_paths = sorted(
|
| 488 |
+
{
|
| 489 |
+
line.rsplit(None, 1)[-1]
|
| 490 |
+
for line in maps_text.splitlines()
|
| 491 |
+
if line.strip() and "/" in line.rsplit(None, 1)[-1]
|
| 492 |
+
}
|
| 493 |
+
)
|
| 494 |
+
nvidia_egl_libraries = [
|
| 495 |
+
path
|
| 496 |
+
for path in mapped_paths
|
| 497 |
+
if (
|
| 498 |
+
"libegl_nvidia" in path.lower()
|
| 499 |
+
or "libnvidia-eglcore" in path.lower()
|
| 500 |
+
)
|
| 501 |
+
]
|
| 502 |
+
software_library_markers = sorted(
|
| 503 |
+
{
|
| 504 |
+
marker
|
| 505 |
+
for path in mapped_paths
|
| 506 |
+
for marker in (
|
| 507 |
+
"libegl_mesa",
|
| 508 |
+
"llvmpipe",
|
| 509 |
+
"softpipe",
|
| 510 |
+
"swrast",
|
| 511 |
+
"lavapipe",
|
| 512 |
+
"swiftshader",
|
| 513 |
+
)
|
| 514 |
+
if marker in path.lower()
|
| 515 |
+
}
|
| 516 |
+
)
|
| 517 |
+
gpu_device_fds = sorted(
|
| 518 |
+
{target for target in fd_targets if target.startswith("/dev/nvidia")}
|
| 519 |
+
)
|
| 520 |
+
physical_gpu_fds = [
|
| 521 |
+
target
|
| 522 |
+
for target in gpu_device_fds
|
| 523 |
+
if re.fullmatch(r"/dev/nvidia\d+", target)
|
| 524 |
+
]
|
| 525 |
+
nvidia_backend_verified = bool(
|
| 526 |
+
nvidia_egl_libraries
|
| 527 |
+
and physical_gpu_fds
|
| 528 |
+
and not software_library_markers
|
| 529 |
+
)
|
| 530 |
+
return {
|
| 531 |
+
"inspection_api": "/proc/self/maps + /proc/self/fd",
|
| 532 |
+
"nvidia_egl_libraries": nvidia_egl_libraries,
|
| 533 |
+
"software_library_markers": software_library_markers,
|
| 534 |
+
"gpu_device_fds": gpu_device_fds,
|
| 535 |
+
"physical_gpu_fds": physical_gpu_fds,
|
| 536 |
+
"forced_egl_vendor_json": env.get(
|
| 537 |
+
"__EGL_VENDOR_LIBRARY_FILENAMES"
|
| 538 |
+
),
|
| 539 |
+
"nvidia_process_backend_verified": nvidia_backend_verified,
|
| 540 |
+
"verification_status": (
|
| 541 |
+
"nvidia_egl_and_physical_device_fd_verified"
|
| 542 |
+
if nvidia_backend_verified
|
| 543 |
+
else "process_graphics_backend_unverified"
|
| 544 |
+
),
|
| 545 |
+
"errors": errors,
|
| 546 |
+
}
|
| 547 |
+
|
| 548 |
+
|
| 549 |
+
def enforce_opengl_context_policy(
|
| 550 |
+
opengl_context: Mapping[str, Any],
|
| 551 |
+
*,
|
| 552 |
+
cpu_fallback: bool,
|
| 553 |
+
process_backend: Mapping[str, Any] | None = None,
|
| 554 |
+
) -> dict[str, Any]:
|
| 555 |
+
"""Apply the fail-closed GPU policy to actual active-context evidence."""
|
| 556 |
+
|
| 557 |
+
query_succeeded = bool(opengl_context.get("query_succeeded"))
|
| 558 |
+
software_detected = bool(opengl_context.get("software_renderer_detected"))
|
| 559 |
+
renderer_name = opengl_context.get("GL_RENDERER")
|
| 560 |
+
nvidia_process_verified = bool(
|
| 561 |
+
process_backend
|
| 562 |
+
and process_backend.get("nvidia_process_backend_verified")
|
| 563 |
+
)
|
| 564 |
+
|
| 565 |
+
if cpu_fallback:
|
| 566 |
+
if not query_succeeded:
|
| 567 |
+
status = "cpu_fallback_requested_context_unverified"
|
| 568 |
+
reason = (
|
| 569 |
+
"Explicit --cpu-fallback permits diagnostic rendering even though "
|
| 570 |
+
"the active OpenGL strings could not be verified."
|
| 571 |
+
)
|
| 572 |
+
elif software_detected:
|
| 573 |
+
status = "software_fallback_active_context_verified"
|
| 574 |
+
reason = (
|
| 575 |
+
"Explicit --cpu-fallback was requested and the active OpenGL "
|
| 576 |
+
"context identifies a software renderer."
|
| 577 |
+
)
|
| 578 |
+
else:
|
| 579 |
+
status = "cpu_fallback_requested_non_software_context_verified"
|
| 580 |
+
reason = (
|
| 581 |
+
"Explicit --cpu-fallback was requested, but the active OpenGL "
|
| 582 |
+
"strings do not identify a known software renderer."
|
| 583 |
+
)
|
| 584 |
+
return {
|
| 585 |
+
"status": status,
|
| 586 |
+
"reason": reason,
|
| 587 |
+
"active_context_verified": query_succeeded,
|
| 588 |
+
"software_renderer_detected": software_detected,
|
| 589 |
+
"physical_gpu_identity_proven": False,
|
| 590 |
+
}
|
| 591 |
+
|
| 592 |
+
if software_detected:
|
| 593 |
+
markers = ", ".join(
|
| 594 |
+
str(value) for value in opengl_context.get("software_markers", [])
|
| 595 |
+
)
|
| 596 |
+
raise RuntimeError(
|
| 597 |
+
"The active OpenGL context is a software OpenGL renderer "
|
| 598 |
+
f"(GL_RENDERER={renderer_name!r}; markers={markers}); refusing the "
|
| 599 |
+
"default GPU render. Use --cpu-fallback to opt in explicitly."
|
| 600 |
+
)
|
| 601 |
+
if nvidia_process_verified:
|
| 602 |
+
physical_gpu_fds = list(
|
| 603 |
+
process_backend.get("physical_gpu_fds", [])
|
| 604 |
+
if process_backend is not None
|
| 605 |
+
else []
|
| 606 |
+
)
|
| 607 |
+
return {
|
| 608 |
+
"status": "active_nvidia_egl_process_backend_verified",
|
| 609 |
+
"reason": (
|
| 610 |
+
(
|
| 611 |
+
"The active OpenGL context contains no software-renderer "
|
| 612 |
+
"marker, and the renderer process loaded NVIDIA EGL without "
|
| 613 |
+
"a known software library and opened a physical NVIDIA GPU "
|
| 614 |
+
"device."
|
| 615 |
+
)
|
| 616 |
+
if query_succeeded
|
| 617 |
+
else (
|
| 618 |
+
"Filament's render-thread GL strings were not visible to "
|
| 619 |
+
"Python, but the renderer process loaded NVIDIA EGL without "
|
| 620 |
+
"a known software library and opened a physical NVIDIA GPU "
|
| 621 |
+
"device."
|
| 622 |
+
)
|
| 623 |
+
),
|
| 624 |
+
"active_context_verified": query_succeeded,
|
| 625 |
+
"process_backend_verified": True,
|
| 626 |
+
"software_renderer_detected": False,
|
| 627 |
+
"physical_gpu_identity_proven": True,
|
| 628 |
+
"physical_gpu_fds": physical_gpu_fds,
|
| 629 |
+
}
|
| 630 |
+
if not query_succeeded:
|
| 631 |
+
detail = str(opengl_context.get("error") or "no GL strings were returned")
|
| 632 |
+
raise RuntimeError(
|
| 633 |
+
"Could not verify the active OpenGL context after creating "
|
| 634 |
+
f"OffscreenRenderer; refusing the default GPU render ({detail}). "
|
| 635 |
+
"Use --cpu-fallback only for an explicit diagnostic software render."
|
| 636 |
+
)
|
| 637 |
+
raise RuntimeError(
|
| 638 |
+
"The active OpenGL strings contain no known software marker, but the "
|
| 639 |
+
"renderer process did not prove an NVIDIA EGL backend plus a physical "
|
| 640 |
+
"/dev/nvidiaN device; refusing the default GPU render. Use "
|
| 641 |
+
"--cpu-fallback only for an explicit diagnostic software render."
|
| 642 |
+
)
|
| 643 |
+
|
| 644 |
+
|
| 645 |
+
def _merge_runtime_graphics_evidence(
|
| 646 |
+
environment_evidence: Mapping[str, Any],
|
| 647 |
+
render_result: Mapping[str, Any],
|
| 648 |
+
) -> dict[str, Any]:
|
| 649 |
+
"""Combine pre-import environment facts with post-context GL evidence."""
|
| 650 |
+
|
| 651 |
+
merged = dict(environment_evidence)
|
| 652 |
+
opengl_context = render_result.get("opengl_context")
|
| 653 |
+
process_backend = render_result.get("process_graphics_backend")
|
| 654 |
+
hardware_acceleration = render_result.get("hardware_acceleration")
|
| 655 |
+
if isinstance(opengl_context, Mapping):
|
| 656 |
+
merged["opengl_context"] = dict(opengl_context)
|
| 657 |
+
if isinstance(process_backend, Mapping):
|
| 658 |
+
merged["process_graphics_backend"] = dict(process_backend)
|
| 659 |
+
if isinstance(hardware_acceleration, Mapping):
|
| 660 |
+
merged["hardware_acceleration"] = dict(hardware_acceleration)
|
| 661 |
+
return merged
|
| 662 |
+
|
| 663 |
+
|
| 664 |
+
def relative_path(path: Path, anchor: Path) -> str:
|
| 665 |
+
"""Return a portable relative path, even for an input outside ``anchor``."""
|
| 666 |
+
return Path(os.path.relpath(path.resolve(), anchor.resolve())).as_posix()
|
| 667 |
+
|
| 668 |
+
|
| 669 |
+
def file_record(path: Path, anchor: Path) -> dict[str, Any]:
|
| 670 |
+
import hashlib
|
| 671 |
+
|
| 672 |
+
digest = hashlib.sha256()
|
| 673 |
+
with path.open("rb") as handle:
|
| 674 |
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
| 675 |
+
digest.update(chunk)
|
| 676 |
+
return {
|
| 677 |
+
"path": relative_path(path, anchor),
|
| 678 |
+
"sha256": digest.hexdigest(),
|
| 679 |
+
"bytes": path.stat().st_size,
|
| 680 |
+
}
|
| 681 |
+
|
| 682 |
+
|
| 683 |
+
def build_render_manifest(
|
| 684 |
+
*,
|
| 685 |
+
mesh_path: Path,
|
| 686 |
+
output_dir: Path,
|
| 687 |
+
sample_id: str,
|
| 688 |
+
category: str,
|
| 689 |
+
render_settings: Mapping[str, Any],
|
| 690 |
+
render_result: Mapping[str, Any],
|
| 691 |
+
environment_evidence: Mapping[str, Any],
|
| 692 |
+
cpu_fallback: bool,
|
| 693 |
+
) -> dict[str, Any]:
|
| 694 |
+
"""Build a manifest whose filesystem paths are all relative."""
|
| 695 |
+
runtime_evidence = _merge_runtime_graphics_evidence(
|
| 696 |
+
environment_evidence,
|
| 697 |
+
render_result,
|
| 698 |
+
)
|
| 699 |
+
opengl_context = dict(runtime_evidence.get("opengl_context", {}))
|
| 700 |
+
process_backend = dict(
|
| 701 |
+
runtime_evidence.get("process_graphics_backend", {})
|
| 702 |
+
)
|
| 703 |
+
outputs = {
|
| 704 |
+
role: file_record(output_dir / filename, output_dir)
|
| 705 |
+
for role, filename in OUTPUT_NAMES.items()
|
| 706 |
+
}
|
| 707 |
+
return {
|
| 708 |
+
"schema_version": "accessibilityamodal_turntable_v2",
|
| 709 |
+
"created_at_utc": datetime.now(timezone.utc).isoformat(),
|
| 710 |
+
"sample_id": sample_id,
|
| 711 |
+
"category": category,
|
| 712 |
+
"requested_renderer": REQUESTED_RENDERER,
|
| 713 |
+
"renderer": {
|
| 714 |
+
"requested": REQUESTED_RENDERER,
|
| 715 |
+
"implementation": RENDERER_IMPLEMENTATION,
|
| 716 |
+
"production_render_device": PRODUCTION_RENDER_DEVICE,
|
| 717 |
+
"production_gpu_fail_closed": True,
|
| 718 |
+
"requested_execution": (
|
| 719 |
+
"cpu_fallback_explicit"
|
| 720 |
+
if cpu_fallback
|
| 721 |
+
else "slurm_gpu_required_egl"
|
| 722 |
+
),
|
| 723 |
+
"cpu_fallback_explicit": cpu_fallback,
|
| 724 |
+
"gpu_required_by_default": True,
|
| 725 |
+
"render_call_succeeded": True,
|
| 726 |
+
"hardware_acceleration_status": runtime_evidence[
|
| 727 |
+
"hardware_acceleration"
|
| 728 |
+
]["status"],
|
| 729 |
+
"opengl_context": opengl_context,
|
| 730 |
+
"process_graphics_backend": process_backend,
|
| 731 |
+
},
|
| 732 |
+
"environment_evidence": runtime_evidence,
|
| 733 |
+
"input": {
|
| 734 |
+
"mesh": file_record(mesh_path, output_dir),
|
| 735 |
+
},
|
| 736 |
+
"outputs": outputs,
|
| 737 |
+
"render_settings": dict(render_settings),
|
| 738 |
+
"mesh_summary": {
|
| 739 |
+
"vertices": int(render_result["vertices"]),
|
| 740 |
+
"renderable_vertices": int(
|
| 741 |
+
render_result.get("renderable_vertices", render_result["vertices"])
|
| 742 |
+
),
|
| 743 |
+
"unreferenced_vertices_excluded_from_camera_fit": int(
|
| 744 |
+
render_result.get(
|
| 745 |
+
"unreferenced_vertices_excluded_from_camera_fit",
|
| 746 |
+
0,
|
| 747 |
+
)
|
| 748 |
+
),
|
| 749 |
+
"faces": int(render_result["faces"]),
|
| 750 |
+
"camera_framing": dict(render_result.get("camera_framing", {})),
|
| 751 |
+
"camera_motion": dict(render_result.get("camera", {})),
|
| 752 |
+
"vertex_colors": dict(render_result.get("vertex_colors", {})),
|
| 753 |
+
},
|
| 754 |
+
"scene_review": dict(render_result.get("orbit", {})),
|
| 755 |
+
"limitations": [
|
| 756 |
+
"GPU mode rejects known software GL strings; when Filament keeps its context on a render thread, NVIDIA EGL mappings plus a physical /dev/nvidiaN handle provide process-backend proof.",
|
| 757 |
+
"When one connected surface contains at least 90% of faces, camera framing prioritizes it; detached islands remain in the mesh but may fall outside a close-up view.",
|
| 758 |
+
"front_arc shows the observed scene-facing hemisphere and intentionally does not claim an inferred back side.",
|
| 759 |
+
"This rotating geometry is review evidence, not metric accessibility or navigation certification.",
|
| 760 |
+
],
|
| 761 |
+
"path_policy": "All filesystem paths in this manifest are relative to manifest.json.",
|
| 762 |
+
}
|
| 763 |
+
|
| 764 |
+
|
| 765 |
+
def _camera_pose(
|
| 766 |
+
center: np.ndarray,
|
| 767 |
+
camera_distance: float,
|
| 768 |
+
yaw_degrees: float,
|
| 769 |
+
elevation_degrees: float,
|
| 770 |
+
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
| 771 |
+
yaw = math.radians(yaw_degrees)
|
| 772 |
+
elevation = math.radians(elevation_degrees)
|
| 773 |
+
# ``camera_distance`` already includes the requested framing margin. Keep
|
| 774 |
+
# that margin in one place so --distance-scale is not applied twice.
|
| 775 |
+
distance = max(camera_distance, 1e-3)
|
| 776 |
+
eye = center + np.array(
|
| 777 |
+
[
|
| 778 |
+
distance * math.sin(yaw) * math.cos(elevation),
|
| 779 |
+
-distance * math.cos(yaw) * math.cos(elevation),
|
| 780 |
+
distance * math.sin(elevation),
|
| 781 |
+
],
|
| 782 |
+
dtype=np.float64,
|
| 783 |
+
)
|
| 784 |
+
return eye, center, np.array([0.0, 0.0, 1.0], dtype=np.float64)
|
| 785 |
+
|
| 786 |
+
|
| 787 |
+
def _orbit_yaws(
|
| 788 |
+
frames: int,
|
| 789 |
+
*,
|
| 790 |
+
mode: str,
|
| 791 |
+
span_degrees: float,
|
| 792 |
+
) -> list[float]:
|
| 793 |
+
"""Return deterministic yaw positions for object or open-scene review."""
|
| 794 |
+
|
| 795 |
+
if frames < 2:
|
| 796 |
+
raise ValueError("orbit requires at least two frames")
|
| 797 |
+
if mode == "full_360":
|
| 798 |
+
return [360.0 * index / frames for index in range(frames)]
|
| 799 |
+
if mode != "front_arc":
|
| 800 |
+
raise ValueError(f"Unsupported orbit mode: {mode}")
|
| 801 |
+
half_span = 0.5 * float(span_degrees)
|
| 802 |
+
# Begin at the input-facing view, sweep right, return through the front,
|
| 803 |
+
# sweep left, and finish near the front. This loops without a 360-degree
|
| 804 |
+
# discontinuity and never claims an unobserved back-side reconstruction.
|
| 805 |
+
return [
|
| 806 |
+
half_span * math.sin(2.0 * math.pi * index / frames)
|
| 807 |
+
for index in range(frames)
|
| 808 |
+
]
|
| 809 |
+
|
| 810 |
+
|
| 811 |
+
def _fitted_camera_distance(
|
| 812 |
+
bbox_extent: np.ndarray,
|
| 813 |
+
vertical_fov_degrees: float,
|
| 814 |
+
aspect_ratio: float,
|
| 815 |
+
) -> float:
|
| 816 |
+
radius = max(float(np.linalg.norm(bbox_extent)) * 0.5, 1e-3)
|
| 817 |
+
vertical_fov = math.radians(vertical_fov_degrees)
|
| 818 |
+
horizontal_fov = 2.0 * math.atan(math.tan(vertical_fov * 0.5) * aspect_ratio)
|
| 819 |
+
limiting_fov = min(vertical_fov, horizontal_fov)
|
| 820 |
+
return radius / max(math.sin(limiting_fov * 0.5), 1e-3)
|
| 821 |
+
|
| 822 |
+
|
| 823 |
+
def _fitted_camera_distance_for_view(
|
| 824 |
+
vertices: np.ndarray,
|
| 825 |
+
center: np.ndarray,
|
| 826 |
+
*,
|
| 827 |
+
yaw_degrees: float,
|
| 828 |
+
elevation_degrees: float,
|
| 829 |
+
vertical_fov_degrees: float,
|
| 830 |
+
aspect_ratio: float,
|
| 831 |
+
distance_scale: float,
|
| 832 |
+
) -> float:
|
| 833 |
+
"""Fit an elongated surface for one view without a global-sphere zoom-out."""
|
| 834 |
+
|
| 835 |
+
yaw = math.radians(yaw_degrees)
|
| 836 |
+
elevation = math.radians(elevation_degrees)
|
| 837 |
+
eye_direction = np.array(
|
| 838 |
+
[
|
| 839 |
+
math.sin(yaw) * math.cos(elevation),
|
| 840 |
+
-math.cos(yaw) * math.cos(elevation),
|
| 841 |
+
math.sin(elevation),
|
| 842 |
+
],
|
| 843 |
+
dtype=np.float64,
|
| 844 |
+
)
|
| 845 |
+
forward = -eye_direction
|
| 846 |
+
world_up = np.array([0.0, 0.0, 1.0], dtype=np.float64)
|
| 847 |
+
right = np.cross(forward, world_up)
|
| 848 |
+
right_norm = float(np.linalg.norm(right))
|
| 849 |
+
if right_norm < 1e-8:
|
| 850 |
+
right = np.array([1.0, 0.0, 0.0], dtype=np.float64)
|
| 851 |
+
else:
|
| 852 |
+
right /= right_norm
|
| 853 |
+
view_up = np.cross(right, forward)
|
| 854 |
+
view_up /= max(float(np.linalg.norm(view_up)), 1e-8)
|
| 855 |
+
|
| 856 |
+
relative = np.asarray(vertices, dtype=np.float64) - center
|
| 857 |
+
projected_x = np.abs(relative @ right)
|
| 858 |
+
projected_y = np.abs(relative @ view_up)
|
| 859 |
+
toward_camera = relative @ eye_direction
|
| 860 |
+
vertical_half_fov = math.radians(vertical_fov_degrees) * 0.5
|
| 861 |
+
horizontal_half_fov = math.atan(
|
| 862 |
+
math.tan(vertical_half_fov) * aspect_ratio
|
| 863 |
+
)
|
| 864 |
+
required_x = toward_camera + projected_x / max(
|
| 865 |
+
math.tan(horizontal_half_fov),
|
| 866 |
+
1e-6,
|
| 867 |
+
)
|
| 868 |
+
required_y = toward_camera + projected_y / max(
|
| 869 |
+
math.tan(vertical_half_fov),
|
| 870 |
+
1e-6,
|
| 871 |
+
)
|
| 872 |
+
fitted = max(float(np.max(required_x)), float(np.max(required_y)), 1e-3)
|
| 873 |
+
return fitted * distance_scale
|
| 874 |
+
|
| 875 |
+
|
| 876 |
+
def _fixed_camera_distance_for_views(
|
| 877 |
+
vertices: np.ndarray,
|
| 878 |
+
center: np.ndarray,
|
| 879 |
+
*,
|
| 880 |
+
yaw_degrees: Sequence[float],
|
| 881 |
+
elevation_degrees: float,
|
| 882 |
+
vertical_fov_degrees: float,
|
| 883 |
+
aspect_ratio: float,
|
| 884 |
+
distance_scale: float,
|
| 885 |
+
) -> tuple[float, dict[str, Any]]:
|
| 886 |
+
"""Fit every requested view once and use the largest distance for all frames."""
|
| 887 |
+
|
| 888 |
+
unique_yaws = tuple(dict.fromkeys(float(yaw) for yaw in yaw_degrees))
|
| 889 |
+
if not unique_yaws:
|
| 890 |
+
raise ValueError("fixed camera fitting requires at least one yaw")
|
| 891 |
+
if any(not math.isfinite(yaw) for yaw in unique_yaws):
|
| 892 |
+
raise ValueError("fixed camera fitting requires finite yaws")
|
| 893 |
+
required_distances = [
|
| 894 |
+
_fitted_camera_distance_for_view(
|
| 895 |
+
vertices,
|
| 896 |
+
center,
|
| 897 |
+
yaw_degrees=yaw,
|
| 898 |
+
elevation_degrees=elevation_degrees,
|
| 899 |
+
vertical_fov_degrees=vertical_fov_degrees,
|
| 900 |
+
aspect_ratio=aspect_ratio,
|
| 901 |
+
distance_scale=distance_scale,
|
| 902 |
+
)
|
| 903 |
+
for yaw in unique_yaws
|
| 904 |
+
]
|
| 905 |
+
fixed_distance = max(required_distances)
|
| 906 |
+
return fixed_distance, {
|
| 907 |
+
"distance_policy": "fixed_max_over_orbit_and_multiview",
|
| 908 |
+
"fixed_distance": float(fixed_distance),
|
| 909 |
+
"fit_yaw_count": len(unique_yaws),
|
| 910 |
+
"fit_yaw_min": float(min(unique_yaws)),
|
| 911 |
+
"fit_yaw_max": float(max(unique_yaws)),
|
| 912 |
+
"per_view_required_distance_min": float(min(required_distances)),
|
| 913 |
+
"per_view_required_distance_max": float(max(required_distances)),
|
| 914 |
+
}
|
| 915 |
+
|
| 916 |
+
|
| 917 |
+
def _summarize_vertex_colors(
|
| 918 |
+
vertex_colors: np.ndarray,
|
| 919 |
+
vertex_count: int,
|
| 920 |
+
*,
|
| 921 |
+
required: bool,
|
| 922 |
+
) -> dict[str, Any]:
|
| 923 |
+
"""Return auditable texture evidence without inferring visual quality."""
|
| 924 |
+
|
| 925 |
+
colors = np.asarray(vertex_colors, dtype=np.float64)
|
| 926 |
+
complete = (
|
| 927 |
+
vertex_count > 0
|
| 928 |
+
and colors.ndim == 2
|
| 929 |
+
and colors.shape == (vertex_count, 3)
|
| 930 |
+
)
|
| 931 |
+
finite = bool(complete and np.isfinite(colors).all())
|
| 932 |
+
summary: dict[str, Any] = {
|
| 933 |
+
"required": bool(required),
|
| 934 |
+
"present": bool(complete),
|
| 935 |
+
"finite": finite,
|
| 936 |
+
"colored_vertices": int(colors.shape[0]) if colors.ndim == 2 else 0,
|
| 937 |
+
"coverage_fraction": 1.0 if complete else 0.0,
|
| 938 |
+
}
|
| 939 |
+
if finite:
|
| 940 |
+
summary.update(
|
| 941 |
+
{
|
| 942 |
+
"channel_mean": [float(value) for value in np.mean(colors, axis=0)],
|
| 943 |
+
"channel_std": [float(value) for value in np.std(colors, axis=0)],
|
| 944 |
+
"channel_min": [float(value) for value in np.min(colors, axis=0)],
|
| 945 |
+
"channel_max": [float(value) for value in np.max(colors, axis=0)],
|
| 946 |
+
}
|
| 947 |
+
)
|
| 948 |
+
return summary
|
| 949 |
+
|
| 950 |
+
|
| 951 |
+
def _referenced_vertex_indices(
|
| 952 |
+
triangles: np.ndarray,
|
| 953 |
+
vertex_count: int,
|
| 954 |
+
) -> np.ndarray:
|
| 955 |
+
"""Return the vertices that can contribute pixels to a triangle render.
|
| 956 |
+
|
| 957 |
+
Some reconstruction meshes retain point-cloud vertices that are not
|
| 958 |
+
referenced by any triangle. Those vertices are invisible, and a single
|
| 959 |
+
far-away orphan must not expand the camera bounds and shrink the actual
|
| 960 |
+
surface to a few pixels.
|
| 961 |
+
"""
|
| 962 |
+
|
| 963 |
+
faces = np.asarray(triangles, dtype=np.int64)
|
| 964 |
+
if faces.ndim != 2 or faces.shape[1] != 3 or faces.size == 0:
|
| 965 |
+
raise ValueError("camera framing requires at least one triangular face")
|
| 966 |
+
referenced = np.unique(faces.reshape(-1))
|
| 967 |
+
if int(referenced[0]) < 0 or int(referenced[-1]) >= vertex_count:
|
| 968 |
+
raise ValueError("triangle index is outside the mesh vertex array")
|
| 969 |
+
return referenced
|
| 970 |
+
|
| 971 |
+
|
| 972 |
+
def _camera_framing_vertex_indices(
|
| 973 |
+
triangles: np.ndarray,
|
| 974 |
+
triangle_component_labels: np.ndarray,
|
| 975 |
+
*,
|
| 976 |
+
dominant_component_min_face_fraction: float = DOMINANT_COMPONENT_MIN_FACE_FRACTION,
|
| 977 |
+
) -> tuple[np.ndarray, dict[str, Any]]:
|
| 978 |
+
"""Choose a review framing surface without letting tiny islands zoom out.
|
| 979 |
+
|
| 980 |
+
A strongly dominant connected triangle surface represents the primary
|
| 981 |
+
accessibility target in these reconstructions. Small detached islands can
|
| 982 |
+
still be rendered, but they should not make the target unreadably small.
|
| 983 |
+
When no component owns at least 90% of faces, all triangle vertices remain
|
| 984 |
+
in the camera fit so legitimate multi-part geometry is not guessed away.
|
| 985 |
+
"""
|
| 986 |
+
|
| 987 |
+
faces = np.asarray(triangles, dtype=np.int64)
|
| 988 |
+
labels = np.asarray(triangle_component_labels, dtype=np.int64).reshape(-1)
|
| 989 |
+
referenced = _referenced_vertex_indices(faces, int(faces.max()) + 1)
|
| 990 |
+
if labels.shape[0] != faces.shape[0] or np.any(labels < 0):
|
| 991 |
+
raise ValueError("triangle component labels must match triangular faces")
|
| 992 |
+
if not (0.0 < dominant_component_min_face_fraction <= 1.0):
|
| 993 |
+
raise ValueError("dominant component face fraction must be in (0, 1]")
|
| 994 |
+
|
| 995 |
+
counts = np.bincount(labels)
|
| 996 |
+
dominant_label = int(np.argmax(counts))
|
| 997 |
+
dominant_faces = int(counts[dominant_label])
|
| 998 |
+
face_count = int(faces.shape[0])
|
| 999 |
+
dominant_fraction = dominant_faces / face_count
|
| 1000 |
+
use_dominant = dominant_fraction >= dominant_component_min_face_fraction
|
| 1001 |
+
if use_dominant:
|
| 1002 |
+
framing = np.unique(faces[labels == dominant_label].reshape(-1))
|
| 1003 |
+
scope = "dominant_connected_surface"
|
| 1004 |
+
else:
|
| 1005 |
+
framing = referenced
|
| 1006 |
+
scope = "all_connected_surfaces"
|
| 1007 |
+
return framing, {
|
| 1008 |
+
"scope": scope,
|
| 1009 |
+
"framing_faces": dominant_faces if use_dominant else face_count,
|
| 1010 |
+
"framing_vertices": int(len(framing)),
|
| 1011 |
+
"dominant_component_faces": dominant_faces,
|
| 1012 |
+
"dominant_component_face_fraction": float(dominant_fraction),
|
| 1013 |
+
"component_count": int(len(counts)),
|
| 1014 |
+
"minimum_dominant_face_fraction": float(
|
| 1015 |
+
dominant_component_min_face_fraction
|
| 1016 |
+
),
|
| 1017 |
+
}
|
| 1018 |
+
|
| 1019 |
+
|
| 1020 |
+
def _contact_sheet(images: Sequence[np.ndarray], labels: Sequence[str], path: Path) -> None:
|
| 1021 |
+
columns = 2 if len(images) > 1 else 1
|
| 1022 |
+
rows = math.ceil(len(images) / columns)
|
| 1023 |
+
panel_width = 480
|
| 1024 |
+
panel_height = 390
|
| 1025 |
+
label_height = 34
|
| 1026 |
+
canvas = Image.new(
|
| 1027 |
+
"RGB",
|
| 1028 |
+
(panel_width * columns, panel_height * rows),
|
| 1029 |
+
(235, 235, 232),
|
| 1030 |
+
)
|
| 1031 |
+
for index, (array, label) in enumerate(zip(images, labels)):
|
| 1032 |
+
image = Image.fromarray(array).convert("RGB")
|
| 1033 |
+
body = ImageOps.contain(image, (panel_width, panel_height - label_height))
|
| 1034 |
+
panel = Image.new("RGB", (panel_width, panel_height), "white")
|
| 1035 |
+
draw = ImageDraw.Draw(panel)
|
| 1036 |
+
draw.rectangle((0, 0, panel_width, label_height), fill=(242, 242, 239))
|
| 1037 |
+
draw.text((12, 11), label, fill=(20, 20, 20))
|
| 1038 |
+
panel.paste(
|
| 1039 |
+
body,
|
| 1040 |
+
(
|
| 1041 |
+
(panel_width - body.width) // 2,
|
| 1042 |
+
label_height + (panel_height - label_height - body.height) // 2,
|
| 1043 |
+
),
|
| 1044 |
+
)
|
| 1045 |
+
canvas.paste(panel, ((index % columns) * panel_width, (index // columns) * panel_height))
|
| 1046 |
+
canvas.save(path, quality=94, subsampling=0)
|
| 1047 |
+
|
| 1048 |
+
|
| 1049 |
+
def render_outputs(
|
| 1050 |
+
mesh_path: Path,
|
| 1051 |
+
output_dir: Path,
|
| 1052 |
+
*,
|
| 1053 |
+
width: int,
|
| 1054 |
+
height: int,
|
| 1055 |
+
frames: int,
|
| 1056 |
+
fps: int,
|
| 1057 |
+
fov: float,
|
| 1058 |
+
elevation: float,
|
| 1059 |
+
distance_scale: float,
|
| 1060 |
+
shader: str,
|
| 1061 |
+
background: Sequence[float],
|
| 1062 |
+
smooth_iterations: int,
|
| 1063 |
+
multiview_yaws: Sequence[float],
|
| 1064 |
+
orbit_mode: str = "front_arc",
|
| 1065 |
+
orbit_span: float = 140.0,
|
| 1066 |
+
require_vertex_colors: bool = False,
|
| 1067 |
+
vertex_colors_srgb: bool = True,
|
| 1068 |
+
cpu_fallback: bool = False,
|
| 1069 |
+
) -> dict[str, Any]:
|
| 1070 |
+
"""Render all outputs through Open3D's OffscreenRenderer path."""
|
| 1071 |
+
import open3d as o3d
|
| 1072 |
+
from open3d.visualization import rendering
|
| 1073 |
+
|
| 1074 |
+
mesh = o3d.io.read_triangle_mesh(str(mesh_path))
|
| 1075 |
+
if len(mesh.vertices) == 0 or len(mesh.triangles) == 0:
|
| 1076 |
+
raise ValueError(f"Input is not a non-empty triangle mesh: {mesh_path}")
|
| 1077 |
+
|
| 1078 |
+
source_vertex_count = len(mesh.vertices)
|
| 1079 |
+
referenced_indices = _referenced_vertex_indices(
|
| 1080 |
+
np.asarray(mesh.triangles, dtype=np.int64),
|
| 1081 |
+
source_vertex_count,
|
| 1082 |
+
)
|
| 1083 |
+
unreferenced_vertex_count = source_vertex_count - len(referenced_indices)
|
| 1084 |
+
# Open3D's scene bounds can include vertices that have no incident face,
|
| 1085 |
+
# even though rasterization cannot display them. Remove only these
|
| 1086 |
+
# topologically inert points before both rendering and camera fitting.
|
| 1087 |
+
if unreferenced_vertex_count:
|
| 1088 |
+
mesh.remove_unreferenced_vertices()
|
| 1089 |
+
|
| 1090 |
+
# Geometry produced from image-camera coordinates is easier to inspect after
|
| 1091 |
+
# mapping x/right, z/up, and -y/depth into a conventional scene frame.
|
| 1092 |
+
vertices = np.asarray(mesh.vertices, dtype=np.float64)
|
| 1093 |
+
mesh.vertices = o3d.utility.Vector3dVector(
|
| 1094 |
+
np.stack([vertices[:, 0], vertices[:, 2], -vertices[:, 1]], axis=1)
|
| 1095 |
+
)
|
| 1096 |
+
if smooth_iterations:
|
| 1097 |
+
mesh = mesh.filter_smooth_laplacian(number_of_iterations=smooth_iterations)
|
| 1098 |
+
mesh.compute_vertex_normals()
|
| 1099 |
+
vertex_color_summary = _summarize_vertex_colors(
|
| 1100 |
+
np.asarray(mesh.vertex_colors, dtype=np.float64),
|
| 1101 |
+
len(mesh.vertices),
|
| 1102 |
+
required=require_vertex_colors,
|
| 1103 |
+
)
|
| 1104 |
+
if require_vertex_colors and not (
|
| 1105 |
+
vertex_color_summary["present"] and vertex_color_summary["finite"]
|
| 1106 |
+
):
|
| 1107 |
+
raise ValueError(
|
| 1108 |
+
"Input mesh does not provide complete finite vertex colors required "
|
| 1109 |
+
f"for texture-preserving rendering: {mesh_path}"
|
| 1110 |
+
)
|
| 1111 |
+
|
| 1112 |
+
transformed_vertices = np.asarray(mesh.vertices, dtype=np.float64)
|
| 1113 |
+
triangle_component_labels, _, _ = mesh.cluster_connected_triangles()
|
| 1114 |
+
framing_indices, framing_summary = _camera_framing_vertex_indices(
|
| 1115 |
+
np.asarray(mesh.triangles, dtype=np.int64),
|
| 1116 |
+
np.asarray(triangle_component_labels, dtype=np.int64),
|
| 1117 |
+
)
|
| 1118 |
+
framing_vertices = transformed_vertices[framing_indices]
|
| 1119 |
+
framing_min = np.min(framing_vertices, axis=0)
|
| 1120 |
+
framing_max = np.max(framing_vertices, axis=0)
|
| 1121 |
+
center = (framing_min + framing_max) * 0.5
|
| 1122 |
+
orbit_yaws = _orbit_yaws(
|
| 1123 |
+
frames,
|
| 1124 |
+
mode=orbit_mode,
|
| 1125 |
+
span_degrees=orbit_span,
|
| 1126 |
+
)
|
| 1127 |
+
fixed_camera_distance, camera_summary = _fixed_camera_distance_for_views(
|
| 1128 |
+
framing_vertices,
|
| 1129 |
+
center,
|
| 1130 |
+
yaw_degrees=tuple(orbit_yaws) + tuple(multiview_yaws),
|
| 1131 |
+
elevation_degrees=elevation,
|
| 1132 |
+
vertical_fov_degrees=fov,
|
| 1133 |
+
aspect_ratio=width / height,
|
| 1134 |
+
distance_scale=distance_scale,
|
| 1135 |
+
)
|
| 1136 |
+
|
| 1137 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 1138 |
+
renderer = rendering.OffscreenRenderer(width, height)
|
| 1139 |
+
try:
|
| 1140 |
+
opengl_context = query_current_opengl_context()
|
| 1141 |
+
process_graphics_backend = inspect_process_graphics_backend()
|
| 1142 |
+
hardware_acceleration = enforce_opengl_context_policy(
|
| 1143 |
+
opengl_context,
|
| 1144 |
+
cpu_fallback=cpu_fallback,
|
| 1145 |
+
process_backend=process_graphics_backend,
|
| 1146 |
+
)
|
| 1147 |
+
renderer.scene.set_background(list(background))
|
| 1148 |
+
renderer.scene.scene.set_sun_light(
|
| 1149 |
+
[0.35, -0.5, -0.75],
|
| 1150 |
+
[1.0, 0.97, 0.92],
|
| 1151 |
+
65000,
|
| 1152 |
+
)
|
| 1153 |
+
renderer.scene.scene.enable_sun_light(True)
|
| 1154 |
+
renderer.scene.show_axes(False)
|
| 1155 |
+
material = rendering.MaterialRecord()
|
| 1156 |
+
material.shader = shader
|
| 1157 |
+
material.sRGB_color = bool(vertex_colors_srgb)
|
| 1158 |
+
material.base_color = [1.0, 1.0, 1.0, 1.0]
|
| 1159 |
+
material.base_roughness = 0.68
|
| 1160 |
+
material.base_reflectance = 0.18
|
| 1161 |
+
renderer.scene.add_geometry("accessibility_geometry", mesh, material)
|
| 1162 |
+
|
| 1163 |
+
def render_yaw(yaw: float) -> np.ndarray:
|
| 1164 |
+
eye, look_at, up = _camera_pose(
|
| 1165 |
+
center,
|
| 1166 |
+
fixed_camera_distance,
|
| 1167 |
+
yaw,
|
| 1168 |
+
elevation,
|
| 1169 |
+
)
|
| 1170 |
+
renderer.setup_camera(fov, look_at, eye, up)
|
| 1171 |
+
array = np.asarray(renderer.render_to_image())
|
| 1172 |
+
if array.ndim != 3 or array.shape[2] not in (3, 4):
|
| 1173 |
+
raise RuntimeError(f"Unexpected Open3D render shape: {array.shape}")
|
| 1174 |
+
return array[..., :3].astype(np.uint8, copy=False)
|
| 1175 |
+
|
| 1176 |
+
video_frames = [render_yaw(yaw) for yaw in orbit_yaws]
|
| 1177 |
+
imageio.mimsave(
|
| 1178 |
+
output_dir / OUTPUT_NAMES["video"],
|
| 1179 |
+
video_frames,
|
| 1180 |
+
fps=fps,
|
| 1181 |
+
quality=8,
|
| 1182 |
+
macro_block_size=1,
|
| 1183 |
+
)
|
| 1184 |
+
imageio.mimsave(
|
| 1185 |
+
output_dir / OUTPUT_NAMES["gif"],
|
| 1186 |
+
video_frames,
|
| 1187 |
+
duration=1000.0 / fps,
|
| 1188 |
+
loop=0,
|
| 1189 |
+
)
|
| 1190 |
+
Image.fromarray(video_frames[0]).save(output_dir / OUTPUT_NAMES["poster"])
|
| 1191 |
+
|
| 1192 |
+
views = [render_yaw(yaw) for yaw in multiview_yaws]
|
| 1193 |
+
labels = [f"Geometry view {yaw:g} deg" for yaw in multiview_yaws]
|
| 1194 |
+
_contact_sheet(views, labels, output_dir / OUTPUT_NAMES["multiview"])
|
| 1195 |
+
finally:
|
| 1196 |
+
release = getattr(renderer, "release_resources", None)
|
| 1197 |
+
if callable(release):
|
| 1198 |
+
release()
|
| 1199 |
+
|
| 1200 |
+
return {
|
| 1201 |
+
"vertices": source_vertex_count,
|
| 1202 |
+
"renderable_vertices": len(mesh.vertices),
|
| 1203 |
+
"unreferenced_vertices_excluded_from_camera_fit": unreferenced_vertex_count,
|
| 1204 |
+
"faces": len(mesh.triangles),
|
| 1205 |
+
"camera_framing": framing_summary,
|
| 1206 |
+
"camera": camera_summary,
|
| 1207 |
+
"vertex_colors": vertex_color_summary,
|
| 1208 |
+
"opengl_context": opengl_context,
|
| 1209 |
+
"process_graphics_backend": process_graphics_backend,
|
| 1210 |
+
"hardware_acceleration": hardware_acceleration,
|
| 1211 |
+
"orbit": {
|
| 1212 |
+
"mode": orbit_mode,
|
| 1213 |
+
"span_degrees": float(orbit_span),
|
| 1214 |
+
"yaw_min": float(min(orbit_yaws)),
|
| 1215 |
+
"yaw_max": float(max(orbit_yaws)),
|
| 1216 |
+
"multiview_yaws": [float(yaw) for yaw in multiview_yaws],
|
| 1217 |
+
"open_world_scene_review": orbit_mode == "front_arc",
|
| 1218 |
+
},
|
| 1219 |
+
}
|
| 1220 |
+
|
| 1221 |
+
|
| 1222 |
+
def write_json(path: Path, payload: Mapping[str, Any]) -> None:
|
| 1223 |
+
path.write_text(
|
| 1224 |
+
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
| 1225 |
+
encoding="utf-8",
|
| 1226 |
+
)
|
| 1227 |
+
|
| 1228 |
+
|
| 1229 |
+
def main(argv: Sequence[str] | None = None) -> int:
|
| 1230 |
+
parser = build_argument_parser()
|
| 1231 |
+
args = parser.parse_args(argv)
|
| 1232 |
+
validate_arguments(args)
|
| 1233 |
+
|
| 1234 |
+
mesh_path = Path(args.mesh).expanduser().resolve()
|
| 1235 |
+
if not mesh_path.is_file():
|
| 1236 |
+
raise FileNotFoundError(mesh_path)
|
| 1237 |
+
output_dir = Path(args.output_dir).expanduser().resolve()
|
| 1238 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 1239 |
+
sample_id = str(args.sample_id or mesh_path.stem)
|
| 1240 |
+
category = str(args.category).strip().lower() or "unknown"
|
| 1241 |
+
|
| 1242 |
+
environment_evidence = configure_render_environment(args.cpu_fallback)
|
| 1243 |
+
render_settings = {
|
| 1244 |
+
"resolution": [args.width, args.height],
|
| 1245 |
+
"frames": args.frames,
|
| 1246 |
+
"fps": args.fps,
|
| 1247 |
+
"fov_degrees": args.fov,
|
| 1248 |
+
"elevation_degrees": args.elevation,
|
| 1249 |
+
"distance_scale": args.distance_scale,
|
| 1250 |
+
"camera_fit": "fixed_max_projected_primary_surface_v4",
|
| 1251 |
+
"camera_distance_policy": "fixed_max_over_orbit_and_multiview",
|
| 1252 |
+
"shader": args.shader,
|
| 1253 |
+
"background": list(args.background),
|
| 1254 |
+
"smooth_iterations": args.smooth_iterations,
|
| 1255 |
+
"require_vertex_colors": args.require_vertex_colors,
|
| 1256 |
+
"vertex_colors_srgb": args.vertex_colors_srgb,
|
| 1257 |
+
"multiview_yaws_degrees": list(args.multiview_yaws),
|
| 1258 |
+
"orbit_mode": args.orbit_mode,
|
| 1259 |
+
"orbit_span_degrees": args.orbit_span,
|
| 1260 |
+
"scene_interpretation": (
|
| 1261 |
+
"open_world_accessibility_surface_review"
|
| 1262 |
+
if args.orbit_mode == "front_arc"
|
| 1263 |
+
else "full_object_turntable"
|
| 1264 |
+
),
|
| 1265 |
+
}
|
| 1266 |
+
result = render_outputs(
|
| 1267 |
+
mesh_path,
|
| 1268 |
+
output_dir,
|
| 1269 |
+
width=args.width,
|
| 1270 |
+
height=args.height,
|
| 1271 |
+
frames=args.frames,
|
| 1272 |
+
fps=args.fps,
|
| 1273 |
+
fov=args.fov,
|
| 1274 |
+
elevation=args.elevation,
|
| 1275 |
+
distance_scale=args.distance_scale,
|
| 1276 |
+
shader=args.shader,
|
| 1277 |
+
background=args.background,
|
| 1278 |
+
smooth_iterations=args.smooth_iterations,
|
| 1279 |
+
multiview_yaws=args.multiview_yaws,
|
| 1280 |
+
orbit_mode=args.orbit_mode,
|
| 1281 |
+
orbit_span=args.orbit_span,
|
| 1282 |
+
require_vertex_colors=args.require_vertex_colors,
|
| 1283 |
+
vertex_colors_srgb=args.vertex_colors_srgb,
|
| 1284 |
+
cpu_fallback=args.cpu_fallback,
|
| 1285 |
+
)
|
| 1286 |
+
manifest = build_render_manifest(
|
| 1287 |
+
mesh_path=mesh_path,
|
| 1288 |
+
output_dir=output_dir,
|
| 1289 |
+
sample_id=sample_id,
|
| 1290 |
+
category=category,
|
| 1291 |
+
render_settings=render_settings,
|
| 1292 |
+
render_result=result,
|
| 1293 |
+
environment_evidence=environment_evidence,
|
| 1294 |
+
cpu_fallback=args.cpu_fallback,
|
| 1295 |
+
)
|
| 1296 |
+
write_json(output_dir / "manifest.json", manifest)
|
| 1297 |
+
print(
|
| 1298 |
+
json.dumps(
|
| 1299 |
+
{
|
| 1300 |
+
"status": "rendered",
|
| 1301 |
+
"sample_id": sample_id,
|
| 1302 |
+
"output_dir": str(output_dir),
|
| 1303 |
+
"cpu_fallback_explicit": args.cpu_fallback,
|
| 1304 |
+
"hardware_acceleration_status": manifest["renderer"][
|
| 1305 |
+
"hardware_acceleration_status"
|
| 1306 |
+
],
|
| 1307 |
+
},
|
| 1308 |
+
ensure_ascii=False,
|
| 1309 |
+
sort_keys=True,
|
| 1310 |
+
)
|
| 1311 |
+
)
|
| 1312 |
+
return 0
|
| 1313 |
+
|
| 1314 |
+
|
| 1315 |
+
if __name__ == "__main__":
|
| 1316 |
+
raise SystemExit(main())
|
tools/sanitize_log_stream.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Filter a text stream so Slurm logs do not expose local identity details."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import sys
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
| 12 |
+
if str(PROJECT_ROOT) not in sys.path:
|
| 13 |
+
sys.path.insert(0, str(PROJECT_ROOT))
|
| 14 |
+
|
| 15 |
+
from accesspath3r.privacy import sanitize_text # noqa: E402
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def build_parser() -> argparse.ArgumentParser:
|
| 19 |
+
parser = argparse.ArgumentParser()
|
| 20 |
+
parser.add_argument("--project-root", type=Path, required=True)
|
| 21 |
+
parser.add_argument("--output-root", type=Path, default=None)
|
| 22 |
+
parser.add_argument("--sensitive-path", action="append", default=[])
|
| 23 |
+
return parser
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def main() -> int:
|
| 27 |
+
args = build_parser().parse_args()
|
| 28 |
+
for chunk in sys.stdin:
|
| 29 |
+
sys.stdout.write(
|
| 30 |
+
sanitize_text(
|
| 31 |
+
chunk,
|
| 32 |
+
project_root=args.project_root,
|
| 33 |
+
output_root=args.output_root,
|
| 34 |
+
sensitive_paths=args.sensitive_path,
|
| 35 |
+
)
|
| 36 |
+
)
|
| 37 |
+
sys.stdout.flush()
|
| 38 |
+
return 0
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
if __name__ == "__main__":
|
| 42 |
+
raise SystemExit(main())
|
tools/sanitize_output_metadata.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Remove workstation identity and absolute prefixes from text metadata."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import os
|
| 8 |
+
import sys
|
| 9 |
+
import tempfile
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
| 14 |
+
if str(PROJECT_ROOT) not in sys.path:
|
| 15 |
+
sys.path.insert(0, str(PROJECT_ROOT))
|
| 16 |
+
|
| 17 |
+
from accesspath3r.privacy import sanitize_text # noqa: E402
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
TEXT_SUFFIXES = {".err", ".json", ".jsonl", ".log", ".md", ".out", ".txt"}
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def sanitize_file(
|
| 24 |
+
path: Path,
|
| 25 |
+
*,
|
| 26 |
+
project_root: Path,
|
| 27 |
+
output_root: Path,
|
| 28 |
+
sensitive_paths: list[str],
|
| 29 |
+
) -> bool:
|
| 30 |
+
try:
|
| 31 |
+
original = path.read_text(encoding="utf-8")
|
| 32 |
+
except (OSError, UnicodeDecodeError):
|
| 33 |
+
return False
|
| 34 |
+
sanitized = sanitize_text(
|
| 35 |
+
original,
|
| 36 |
+
project_root=project_root,
|
| 37 |
+
output_root=output_root,
|
| 38 |
+
sensitive_paths=sensitive_paths,
|
| 39 |
+
)
|
| 40 |
+
if sanitized == original:
|
| 41 |
+
return False
|
| 42 |
+
|
| 43 |
+
original_mode = path.stat().st_mode & 0o7777
|
| 44 |
+
file_descriptor, temporary_name = tempfile.mkstemp(
|
| 45 |
+
prefix=f".{path.name}.",
|
| 46 |
+
suffix=".tmp",
|
| 47 |
+
dir=path.parent,
|
| 48 |
+
)
|
| 49 |
+
try:
|
| 50 |
+
os.fchmod(file_descriptor, original_mode)
|
| 51 |
+
with os.fdopen(file_descriptor, "w", encoding="utf-8") as handle:
|
| 52 |
+
handle.write(sanitized)
|
| 53 |
+
handle.flush()
|
| 54 |
+
os.fsync(handle.fileno())
|
| 55 |
+
os.replace(temporary_name, path)
|
| 56 |
+
except BaseException:
|
| 57 |
+
try:
|
| 58 |
+
os.unlink(temporary_name)
|
| 59 |
+
except OSError:
|
| 60 |
+
pass
|
| 61 |
+
raise
|
| 62 |
+
return True
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def build_parser() -> argparse.ArgumentParser:
|
| 66 |
+
parser = argparse.ArgumentParser()
|
| 67 |
+
parser.add_argument("--root", type=Path, required=True)
|
| 68 |
+
parser.add_argument("--project-root", type=Path, required=True)
|
| 69 |
+
parser.add_argument("--sensitive-path", action="append", default=[])
|
| 70 |
+
return parser
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def main() -> int:
|
| 74 |
+
args = build_parser().parse_args()
|
| 75 |
+
if args.root.is_file():
|
| 76 |
+
paths = [args.root]
|
| 77 |
+
output_root = args.root.parent
|
| 78 |
+
elif args.root.is_dir():
|
| 79 |
+
paths = args.root.rglob("*")
|
| 80 |
+
output_root = args.root
|
| 81 |
+
else:
|
| 82 |
+
return 0
|
| 83 |
+
for path in paths:
|
| 84 |
+
if path.is_file() and path.suffix.lower() in TEXT_SUFFIXES:
|
| 85 |
+
sanitize_file(
|
| 86 |
+
path,
|
| 87 |
+
project_root=args.project_root,
|
| 88 |
+
output_root=output_root,
|
| 89 |
+
sensitive_paths=args.sensitive_path,
|
| 90 |
+
)
|
| 91 |
+
return 0
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
if __name__ == "__main__":
|
| 95 |
+
raise SystemExit(main())
|
tools/train_accessibility_amodal_adapter.py
ADDED
|
@@ -0,0 +1,462 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Train a small 2D hidden-mask adapter on the frozen accessibility split."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
import random
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from typing import Any
|
| 11 |
+
|
| 12 |
+
import numpy as np
|
| 13 |
+
import torch
|
| 14 |
+
import torch.nn as nn
|
| 15 |
+
import torch.nn.functional as F
|
| 16 |
+
from PIL import Image, ImageDraw, ImageOps
|
| 17 |
+
from torch.utils.data import DataLoader, Dataset
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def read_jsonl(path: Path) -> list[dict[str, Any]]:
|
| 21 |
+
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
CANONICAL_CATEGORIES = ("curb_cut", "ramp", "stairs", "tactile_paving", "walkway")
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def row_image_path(row: dict[str, Any]) -> Path:
|
| 28 |
+
if row.get("image_path"):
|
| 29 |
+
path = Path(str(row["image_path"]))
|
| 30 |
+
if path.is_file():
|
| 31 |
+
return path
|
| 32 |
+
sample_dir = Path(row["sample_dir"])
|
| 33 |
+
candidates = [path for path in (sample_dir / "image.jpg", sample_dir / "image.png") if path.is_file()]
|
| 34 |
+
if len(candidates) != 1:
|
| 35 |
+
raise FileNotFoundError(f"Expected exactly one RGB image in {sample_dir}")
|
| 36 |
+
return candidates[0]
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def load_rgb(path: Path, size: int) -> np.ndarray:
|
| 40 |
+
# Ignore EXIF display orientation so RGB stays aligned with raw PNG masks.
|
| 41 |
+
image = Image.open(path).convert("RGB")
|
| 42 |
+
return np.asarray(image.resize((size, size), Image.Resampling.BILINEAR), dtype=np.float32) / 255.0
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def load_mask(path: Path, size: int | None = None) -> np.ndarray:
|
| 46 |
+
image = Image.open(path).convert("L")
|
| 47 |
+
if size is not None:
|
| 48 |
+
image = image.resize((size, size), Image.Resampling.NEAREST)
|
| 49 |
+
return np.asarray(image) > 127
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def category_planes(category: str, size: int) -> np.ndarray:
|
| 53 |
+
planes = np.zeros((len(CANONICAL_CATEGORIES), size, size), dtype=np.float32)
|
| 54 |
+
index = CANONICAL_CATEGORIES.index(category) if category in CANONICAL_CATEGORIES else -1
|
| 55 |
+
if index >= 0:
|
| 56 |
+
planes[index] = 1.0
|
| 57 |
+
return planes
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
class AccessibilityMaskDataset(Dataset):
|
| 61 |
+
def __init__(self, rows: list[dict[str, Any]], size: int, augment: bool):
|
| 62 |
+
self.rows = rows
|
| 63 |
+
self.size = size
|
| 64 |
+
self.augment = augment
|
| 65 |
+
|
| 66 |
+
def __len__(self) -> int:
|
| 67 |
+
return len(self.rows)
|
| 68 |
+
|
| 69 |
+
def __getitem__(self, index: int) -> dict[str, Any]:
|
| 70 |
+
row = self.rows[index]
|
| 71 |
+
sample_dir = Path(row["sample_dir"])
|
| 72 |
+
rgb = load_rgb(row_image_path(row), self.size)
|
| 73 |
+
visible = load_mask(sample_dir / "target_visible.png", self.size)
|
| 74 |
+
obstacle = load_mask(sample_dir / "obstacle.png", self.size)
|
| 75 |
+
hidden = load_mask(sample_dir / "hidden.png", self.size)
|
| 76 |
+
if self.augment and random.random() < 0.5:
|
| 77 |
+
rgb = rgb[:, ::-1].copy()
|
| 78 |
+
visible = visible[:, ::-1].copy()
|
| 79 |
+
obstacle = obstacle[:, ::-1].copy()
|
| 80 |
+
hidden = hidden[:, ::-1].copy()
|
| 81 |
+
if self.augment:
|
| 82 |
+
gain = random.uniform(0.88, 1.12)
|
| 83 |
+
bias = random.uniform(-0.05, 0.05)
|
| 84 |
+
rgb = np.clip(rgb * gain + bias, 0.0, 1.0)
|
| 85 |
+
inputs = np.concatenate(
|
| 86 |
+
[
|
| 87 |
+
rgb.transpose(2, 0, 1),
|
| 88 |
+
visible[None].astype(np.float32),
|
| 89 |
+
obstacle[None].astype(np.float32),
|
| 90 |
+
category_planes(row["category"], self.size),
|
| 91 |
+
],
|
| 92 |
+
axis=0,
|
| 93 |
+
)
|
| 94 |
+
return {
|
| 95 |
+
"input": torch.from_numpy(inputs.astype(np.float32)),
|
| 96 |
+
"hidden": torch.from_numpy(hidden[None].astype(np.float32)),
|
| 97 |
+
"obstacle": torch.from_numpy(obstacle[None].astype(np.float32)),
|
| 98 |
+
"sample_id": row["sample_id"],
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
class ConvBlock(nn.Module):
|
| 103 |
+
def __init__(self, in_channels: int, out_channels: int):
|
| 104 |
+
super().__init__()
|
| 105 |
+
groups = min(8, out_channels)
|
| 106 |
+
self.block = nn.Sequential(
|
| 107 |
+
nn.Conv2d(in_channels, out_channels, 3, padding=1),
|
| 108 |
+
nn.GroupNorm(groups, out_channels),
|
| 109 |
+
nn.SiLU(),
|
| 110 |
+
nn.Conv2d(out_channels, out_channels, 3, padding=1),
|
| 111 |
+
nn.GroupNorm(groups, out_channels),
|
| 112 |
+
nn.SiLU(),
|
| 113 |
+
)
|
| 114 |
+
|
| 115 |
+
def forward(self, inputs: torch.Tensor) -> torch.Tensor:
|
| 116 |
+
return self.block(inputs)
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
class TinyAmodalUNet(nn.Module):
|
| 120 |
+
def __init__(self, base: int = 24, in_channels: int = 5 + len(CANONICAL_CATEGORIES)):
|
| 121 |
+
super().__init__()
|
| 122 |
+
self.enc1 = ConvBlock(in_channels, base)
|
| 123 |
+
self.enc2 = ConvBlock(base, base * 2)
|
| 124 |
+
self.bottleneck = ConvBlock(base * 2, base * 4)
|
| 125 |
+
self.dec2 = ConvBlock(base * 4 + base * 2, base * 2)
|
| 126 |
+
self.dec1 = ConvBlock(base * 2 + base, base)
|
| 127 |
+
self.head = nn.Conv2d(base, 1, 1)
|
| 128 |
+
|
| 129 |
+
def forward(self, inputs: torch.Tensor) -> torch.Tensor:
|
| 130 |
+
first = self.enc1(inputs)
|
| 131 |
+
second = self.enc2(F.max_pool2d(first, 2))
|
| 132 |
+
bottleneck = self.bottleneck(F.max_pool2d(second, 2))
|
| 133 |
+
up_second = F.interpolate(bottleneck, size=second.shape[-2:], mode="bilinear", align_corners=False)
|
| 134 |
+
decoded_second = self.dec2(torch.cat([up_second, second], dim=1))
|
| 135 |
+
up_first = F.interpolate(decoded_second, size=first.shape[-2:], mode="bilinear", align_corners=False)
|
| 136 |
+
return self.head(self.dec1(torch.cat([up_first, first], dim=1)))
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def training_loss(logits: torch.Tensor, target: torch.Tensor, obstacle: torch.Tensor) -> torch.Tensor:
|
| 140 |
+
positive_weight = torch.tensor(8.0, device=logits.device)
|
| 141 |
+
bce = F.binary_cross_entropy_with_logits(logits, target, pos_weight=positive_weight)
|
| 142 |
+
probabilities = torch.sigmoid(logits)
|
| 143 |
+
intersection = (probabilities * target).sum(dim=(1, 2, 3))
|
| 144 |
+
denominator = probabilities.sum(dim=(1, 2, 3)) + target.sum(dim=(1, 2, 3))
|
| 145 |
+
dice_loss = 1.0 - ((2.0 * intersection + 1.0) / (denominator + 1.0)).mean()
|
| 146 |
+
outside_obstacle = (probabilities * (1.0 - obstacle)).mean()
|
| 147 |
+
return bce + dice_loss + 0.20 * outside_obstacle
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def iou(left: np.ndarray, right: np.ndarray, empty_value: float = 1.0) -> float:
|
| 151 |
+
union = left | right
|
| 152 |
+
return float((left & right).sum() / union.sum()) if np.any(union) else empty_value
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
@torch.no_grad()
|
| 156 |
+
def predict_probability(
|
| 157 |
+
model: nn.Module, row: dict[str, Any], size: int, device: torch.device
|
| 158 |
+
) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
|
| 159 |
+
sample_dir = Path(row["sample_dir"])
|
| 160 |
+
image_path = row_image_path(row)
|
| 161 |
+
original = Image.open(image_path).convert("RGB")
|
| 162 |
+
width, height = original.size
|
| 163 |
+
rgb = load_rgb(image_path, size)
|
| 164 |
+
visible_small = load_mask(sample_dir / "target_visible.png", size)
|
| 165 |
+
obstacle_small = load_mask(sample_dir / "obstacle.png", size)
|
| 166 |
+
inputs = np.concatenate(
|
| 167 |
+
[
|
| 168 |
+
rgb.transpose(2, 0, 1),
|
| 169 |
+
visible_small[None].astype(np.float32),
|
| 170 |
+
obstacle_small[None].astype(np.float32),
|
| 171 |
+
category_planes(row["category"], size),
|
| 172 |
+
],
|
| 173 |
+
axis=0,
|
| 174 |
+
)
|
| 175 |
+
logits = model(torch.from_numpy(inputs[None]).to(device)).sigmoid()[0, 0].cpu().numpy()
|
| 176 |
+
probability = np.asarray(
|
| 177 |
+
Image.fromarray(logits.astype(np.float32), mode="F").resize((width, height), Image.Resampling.BILINEAR)
|
| 178 |
+
).copy()
|
| 179 |
+
visible = load_mask(sample_dir / "target_visible.png")
|
| 180 |
+
obstacle = load_mask(sample_dir / "obstacle.png")
|
| 181 |
+
hidden = load_mask(sample_dir / "hidden.png")
|
| 182 |
+
probability *= (obstacle & ~visible).astype(np.float32)
|
| 183 |
+
return probability, visible, obstacle, hidden
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
@torch.no_grad()
|
| 187 |
+
def evaluate_thresholds(
|
| 188 |
+
model: nn.Module,
|
| 189 |
+
rows: list[dict[str, Any]],
|
| 190 |
+
size: int,
|
| 191 |
+
device: torch.device,
|
| 192 |
+
thresholds: list[float],
|
| 193 |
+
) -> tuple[float, dict[str, Any], dict[str, tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]]]:
|
| 194 |
+
cached = {row["sample_id"]: predict_probability(model, row, size, device) for row in rows}
|
| 195 |
+
reports = []
|
| 196 |
+
for threshold in thresholds:
|
| 197 |
+
sample_rows = []
|
| 198 |
+
for row in rows:
|
| 199 |
+
probability, visible, _, hidden = cached[row["sample_id"]]
|
| 200 |
+
predicted_hidden = probability >= threshold
|
| 201 |
+
predicted_amodal = visible | predicted_hidden
|
| 202 |
+
target_amodal = visible | hidden
|
| 203 |
+
sample_rows.append(
|
| 204 |
+
{
|
| 205 |
+
"sample_id": row["sample_id"],
|
| 206 |
+
"category": row["category"],
|
| 207 |
+
"hidden_gt_pixels": int(hidden.sum()),
|
| 208 |
+
"hidden_pred_pixels": int(predicted_hidden.sum()),
|
| 209 |
+
"hidden_iou": iou(predicted_hidden, hidden),
|
| 210 |
+
"amodal_iou": iou(predicted_amodal, target_amodal),
|
| 211 |
+
}
|
| 212 |
+
)
|
| 213 |
+
nonempty = [item for item in sample_rows if item["hidden_gt_pixels"] > 0]
|
| 214 |
+
reports.append(
|
| 215 |
+
{
|
| 216 |
+
"threshold": threshold,
|
| 217 |
+
"mean_hidden_iou_nonempty": float(np.mean([item["hidden_iou"] for item in nonempty])) if nonempty else 1.0,
|
| 218 |
+
"mean_amodal_iou": float(np.mean([item["amodal_iou"] for item in sample_rows])),
|
| 219 |
+
"negative_control_false_positive_pixels": int(
|
| 220 |
+
sum(item["hidden_pred_pixels"] for item in sample_rows if item["hidden_gt_pixels"] == 0)
|
| 221 |
+
),
|
| 222 |
+
"rows": sample_rows,
|
| 223 |
+
}
|
| 224 |
+
)
|
| 225 |
+
best = max(
|
| 226 |
+
reports,
|
| 227 |
+
key=lambda item: (
|
| 228 |
+
item["mean_hidden_iou_nonempty"],
|
| 229 |
+
item["mean_amodal_iou"],
|
| 230 |
+
-item["negative_control_false_positive_pixels"],
|
| 231 |
+
),
|
| 232 |
+
)
|
| 233 |
+
return float(best["threshold"]), best, cached
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
def save_mask(path: Path, mask: np.ndarray) -> None:
|
| 237 |
+
Image.fromarray(mask.astype(np.uint8) * 255, mode="L").save(path)
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def save_predictions(
|
| 241 |
+
output: Path,
|
| 242 |
+
rows: list[dict[str, Any]],
|
| 243 |
+
cached: dict[str, tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]],
|
| 244 |
+
threshold: float,
|
| 245 |
+
) -> None:
|
| 246 |
+
output.mkdir(parents=True, exist_ok=True)
|
| 247 |
+
for row in rows:
|
| 248 |
+
sample_output = output / row["sample_id"]
|
| 249 |
+
sample_output.mkdir(parents=True, exist_ok=True)
|
| 250 |
+
probability, visible, _, _ = cached[row["sample_id"]]
|
| 251 |
+
predicted_hidden = probability >= threshold
|
| 252 |
+
save_mask(sample_output / "target_visible_mask.png", visible)
|
| 253 |
+
save_mask(sample_output / "hidden_completion_mask.png", predicted_hidden)
|
| 254 |
+
save_mask(sample_output / "amodal_accessibility_mask.png", visible | predicted_hidden)
|
| 255 |
+
Image.fromarray(np.clip(probability * 255.0, 0, 255).astype(np.uint8), mode="L").save(
|
| 256 |
+
sample_output / "hidden_probability.png"
|
| 257 |
+
)
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def color_overlay(image: Image.Image, masks: list[tuple[np.ndarray, tuple[int, int, int], float]]) -> Image.Image:
|
| 261 |
+
array = np.asarray(image.convert("RGB"), dtype=np.float32).copy()
|
| 262 |
+
for mask, color, alpha in masks:
|
| 263 |
+
array[mask] = array[mask] * (1.0 - alpha) + np.asarray(color) * alpha
|
| 264 |
+
return Image.fromarray(np.clip(array, 0, 255).astype(np.uint8))
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def build_contact_sheet(
|
| 268 |
+
path: Path,
|
| 269 |
+
rows: list[dict[str, Any]],
|
| 270 |
+
cached: dict[str, tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]],
|
| 271 |
+
threshold: float,
|
| 272 |
+
) -> None:
|
| 273 |
+
panel_size = (320, 180)
|
| 274 |
+
label_height = 30
|
| 275 |
+
sheet = Image.new("RGB", (panel_size[0] * 3, (panel_size[1] + label_height) * len(rows)), "white")
|
| 276 |
+
draw = ImageDraw.Draw(sheet)
|
| 277 |
+
for row_index, row in enumerate(rows):
|
| 278 |
+
image = Image.open(row_image_path(row)).convert("RGB")
|
| 279 |
+
probability, visible, obstacle, hidden = cached[row["sample_id"]]
|
| 280 |
+
predicted = probability >= threshold
|
| 281 |
+
panels = (
|
| 282 |
+
(row["sample_id"], image),
|
| 283 |
+
("GT: green visible / blue hidden / red obstacle", color_overlay(image, [(visible, (20, 210, 70), 0.38), (hidden, (40, 100, 245), 0.72), (obstacle, (235, 40, 40), 0.25)])),
|
| 284 |
+
("prediction: green visible / magenta hidden", color_overlay(image, [(visible, (20, 210, 70), 0.38), (predicted, (235, 40, 200), 0.75)])),
|
| 285 |
+
)
|
| 286 |
+
y = row_index * (panel_size[1] + label_height)
|
| 287 |
+
for column, (label, panel) in enumerate(panels):
|
| 288 |
+
x = column * panel_size[0]
|
| 289 |
+
draw.text((x + 5, y + 7), label, fill="black")
|
| 290 |
+
fitted = ImageOps.fit(panel, panel_size, method=Image.Resampling.LANCZOS)
|
| 291 |
+
sheet.paste(fitted, (x, y + label_height))
|
| 292 |
+
sheet.save(path, quality=92, subsampling=0)
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
def main() -> int:
|
| 296 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 297 |
+
parser.add_argument("--split", default="output/accessibility_training_split_v1")
|
| 298 |
+
parser.add_argument("--output", default="output/accessibility_amodal_adapter_v1")
|
| 299 |
+
parser.add_argument("--epochs", type=int, default=100)
|
| 300 |
+
parser.add_argument("--patience", type=int, default=25)
|
| 301 |
+
parser.add_argument("--batch-size", type=int, default=4)
|
| 302 |
+
parser.add_argument("--image-size", type=int, default=256)
|
| 303 |
+
parser.add_argument("--learning-rate", type=float, default=3e-4)
|
| 304 |
+
parser.add_argument("--num-workers", type=int, default=2)
|
| 305 |
+
parser.add_argument("--seed", type=int, default=20260623)
|
| 306 |
+
parser.add_argument("--device", choices=["auto", "cuda", "cpu"], default="auto")
|
| 307 |
+
args = parser.parse_args()
|
| 308 |
+
|
| 309 |
+
random.seed(args.seed)
|
| 310 |
+
np.random.seed(args.seed)
|
| 311 |
+
torch.manual_seed(args.seed)
|
| 312 |
+
if torch.cuda.is_available():
|
| 313 |
+
torch.cuda.manual_seed_all(args.seed)
|
| 314 |
+
split = Path(args.split)
|
| 315 |
+
split_summary = json.loads((split / "summary.json").read_text(encoding="utf-8"))
|
| 316 |
+
train_rows = read_jsonl(split / "train.jsonl") + read_jsonl(split / "auxiliary_train.jsonl")
|
| 317 |
+
validation_rows = read_jsonl(split / "validation.jsonl")
|
| 318 |
+
test_rows = read_jsonl(split / "test.jsonl")
|
| 319 |
+
if any(row["tier"] == "gold" for row in train_rows):
|
| 320 |
+
raise RuntimeError("Gold sample detected in training rows")
|
| 321 |
+
selected_device = (
|
| 322 |
+
"cuda" if args.device == "auto" and torch.cuda.is_available()
|
| 323 |
+
else "cpu" if args.device == "auto"
|
| 324 |
+
else args.device
|
| 325 |
+
)
|
| 326 |
+
device = torch.device(selected_device)
|
| 327 |
+
if device.type == "cuda" and not torch.cuda.is_available():
|
| 328 |
+
raise RuntimeError("CUDA requested but unavailable")
|
| 329 |
+
|
| 330 |
+
output = Path(args.output)
|
| 331 |
+
output.mkdir(parents=True, exist_ok=True)
|
| 332 |
+
loader = DataLoader(
|
| 333 |
+
AccessibilityMaskDataset(train_rows, args.image_size, augment=True),
|
| 334 |
+
batch_size=args.batch_size,
|
| 335 |
+
shuffle=True,
|
| 336 |
+
num_workers=args.num_workers,
|
| 337 |
+
pin_memory=device.type == "cuda",
|
| 338 |
+
generator=torch.Generator().manual_seed(args.seed),
|
| 339 |
+
)
|
| 340 |
+
model = TinyAmodalUNet().to(device)
|
| 341 |
+
optimizer = torch.optim.AdamW(model.parameters(), lr=args.learning_rate, weight_decay=1e-4)
|
| 342 |
+
scaler = torch.amp.GradScaler("cuda", enabled=device.type == "cuda")
|
| 343 |
+
thresholds = [round(value, 2) for value in np.arange(0.20, 0.81, 0.05)]
|
| 344 |
+
best_score = (-1.0, -1.0)
|
| 345 |
+
best_epoch = -1
|
| 346 |
+
stale_epochs = 0
|
| 347 |
+
history = []
|
| 348 |
+
for epoch in range(1, args.epochs + 1):
|
| 349 |
+
model.train()
|
| 350 |
+
losses = []
|
| 351 |
+
for batch in loader:
|
| 352 |
+
inputs = batch["input"].to(device, non_blocking=True)
|
| 353 |
+
target = batch["hidden"].to(device, non_blocking=True)
|
| 354 |
+
obstacle = batch["obstacle"].to(device, non_blocking=True)
|
| 355 |
+
optimizer.zero_grad(set_to_none=True)
|
| 356 |
+
with torch.autocast(device_type=device.type, dtype=torch.float16, enabled=device.type == "cuda"):
|
| 357 |
+
logits = model(inputs)
|
| 358 |
+
loss = training_loss(logits, target, obstacle)
|
| 359 |
+
scaler.scale(loss).backward()
|
| 360 |
+
scaler.step(optimizer)
|
| 361 |
+
scaler.update()
|
| 362 |
+
losses.append(float(loss.detach().cpu()))
|
| 363 |
+
model.eval()
|
| 364 |
+
threshold, validation, _ = evaluate_thresholds(
|
| 365 |
+
model, validation_rows, args.image_size, device, thresholds
|
| 366 |
+
)
|
| 367 |
+
record = {
|
| 368 |
+
"epoch": epoch,
|
| 369 |
+
"train_loss": float(np.mean(losses)),
|
| 370 |
+
"validation_threshold": threshold,
|
| 371 |
+
"validation_hidden_iou_nonempty": validation["mean_hidden_iou_nonempty"],
|
| 372 |
+
"validation_amodal_iou": validation["mean_amodal_iou"],
|
| 373 |
+
}
|
| 374 |
+
history.append(record)
|
| 375 |
+
print(json.dumps(record, ensure_ascii=False), flush=True)
|
| 376 |
+
score = (validation["mean_hidden_iou_nonempty"], validation["mean_amodal_iou"])
|
| 377 |
+
if score > best_score:
|
| 378 |
+
best_score = score
|
| 379 |
+
best_epoch = epoch
|
| 380 |
+
stale_epochs = 0
|
| 381 |
+
torch.save(
|
| 382 |
+
{
|
| 383 |
+
"model_state": model.state_dict(),
|
| 384 |
+
"epoch": epoch,
|
| 385 |
+
"validation_threshold": threshold,
|
| 386 |
+
"validation_metrics": validation,
|
| 387 |
+
"split_fingerprint": split_summary["fingerprint"],
|
| 388 |
+
"architecture": "TinyAmodalUNet",
|
| 389 |
+
"input_channels": [
|
| 390 |
+
"rgb",
|
| 391 |
+
"target_visible",
|
| 392 |
+
"occluding_obstacle",
|
| 393 |
+
*CANONICAL_CATEGORIES,
|
| 394 |
+
],
|
| 395 |
+
"model_input_channels": 5 + len(CANONICAL_CATEGORIES),
|
| 396 |
+
"pseudo_depth_supervision": False,
|
| 397 |
+
},
|
| 398 |
+
output / "best.pt",
|
| 399 |
+
)
|
| 400 |
+
else:
|
| 401 |
+
stale_epochs += 1
|
| 402 |
+
if stale_epochs >= args.patience:
|
| 403 |
+
break
|
| 404 |
+
|
| 405 |
+
checkpoint = torch.load(output / "best.pt", map_location=device, weights_only=False)
|
| 406 |
+
model.load_state_dict(checkpoint["model_state"])
|
| 407 |
+
model.eval()
|
| 408 |
+
threshold, validation_report, validation_cache = evaluate_thresholds(
|
| 409 |
+
model, validation_rows, args.image_size, device, thresholds
|
| 410 |
+
)
|
| 411 |
+
_, test_report, test_cache = evaluate_thresholds(
|
| 412 |
+
model, test_rows, args.image_size, device, [threshold]
|
| 413 |
+
)
|
| 414 |
+
save_predictions(output / "predictions" / "validation", validation_rows, validation_cache, threshold)
|
| 415 |
+
save_predictions(output / "predictions" / "test", test_rows, test_cache, threshold)
|
| 416 |
+
build_contact_sheet(output / "validation_contact_sheet.jpg", validation_rows, validation_cache, threshold)
|
| 417 |
+
build_contact_sheet(output / "test_contact_sheet.jpg", test_rows, test_cache, threshold)
|
| 418 |
+
(output / "history.jsonl").write_text(
|
| 419 |
+
"".join(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n" for row in history),
|
| 420 |
+
encoding="utf-8",
|
| 421 |
+
)
|
| 422 |
+
baseline = {
|
| 423 |
+
"validation_mean_amodal_iou_visible_only": float(
|
| 424 |
+
np.mean(
|
| 425 |
+
[
|
| 426 |
+
iou(load_mask(Path(row["sample_dir"]) / "target_visible.png"), load_mask(Path(row["sample_dir"]) / "target_amodal.png"))
|
| 427 |
+
for row in validation_rows
|
| 428 |
+
]
|
| 429 |
+
)
|
| 430 |
+
),
|
| 431 |
+
"validation_mean_hidden_iou_nonempty_visible_only": 0.0,
|
| 432 |
+
}
|
| 433 |
+
summary = {
|
| 434 |
+
"status": "complete",
|
| 435 |
+
"model": "small_2d_hidden_mask_adapter_not_original_amodal3d",
|
| 436 |
+
"best_epoch": best_epoch,
|
| 437 |
+
"epochs_run": len(history),
|
| 438 |
+
"threshold_selected_on_validation": threshold,
|
| 439 |
+
"split_fingerprint": split_summary["fingerprint"],
|
| 440 |
+
"training_real_silver_count": 0,
|
| 441 |
+
"training_strict_gt_count": len(read_jsonl(split / "train.jsonl")),
|
| 442 |
+
"training_annotation_tier": "human_reviewed_strict_gt",
|
| 443 |
+
"strict_gt_used_for_training": True,
|
| 444 |
+
"training_auxiliary_synthetic_silver_count": len(read_jsonl(split / "auxiliary_train.jsonl")),
|
| 445 |
+
"gold_used_for_training": False,
|
| 446 |
+
"pseudo_depth_supervision": False,
|
| 447 |
+
"baseline": baseline,
|
| 448 |
+
"validation": {key: value for key, value in validation_report.items() if key != "rows"},
|
| 449 |
+
"test": {key: value for key, value in test_report.items() if key != "rows"},
|
| 450 |
+
"validation_rows": validation_report["rows"],
|
| 451 |
+
"test_rows": test_report["rows"],
|
| 452 |
+
}
|
| 453 |
+
(output / "summary.json").write_text(
|
| 454 |
+
json.dumps(summary, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
| 455 |
+
encoding="utf-8",
|
| 456 |
+
)
|
| 457 |
+
print(json.dumps(summary, ensure_ascii=False, indent=2), flush=True)
|
| 458 |
+
return 0
|
| 459 |
+
|
| 460 |
+
|
| 461 |
+
if __name__ == "__main__":
|
| 462 |
+
raise SystemExit(main())
|
tools/update_accessibility_inference_progress.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Atomically update the Accessibility3R inference status file.
|
| 3 |
+
|
| 4 |
+
This helper deliberately uses only the Python standard library so it can run
|
| 5 |
+
inside the existing Accessibility3D environment without adding a dependency.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import argparse
|
| 11 |
+
import json
|
| 12 |
+
import os
|
| 13 |
+
from datetime import datetime
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def now_iso() -> str:
|
| 18 |
+
return datetime.now().astimezone().isoformat(timespec="seconds")
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def load_existing(path: Path) -> dict:
|
| 22 |
+
try:
|
| 23 |
+
value = json.loads(path.read_text(encoding="utf-8"))
|
| 24 |
+
except (FileNotFoundError, json.JSONDecodeError, OSError):
|
| 25 |
+
return {}
|
| 26 |
+
return value if isinstance(value, dict) else {}
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def main() -> int:
|
| 30 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 31 |
+
parser.add_argument("--path", type=Path, required=True)
|
| 32 |
+
parser.add_argument("--status", choices=("running", "completed", "failed"), required=True)
|
| 33 |
+
parser.add_argument("--stage-key")
|
| 34 |
+
parser.add_argument("--stage-label")
|
| 35 |
+
parser.add_argument("--percent", type=float)
|
| 36 |
+
parser.add_argument("--stage-end-percent", type=float)
|
| 37 |
+
parser.add_argument("--stage-expected-seconds", type=float)
|
| 38 |
+
parser.add_argument("--total-estimated-seconds", type=float)
|
| 39 |
+
parser.add_argument("--message")
|
| 40 |
+
parser.add_argument("--exit-code", type=int)
|
| 41 |
+
args = parser.parse_args()
|
| 42 |
+
|
| 43 |
+
path = args.path.expanduser().resolve()
|
| 44 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 45 |
+
current = load_existing(path)
|
| 46 |
+
timestamp = now_iso()
|
| 47 |
+
current.setdefault("started_at", timestamp)
|
| 48 |
+
current["updated_at"] = timestamp
|
| 49 |
+
current["status"] = args.status
|
| 50 |
+
|
| 51 |
+
if args.stage_key is not None:
|
| 52 |
+
current["stage_key"] = args.stage_key
|
| 53 |
+
current["stage_started_at"] = timestamp
|
| 54 |
+
if args.stage_label is not None:
|
| 55 |
+
current["stage_label"] = args.stage_label
|
| 56 |
+
if args.percent is not None:
|
| 57 |
+
current["percent"] = max(0.0, min(100.0, args.percent))
|
| 58 |
+
if args.stage_end_percent is not None:
|
| 59 |
+
current["stage_end_percent"] = max(0.0, min(100.0, args.stage_end_percent))
|
| 60 |
+
if args.stage_expected_seconds is not None:
|
| 61 |
+
current["stage_expected_seconds"] = max(0.0, args.stage_expected_seconds)
|
| 62 |
+
if args.total_estimated_seconds is not None:
|
| 63 |
+
current["total_estimated_seconds"] = max(1.0, args.total_estimated_seconds)
|
| 64 |
+
if args.message is not None:
|
| 65 |
+
current["message"] = args.message
|
| 66 |
+
if args.exit_code is not None:
|
| 67 |
+
current["exit_code"] = args.exit_code
|
| 68 |
+
|
| 69 |
+
if args.status == "completed":
|
| 70 |
+
current.pop("failed_at", None)
|
| 71 |
+
current.pop("exit_code", None)
|
| 72 |
+
current["percent"] = 100.0
|
| 73 |
+
current["stage_end_percent"] = 100.0
|
| 74 |
+
current["completed_at"] = timestamp
|
| 75 |
+
elif args.status == "failed":
|
| 76 |
+
current.pop("completed_at", None)
|
| 77 |
+
current["failed_at"] = timestamp
|
| 78 |
+
else:
|
| 79 |
+
current.pop("completed_at", None)
|
| 80 |
+
current.pop("failed_at", None)
|
| 81 |
+
current.pop("exit_code", None)
|
| 82 |
+
|
| 83 |
+
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
| 84 |
+
temporary.write_text(
|
| 85 |
+
json.dumps(current, ensure_ascii=False, indent=2) + "\n",
|
| 86 |
+
encoding="utf-8",
|
| 87 |
+
)
|
| 88 |
+
os.replace(temporary, path)
|
| 89 |
+
return 0
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
if __name__ == "__main__":
|
| 93 |
+
raise SystemExit(main())
|
tools/validate_accessibility_reviewed_visible_workspace.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Validate and materialize one human-reviewed visible accessibility mask.
|
| 3 |
+
|
| 4 |
+
This deliberately supports *visible-mask-only* review. It does not upgrade a
|
| 5 |
+
reviewed visible mask into amodal ground truth: the hidden/obstacle layers are
|
| 6 |
+
still generated as review candidates by the normal pipeline. The tool is
|
| 7 |
+
used by the Slurm launcher before it bypasses SAM3 for a reviewed mask.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import argparse
|
| 13 |
+
import hashlib
|
| 14 |
+
import json
|
| 15 |
+
import shutil
|
| 16 |
+
from datetime import datetime, timezone
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
from typing import Any
|
| 19 |
+
|
| 20 |
+
import numpy as np
|
| 21 |
+
from PIL import Image, ImageOps
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def file_sha256(path: Path) -> str:
|
| 25 |
+
digest = hashlib.sha256()
|
| 26 |
+
with path.open("rb") as handle:
|
| 27 |
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
| 28 |
+
digest.update(chunk)
|
| 29 |
+
return digest.hexdigest()
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def read_json(path: Path) -> dict[str, Any]:
|
| 33 |
+
return json.loads(path.read_text(encoding="utf-8"))
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def write_json(path: Path, value: dict[str, Any]) -> None:
|
| 37 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 38 |
+
temporary = path.with_suffix(path.suffix + ".tmp")
|
| 39 |
+
temporary.write_text(
|
| 40 |
+
json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
| 41 |
+
encoding="utf-8",
|
| 42 |
+
)
|
| 43 |
+
temporary.replace(path)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def display_rgb(path: Path) -> Image.Image:
|
| 47 |
+
return ImageOps.exif_transpose(Image.open(path)).convert("RGB")
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def read_binary_mask(path: Path, size: tuple[int, int]) -> np.ndarray:
|
| 51 |
+
mask = ImageOps.exif_transpose(Image.open(path)).convert("L")
|
| 52 |
+
if mask.size != size:
|
| 53 |
+
raise ValueError(
|
| 54 |
+
f"Reviewed visible mask size {mask.size} does not match normalized source RGB {size}."
|
| 55 |
+
)
|
| 56 |
+
return np.asarray(mask) > 127
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def validate_workspace(
|
| 60 |
+
workspace: Path,
|
| 61 |
+
source_image: Path,
|
| 62 |
+
sample_id: str,
|
| 63 |
+
category: str,
|
| 64 |
+
) -> tuple[dict[str, Any], np.ndarray, tuple[int, int]]:
|
| 65 |
+
"""Validate the reviewer identity, provenance, category, and exact raster grid."""
|
| 66 |
+
metadata_path = workspace / "metadata.json"
|
| 67 |
+
target_path = workspace / "target_visible.png"
|
| 68 |
+
image_path = workspace / "image.png"
|
| 69 |
+
for path in (metadata_path, target_path, image_path):
|
| 70 |
+
if not path.is_file():
|
| 71 |
+
raise ValueError(f"Reviewed visible-mask workspace is missing {path.name}: {workspace}")
|
| 72 |
+
metadata = read_json(metadata_path)
|
| 73 |
+
errors: list[str] = []
|
| 74 |
+
if str(metadata.get("sample_id", "")) != sample_id:
|
| 75 |
+
errors.append("sample_id does not match the requested inference sample")
|
| 76 |
+
if str(metadata.get("category", "")) != category:
|
| 77 |
+
errors.append("category does not match the requested inference category")
|
| 78 |
+
if str(metadata.get("review_status", "")) != "approved":
|
| 79 |
+
errors.append("review_status must be approved")
|
| 80 |
+
if not bool(metadata.get("review_is_human", False)):
|
| 81 |
+
errors.append("review_is_human must be true")
|
| 82 |
+
if not bool(metadata.get("visible_confirmed", False)):
|
| 83 |
+
errors.append("visible_confirmed must be true")
|
| 84 |
+
if not str(metadata.get("annotator", "")).strip():
|
| 85 |
+
errors.append("approved review requires a non-empty annotator")
|
| 86 |
+
if bool(metadata.get("target_amodal_is_ground_truth", False)):
|
| 87 |
+
errors.append("visible-only workspace must not claim target_amodal ground truth")
|
| 88 |
+
source_hash = file_sha256(source_image)
|
| 89 |
+
recorded_hash = str(metadata.get("source_image_sha256", ""))
|
| 90 |
+
if not recorded_hash or recorded_hash != source_hash:
|
| 91 |
+
errors.append("source image SHA-256 does not match the reviewed workspace")
|
| 92 |
+
source = display_rgb(source_image)
|
| 93 |
+
workspace_image = Image.open(image_path).convert("RGB")
|
| 94 |
+
if workspace_image.size != source.size:
|
| 95 |
+
errors.append(
|
| 96 |
+
f"workspace normalized RGB size {workspace_image.size} does not match source {source.size}"
|
| 97 |
+
)
|
| 98 |
+
if errors:
|
| 99 |
+
raise ValueError("Invalid reviewed visible-mask workspace: " + "; ".join(errors))
|
| 100 |
+
target = read_binary_mask(target_path, source.size)
|
| 101 |
+
if not target.any():
|
| 102 |
+
raise ValueError("Reviewed visible target mask is empty")
|
| 103 |
+
return metadata, target, source.size
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def materialize_workspace(
|
| 107 |
+
workspace: Path,
|
| 108 |
+
source_image: Path,
|
| 109 |
+
sample_id: str,
|
| 110 |
+
category: str,
|
| 111 |
+
output_dir: Path,
|
| 112 |
+
) -> dict[str, Any]:
|
| 113 |
+
metadata, target, size = validate_workspace(workspace, source_image, sample_id, category)
|
| 114 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 115 |
+
target_path = output_dir / "target_visible.png"
|
| 116 |
+
obstacle_path = output_dir / "obstacle.png"
|
| 117 |
+
Image.fromarray(target.astype(np.uint8) * 255, mode="L").save(target_path)
|
| 118 |
+
Image.fromarray(np.zeros(target.shape, dtype=np.uint8), mode="L").save(obstacle_path)
|
| 119 |
+
result = {
|
| 120 |
+
"schema_version": 1,
|
| 121 |
+
"source_kind": "human_reviewed_visible_mask_only",
|
| 122 |
+
"sample_id": sample_id,
|
| 123 |
+
"category": category,
|
| 124 |
+
"review_status": "approved",
|
| 125 |
+
"review_is_human": True,
|
| 126 |
+
"visible_confirmed": True,
|
| 127 |
+
"annotator": str(metadata["annotator"]).strip(),
|
| 128 |
+
"reviewed_at_utc": metadata.get("reviewed_at_utc"),
|
| 129 |
+
"source_image_sha256": file_sha256(source_image),
|
| 130 |
+
"workspace_metadata_sha256": file_sha256(workspace / "metadata.json"),
|
| 131 |
+
"target_visible_sha256": file_sha256(target_path),
|
| 132 |
+
"target_visible_pixels": int(target.sum()),
|
| 133 |
+
"raster_size": {"width": size[0], "height": size[1]},
|
| 134 |
+
"hidden_and_obstacle_policy": "No reviewed hidden/obstacle layer was supplied; an empty obstacle candidate is passed to the adapter and all derived layers remain review candidates.",
|
| 135 |
+
"automatic_mask_is_ground_truth": False,
|
| 136 |
+
"target_amodal_is_ground_truth": False,
|
| 137 |
+
"created_at_utc": datetime.now(timezone.utc).isoformat(),
|
| 138 |
+
}
|
| 139 |
+
write_json(output_dir / "metadata.json", result)
|
| 140 |
+
return result
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def main() -> int:
|
| 144 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 145 |
+
parser.add_argument("--workspace", required=True)
|
| 146 |
+
parser.add_argument("--image", required=True)
|
| 147 |
+
parser.add_argument("--sample-id", required=True)
|
| 148 |
+
parser.add_argument("--category", required=True, choices=("curb_cut", "ramp", "stairs", "tactile_paving", "walkway"))
|
| 149 |
+
parser.add_argument("--output-dir", required=True)
|
| 150 |
+
args = parser.parse_args()
|
| 151 |
+
result = materialize_workspace(
|
| 152 |
+
Path(args.workspace).expanduser().resolve(),
|
| 153 |
+
Path(args.image).expanduser().resolve(),
|
| 154 |
+
args.sample_id,
|
| 155 |
+
args.category,
|
| 156 |
+
Path(args.output_dir).expanduser().resolve(),
|
| 157 |
+
)
|
| 158 |
+
print(json.dumps(result, ensure_ascii=False))
|
| 159 |
+
return 0
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
if __name__ == "__main__":
|
| 163 |
+
raise SystemExit(main())
|