Spaces:
Running
Running
Upload 19 files
Browse files- Dockerfile +5 -1
- app.py +39 -1
- docs/VALIDATION_VESUVIUS_4.14.0.json +69 -0
- docs/VESUVIUS_LAB.md +66 -0
- mouh/learning/manifest.json +17 -6
- requirements-vesuvius.txt +7 -0
- requirements.txt +3 -0
- src/__init__.py +1 -0
- src/controller.py +89 -0
- src/exporter.py +90 -0
- src/loader.py +101 -0
- src/scanner.py +32 -0
- src/storage.py +73 -0
- src/trainer.py +110 -0
- src/ui.py +88 -0
- src/unwrapper.py +78 -0
- src/worker.py +50 -0
- tests/test_vesuvius.py +234 -0
- unique_ext/__init__.py +1 -1
Dockerfile
CHANGED
|
@@ -8,6 +8,8 @@
|
|
| 8 |
FROM python:3.12-slim
|
| 9 |
|
| 10 |
ARG TORCH_VARIANT=gpu
|
|
|
|
|
|
|
| 11 |
ENV PYTHONUNBUFFERED=1 \
|
| 12 |
PYTHONDONTWRITEBYTECODE=1 \
|
| 13 |
PIP_NO_CACHE_DIR=1 \
|
|
@@ -26,9 +28,11 @@ RUN apt-get update \
|
|
| 26 |
RUN useradd -m -u 1000 user
|
| 27 |
WORKDIR /home/user/app
|
| 28 |
|
| 29 |
-
COPY --chown=user requirements.txt requirements-dev.txt ./
|
| 30 |
RUN if [ "$TORCH_VARIANT" = "cpu" ]; then \
|
| 31 |
pip install --index-url https://download.pytorch.org/whl/cpu torch ; \
|
|
|
|
|
|
|
| 32 |
else \
|
| 33 |
pip install "torch>=2.4,<2.16" ; \
|
| 34 |
fi \
|
|
|
|
| 8 |
FROM python:3.12-slim
|
| 9 |
|
| 10 |
ARG TORCH_VARIANT=gpu
|
| 11 |
+
# Option explicite CUDA 12.4 ; la valeur par défaut conserve le choix de PyTorch du noyau.
|
| 12 |
+
ARG VESUVIUS_CUDA12=0
|
| 13 |
ENV PYTHONUNBUFFERED=1 \
|
| 14 |
PYTHONDONTWRITEBYTECODE=1 \
|
| 15 |
PIP_NO_CACHE_DIR=1 \
|
|
|
|
| 28 |
RUN useradd -m -u 1000 user
|
| 29 |
WORKDIR /home/user/app
|
| 30 |
|
| 31 |
+
COPY --chown=user requirements.txt requirements-dev.txt requirements-vesuvius.txt ./
|
| 32 |
RUN if [ "$TORCH_VARIANT" = "cpu" ]; then \
|
| 33 |
pip install --index-url https://download.pytorch.org/whl/cpu torch ; \
|
| 34 |
+
elif [ "$VESUVIUS_CUDA12" = "1" ]; then \
|
| 35 |
+
pip install --index-url https://download.pytorch.org/whl/cu124 "torch==2.6.0" ; \
|
| 36 |
else \
|
| 37 |
pip install "torch>=2.4,<2.16" ; \
|
| 38 |
fi \
|
app.py
CHANGED
|
@@ -727,6 +727,9 @@ def mission_gpu_reason() -> str | None:
|
|
| 727 |
return "une passe GRPO/DPO occupe le GPU"
|
| 728 |
if TRAINER.running:
|
| 729 |
return "l'entraînement continu occupe le GPU"
|
|
|
|
|
|
|
|
|
|
| 730 |
return None
|
| 731 |
|
| 732 |
|
|
@@ -886,7 +889,32 @@ def learn_configure(key, actif=None, seuil=None, max_passes_jour=None, phases=No
|
|
| 886 |
if state["actif"] else "Apprentissage automatique éteint : aucune passe ne partira toute seule.")
|
| 887 |
|
| 888 |
|
| 889 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 890 |
|
| 891 |
|
| 892 |
with gr.Blocks(title="UNIQUE") as demo:
|
|
@@ -996,6 +1024,16 @@ with gr.Blocks(title="UNIQUE") as demo:
|
|
| 996 |
sp_btn.click(selfplay_cmd, [sp_key, sp_limit, sp_per], [sp_msg, a_journal])
|
| 997 |
rl_go.click(rl_start, [rl_key, rl_phases], rl_msg)
|
| 998 |
rl_halt.click(rl_stop, rl_key, rl_msg)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 999 |
timer = gr.Timer(10)
|
| 1000 |
timer.tick(refresh, None, [status, pass_plot, loss_plot, history, console])
|
| 1001 |
timer.tick(agents_refresh, None, [agents_status, a_journal])
|
|
|
|
| 727 |
return "une passe GRPO/DPO occupe le GPU"
|
| 728 |
if TRAINER.running:
|
| 729 |
return "l'entraînement continu occupe le GPU"
|
| 730 |
+
laboratory = globals().get("VESUVIUS")
|
| 731 |
+
if laboratory is not None and laboratory.gpu_active:
|
| 732 |
+
return "le laboratoire Vesuvius libère le GPU pour la mission"
|
| 733 |
return None
|
| 734 |
|
| 735 |
|
|
|
|
| 889 |
if state["actif"] else "Apprentissage automatique éteint : aucune passe ne partira toute seule.")
|
| 890 |
|
| 891 |
|
| 892 |
+
# Laboratoire optionnel : initialisation légère, aucun I/O ni job au démarrage.
|
| 893 |
+
VESUVIUS = None
|
| 894 |
+
VESUVIUS_ERROR = ""
|
| 895 |
+
try:
|
| 896 |
+
from src.controller import Controller as VesuviusController
|
| 897 |
+
VESUVIUS = VesuviusController(
|
| 898 |
+
busy=lambda: ("mission active" if mission_busy() else "entraînement actif" if training_active()
|
| 899 |
+
else "modèle en génération" if service.models_in_use() else None),
|
| 900 |
+
preempt=lambda: mission_active() or training_active(),
|
| 901 |
+
gpu_context=service.model_in_use, concurrent_uses=service.models_in_use)
|
| 902 |
+
service.set_gpu_guard(lambda: "laboratoire Vesuvius : GPU réservé" if VESUVIUS.gpu_active
|
| 903 |
+
else RL_BUSY if rl_reserved() else None)
|
| 904 |
+
except Exception as exc:
|
| 905 |
+
VESUVIUS_ERROR = type(exc).__name__ + ": " + str(exc)
|
| 906 |
+
|
| 907 |
+
# Conserve le branchement distant 4.13 s'il est déjà installé, sans l'imposer au noyau 4.12.
|
| 908 |
+
try:
|
| 909 |
+
from agents.remote_executor import RemoteExecutor
|
| 910 |
+
if hasattr(PIPELINE, "candidate_observer"):
|
| 911 |
+
REMOTE_EXECUTOR = RemoteExecutor(DISCOVERY.registry)
|
| 912 |
+
PIPELINE.candidate_observer = REMOTE_EXECUTOR.check
|
| 913 |
+
except ImportError:
|
| 914 |
+
pass
|
| 915 |
+
|
| 916 |
+
TRAINER.gate = lambda: ("une mission utilise le GPU" if mission_active() else
|
| 917 |
+
"laboratoire Vesuvius actif" if VESUVIUS is not None and VESUVIUS.gpu_active else None)
|
| 918 |
|
| 919 |
|
| 920 |
with gr.Blocks(title="UNIQUE") as demo:
|
|
|
|
| 1024 |
sp_btn.click(selfplay_cmd, [sp_key, sp_limit, sp_per], [sp_msg, a_journal])
|
| 1025 |
rl_go.click(rl_start, [rl_key, rl_phases], rl_msg)
|
| 1026 |
rl_halt.click(rl_stop, rl_key, rl_msg)
|
| 1027 |
+
if VESUVIUS is not None:
|
| 1028 |
+
try:
|
| 1029 |
+
from src.ui import build_tab as build_vesuvius_tab
|
| 1030 |
+
build_vesuvius_tab(VESUVIUS, authorized)
|
| 1031 |
+
except Exception as exc:
|
| 1032 |
+
with gr.Tab("Laboratoire Vesuvius — indisponible"):
|
| 1033 |
+
gr.Markdown("Extension désactivée : " + type(exc).__name__)
|
| 1034 |
+
else:
|
| 1035 |
+
with gr.Tab("Laboratoire Vesuvius — indisponible"):
|
| 1036 |
+
gr.Markdown("Le module isolé n'a pas pu être chargé. Les autres onglets restent disponibles.")
|
| 1037 |
timer = gr.Timer(10)
|
| 1038 |
timer.tick(refresh, None, [status, pass_plot, loss_plot, history, console])
|
| 1039 |
timer.tick(agents_refresh, None, [agents_status, a_journal])
|
docs/VALIDATION_VESUVIUS_4.14.0.json
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"release": "4.14.0",
|
| 3 |
+
"date": "2026-11-13",
|
| 4 |
+
"tests": {
|
| 5 |
+
"laboratory": {
|
| 6 |
+
"tests": 22,
|
| 7 |
+
"failures": 0,
|
| 8 |
+
"errors": 0,
|
| 9 |
+
"skipped": 0
|
| 10 |
+
},
|
| 11 |
+
"unique_regression": {
|
| 12 |
+
"tests": 109,
|
| 13 |
+
"failures": 0,
|
| 14 |
+
"errors": 0,
|
| 15 |
+
"skipped": 0
|
| 16 |
+
}
|
| 17 |
+
},
|
| 18 |
+
"live_s3": {
|
| 19 |
+
"ok": true,
|
| 20 |
+
"shape": [
|
| 21 |
+
75784,
|
| 22 |
+
32693,
|
| 23 |
+
32693
|
| 24 |
+
],
|
| 25 |
+
"chunks": [
|
| 26 |
+
128,
|
| 27 |
+
128,
|
| 28 |
+
128
|
| 29 |
+
],
|
| 30 |
+
"dtype": "uint8",
|
| 31 |
+
"roi_shape": [
|
| 32 |
+
8,
|
| 33 |
+
8,
|
| 34 |
+
8
|
| 35 |
+
],
|
| 36 |
+
"origin": [
|
| 37 |
+
37892,
|
| 38 |
+
16346,
|
| 39 |
+
16346
|
| 40 |
+
],
|
| 41 |
+
"min": 33.0,
|
| 42 |
+
"max": 134.0,
|
| 43 |
+
"seconds": 8.105108438999991
|
| 44 |
+
},
|
| 45 |
+
"gpu_validation": "not_run_no_GPU_in_executor",
|
| 46 |
+
"persistence_validation": "path and mount refusal tested; actual Space /data restart persistence not tested",
|
| 47 |
+
"ui_validation": "Gradio tab built; isolated ASGI /lab/config returned 200; authorization callbacks tested",
|
| 48 |
+
"geometry_validation": "plane metric preserved; TIFXYZ XYZ/mask/scale/bbox exported; VC3D roundtrip not run",
|
| 49 |
+
"torch": "CPU microcycle and disjoint prediction executed; no ink quality claim",
|
| 50 |
+
"docker": "syntax inspected, image not built; docker executable unavailable",
|
| 51 |
+
"existing_core": "No agents/, engine/ or mouh/service.py files in archive. app glue adds priority guards and optional tab.",
|
| 52 |
+
"dependencies": {
|
| 53 |
+
"torch": "2.6.0+cpu",
|
| 54 |
+
"numpy": "2.5.4",
|
| 55 |
+
"scipy": "1.18.1",
|
| 56 |
+
"zarr": "2.18.7",
|
| 57 |
+
"s3fs": "2025.12.0",
|
| 58 |
+
"tifffile": "2026.11.8",
|
| 59 |
+
"gradio": "5.50.0",
|
| 60 |
+
"safetensors": "0.8.0"
|
| 61 |
+
},
|
| 62 |
+
"limits": [
|
| 63 |
+
"Not a validated full-scroll unrolling pipeline",
|
| 64 |
+
"No pretrained ink checkpoint supplied",
|
| 65 |
+
"Research exports have submission_ready=false",
|
| 66 |
+
"64x64 is an internal guard, not a general official rule",
|
| 67 |
+
"CUDA12 build option changes the global torch stack only if explicitly enabled"
|
| 68 |
+
]
|
| 69 |
+
}
|
docs/VESUVIUS_LAB.md
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# UNIQUE 4.14.0 — laboratoire Vesuvius isolé
|
| 2 |
+
|
| 3 |
+
## Installation et portée
|
| 4 |
+
|
| 5 |
+
Décompresser le lot et uploader tous ses fichiers à la racine du Space en conservant les dossiers, dans un seul commit. Le lot est fondé sur les sources du dépôt TARGET_SPACE lues le 13 novembre 2026, dont le manifeste annonce 4.12.3. Aucun fichier `agents/`, `engine/` ou `mouh/service.py` n'est remplacé. Si le branchement distant 4.13 est déjà installé, le crochet est conservé de manière optionnelle. Le laboratoire ne l'utilise jamais pour envoyer les volumes CT.
|
| 6 |
+
|
| 7 |
+
L'onglet **Laboratoire Vesuvius 3D** est dans l'interface Gradio, accessible sur `/gradio/`. La page Symbioza et ses panneaux ne sont pas redessinés. Le chargement de l'onglet n'importe pas torch et ne démarre aucun calcul ni téléchargement.
|
| 8 |
+
|
| 9 |
+
Le laboratoire est une base de recherche locale fonctionnelle : petites ROI Zarr, score spectral, surface candidate ou fournie, paramétrisation métrique, petit modèle hybride et paquets traçables. Il ne constitue ni un lecteur validé de textes anciens, ni un pipeline complet d'unrolling d'un rouleau, ni une méthode prête à gagner un prix.
|
| 10 |
+
|
| 11 |
+
## Stockage obligatoire
|
| 12 |
+
|
| 13 |
+
Le bouton **Vérifier /data et l'état du laboratoire** doit confirmer que `/data` est monté et accessible en écriture. Sans montage, tous les jobs refusent de démarrer ; aucun repli vers `/tmp`, le dépôt ou le disque éphémère n'est utilisé pour les artefacts du laboratoire.
|
| 14 |
+
|
| 15 |
+
Tous les fichiers métier sont sous :
|
| 16 |
+
|
| 17 |
+
- `/data/chunks/vesuvius/` : ROI, requêtes et résultats de jobs, logs, masques et prédictions ;
|
| 18 |
+
- `/data/meshes/vesuvius/` : surfaces XYZ et charts UV ;
|
| 19 |
+
- `/data/weights/vesuvius/` : checkpoints safetensors, suivi d'expériences et caches de bibliothèques du worker ;
|
| 20 |
+
- `/data/submissions/vesuvius/` : paquets de recherche exportés.
|
| 21 |
+
|
| 22 |
+
Les sous-dossiers `vesuvius` évitent de toucher les autres artefacts d'UNIQUE. Le cache de ROI est plafonné à 512 Mio ; les plus anciennes entrées de ce seul cache sont évincées. Une vérification de place libre bloque les nouveaux jobs sous 128 Mio disponibles. Les autres sorties se conservent jusqu'au nettoyage manuel.
|
| 23 |
+
|
| 24 |
+
Le test local utilise un répertoire temporaire uniquement via l'argument explicite `require_mount=False` des tests. Le serveur et le worker de production utilisent toujours `Storage()` avec `/data` obligatoire. Le montage réel du Space et sa survie à un redémarrage doivent être contrôlés sur le Space ; ils ne sont pas déduits d'un test local.
|
| 25 |
+
|
| 26 |
+
## GPU et isolation
|
| 27 |
+
|
| 28 |
+
Chaque job tourne dans un nouveau sous-processus. Les tâches CPU sont limitées à un thread de calcul et lancées avec une priorité réduite. Les caches et temporaires du worker sont redirigés sous `/data`. Un job CPU a un budget de 120 secondes, un job CUDA de 45 secondes ; annulation et préemption peuvent ajouter jusqu'à trois secondes d'arrêt.
|
| 29 |
+
|
| 30 |
+
Un microcycle CUDA démarre uniquement si aucune mission, passe d'entraînement ou génération d'UNIQUE n'est active. Il utilise le mécanisme existant `model_in_use` comme réservation, sans modifier son code. La garde GPU empêche un nouveau chargement concurrent du Codeur et le superviseur donne priorité à une nouvelle mission : il arrête le job du laboratoire, puis libère sa réservation. L'entraînement continu attend ce créneau court. L'onglet n'arrête et ne décharge jamais lui-même le modèle d'UNIQUE.
|
| 31 |
+
|
| 32 |
+
Le microcycle prend bf16 sur CUDA quand PyTorch le supporte, sinon float32. Une A10G n'a pas été disponible dans l'environnement de validation : le chemin bf16/CUDA, l'utilisation réelle de VRAM et le démarrage de l'image complète restent à contrôler sur le Space. Aucun résultat CPU n'est présenté comme un benchmark A10G.
|
| 33 |
+
|
| 34 |
+
Le Dockerfile conserve par défaut le choix de PyTorch du noyau. Pour reconstruire explicitement avec les roues CUDA 12.4 / PyTorch 2.6, utiliser l'argument de build `VESUVIUS_CUDA12=1`. Cette option change la pile globale PyTorch et demande une nouvelle validation du modèle principal. La valeur `0` évite cette modification implicite. Les versions de transformers, peft et trl ne sont pas modifiées ; les ajouts sont regroupés dans `requirements-vesuvius.txt`.
|
| 35 |
+
|
| 36 |
+
## Parcours de recherche
|
| 37 |
+
|
| 38 |
+
1. Vérifier `/data`, avec la clé d'administration existante. Tous les boutons qui déclenchent un traitement vérifient cette clé.
|
| 39 |
+
2. Dans **Volume et footholds**, choisir le chemin S3, le tableau/niveau, l'origine `z,y,x`, une ROI de 2 à 64 voxels par axe et le pas isotrope exact du niveau choisi. Le chemin d'exemple provient du tutoriel officiel et a répondu lors du contrôle ; il n'est pas présenté comme un volume éligible à tous les prix.
|
| 40 |
+
3. Copier le chemin `.npz` retourné dans le champ chunk. Le scanner FFT traite des fenêtres 2D et conserve le meilleur 1 % des candidats. Le score est une anisotropie spectrale ; il n'atteste ni une surface de papyrus ni de l'encre.
|
| 41 |
+
4. Fournir, si disponible, une surface `H×W×3` en `.npy`, coordonnées `x,y,z` en voxels du niveau choisi, stockée sous `/data/meshes/vesuvius/`. Sans surface, le gradient produit un candidat non validé. La fenêtre est bornée à 64×64 points, puis l'aplatissement minimise les erreurs de longueur d'arête. Le résultat rapporte le maximum et le RMS de distorsion. Les plis sont refusés. Une isométrie exacte d'une surface courbe quelconque n'est pas garantie.
|
| 42 |
+
5. Dans **Microcycles**, choisir reconstruction ou encre supervisée. Sans checkpoint initial, les poids sont explicitement aléatoires ; reprendre un checkpoint du laboratoire effectue un fine-tuning réel. Le modèle est un petit encodeur Conv3D, agrégation en profondeur et tête Conv2D. Le mode reconstruction réalise un débruitage masqué, pas une détection d'encre. Le mode encre exige un véritable masque annoté `.npy` H×W dans [0,1]. Aucun masque de vérité artificiel n'est créé par le programme.
|
| 43 |
+
6. Pour prédire, charger une ROI disjointe. Les régions d'entraînement sont enregistrées dans le checkpoint et tout recouvrement est refusé, y compris en changeant de niveau du même volume lorsque la transformation n'est pas certifiée. Le seed, la loss, les versions des bibliothèques et l'empreinte du chunk sont enregistrés. La validation indépendante de la qualité des cartes d'encre reste à faire.
|
| 44 |
+
7. Construire le chart de cette ROI de prédiction, puis exporter avec le mesh et la carte correspondants. Sans prédiction, l'image est un rendu de densité CT, clairement identifié comme tel. Une carte d'un autre chunk est refusée.
|
| 45 |
+
|
| 46 |
+
Une panne S3 réutilise uniquement le cache de la requête exacte, avec sa date et `cache_fallback=true`. Sans cache exact, le job échoue proprement. Aucune donnée synthétique n'est substituée silencieusement à un scan réel. Les accès S3 sont anonymes, en lecture seule, limités au bucket public `vesuvius-challenge-open-data` ; aucun téléchargement de volume entier ni jeton HF transmis.
|
| 47 |
+
|
| 48 |
+
## Export et règles officielles
|
| 49 |
+
|
| 50 |
+
Chaque ZIP contient les TIFF float32 `x.tif`, `y.tif`, `z.tif`, un masque, `meta.json`, une PNG 16 bits, le rendu brut float32, une prévisualisation avec échelle et les paramètres de provenance. La géométrie est rééchantillonnée sur la même grille UV régulière que l'image. Le facteur `scale` exprime les pixels par voxel du niveau choisi. Les champs `format=tifxyz`, `type=seg`, `uuid`, `scale` et `bbox` suivent un exemple de métadonnées publiques officielles. Un aller-retour complet dans VC3D n'a pas été exécuté ici.
|
| 51 |
+
|
| 52 |
+
Le rééchantillonnage jusqu'à 2048² pixels n'ajoute pas de résolution CT. L'échelle est calculée à partir du pas renseigné, pas d'une calibration inventée. Une barre de 1 cm est produite seulement si le champ physique le permet ; sinon une barre plus courte et sa longueur sont indiquées honnêtement. Un patch de 64 voxels à 2,4 µm ne couvre que 0,154 mm : lui attribuer une barre de 1 cm serait faux.
|
| 53 |
+
|
| 54 |
+
Le champ `submission_ready` reste **false**. Les règles actuelles du Grand Prize exigent notamment des colonnes complètes, une géométrie TIFXYZ traçable, des images lisibles avec échelles de 1 cm, un panorama, une méthodologie reproductible et une évaluation indépendante sans recouvrement entraînement/prédiction. Ce petit laboratoire ne certifie ni la couverture d'un rouleau ni la lisibilité papyrologique. Il n'envoie rien aux organisateurs et ne publie aucun dataset ou checkpoint.
|
| 55 |
+
|
| 56 |
+
La limite 64×64 est un garde-fou interne demandé pour le laboratoire, pas une limite réglementaire générale trouvée dans les règles actuelles. Une adaptation transductive sur la région à soumettre ne doit pas être confondue avec une validation admissible ; le verrou de recouvrement empêche cet usage dans ce pipeline.
|
| 57 |
+
|
| 58 |
+
## Validation et sources
|
| 59 |
+
|
| 60 |
+
Voir `VALIDATION_VESUVIUS_4.14.0.json` pour les résultats exacts, les versions de test et les limites. La compatibilité Gradio est vérifiée par construction de l'onglet et montage ASGI isolé. Le noyau n'est pas entièrement démarré avec son modèle GPU dans cet environnement.
|
| 61 |
+
|
| 62 |
+
- Règles actuelles : https://scrollprize.org/prizes
|
| 63 |
+
- Formats : https://scrollprize.org/data_segments
|
| 64 |
+
- Données et échelle des volumes : https://scrollprize.org/from_ash_to_text
|
| 65 |
+
- Tutoriel et volume d'exemple : https://github.com/ScrollPrize/villa/blob/main/scrollprize.org/docs/07_tutorial5.md
|
| 66 |
+
- Exemple TIFXYZ lu : https://vesuvius-challenge-open-data.s3.amazonaws.com/PHerc0500P2/segments/20250716055236-z_dbg_gen_00356_inp_hr/mesh/intermediate/tifxyz_original/meta.json
|
mouh/learning/manifest.json
CHANGED
|
@@ -1,14 +1,14 @@
|
|
| 1 |
{
|
| 2 |
-
"version": "v4.
|
| 3 |
"files": {
|
| 4 |
-
"Dockerfile": "
|
| 5 |
"README.md": "ed433a59c0e4fb6d",
|
| 6 |
"agents/__init__.py": "c726c5c5cec54f5b",
|
| 7 |
"agents/autonomy.py": "6c73a4b5b49aeed8",
|
| 8 |
"agents/contract.py": "925f5ac350a90ea0",
|
| 9 |
"agents/core.py": "24e49461b79764ba",
|
| 10 |
"agents/mission.py": "9b760c2378774b7b",
|
| 11 |
-
"app.py": "
|
| 12 |
"config.json": "e75d8b151c4de3d1",
|
| 13 |
"engine/__init__.py": "9ed43b8c6346ad0e",
|
| 14 |
"engine/exam.py": "b9d7eeac20ff9a12",
|
|
@@ -84,9 +84,9 @@
|
|
| 84 |
"mouh/world.py": "b20550f63e2d7cc1",
|
| 85 |
"pyproject.toml": "5dff2198898efe09",
|
| 86 |
"requirements-dev.txt": "9a03d0d2ca94a0d2",
|
| 87 |
-
"requirements.txt": "
|
| 88 |
"unique_ext/README.md": "0f0be41551e6bacb",
|
| 89 |
-
"unique_ext/__init__.py": "
|
| 90 |
"unique_ext/backends/__init__.py": "e3b0c44298fc1c14",
|
| 91 |
"unique_ext/backends/hf.py": "506d29e81c127e1e",
|
| 92 |
"unique_ext/contracts.py": "4ce5632e921dfcdc",
|
|
@@ -138,7 +138,18 @@
|
|
| 138 |
"agents/discovery.py": "9c688f744e888102",
|
| 139 |
"agents/discovery_api.py": "ea8ba886909ef1ea",
|
| 140 |
"agents/discovery_capabilities.py": "41a76670ed0553c1",
|
| 141 |
-
"config/spaces_curated.json": "9a9f54dceab9cfbf"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
},
|
| 143 |
"removed": [
|
| 144 |
"mouh/learning/autopilot.py",
|
|
|
|
| 1 |
{
|
| 2 |
+
"version": "v4.14.0",
|
| 3 |
"files": {
|
| 4 |
+
"Dockerfile": "f68259170ec47222",
|
| 5 |
"README.md": "ed433a59c0e4fb6d",
|
| 6 |
"agents/__init__.py": "c726c5c5cec54f5b",
|
| 7 |
"agents/autonomy.py": "6c73a4b5b49aeed8",
|
| 8 |
"agents/contract.py": "925f5ac350a90ea0",
|
| 9 |
"agents/core.py": "24e49461b79764ba",
|
| 10 |
"agents/mission.py": "9b760c2378774b7b",
|
| 11 |
+
"app.py": "bad902c73a24d67d",
|
| 12 |
"config.json": "e75d8b151c4de3d1",
|
| 13 |
"engine/__init__.py": "9ed43b8c6346ad0e",
|
| 14 |
"engine/exam.py": "b9d7eeac20ff9a12",
|
|
|
|
| 84 |
"mouh/world.py": "b20550f63e2d7cc1",
|
| 85 |
"pyproject.toml": "5dff2198898efe09",
|
| 86 |
"requirements-dev.txt": "9a03d0d2ca94a0d2",
|
| 87 |
+
"requirements.txt": "f6e0d34c6f292b9a",
|
| 88 |
"unique_ext/README.md": "0f0be41551e6bacb",
|
| 89 |
+
"unique_ext/__init__.py": "179aa19888358397",
|
| 90 |
"unique_ext/backends/__init__.py": "e3b0c44298fc1c14",
|
| 91 |
"unique_ext/backends/hf.py": "506d29e81c127e1e",
|
| 92 |
"unique_ext/contracts.py": "4ce5632e921dfcdc",
|
|
|
|
| 138 |
"agents/discovery.py": "9c688f744e888102",
|
| 139 |
"agents/discovery_api.py": "ea8ba886909ef1ea",
|
| 140 |
"agents/discovery_capabilities.py": "41a76670ed0553c1",
|
| 141 |
+
"config/spaces_curated.json": "9a9f54dceab9cfbf",
|
| 142 |
+
"src/__init__.py": "ef77ad4265bda616",
|
| 143 |
+
"src/controller.py": "b4c9586d5de615d3",
|
| 144 |
+
"src/exporter.py": "a6fdfcce4e1135f5",
|
| 145 |
+
"src/loader.py": "1d7ecba8594d015a",
|
| 146 |
+
"src/scanner.py": "3746e1d337810a54",
|
| 147 |
+
"src/storage.py": "e5feb00026c0dbc5",
|
| 148 |
+
"src/trainer.py": "3c0abcbc27d70514",
|
| 149 |
+
"src/ui.py": "2784fe94110157a8",
|
| 150 |
+
"src/unwrapper.py": "6f556fa3fb15fd9c",
|
| 151 |
+
"src/worker.py": "7104681fcf9e81cf",
|
| 152 |
+
"requirements-vesuvius.txt": "61384af45af4799c"
|
| 153 |
},
|
| 154 |
"removed": [
|
| 155 |
"mouh/learning/autopilot.py",
|
requirements-vesuvius.txt
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Dépendances optionnelles du laboratoire ; aucun changement de transformers/peft/trl.
|
| 2 |
+
s3fs>=2024.6,<2026
|
| 3 |
+
zarr>=2.18,<3
|
| 4 |
+
numcodecs>=0.12,<0.16
|
| 5 |
+
tifffile>=2024.8,<2027
|
| 6 |
+
Pillow>=10,<12
|
| 7 |
+
plotly>=5,<7
|
requirements.txt
CHANGED
|
@@ -25,3 +25,6 @@ manifold3d>=2.4,<4 # géométrie solide constructive (CSG)
|
|
| 25 |
scikit-fem>=9.0,<13 # éléments finis (volumes, plaques)
|
| 26 |
control>=0.10,<1 # automatique : fonctions de transfert, stabilité
|
| 27 |
tetgen>=0.8,<1 # v4.9 : maillage en tétraèdres des pièces solides (atelier mécanique) ; roue abi3, numpy seul
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
scikit-fem>=9.0,<13 # éléments finis (volumes, plaques)
|
| 26 |
control>=0.10,<1 # automatique : fonctions de transfert, stabilité
|
| 27 |
tetgen>=0.8,<1 # v4.9 : maillage en tétraèdres des pièces solides (atelier mécanique) ; roue abi3, numpy seul
|
| 28 |
+
|
| 29 |
+
# Laboratoire Vesuvius, imports lourds uniquement dans les jobs manuels.
|
| 30 |
+
-r requirements-vesuvius.txt
|
src/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Laboratoire Vesuvius isolé : aucun import lourd, accès disque ou réseau ici."""
|
src/controller.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Jobs manuels et préemptibles ; aucune tâche ni import torch au bootstrap."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
from contextlib import nullcontext
|
| 4 |
+
import json
|
| 5 |
+
import os
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
import subprocess
|
| 8 |
+
import sys
|
| 9 |
+
import threading
|
| 10 |
+
import time
|
| 11 |
+
import uuid
|
| 12 |
+
|
| 13 |
+
from .storage import Storage
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class Controller:
|
| 17 |
+
def __init__(self, storage=None, busy=lambda:None, preempt=lambda:False,
|
| 18 |
+
gpu_context=nullcontext, concurrent_uses=lambda:0):
|
| 19 |
+
self.storage=storage or Storage()
|
| 20 |
+
self.busy,self.preempt=busy,preempt
|
| 21 |
+
self.gpu_context,self.concurrent_uses=gpu_context,concurrent_uses
|
| 22 |
+
self.lock=threading.Lock();self.proc=None;self.gpu_active=False
|
| 23 |
+
self.state={'status':'idle','result':None,'error':'','gpu':False}
|
| 24 |
+
self._cancel=False
|
| 25 |
+
|
| 26 |
+
def status(self):
|
| 27 |
+
with self.lock:return dict(self.state)
|
| 28 |
+
|
| 29 |
+
def start(self, action, payload):
|
| 30 |
+
if action not in ('load','scan','unwrap','train','predict','export'):
|
| 31 |
+
raise ValueError('action inconnue')
|
| 32 |
+
with self.lock:
|
| 33 |
+
if self.state['status']=='running':raise RuntimeError('un job du laboratoire est déjà actif')
|
| 34 |
+
self.storage.initialize()
|
| 35 |
+
gpu=action=='train' and payload.get('device')=='cuda'
|
| 36 |
+
if gpu and self.busy():raise RuntimeError('GPU prioritaire occupé par UNIQUE : '+str(self.busy()))
|
| 37 |
+
job=uuid.uuid4().hex
|
| 38 |
+
request=self.storage.json('chunks',job+'.request.json',{'action':action,'payload':payload})
|
| 39 |
+
self.gpu_active=gpu;self._cancel=False
|
| 40 |
+
self.state={'status':'running','job':job,'action':action,'gpu':gpu,'result':None,'error':''}
|
| 41 |
+
thread=threading.Thread(target=self._run,args=(request,gpu),daemon=True,name='vesuvius-job')
|
| 42 |
+
thread.start()
|
| 43 |
+
return dict(self.state)
|
| 44 |
+
|
| 45 |
+
def _run(self, request, gpu):
|
| 46 |
+
context=self.gpu_context() if gpu else nullcontext()
|
| 47 |
+
result={'ok':False,'error':'job interrompu'}
|
| 48 |
+
try:
|
| 49 |
+
with context:
|
| 50 |
+
if gpu and self.concurrent_uses()>1:raise RuntimeError('une génération UNIQUE a priorité')
|
| 51 |
+
cache=self.storage.folder('weights')/'cache';cache.mkdir(exist_ok=True)
|
| 52 |
+
temp=self.storage.folder('chunks')/'tmp';temp.mkdir(exist_ok=True)
|
| 53 |
+
env=os.environ.copy()
|
| 54 |
+
env.update(TMPDIR=str(temp),TMP=str(temp),TEMP=str(temp),TORCH_HOME=str(cache),
|
| 55 |
+
XDG_CACHE_HOME=str(cache),CUDA_CACHE_PATH=str(cache/'cuda'),HF_HOME=str(cache/'hf'),
|
| 56 |
+
OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',PYTHONDONTWRITEBYTECODE='1',
|
| 57 |
+
CUBLAS_WORKSPACE_CONFIG=':4096:8')
|
| 58 |
+
if not gpu:env['CUDA_VISIBLE_DEVICES']=''
|
| 59 |
+
log=request.with_suffix('.log')
|
| 60 |
+
with log.open('wb') as stream:
|
| 61 |
+
self.proc=subprocess.Popen([sys.executable,'-m','src.worker',str(request)],
|
| 62 |
+
cwd=str(Path(__file__).resolve().parent.parent),env=env,stdout=stream,stderr=subprocess.STDOUT,
|
| 63 |
+
start_new_session=True)
|
| 64 |
+
deadline=time.monotonic()+(45 if gpu else 120)
|
| 65 |
+
while self.proc.poll() is None:
|
| 66 |
+
if self._cancel or time.monotonic()>deadline or (gpu and self.preempt()):
|
| 67 |
+
self.proc.terminate()
|
| 68 |
+
try:self.proc.wait(timeout=3)
|
| 69 |
+
except subprocess.TimeoutExpired:self.proc.kill();self.proc.wait()
|
| 70 |
+
raise RuntimeError('job annulé, préempté par UNIQUE ou budget épuisé')
|
| 71 |
+
time.sleep(0.2)
|
| 72 |
+
output=request.with_name(request.stem+'.result.json')
|
| 73 |
+
if output.is_file() and output.stat().st_size<1024**2:
|
| 74 |
+
result=json.loads(output.read_text())
|
| 75 |
+
else:result={'ok':False,'error':'worker indisponible ; voir '+str(log)}
|
| 76 |
+
except Exception as exc:result={'ok':False,'error':type(exc).__name__+': '+str(exc)[:1000]}
|
| 77 |
+
finally:
|
| 78 |
+
with self.lock:
|
| 79 |
+
self.gpu_active=False;self.proc=None
|
| 80 |
+
self.state.update(status='done' if result.get('ok') else 'error',gpu=False,
|
| 81 |
+
result=result.get('result'),error=result.get('error',''))
|
| 82 |
+
|
| 83 |
+
def stop(self):
|
| 84 |
+
self._cancel=True
|
| 85 |
+
return 'Arrêt demandé au laboratoire uniquement.'
|
| 86 |
+
|
| 87 |
+
def diagnostic(self):
|
| 88 |
+
try:return {'storage':self.storage.initialize(),'job':self.status()}
|
| 89 |
+
except Exception as exc:return {'storage':{'ok':False,'error':str(exc)},'job':self.status()}
|
src/exporter.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Paquet de recherche traçable PNG + géométrie TIFXYZ + métadonnées."""
|
| 2 |
+
import json
|
| 3 |
+
import time
|
| 4 |
+
import uuid
|
| 5 |
+
import zipfile
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def export(storage, mesh_path, prediction_path=None, pixels=1024):
|
| 9 |
+
import numpy as np
|
| 10 |
+
from scipy.interpolate import LinearNDInterpolator
|
| 11 |
+
from scipy.ndimage import map_coordinates
|
| 12 |
+
from PIL import Image,ImageDraw
|
| 13 |
+
import tifffile
|
| 14 |
+
from .loader import read_chunk
|
| 15 |
+
from .trainer import overlaps
|
| 16 |
+
if not 64<=int(pixels)<=2048:raise ValueError('résolution de 64 à 2048 pixels par axe')
|
| 17 |
+
path=storage.checked(mesh_path,'meshes')
|
| 18 |
+
with np.load(path,allow_pickle=False) as data:
|
| 19 |
+
xyz=data['xyz'];uv=data['uv'];meta=json.loads(str(data['metadata']))
|
| 20 |
+
if xyz.shape[:2]!=uv.shape[:2] or max(xyz.shape[:2])>64 or meta['quality']['folds']:
|
| 21 |
+
raise ValueError('chart invalide')
|
| 22 |
+
if meta['quality']['max_relative_stretch']>0.15:raise ValueError('distorsion supérieure à 15 % : réviser la surface')
|
| 23 |
+
volume,chunk_meta=read_chunk(storage,meta['chunk'])
|
| 24 |
+
if chunk_meta['request']!=meta['request']:raise ValueError('provenance du chunk modifiée')
|
| 25 |
+
coordinates=xyz[...,::-1]-np.array(meta['request']['origin'])
|
| 26 |
+
if np.any(coordinates<0) or np.any(coordinates>np.array(volume.shape)-1):
|
| 27 |
+
raise ValueError('surface hors de la ROI chargée')
|
| 28 |
+
values=map_coordinates(volume.astype('float32'),coordinates.reshape(-1,3).T,order=1,mode='nearest').reshape(xyz.shape[:2])
|
| 29 |
+
prediction_meta=None
|
| 30 |
+
if prediction_path:
|
| 31 |
+
pp=storage.checked(prediction_path,'chunks')
|
| 32 |
+
with np.load(pp,allow_pickle=False) as data:
|
| 33 |
+
predicted=data['prediction'];prediction_meta=json.loads(str(data['metadata']))
|
| 34 |
+
if prediction_meta['request']!=meta['request'] or predicted.shape!=values.shape:
|
| 35 |
+
raise ValueError('prédiction et mesh ne correspondent pas')
|
| 36 |
+
if any(overlaps(meta['request'],r) for r in prediction_meta['training']['training_regions']):
|
| 37 |
+
raise ValueError('entraînement et export se recouvrent')
|
| 38 |
+
if not np.isfinite(predicted).all():raise ValueError('prédiction non finie')
|
| 39 |
+
values=predicted
|
| 40 |
+
low=uv.reshape(-1,2).min(0);high=uv.reshape(-1,2).max(0)
|
| 41 |
+
if np.min(high-low)<=1e-8:raise ValueError('chart sans étendue')
|
| 42 |
+
# Grille UV régulière ; la carte XYZ est rééchantillonnée pour conserver la correspondance.
|
| 43 |
+
n=int(pixels);xs=np.linspace(low[0],high[0],n);ys=np.linspace(low[1],high[1],n)
|
| 44 |
+
gx,gy=np.meshgrid(xs,ys)
|
| 45 |
+
interpolator=LinearNDInterpolator(uv.reshape(-1,2),np.c_[xyz.reshape(-1,3),values.ravel()])
|
| 46 |
+
mapped=interpolator(gx,gy);valid=np.isfinite(mapped).all(axis=-1)
|
| 47 |
+
if valid.sum()==0:raise ValueError('paramétrisation vide')
|
| 48 |
+
raw=np.where(valid,mapped[...,3],0)
|
| 49 |
+
lo,hi=np.percentile(raw[valid],[1,99])
|
| 50 |
+
normalized=np.where(valid,np.clip((raw-lo)/max(float(hi-lo),1e-6),0,1),0)
|
| 51 |
+
name='patch_'+uuid.uuid4().hex
|
| 52 |
+
folder=storage.path('submissions',name);folder.mkdir()
|
| 53 |
+
mesh=folder/(name+'.tifxyz');mesh.mkdir()
|
| 54 |
+
for i,axis in enumerate('xyz'):
|
| 55 |
+
tifffile.imwrite(mesh/(axis+'.tif'),np.where(valid,mapped[...,i],-1).astype('float32'))
|
| 56 |
+
Image.fromarray((valid*255).astype('uint8')).save(mesh/'mask.png')
|
| 57 |
+
tifffile.imwrite(mesh/'mask.tif',(valid*255).astype('uint8'))
|
| 58 |
+
tifffile.imwrite(folder/(name+'.raw.tif'),raw.astype('float32'))
|
| 59 |
+
image=Image.fromarray((normalized*65535).astype('uint16'));image.save(folder/(name+'.png'))
|
| 60 |
+
voxel_um=meta['request']['voxel_um'];um_per_pixel=float((high[0]-low[0])*voxel_um/(n-1))
|
| 61 |
+
possible_mm=float((high[0]-low[0])*voxel_um/1000)
|
| 62 |
+
bar_mm=float(10.0 if possible_mm>=12 else max(0.001,possible_mm/4))
|
| 63 |
+
bar_pixels=max(1,int(round(bar_mm*1000/um_per_pixel)))
|
| 64 |
+
preview=Image.fromarray((normalized*255).astype('uint8')).convert('RGB')
|
| 65 |
+
canvas=Image.new('RGB',(n,n+48),'black');canvas.paste(preview,(0,0));draw=ImageDraw.Draw(canvas)
|
| 66 |
+
draw.line((8,n+12,8+bar_pixels,n+12),fill='white',width=3)
|
| 67 |
+
draw.text((8,n+22),f'{bar_mm:g} mm',fill='white');canvas.save(folder/(name+'.scale.png'))
|
| 68 |
+
report={'schema':'unique-vesuvius-research/1','created_at':time.time(),'submission_ready':False,
|
| 69 |
+
'reasons':['local_patch_not_full_column','held_out_and_papyrological_validation_missing',
|
| 70 |
+
'VC3D_roundtrip_not_certified'],
|
| 71 |
+
'source':meta['request'],'mesh':name+'.tifxyz','coordinate_units':'selected_level_voxels',
|
| 72 |
+
'axes':'xyz','shape':[n,n],'uv_bounds':[low.tolist(),high.tolist()],
|
| 73 |
+
'micrometres_per_pixel_x':um_per_pixel,
|
| 74 |
+
'scale_bar':{'length_mm':bar_mm,'pixels':bar_pixels,'official_1cm_fits':bar_mm==10},
|
| 75 |
+
'chunk_sha256':storage.digest(meta['chunk'],'chunks'),'geometry':meta['quality'],'surface_kind':meta['surface_kind'],
|
| 76 |
+
'render_kind':prediction_meta['kind'] if prediction_meta else 'CT_density_not_ink',
|
| 77 |
+
'prediction':prediction_meta,'mesh_sha256':storage.digest(path,'meshes'),
|
| 78 |
+
'note':'Une grande image rééchantillonnée ne crée pas de résolution CT supplémentaire.'}
|
| 79 |
+
valid_xyz=mapped[...,:3][valid]
|
| 80 |
+
(mesh/'meta.json').write_text(json.dumps({'format':'tifxyz','type':'seg','uuid':name,
|
| 81 |
+
'scale':[float((n-1)/(high[0]-low[0])),float((n-1)/(high[1]-low[1]))],
|
| 82 |
+
'bbox':[valid_xyz.min(0).tolist(),valid_xyz.max(0).tolist()],
|
| 83 |
+
'target_volume':meta['request']['uri']+'#vc-base-scale='+meta['request']['level'],
|
| 84 |
+
'coordinate_system':'selected_level_voxels','source':meta['request']},indent=2))
|
| 85 |
+
(folder/'metadata.json').write_text(json.dumps(report,ensure_ascii=False,allow_nan=False,indent=2))
|
| 86 |
+
archive=storage.path('submissions',name+'.zip')
|
| 87 |
+
with zipfile.ZipFile(archive,'w',zipfile.ZIP_DEFLATED) as z:
|
| 88 |
+
for file in sorted(folder.rglob('*')):
|
| 89 |
+
if file.is_file():z.write(file,str(file.relative_to(folder)))
|
| 90 |
+
return {'path':str(archive),'metadata':report,'image':str(folder/(name+'.scale.png'))}
|
src/loader.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Lecture S3 anonyme, ROI bornée, cache exact ; jamais de volume synthétique de repli."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import asyncio
|
| 5 |
+
import hashlib
|
| 6 |
+
import json
|
| 7 |
+
import math
|
| 8 |
+
import re
|
| 9 |
+
import time
|
| 10 |
+
from urllib.parse import urlsplit
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def request_spec(uri, level, origin, shape, voxel_um):
|
| 14 |
+
p = urlsplit(str(uri))
|
| 15 |
+
if (p.scheme != 's3' or p.netloc != 'vesuvius-challenge-open-data' or p.query or p.fragment
|
| 16 |
+
or '..' in p.path.split('/') or not p.path.endswith(('.zarr', '.zarr/'))):
|
| 17 |
+
raise ValueError('seul le bucket public Vesuvius et un chemin .zarr sont autorisés')
|
| 18 |
+
if not re.fullmatch(r'[A-Za-z0-9_.-]{1,64}', str(level)) or str(level) in ('.','..'):
|
| 19 |
+
raise ValueError('niveau Zarr invalide')
|
| 20 |
+
if len(origin) != 3 or len(shape) != 3 or any(type(v) is not int or v < 0 for v in origin):
|
| 21 |
+
raise ValueError('origine attendue : trois entiers z,y,x positifs')
|
| 22 |
+
if any(type(v) is not int or not 2 <= v <= 64 for v in shape):
|
| 23 |
+
raise ValueError('ROI de 2 à 64 voxels par axe')
|
| 24 |
+
if not math.isfinite(float(voxel_um)) or not 0 < float(voxel_um) <= 1000:
|
| 25 |
+
raise ValueError('pas de voxel isotrope en µm requis')
|
| 26 |
+
return dict(uri=str(uri).rstrip('/'), level=str(level), origin=list(origin), shape=list(shape), voxel_um=float(voxel_um), axes='zyx')
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class Loader:
|
| 30 |
+
def __init__(self, storage, opener=None):
|
| 31 |
+
self.storage, self.opener = storage, opener or self.open_array
|
| 32 |
+
|
| 33 |
+
@staticmethod
|
| 34 |
+
def open_array(spec):
|
| 35 |
+
import s3fs
|
| 36 |
+
import zarr
|
| 37 |
+
fs = s3fs.S3FileSystem(anon=True, config_kwargs={'connect_timeout':5, 'read_timeout':8,
|
| 38 |
+
'retries':{'max_attempts':1}}, default_fill_cache=False)
|
| 39 |
+
store = fs.get_mapper(spec['uri'], check=False)
|
| 40 |
+
# mode r : pas de création de métadonnées ni d'objets S3.
|
| 41 |
+
return zarr.open_array(store=store, path=spec['level'], mode='r')
|
| 42 |
+
|
| 43 |
+
def load(self, spec):
|
| 44 |
+
import numpy as np
|
| 45 |
+
self.storage.initialize()
|
| 46 |
+
spec = request_spec(spec['uri'], spec['level'], spec['origin'], spec['shape'], spec['voxel_um'])
|
| 47 |
+
key = hashlib.sha256(json.dumps(spec,sort_keys=True).encode()).hexdigest()[:24]
|
| 48 |
+
cache = self.storage.path('chunks', key+'.npz')
|
| 49 |
+
def cached():
|
| 50 |
+
with np.load(cache, allow_pickle=False) as saved:
|
| 51 |
+
volume = saved['volume']
|
| 52 |
+
metadata = json.loads(str(saved['metadata']))
|
| 53 |
+
if list(volume.shape) != spec['shape'] or metadata['request'] != spec:
|
| 54 |
+
raise ValueError('cache incompatible')
|
| 55 |
+
return volume, metadata
|
| 56 |
+
try:
|
| 57 |
+
array = self.opener(spec)
|
| 58 |
+
if len(array.shape) != 3 or any(o+n>s for o,n,s in zip(spec['origin'],spec['shape'],array.shape)):
|
| 59 |
+
raise ValueError('ROI hors volume ou axes non zyx : sélectionner un tableau 3D')
|
| 60 |
+
if array.dtype.kind not in 'uif' or math.prod(array.chunks)*array.dtype.itemsize > 64*1024**2:
|
| 61 |
+
raise ValueError('type ou chunk distant trop volumineux')
|
| 62 |
+
volume = np.asarray(array[tuple(slice(o,o+n) for o,n in zip(spec['origin'],spec['shape']))])
|
| 63 |
+
if not np.isfinite(volume).all():
|
| 64 |
+
raise ValueError('voxels non finis')
|
| 65 |
+
metadata = {'request':spec, 'source_shape':list(array.shape), 'loaded_at':time.time(),
|
| 66 |
+
'provenance':'public_CT', 'cache_fallback':False}
|
| 67 |
+
tmp = cache.with_suffix('.tmp.npz')
|
| 68 |
+
np.savez_compressed(tmp, volume=volume, metadata=json.dumps(metadata))
|
| 69 |
+
tmp.replace(cache)
|
| 70 |
+
# Quota du laboratoire uniquement, les autres dossiers restent intacts.
|
| 71 |
+
entries = sorted(self.storage.folder('chunks').glob('*.npz'), key=lambda p:p.stat().st_mtime)
|
| 72 |
+
total = sum(p.stat().st_size for p in entries)
|
| 73 |
+
for p in entries:
|
| 74 |
+
if total <= 512*1024**2: break
|
| 75 |
+
if p != cache:
|
| 76 |
+
total -= p.stat().st_size; p.unlink()
|
| 77 |
+
except ValueError:
|
| 78 |
+
raise
|
| 79 |
+
except Exception:
|
| 80 |
+
if not cache.is_file():
|
| 81 |
+
raise RuntimeError('S3 indisponible et aucun cache exact pour cette ROI') from None
|
| 82 |
+
volume, metadata = cached()
|
| 83 |
+
metadata = {**metadata, 'cache_fallback':True}
|
| 84 |
+
return {'path':str(cache), 'metadata':metadata, 'shape':list(volume.shape)}
|
| 85 |
+
|
| 86 |
+
async def load_async(self, spec):
|
| 87 |
+
return await asyncio.to_thread(self.load, spec)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def read_chunk(storage, path):
|
| 91 |
+
import numpy as np
|
| 92 |
+
path = storage.checked(path, 'chunks')
|
| 93 |
+
if path.stat().st_size > 16*1024**2:
|
| 94 |
+
raise ValueError('chunk local trop volumineux')
|
| 95 |
+
with np.load(path, allow_pickle=False) as saved:
|
| 96 |
+
volume = saved['volume'].copy()
|
| 97 |
+
metadata = json.loads(str(saved['metadata']))
|
| 98 |
+
spec = request_spec(**{k:metadata['request'][k] for k in ('uri','level','origin','shape','voxel_um')})
|
| 99 |
+
if list(volume.shape) != spec['shape'] or not np.isfinite(volume).all():
|
| 100 |
+
raise ValueError('chunk incohérent')
|
| 101 |
+
return volume, metadata
|
src/scanner.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Score spectral d'anisotropie ; sélection top 1 %, pas une preuve d'encre."""
|
| 2 |
+
import math
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def scan(volume, tile=16):
|
| 6 |
+
import numpy as np
|
| 7 |
+
volume = np.asarray(volume)
|
| 8 |
+
if volume.ndim != 3 or not np.isfinite(volume).all() or not 4 <= int(tile) <= 64:
|
| 9 |
+
raise ValueError('volume ou taille de fenêtre invalide')
|
| 10 |
+
tile = int(tile)
|
| 11 |
+
if min(volume.shape[1:]) < tile:
|
| 12 |
+
raise ValueError('fenêtre plus grande que la ROI')
|
| 13 |
+
scores=[]
|
| 14 |
+
fy,fx=np.meshgrid(np.fft.fftshift(np.fft.fftfreq(tile)),np.fft.fftshift(np.fft.fftfreq(tile)),indexing='ij')
|
| 15 |
+
radius=fx*fx+fy*fy
|
| 16 |
+
band=(radius>0.01)&(radius<0.2)
|
| 17 |
+
window=np.outer(np.hanning(tile),np.hanning(tile))
|
| 18 |
+
for z in range(volume.shape[0]):
|
| 19 |
+
for y in range(0,volume.shape[1]-tile+1,max(1,tile//2)):
|
| 20 |
+
for x in range(0,volume.shape[2]-tile+1,max(1,tile//2)):
|
| 21 |
+
patch=volume[z,y:y+tile,x:x+tile].astype('float32')
|
| 22 |
+
variance=float(np.var(patch))
|
| 23 |
+
power=np.abs(np.fft.fftshift(np.fft.fft2((patch-patch.mean())*window)))**2*band
|
| 24 |
+
energy=float(power.sum())
|
| 25 |
+
if energy<=1e-12 or variance<=1e-12: continue
|
| 26 |
+
tensor=np.array([[np.sum(power*fx*fx),np.sum(power*fx*fy)],[np.sum(power*fx*fy),np.sum(power*fy*fy)]])
|
| 27 |
+
eigen=np.linalg.eigvalsh(tensor)
|
| 28 |
+
score=float((eigen[1]-eigen[0])/(eigen.sum()+1e-12))
|
| 29 |
+
scores.append({'zyx':[z,y+tile//2,x+tile//2],'score':score,'variance':variance,'window':tile})
|
| 30 |
+
scores.sort(key=lambda v:(-v['score'],v['zyx']))
|
| 31 |
+
return {'method':'FFT_anisotropy', 'ink_detected':False,'candidates':len(scores),
|
| 32 |
+
'selected':scores[:max(1,math.ceil(len(scores)*0.01))] if scores else []}
|
src/storage.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tous les artefacts du laboratoire restent dans les quatre dossiers /data."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import hashlib
|
| 5 |
+
import json
|
| 6 |
+
import os
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
import re
|
| 9 |
+
import shutil
|
| 10 |
+
import uuid
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class Storage:
|
| 14 |
+
KINDS = ('chunks', 'meshes', 'weights', 'submissions')
|
| 15 |
+
|
| 16 |
+
def __init__(self, root=Path('/data'), require_mount=True):
|
| 17 |
+
self.root = Path(root).resolve()
|
| 18 |
+
self.require_mount = require_mount
|
| 19 |
+
if require_mount and self.root != Path('/data'):
|
| 20 |
+
raise ValueError('le laboratoire exige /data')
|
| 21 |
+
|
| 22 |
+
def folder(self, kind):
|
| 23 |
+
if kind not in self.KINDS:
|
| 24 |
+
raise ValueError('catégorie inconnue')
|
| 25 |
+
path = self.root / kind / 'vesuvius'
|
| 26 |
+
if not path.resolve().is_relative_to(self.root):
|
| 27 |
+
raise ValueError('lien symbolique hors /data')
|
| 28 |
+
return path
|
| 29 |
+
|
| 30 |
+
def initialize(self):
|
| 31 |
+
if self.require_mount and not os.path.ismount(self.root):
|
| 32 |
+
raise RuntimeError('/data non monté : laboratoire indisponible, aucun repli local')
|
| 33 |
+
for kind in self.KINDS:
|
| 34 |
+
self.folder(kind).mkdir(parents=True, exist_ok=True)
|
| 35 |
+
if shutil.disk_usage(self.root).free < 128*1024**2:
|
| 36 |
+
raise RuntimeError('moins de 128 Mio disponibles sur /data')
|
| 37 |
+
probe = self.folder('chunks') / ('.probe-'+uuid.uuid4().hex)
|
| 38 |
+
try:
|
| 39 |
+
probe.write_bytes(b'unique-vesuvius')
|
| 40 |
+
if probe.read_bytes() != b'unique-vesuvius':
|
| 41 |
+
raise OSError('lecture après écriture incorrecte')
|
| 42 |
+
finally:
|
| 43 |
+
probe.unlink(missing_ok=True)
|
| 44 |
+
return {'root': str(self.root), 'mounted': os.path.ismount(self.root), 'writable': True}
|
| 45 |
+
|
| 46 |
+
def path(self, kind, name):
|
| 47 |
+
if not re.fullmatch(r'[A-Za-z0-9_.-]{1,160}', str(name)) or name in ('.', '..'):
|
| 48 |
+
raise ValueError('nom invalide')
|
| 49 |
+
return self.checked(self.folder(kind)/name, kind)
|
| 50 |
+
|
| 51 |
+
def checked(self, path, kind):
|
| 52 |
+
path = Path(path).resolve()
|
| 53 |
+
if not path.is_relative_to(self.folder(kind).resolve()):
|
| 54 |
+
raise ValueError('artefact hors du dossier autorisé')
|
| 55 |
+
return path
|
| 56 |
+
|
| 57 |
+
def json(self, kind, name, value):
|
| 58 |
+
path = self.path(kind, name)
|
| 59 |
+
tmp = path.with_name('.'+path.name+'.'+uuid.uuid4().hex)
|
| 60 |
+
try:
|
| 61 |
+
tmp.write_text(json.dumps(value, ensure_ascii=False, allow_nan=False, indent=2), encoding='utf-8')
|
| 62 |
+
tmp.replace(path)
|
| 63 |
+
finally:
|
| 64 |
+
tmp.unlink(missing_ok=True)
|
| 65 |
+
return path
|
| 66 |
+
|
| 67 |
+
def digest(self, path, kind):
|
| 68 |
+
path = self.checked(path, kind)
|
| 69 |
+
h = hashlib.sha256()
|
| 70 |
+
with path.open('rb') as f:
|
| 71 |
+
for block in iter(lambda:f.read(1024*1024), b''):
|
| 72 |
+
h.update(block)
|
| 73 |
+
return h.hexdigest()
|
src/trainer.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Petit encodeur 3D/2D, adaptation isolée et journal de régions disjointes."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
import json
|
| 4 |
+
import time
|
| 5 |
+
import uuid
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def overlaps(a,b):
|
| 9 |
+
if a['uri']!=b['uri']: return False
|
| 10 |
+
# Sans transformation inter-niveaux certifiée, même volume/niveau différent est refusé.
|
| 11 |
+
if a['level']!=b['level']: return True
|
| 12 |
+
return all(x<y+m and y<x+n for x,n,y,m in zip(a['origin'],a['shape'],b['origin'],b['shape']))
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def build_model():
|
| 16 |
+
import torch.nn as nn
|
| 17 |
+
class Hybrid(nn.Module):
|
| 18 |
+
def __init__(self):
|
| 19 |
+
super().__init__()
|
| 20 |
+
self.volume=nn.Sequential(nn.Conv3d(1,8,3,padding=1),nn.GELU(),nn.Conv3d(8,8,3,padding=1),nn.GELU())
|
| 21 |
+
self.sheet=nn.Sequential(nn.Conv2d(8,8,3,padding=1),nn.GELU(),nn.Conv2d(8,1,1))
|
| 22 |
+
def forward(self,x):return self.sheet(self.volume(x).mean(dim=2))
|
| 23 |
+
return Hybrid()
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def train(storage, chunk_path, steps=4, mode='reconstruction', mask_path=None, checkpoint=None, device='cpu'):
|
| 27 |
+
import numpy as np
|
| 28 |
+
import torch
|
| 29 |
+
from safetensors.torch import load_file,save_file
|
| 30 |
+
from .loader import read_chunk
|
| 31 |
+
if mode not in ('reconstruction','ink_supervised') or not 1<=int(steps)<=32:
|
| 32 |
+
raise ValueError('mode ou budget de microcycle invalide')
|
| 33 |
+
volume,metadata=read_chunk(storage,chunk_path)
|
| 34 |
+
torch.set_num_threads(1)
|
| 35 |
+
with torch.random.fork_rng(devices=[0] if device=='cuda' else []):
|
| 36 |
+
torch.manual_seed(0)
|
| 37 |
+
torch.use_deterministic_algorithms(True)
|
| 38 |
+
if device=='cuda':
|
| 39 |
+
if not torch.cuda.is_available():raise RuntimeError('CUDA indisponible')
|
| 40 |
+
if torch.cuda.mem_get_info()[0]<1024**3:raise RuntimeError('moins de 1 Gio de VRAM libre')
|
| 41 |
+
model=build_model().to(device)
|
| 42 |
+
history=[];initial='random'
|
| 43 |
+
if checkpoint:
|
| 44 |
+
path=storage.checked(checkpoint,'weights')
|
| 45 |
+
if path.stat().st_size>16*1024**2:raise ValueError('checkpoint trop volumineux')
|
| 46 |
+
prior=json.loads(path.with_suffix('.json').read_text())
|
| 47 |
+
if prior['mode']!=mode:raise ValueError('objectif du checkpoint incompatible')
|
| 48 |
+
model.load_state_dict(load_file(str(path),device=device))
|
| 49 |
+
history=list(prior['training_regions']);initial=str(path)
|
| 50 |
+
normalized=volume.astype('float32')
|
| 51 |
+
lo,hi=np.percentile(normalized,[1,99]);normalized=np.clip((normalized-lo)/max(float(hi-lo),1e-6),0,1).astype("float32")
|
| 52 |
+
x=torch.from_numpy(normalized[None,None]).to(device)
|
| 53 |
+
if mode=='ink_supervised':
|
| 54 |
+
if not mask_path:raise ValueError('masque d’encre annoté requis pour un objectif supervisé')
|
| 55 |
+
path=storage.checked(mask_path,'chunks')
|
| 56 |
+
if path.stat().st_size>1024**2:raise ValueError('masque trop volumineux')
|
| 57 |
+
mask=np.load(path,allow_pickle=False)
|
| 58 |
+
if mask.shape!=volume.shape[1:] or not np.isfinite(mask).all() or np.min(mask)<0 or np.max(mask)>1:
|
| 59 |
+
raise ValueError('masque attendu H×W dans [0,1]')
|
| 60 |
+
target=torch.from_numpy(mask.astype('float32')[None,None]).to(device)
|
| 61 |
+
else:target=x.mean(dim=2)
|
| 62 |
+
optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3)
|
| 63 |
+
bf16=device=='cuda' and torch.cuda.is_bf16_supported()
|
| 64 |
+
loss_fn=torch.nn.BCEWithLogitsLoss() if mode=='ink_supervised' else torch.nn.MSELoss()
|
| 65 |
+
loss_values=[]
|
| 66 |
+
model.train()
|
| 67 |
+
for _ in range(int(steps)):
|
| 68 |
+
optimizer.zero_grad(set_to_none=True)
|
| 69 |
+
# Dénoyautage masqué pour la reconstruction ; jamais présenté comme une carte d'encre.
|
| 70 |
+
inp=x*(torch.rand_like(x)>0.15) if mode=='reconstruction' else x
|
| 71 |
+
with torch.autocast(device_type=device,dtype=torch.bfloat16,enabled=bf16):
|
| 72 |
+
output=model(inp);loss=loss_fn(output,target)
|
| 73 |
+
if not torch.isfinite(loss):raise RuntimeError('loss non finie')
|
| 74 |
+
loss.backward();optimizer.step();loss_values.append(float(loss.detach().cpu()))
|
| 75 |
+
name=uuid.uuid4().hex
|
| 76 |
+
path=storage.path('weights',name+'.safetensors')
|
| 77 |
+
save_file({k:v.detach().cpu().contiguous() for k,v in model.state_dict().items()},str(path))
|
| 78 |
+
history.append(metadata['request'])
|
| 79 |
+
result={'mode':mode,'seed':0,'steps':int(steps),'loss':loss_values,'device':device,
|
| 80 |
+
'precision':'bf16' if bf16 else 'float32','initialization':initial,'training_regions':history,
|
| 81 |
+
'held_out_evaluated':False,'ink_quality_measured':False,'created_at':time.time(),
|
| 82 |
+
'architecture':'tiny_hybrid_3d_2d_v1','torch_version':torch.__version__,'numpy_version':np.__version__,
|
| 83 |
+
'training_chunk_sha256':storage.digest(chunk_path,'chunks'),'normalization':{'low':float(lo),'high':float(hi)}}
|
| 84 |
+
storage.json('weights',name+'.json',result)
|
| 85 |
+
return {'path':str(path),'report':result}
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def predict(storage, chunk_path, checkpoint):
|
| 89 |
+
import numpy as np
|
| 90 |
+
import torch
|
| 91 |
+
from safetensors.torch import load_file
|
| 92 |
+
from .loader import read_chunk
|
| 93 |
+
volume,metadata=read_chunk(storage,chunk_path)
|
| 94 |
+
path=storage.checked(checkpoint,'weights')
|
| 95 |
+
if path.stat().st_size>16*1024**2:raise ValueError('checkpoint trop volumineux')
|
| 96 |
+
report=json.loads(path.with_suffix('.json').read_text())
|
| 97 |
+
if any(overlaps(metadata['request'],region) for region in report['training_regions']):
|
| 98 |
+
raise ValueError('région de prédiction en recouvrement avec un entraînement : refus')
|
| 99 |
+
model=build_model();model.load_state_dict(load_file(str(path),device='cpu'));model.eval()
|
| 100 |
+
lo,hi=report['normalization']['low'],report['normalization']['high']
|
| 101 |
+
x=np.clip((volume.astype('float32')-lo)/max(hi-lo,1e-6),0,1).astype("float32")
|
| 102 |
+
with torch.inference_mode():out=model(torch.from_numpy(x[None,None]))
|
| 103 |
+
if report['mode']=='ink_supervised':out=out.sigmoid()
|
| 104 |
+
out=out.squeeze().numpy()
|
| 105 |
+
name=uuid.uuid4().hex
|
| 106 |
+
target=storage.path('chunks',name+'.prediction.npz')
|
| 107 |
+
np.savez_compressed(target,prediction=out,metadata=json.dumps({'request':metadata['request'],
|
| 108 |
+
'checkpoint':str(path),'checkpoint_sha256':storage.digest(path,'weights'),'training':report,
|
| 109 |
+
'kind':'ink_probability_unvalidated' if report['mode']=='ink_supervised' else 'reconstruction_not_ink'}))
|
| 110 |
+
return {'path':str(target),'kind':report['mode'],'held_out_evaluated':False}
|
src/ui.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Onglet additionnel, callbacks authentifiés ; calculs hors du serveur Gradio."""
|
| 2 |
+
import json
|
| 3 |
+
import importlib.util
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def build_tab(controller, authorized):
|
| 7 |
+
import gradio as gr
|
| 8 |
+
with gr.Tab('Laboratoire Vesuvius 3D'):
|
| 9 |
+
gr.Markdown('## Vesuvius — laboratoire de recherche\n'
|
| 10 |
+
'Jobs manuels et isolés. Les modèles et les missions UNIQUE restent prioritaires. '
|
| 11 |
+
'Une structure FFT ou une reconstruction ne prouve pas la présence d’encre. '
|
| 12 |
+
'Les exports de patch sont des paquets de recherche, pas des soumissions complètes.')
|
| 13 |
+
key=gr.Textbox(label='Clé d’administration UNIQUE_ADMIN_KEY',type='password')
|
| 14 |
+
status=gr.JSON(label='État du job et artefacts')
|
| 15 |
+
def call(action,payload,secret):
|
| 16 |
+
if not authorized(secret):return {'error':'clé d’administration requise'}
|
| 17 |
+
try:return controller.start(action,payload)
|
| 18 |
+
except Exception as exc:return {'error':str(exc)}
|
| 19 |
+
def diagnostic(secret):
|
| 20 |
+
return controller.diagnostic() if authorized(secret) else {'error':'clé requise'}
|
| 21 |
+
check=gr.Button('Vérifier /data et l’état du laboratoire')
|
| 22 |
+
check.click(diagnostic,key,status)
|
| 23 |
+
with gr.Tabs():
|
| 24 |
+
with gr.Tab('Volume et footholds'):
|
| 25 |
+
uri=gr.Textbox(label='Volume S3 public .zarr',value='s3://vesuvius-challenge-open-data/PHercParis4/volumes/20260411134726-2.400um-0.2m-78keV-masked.zarr')
|
| 26 |
+
gr.Markdown('Exemple de volume de recherche publié ; vérifier son pas et son niveau dans le catalogue officiel.')
|
| 27 |
+
level=gr.Textbox(label='Tableau / niveau Zarr',value='0')
|
| 28 |
+
origin=gr.Textbox(label='Origine z,y,x (voxels du niveau choisi)',value='37892,16346,16346')
|
| 29 |
+
size=gr.Number(label='Côté de la ROI (2–64)',value=32,precision=0)
|
| 30 |
+
voxel=gr.Number(label='Pas isotrope du niveau choisi (µm)',value=2.4)
|
| 31 |
+
fetch=gr.Button('Lire cette ROI S3')
|
| 32 |
+
def load(secret,u,l,o,n,v):
|
| 33 |
+
try:payload={'uri':u,'level':l,'origin':[int(i.strip()) for i in o.split(',')],
|
| 34 |
+
'shape':[int(n)]*3,'voxel_um':float(v)}
|
| 35 |
+
except Exception:return {'error':'origine ou taille invalide'}
|
| 36 |
+
return call('load',payload,secret)
|
| 37 |
+
fetch.click(load,[key,uri,level,origin,size,voxel],status)
|
| 38 |
+
chunk=gr.Textbox(label='Chemin du chunk .npz affiché dans le résultat (/data/chunks/vesuvius/...)')
|
| 39 |
+
scan=gr.Button('FFT 2D — top 1 % des candidats')
|
| 40 |
+
scan.click(lambda k,c:call('scan',{'chunk':c,'tile':16},k),[key,chunk],status)
|
| 41 |
+
with gr.Tab('Surface et aplatissement'):
|
| 42 |
+
surface=gr.Textbox(label='Surface XYZ .npy sous /data/meshes/vesuvius (facultative)')
|
| 43 |
+
gr.Markdown('Sans surface fournie : estimation par gradient, explicitement non validée. '
|
| 44 |
+
'Fenêtre interne ≤64×64, distorsion mesurée ; aucune isométrie parfaite garantie.')
|
| 45 |
+
unwrap=gr.Button('Construire le chart local')
|
| 46 |
+
unwrap.click(lambda k,c,s:call('unwrap',{'chunk':c,'surface':s},k),[key,chunk,surface],status)
|
| 47 |
+
mesh=gr.Textbox(label='Chemin du mesh .npz produit (/data/meshes/vesuvius/...)')
|
| 48 |
+
plot=gr.Plot(label='Surface 3D — coordonnées du niveau choisi')
|
| 49 |
+
preview=gr.Button('Afficher le mesh 3D')
|
| 50 |
+
def show(secret,path):
|
| 51 |
+
if not authorized(secret):return None
|
| 52 |
+
try:
|
| 53 |
+
import numpy as np
|
| 54 |
+
import plotly.graph_objects as go
|
| 55 |
+
with np.load(controller.storage.checked(path,'meshes'),allow_pickle=False) as data:xyz=data['xyz']
|
| 56 |
+
if xyz.shape[0]>64 or xyz.shape[1]>64:return None
|
| 57 |
+
fig=go.Figure(go.Surface(x=xyz[...,0],y=xyz[...,1],z=xyz[...,2],colorscale='Viridis'))
|
| 58 |
+
fig.update_layout(scene={'aspectmode':'data'},margin=dict(l=0,r=0,t=0,b=0))
|
| 59 |
+
return fig
|
| 60 |
+
except Exception:return None
|
| 61 |
+
preview.click(show,[key,mesh],plot)
|
| 62 |
+
with gr.Tab('Microcycles et prédictions'):
|
| 63 |
+
gr.Markdown('Petit modèle 3D/2D initialisé aléatoirement, ou repris depuis un checkpoint du laboratoire. '
|
| 64 |
+
'La reconstruction auto-supervisée n’est pas une détection d’encre. '
|
| 65 |
+
'Une prédiction sur une région d’entraînement est refusée.')
|
| 66 |
+
mode=gr.Dropdown(['reconstruction','ink_supervised'],value='reconstruction',label='Objectif')
|
| 67 |
+
mask=gr.Textbox(label='Masque .npy H×W annoté, dans /data/chunks/vesuvius (mode supervisé)')
|
| 68 |
+
checkpoint=gr.Textbox(label='Checkpoint .safetensors dans /data/weights/vesuvius (optionnel pour entraîner)')
|
| 69 |
+
steps=gr.Slider(1,32,value=4,step=1,label='Pas du microcycle')
|
| 70 |
+
device=gr.Dropdown(['cpu','cuda'],value='cpu',label='Calcul — CUDA refuse si UNIQUE est occupé')
|
| 71 |
+
train=gr.Button('Lancer le microcycle')
|
| 72 |
+
train.click(lambda k,c,m,ma,cp,s,d:call('train',{'chunk':c,'mode':m,'mask':ma,
|
| 73 |
+
'checkpoint':cp,'steps':int(s),'device':d},k),[key,chunk,mode,mask,checkpoint,steps,device],status)
|
| 74 |
+
predict=gr.Button('Prédire une autre ROI, disjointe')
|
| 75 |
+
predict.click(lambda k,c,cp:call('predict',{'chunk':c,'checkpoint':cp},k),[key,chunk,checkpoint],status)
|
| 76 |
+
with gr.Tab('Export de recherche'):
|
| 77 |
+
prediction=gr.Textbox(label='Prédiction .prediction.npz correspondante (facultative ; sinon rendu CT)')
|
| 78 |
+
pixels=gr.Dropdown([256,512,1024,2048],value=1024,label='Pixels de sortie (rééchantillonnage)')
|
| 79 |
+
export=gr.Button('Exporter PNG, TIFXYZ et provenance en ZIP')
|
| 80 |
+
export.click(lambda k,m,p,n:call('export',{'mesh':m,'prediction':p,'pixels':int(n)},k),
|
| 81 |
+
[key,mesh,prediction,pixels],status)
|
| 82 |
+
gr.Markdown('Le ZIP reste dans `/data/submissions/vesuvius/`. '
|
| 83 |
+
'La conformité Grand Prize n’est pas certifiée : colonnes complètes, validation tenue à part '
|
| 84 |
+
'et vérification VC3D restent nécessaires. Aucun envoi aux organisateurs.')
|
| 85 |
+
halt=gr.Button('Arrêter le job du laboratoire')
|
| 86 |
+
halt.click(lambda k:{'message':controller.stop()} if authorized(k) else {'error':'clé requise'},key,status)
|
| 87 |
+
timer=gr.Timer(2)
|
| 88 |
+
timer.tick(lambda k:controller.status() if authorized(k) else {'status':'lecture protégée'},key,status)
|
src/unwrapper.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Surface candidate et paramétrisation métrique locale avec distorsion mesurée."""
|
| 2 |
+
import json
|
| 3 |
+
import uuid
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def candidate_surface(volume, origin):
|
| 7 |
+
import numpy as np
|
| 8 |
+
from scipy.ndimage import gaussian_filter
|
| 9 |
+
# Candidat géométrique seulement : cette heuristique ne segmente pas un rouleau entier.
|
| 10 |
+
smooth=gaussian_filter(volume.astype('float32'),1)
|
| 11 |
+
z=np.abs(np.gradient(smooth,axis=0)).argmax(axis=0).astype('float32')
|
| 12 |
+
z=gaussian_filter(z,1)
|
| 13 |
+
y,x=np.indices(z.shape,dtype='float32')
|
| 14 |
+
return np.stack([x+origin[2],y+origin[1],z+origin[0]],axis=-1)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def flatten(xyz, iterations=30):
|
| 18 |
+
import numpy as np
|
| 19 |
+
from scipy.optimize import least_squares
|
| 20 |
+
from scipy.sparse import lil_matrix
|
| 21 |
+
xyz=np.asarray(xyz,dtype='float64')
|
| 22 |
+
if xyz.ndim!=3 or xyz.shape[-1]!=3 or not 2<=min(xyz.shape[:2]) or max(xyz.shape[:2])>64 or not np.isfinite(xyz).all():
|
| 23 |
+
raise ValueError('surface finie H×W×3 requise ; fenêtre interne maximale 64×64')
|
| 24 |
+
h,w=xyz.shape[:2]; points=xyz.reshape(-1,3)
|
| 25 |
+
center=points-points.mean(0)
|
| 26 |
+
_,_,basis=np.linalg.svd(center,full_matrices=False)
|
| 27 |
+
uv=center@basis[:2].T
|
| 28 |
+
# Oriente le chart selon les axes de la grille.
|
| 29 |
+
if uv[w-1,0]<uv[0,0]: uv[:,0]*=-1
|
| 30 |
+
if uv[(h-1)*w,1]<uv[0,1]: uv[:,1]*=-1
|
| 31 |
+
uv-=uv[0]
|
| 32 |
+
edges=[]
|
| 33 |
+
for y in range(h):
|
| 34 |
+
for x in range(w):
|
| 35 |
+
a=y*w+x
|
| 36 |
+
if x+1<w:edges.append((a,a+1))
|
| 37 |
+
if y+1<h:edges.append((a,a+w))
|
| 38 |
+
if y+1<h and x+1<w:edges.extend([(a,a+w+1),(a+1,a+w)])
|
| 39 |
+
edge=np.array(edges); lengths=np.linalg.norm(points[edge[:,0]]-points[edge[:,1]],axis=1)
|
| 40 |
+
if np.min(lengths)<=1e-8: raise ValueError('arêtes dégénérées')
|
| 41 |
+
anchor=uv[0].copy(); direction=uv[w-1,1]
|
| 42 |
+
def residual(flat):
|
| 43 |
+
u=flat.reshape(-1,2)
|
| 44 |
+
dist=np.linalg.norm(u[edge[:,0]]-u[edge[:,1]],axis=1)
|
| 45 |
+
return np.r_[(dist-lengths)/lengths, (u[0]-anchor)*10, (u[w-1,1]-direction)*10]
|
| 46 |
+
sparsity=lil_matrix((len(edges)+3,2*h*w),dtype=int)
|
| 47 |
+
for i,(a,b) in enumerate(edges):sparsity[i,[2*a,2*a+1,2*b,2*b+1]]=1
|
| 48 |
+
sparsity[-3,0]=1;sparsity[-2,1]=1;sparsity[-1,2*(w-1)+1]=1
|
| 49 |
+
fit=least_squares(residual,uv.ravel(),jac_sparsity=sparsity.tocsr(),max_nfev=max(1,min(int(iterations),60)),ftol=1e-5,xtol=1e-5,gtol=1e-5)
|
| 50 |
+
uv=fit.x.reshape(h,w,2)
|
| 51 |
+
metric=residual(fit.x)[:-3]
|
| 52 |
+
a=uv[:-1,1:]-uv[:-1,:-1]; b=uv[1:,:-1]-uv[:-1,:-1]
|
| 53 |
+
cross=a[...,0]*b[...,1]-a[...,1]*b[...,0]
|
| 54 |
+
folds=bool(np.any(np.abs(cross)<1e-8) or (np.any(cross>0) and np.any(cross<0)))
|
| 55 |
+
report={'method':'local_edge_length_least_squares','max_relative_stretch':float(np.max(np.abs(metric))),
|
| 56 |
+
'rms_relative_stretch':float(np.sqrt(np.mean(metric**2))), 'folds':folds,
|
| 57 |
+
'converged':bool(fit.success),'isometry_guaranteed':False,'window':[h,w]}
|
| 58 |
+
return uv.astype('float32'),report
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def unwrap(storage, chunk_path, surface_path=None):
|
| 62 |
+
import numpy as np
|
| 63 |
+
from .loader import read_chunk
|
| 64 |
+
volume,metadata=read_chunk(storage,chunk_path)
|
| 65 |
+
if surface_path:
|
| 66 |
+
path=storage.checked(surface_path,'meshes')
|
| 67 |
+
if path.stat().st_size>128000:raise ValueError('surface trop volumineuse')
|
| 68 |
+
xyz=np.load(path,allow_pickle=False)
|
| 69 |
+
kind='provided_surface'
|
| 70 |
+
else:
|
| 71 |
+
xyz=candidate_surface(volume,metadata['request']['origin']);kind='unreviewed_gradient_candidate'
|
| 72 |
+
uv,quality=flatten(xyz)
|
| 73 |
+
if quality['folds']:raise ValueError('pli ou dégénérescence détectée dans le chart')
|
| 74 |
+
artifact=storage.path('meshes',uuid.uuid4().hex+'.npz')
|
| 75 |
+
np.savez_compressed(artifact,xyz=xyz.astype('float32'),uv=uv,
|
| 76 |
+
metadata=json.dumps({'chunk':str(chunk_path),'request':metadata['request'],
|
| 77 |
+
'surface_kind':kind,'quality':quality}))
|
| 78 |
+
return {'path':str(artifact),'quality':quality,'surface_kind':kind}
|
src/worker.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Entrée de calcul séparée du serveur UNIQUE ; requête et résultat sur /data."""
|
| 2 |
+
import json
|
| 3 |
+
import os
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
import sys
|
| 6 |
+
import traceback
|
| 7 |
+
|
| 8 |
+
from .storage import Storage
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def execute(storage, action, payload):
|
| 12 |
+
if action=='load':
|
| 13 |
+
from .loader import Loader
|
| 14 |
+
return Loader(storage).load(payload)
|
| 15 |
+
if action=='scan':
|
| 16 |
+
from .loader import read_chunk
|
| 17 |
+
from .scanner import scan
|
| 18 |
+
return scan(read_chunk(storage,payload['chunk'])[0],payload.get('tile',16))
|
| 19 |
+
if action=='unwrap':
|
| 20 |
+
from .unwrapper import unwrap
|
| 21 |
+
return unwrap(storage,payload['chunk'],payload.get('surface') or None)
|
| 22 |
+
if action=='train':
|
| 23 |
+
from .trainer import train
|
| 24 |
+
return train(storage,payload['chunk'],payload.get('steps',4),payload.get('mode','reconstruction'),
|
| 25 |
+
payload.get('mask') or None,payload.get('checkpoint') or None,payload.get('device','cpu'))
|
| 26 |
+
if action=='predict':
|
| 27 |
+
from .trainer import predict
|
| 28 |
+
return predict(storage,payload['chunk'],payload['checkpoint'])
|
| 29 |
+
if action=='export':
|
| 30 |
+
from .exporter import export
|
| 31 |
+
return export(storage,payload['mesh'],payload.get('prediction') or None,payload.get('pixels',1024))
|
| 32 |
+
raise ValueError('action inconnue')
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def main():
|
| 36 |
+
storage=Storage()
|
| 37 |
+
storage.initialize()
|
| 38 |
+
request=storage.checked(sys.argv[1],'chunks')
|
| 39 |
+
data=json.loads(request.read_text())
|
| 40 |
+
try:
|
| 41 |
+
if hasattr(os,'nice'):os.nice(10)
|
| 42 |
+
result={'ok':True,'result':execute(storage,data['action'],data['payload'])}
|
| 43 |
+
except Exception as exc:
|
| 44 |
+
result={'ok':False,'error':type(exc).__name__+': '+str(exc)[:1000]}
|
| 45 |
+
traceback.print_exc()
|
| 46 |
+
storage.json('chunks',request.stem+'.result.json',result)
|
| 47 |
+
return 0 if result['ok'] else 1
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
if __name__=='__main__':raise SystemExit(main())
|
tests/test_vesuvius.py
ADDED
|
@@ -0,0 +1,234 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
import json
|
| 3 |
+
import math
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
import subprocess
|
| 6 |
+
import sys
|
| 7 |
+
import time
|
| 8 |
+
import zipfile
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
import pytest
|
| 12 |
+
|
| 13 |
+
from src.storage import Storage
|
| 14 |
+
from src.loader import Loader,read_chunk,request_spec
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@pytest.fixture
|
| 18 |
+
def storage(tmp_path):
|
| 19 |
+
s=Storage(tmp_path/'data',require_mount=False);s.initialize();return s
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def spec(origin=(0,0,0),shape=(8,8,8)):
|
| 23 |
+
return request_spec('s3://vesuvius-challenge-open-data/testing.zarr','0',list(origin),list(shape),7.91)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def chunk(storage,origin=(0,0,0)):
|
| 27 |
+
import zarr
|
| 28 |
+
data=np.indices((24,24,24)).sum(0).astype('uint16')
|
| 29 |
+
array=zarr.array(data,chunks=(8,8,8))
|
| 30 |
+
return Loader(storage,opener=lambda s:array).load(spec(origin))['path']
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def test_missing_data_mount_fails_closed(tmp_path):
|
| 34 |
+
with pytest.raises(ValueError):Storage(tmp_path)
|
| 35 |
+
if not Path('/data').is_mount():
|
| 36 |
+
with pytest.raises(RuntimeError,match='non monté'):Storage().initialize()
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def test_storage_blocks_paths_and_symlink_escape(storage,tmp_path):
|
| 40 |
+
with pytest.raises(ValueError):storage.path('chunks','../private')
|
| 41 |
+
with pytest.raises(ValueError):storage.checked(tmp_path/'elsewhere','weights')
|
| 42 |
+
outside=tmp_path/'outside';outside.mkdir()
|
| 43 |
+
link=storage.folder('chunks')/'link';link.symlink_to(outside,target_is_directory=True)
|
| 44 |
+
with pytest.raises(ValueError):storage.checked(link/'x.npy','chunks')
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
@pytest.mark.parametrize('uri',['https://evil.example/v.zarr','s3://private/v.zarr',
|
| 48 |
+
's3://vesuvius-challenge-open-data/a/../v.zarr','s3://vesuvius-challenge-open-data/v.zarr?secret=x'])
|
| 49 |
+
def test_loader_rejects_nonpublic_or_ambiguous_destinations(uri):
|
| 50 |
+
with pytest.raises(ValueError):request_spec(uri,'0',[0,0,0],[8,8,8],7.91)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def test_loader_cache_exact_roi_and_offline_fallback(storage):
|
| 54 |
+
import zarr
|
| 55 |
+
array=zarr.array(np.arange(16**3).reshape(16,16,16).astype('uint16'),chunks=(8,8,8))
|
| 56 |
+
loader=Loader(storage,opener=lambda s:array)
|
| 57 |
+
first=loader.load(spec());data,meta=read_chunk(storage,first['path'])
|
| 58 |
+
assert data.shape==(8,8,8) and not meta['cache_fallback']
|
| 59 |
+
def offline(s):raise OSError('offline')
|
| 60 |
+
fallback=Loader(storage,opener=offline).load(spec())
|
| 61 |
+
assert fallback['metadata']['cache_fallback']
|
| 62 |
+
with pytest.raises(RuntimeError,match='aucun cache exact'):
|
| 63 |
+
Loader(storage,opener=offline).load(spec(origin=(8,0,0)))
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def test_roi_and_huge_remote_chunks_are_bounded(storage):
|
| 67 |
+
with pytest.raises(ValueError):spec(shape=(65,8,8))
|
| 68 |
+
class Huge:
|
| 69 |
+
shape=(100,100,100);chunks=(1024,1024,1024);dtype=np.dtype('uint16')
|
| 70 |
+
def __getitem__(self,x):pytest.fail('huge chunk was materialized')
|
| 71 |
+
with pytest.raises(ValueError,match='volumineux'):Loader(storage,opener=lambda s:Huge()).load(spec())
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def test_async_loading_is_supported(storage):
|
| 75 |
+
import zarr
|
| 76 |
+
array=zarr.array(np.ones((8,8,8),dtype='uint8'),chunks=(4,4,4))
|
| 77 |
+
result=asyncio.run(Loader(storage,opener=lambda s:array).load_async(spec()))
|
| 78 |
+
assert result['shape']==[8,8,8]
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def test_fft_top_one_percent_and_constant_data():
|
| 82 |
+
from src.scanner import scan
|
| 83 |
+
assert scan(np.ones((4,16,16)),8)['selected']==[]
|
| 84 |
+
y,x=np.indices((32,32))
|
| 85 |
+
volume=np.stack([np.sin(x*2*np.pi/4)]*4)
|
| 86 |
+
result=scan(volume,8)
|
| 87 |
+
assert len(result['selected'])==math.ceil(result['candidates']*.01)
|
| 88 |
+
assert not result['ink_detected'] and result['selected'][0]['score']>0.8
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def test_flatten_plane_preserves_metric_and_has_size_guard():
|
| 92 |
+
from src.unwrapper import flatten
|
| 93 |
+
y,x=np.indices((5,6));xyz=np.stack([x,y,np.ones_like(x)],axis=-1)
|
| 94 |
+
uv,report=flatten(xyz)
|
| 95 |
+
assert uv.shape==(5,6,2) and report['max_relative_stretch']<1e-5
|
| 96 |
+
assert not report['folds'] and not report['isometry_guaranteed']
|
| 97 |
+
with pytest.raises(ValueError):flatten(np.zeros((65,5,3)))
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def test_cpu_hybrid_training_disjoint_prediction_and_overlap_guard(storage):
|
| 101 |
+
from src.trainer import train,predict
|
| 102 |
+
first=chunk(storage)
|
| 103 |
+
second=chunk(storage,origin=(8,0,0))
|
| 104 |
+
trained=train(storage,first,steps=1,device='cpu')
|
| 105 |
+
assert trained['report']['precision']=='float32' and trained['report']['initialization']=='random'
|
| 106 |
+
assert len(trained['report']['loss'])==1 and math.isfinite(trained['report']['loss'][0])
|
| 107 |
+
with pytest.raises(ValueError,match='recouvrement'):predict(storage,first,trained['path'])
|
| 108 |
+
predicted=predict(storage,second,trained['path'])
|
| 109 |
+
assert Path(predicted['path']).is_file() and not predicted['held_out_evaluated']
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def test_supervised_mode_needs_real_labels(storage):
|
| 113 |
+
from src.trainer import train
|
| 114 |
+
first=chunk(storage)
|
| 115 |
+
with pytest.raises(ValueError,match='annoté'):train(storage,first,steps=1,mode='ink_supervised')
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def test_research_export_has_matching_xyz_and_provenance(storage):
|
| 119 |
+
from src.unwrapper import unwrap
|
| 120 |
+
from src.exporter import export
|
| 121 |
+
import tifffile
|
| 122 |
+
first=chunk(storage)
|
| 123 |
+
xyz_path=storage.path('meshes','plane.npy')
|
| 124 |
+
y,x=np.indices((8,8),dtype='float32');np.save(xyz_path,np.stack([x,y,np.ones_like(x)*3],axis=-1))
|
| 125 |
+
mesh=unwrap(storage,first,str(xyz_path))
|
| 126 |
+
result=export(storage,mesh['path'],pixels=64)
|
| 127 |
+
assert not result['metadata']['submission_ready']
|
| 128 |
+
assert result['metadata']['render_kind']=='CT_density_not_ink'
|
| 129 |
+
with zipfile.ZipFile(result['path']) as z:
|
| 130 |
+
names=z.namelist()
|
| 131 |
+
assert 'metadata.json' in names and any(n.endswith('/x.tif') for n in names)
|
| 132 |
+
assert any(n.endswith('.png') for n in names)
|
| 133 |
+
folder=Path(result['path']).with_suffix('')
|
| 134 |
+
tiff=next(folder.rglob('x.tif'))
|
| 135 |
+
assert tifffile.imread(tiff).shape==(64,64)
|
| 136 |
+
meta=json.loads((tiff.parent/'meta.json').read_text())
|
| 137 |
+
assert meta['type']=='seg' and meta['scale'][0]>1 and len(meta['bbox'])==2
|
| 138 |
+
assert (tiff.parent/'mask.tif').is_file()
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def test_importing_controller_does_not_load_torch_or_touch_storage(tmp_path):
|
| 142 |
+
code='import sys; from src.controller import Controller; c=Controller(); assert "torch" not in sys.modules; assert c.status()["status"]=="idle"'
|
| 143 |
+
result=subprocess.run([sys.executable,'-c',code],capture_output=True,text=True)
|
| 144 |
+
assert result.returncode==0,result.stderr
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def test_controller_reserves_no_gpu_when_unique_is_busy(storage):
|
| 148 |
+
from src.controller import Controller
|
| 149 |
+
c=Controller(storage,busy=lambda:'mission active')
|
| 150 |
+
with pytest.raises(RuntimeError,match='prioritaire'):
|
| 151 |
+
c.start('train',{'device':'cuda'})
|
| 152 |
+
assert not c.gpu_active
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def test_gradio_tab_constructs_with_no_data_mount():
|
| 156 |
+
import gradio as gr
|
| 157 |
+
from src.controller import Controller
|
| 158 |
+
from src.ui import build_tab
|
| 159 |
+
with gr.Blocks(analytics_enabled=False) as app:build_tab(Controller(),lambda k:False)
|
| 160 |
+
config=app.get_config_file()
|
| 161 |
+
assert any(c.get('props',{}).get('label')=='Laboratoire Vesuvius 3D' for c in config['components'])
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def test_gpu_job_is_preempted_and_resource_context_released(storage,monkeypatch):
|
| 165 |
+
import contextlib
|
| 166 |
+
from src.controller import Controller
|
| 167 |
+
import src.controller as module
|
| 168 |
+
released=[];processes=[]
|
| 169 |
+
class Process:
|
| 170 |
+
def __init__(self,*a,**kw):self.done=False;self.terminated=False;processes.append(self)
|
| 171 |
+
def poll(self):return 0 if self.done else None
|
| 172 |
+
def terminate(self):self.terminated=True;self.done=True
|
| 173 |
+
def wait(self,timeout=None):return 0
|
| 174 |
+
def kill(self):self.done=True
|
| 175 |
+
@contextlib.contextmanager
|
| 176 |
+
def resource():
|
| 177 |
+
try:yield
|
| 178 |
+
finally:released.append(True)
|
| 179 |
+
monkeypatch.setattr(module.subprocess,'Popen',Process)
|
| 180 |
+
c=Controller(storage,busy=lambda:None,preempt=lambda:True,gpu_context=resource,concurrent_uses=lambda:1)
|
| 181 |
+
c.start('train',{'device':'cuda'})
|
| 182 |
+
until=time.monotonic()+3
|
| 183 |
+
while c.status()['status']=='running' and time.monotonic()<until:time.sleep(.01)
|
| 184 |
+
assert c.status()['status']=='error' and not c.gpu_active
|
| 185 |
+
assert processes[0].terminated and released
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def test_gradio_isolated_asgi_config_and_authentication():
|
| 189 |
+
import gradio as gr
|
| 190 |
+
from fastapi import FastAPI
|
| 191 |
+
from fastapi.testclient import TestClient
|
| 192 |
+
from src.controller import Controller
|
| 193 |
+
from src.ui import build_tab
|
| 194 |
+
with gr.Blocks(analytics_enabled=False) as tab:build_tab(Controller(),lambda key:key=='admin')
|
| 195 |
+
app=gr.mount_gradio_app(FastAPI(),tab,path='/lab')
|
| 196 |
+
with TestClient(app) as client:
|
| 197 |
+
response=client.get('/lab/config')
|
| 198 |
+
assert response.status_code==200
|
| 199 |
+
assert any(c.get('type')=='plot' for c in response.json()['components'])
|
| 200 |
+
action=next(f.fn for f in tab.fns.values() if f.fn and f.fn.__name__=='load')
|
| 201 |
+
assert action('invalid','s3://irrelevant','0','0,0,0',8,7.91)['error']=='clé d’administration requise'
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def test_lab_gpu_wait_guard_and_original_ui_are_preserved():
|
| 205 |
+
import ast
|
| 206 |
+
app=Path('app.py').read_text()
|
| 207 |
+
tree=ast.parse(app)
|
| 208 |
+
wait=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='mission_gpu_reason')
|
| 209 |
+
assert 'gpu_active' in ast.unparse(wait)
|
| 210 |
+
for label in ('Pilotage','Données','Vérifier du code','Agents'):
|
| 211 |
+
assert f'with gr.Tab("{label}"):' in app
|
| 212 |
+
assert 'candidate_observer = REMOTE_EXECUTOR.check' in app
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def test_nonmatching_prediction_cannot_be_exported(storage):
|
| 216 |
+
from src.unwrapper import unwrap
|
| 217 |
+
from src.exporter import export
|
| 218 |
+
first=chunk(storage)
|
| 219 |
+
xyz_path=storage.path('meshes','plane.npy')
|
| 220 |
+
y,x=np.indices((8,8),dtype='float32');np.save(xyz_path,np.stack([x,y,np.ones_like(x)*3],axis=-1))
|
| 221 |
+
mesh=unwrap(storage,first,str(xyz_path))
|
| 222 |
+
prediction=storage.path('chunks','invalid.prediction.npz')
|
| 223 |
+
np.savez_compressed(prediction,prediction=np.zeros((8,8)),metadata=json.dumps({'request':spec(origin=(8,0,0))}))
|
| 224 |
+
with pytest.raises(ValueError,match='correspondent'):export(storage,mesh['path'],str(prediction),pixels=64)
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
def test_worker_scan_runs_in_an_isolated_process(storage):
|
| 228 |
+
first=chunk(storage)
|
| 229 |
+
code=('import json;from src.storage import Storage;from src.worker import execute;'
|
| 230 |
+
f's=Storage({str(storage.root)!r},require_mount=False);'
|
| 231 |
+
f'print(json.dumps(execute(s,"scan",{{"chunk":{first!r},"tile":4}})))')
|
| 232 |
+
result=subprocess.run([sys.executable,'-c',code],capture_output=True,text=True,timeout=15)
|
| 233 |
+
assert result.returncode==0,result.stderr
|
| 234 |
+
assert json.loads(result.stdout)['method']=='FFT_anisotropy'
|
unique_ext/__init__.py
CHANGED
|
@@ -13,7 +13,7 @@ from __future__ import annotations
|
|
| 13 |
import os
|
| 14 |
|
| 15 |
API_VERSION = 1 # contrat entre le job d'entraînement et ce paquet : le job refuse une autre valeur
|
| 16 |
-
__version__ = "4.
|
| 17 |
|
| 18 |
|
| 19 |
def enabled(name: str) -> bool:
|
|
|
|
| 13 |
import os
|
| 14 |
|
| 15 |
API_VERSION = 1 # contrat entre le job d'entraînement et ce paquet : le job refuse une autre valeur
|
| 16 |
+
__version__ = "4.14.0"
|
| 17 |
|
| 18 |
|
| 19 |
def enabled(name: str) -> bool:
|