Spaces:
Running
Running
File size: 7,803 Bytes
079985f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 | """The open data's traced segments, for Fiber Checker: which scrolls have segments, on which scans, and where.
Everything comes from the Vesuvius Challenge open data's metadata.json: each segment's tifxyz meshes transformed onto
each scan ("tifxyz-transformed", one per target volume) and each scan's voxel size. Nothing is copied or rehosted;
the Space reads x.tif, y.tif, z.tif and the CT chunks it needs straight from the bucket.
"""
import gzip
import json
import math
import os
import tempfile
import threading
import time
import urllib.error
import urllib.request
from pathlib import Path
BUCKET = "https://vesuvius-challenge-open-data.s3.us-east-1.amazonaws.com"
METADATA = BUCKET + "/metadata.json"
USER_AGENT = "fiber-checker-space (huggingface.co/spaces/SharpMeow/fiber-checker)"
# Scans this Space never reads: PHercParis4 is kept out of every public tool of ours (Chase's rule).
REFUSED_SAMPLES = {"phercparis4"}
# Scrolls on a prize list (scrollprize.org/prizes, snapshot of 11 Oct 2026, kit/data/prizes-2026-10-11.json in
# ChaseHendrick/Scrolls). They can be checked like any other scroll; the examples and guide pictures never use them.
PRIZE_SAMPLES = {"PHerc0125", "PHerc0175A", "PHerc0175B", "PHerc0191", "PHerc0211", "PHerc0257", "PHerc0268",
"PHerc0306B", "PHerc0343", "PHerc0358", "PHerc0483A", "PHerc0483B", "PHerc0490A", "PHerc0490B",
"PHerc0800", "PHerc0813", "PHerc0826", "PHerc0846A", "PHerc0846B", "PHerc1203", "PHerc1218",
"PHerc1447", "PHerc1545", "PHercParis4"}
TARGET_UM = 9.0 # the free judge was trained and tested on renders of about 9 um voxels
TESTED_UM = (2.1, 2.5) # level 2 of the 2.2 to 2.4 um scans: the setting both held-out tests used
CACHE = Path(os.environ.get("FIBERCHECK_CACHE") or Path(tempfile.gettempdir()) / "fiber-checker")
class CatalogError(Exception):
"""A refusal or failure shown to the visitor as is."""
def http_get(url, timeout=60, tries=4):
"""Bytes at `url`, or None for a missing object."""
err = None
for i in range(tries):
try:
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
with urllib.request.urlopen(req, timeout=timeout) as r:
return r.read()
except urllib.error.HTTPError as exc:
if exc.code in (403, 404):
return None
err = exc
except (urllib.error.URLError, OSError) as exc:
err = exc
time.sleep(1.5 * (i + 1))
raise CatalogError(f"could not read {url}: {err}")
def level_for(um):
"""The CT level whose voxels are nearest 9 um (2 for 2.2 to 2.4 um scans, 0 for 7.9 to 9.4 um)."""
return max(0, int(round(math.log2(TARGET_UM / float(um))))) if um else 0
_CAT = {"at": 0.0, "rows": None}
_LOCK = threading.Lock()
def catalog(refresh_s=6 * 3600):
"""{sample: [segment row, ...]}: one row per segment and scan it was transformed onto."""
with _LOCK:
if _CAT["rows"] is not None and time.time() - _CAT["at"] < refresh_s:
return _CAT["rows"]
raw = http_get(METADATA)
if raw is None:
raise CatalogError("the open data's metadata.json is missing")
if raw[:2] == b"\x1f\x8b":
raw = gzip.decompress(raw)
meta = json.loads(raw)
rows, scans = {}, {}
for sid, s in sorted((meta.get("samples") or {}).items()):
if sid.lower() in REFUSED_SAMPLES:
continue
vols = s.get("volumes") or {}
for vid, v in vols.items():
um = (v.get("properties") or {}).get("pixel_size_um")
if um and v.get("long_id"):
scans[vid] = {"sample": sid, "volume": vid, "voxel_um": float(um), "level": level_for(um),
"tested": TESTED_UM[0] <= float(um) <= TESTED_UM[1], "prize": sid in PRIZE_SAMPLES,
"ct": f"{BUCKET}/{sid}/volumes/{v['long_id'].rstrip('/')}/"}
for gid, g in sorted((s.get("segments") or {}).items()):
for d in g.get("data") or []:
if d.get("type") != "tifxyz-transformed":
continue
vid = (d.get("parameters") or {}).get("target_volume")
v = vols.get(vid) or {}
um = (v.get("properties") or {}).get("pixel_size_um")
path = ((d.get("origins") or [{}])[0]).get("path")
if not (vid and um and path and v.get("long_id")):
continue
rows.setdefault(sid, []).append({
"sample": sid, "segment": gid, "name": g.get("long_id") or gid, "volume": vid,
"voxel_um": float(um), "level": level_for(um),
"tested": TESTED_UM[0] <= float(um) <= TESTED_UM[1],
"mesh": f"{BUCKET}/{path.rstrip('/')}/",
"ct": f"{BUCKET}/{sid}/volumes/{v['long_id'].rstrip('/')}/",
"prize": sid in PRIZE_SAMPLES})
_CAT.update(at=time.time(), rows=rows, scans=scans)
return rows
def scan(vid):
"""One CT volume of the open data by its 14-digit id (for a visitor's own mesh)."""
vid = str(vid or "").strip().rstrip("/").split("/")[-1][:14]
catalog()
if vid in _CAT["scans"]:
return _CAT["scans"][vid]
raise CatalogError(f"{vid!r} is not a scan this Space reads; give the 14-digit volume id from the open data, "
"such as 20250526151718 (PHercParis4 is never read)")
def samples():
return sorted(catalog())
def segments(sample):
rows = catalog().get(sample)
if not rows:
raise CatalogError(f"{sample!r} has no traced segments in the open data")
return rows
def find(sample, segment, volume=None):
"""The row for one segment on one scan: the scan given, or the tested setting's scan, or the first."""
rows = [r for r in segments(sample) if segment in (r["segment"], r["name"])]
if not rows:
raise CatalogError(f"{segment!r} is not a traced segment of {sample}")
if volume:
rows_v = [r for r in rows if r["volume"] == str(volume)]
if not rows_v:
raise CatalogError(f"{segment} has no mesh on scan {volume}; it has {', '.join(r['volume'] for r in rows)}")
return rows_v[0]
return sorted(rows, key=lambda r: (not r["tested"], abs(r["voxel_um"] * 2 ** r["level"] - TARGET_UM)))[0]
def fetch_mesh(row):
"""x.tif, y.tif, z.tif (and meta.json) of a segment's mesh, cached by its bucket path."""
folder = CACHE / "meshes" / row["mesh"].split("://")[-1].strip("/").replace("/", "_")
if all((folder / f).is_file() for f in ("x.tif", "y.tif", "z.tif")):
return folder
folder.mkdir(parents=True, exist_ok=True)
for f in ("x.tif", "y.tif", "z.tif", "meta.json"):
data = http_get(row["mesh"] + f, timeout=120)
if data is None:
if f == "meta.json":
continue
raise CatalogError(f"{row['mesh']}{f} is missing in the open data")
tmp = folder / (f + ".part")
tmp.write_bytes(data)
tmp.replace(folder / f)
return folder
def prune(folder, keep_bytes):
"""Delete the oldest files under `folder` until it holds at most `keep_bytes` (the CT chunk cache)."""
files = [p for p in Path(folder).rglob("*") if p.is_file()]
total = sum(p.stat().st_size for p in files)
if total <= keep_bytes:
return 0
freed = 0
for p in sorted(files, key=lambda q: q.stat().st_atime):
size = p.stat().st_size
p.unlink(missing_ok=True)
freed += size
if total - freed <= keep_bytes:
break
return freed
|