Download run_gpu_full.py from Nabidnur/AIP-AD: direct link, hf CLI and curl.
- Browser
- Download file 9.61 kB
-
https://huggingface.co/Nabidnur/AIP-AD/resolve/main/run_gpu_full.py
- Command line
-
hf download hf://Nabidnur/AIP-AD/run_gpu_full.py
-
curl -L -o run_gpu_full.py https://huggingface.co/Nabidnur/AIP-AD/resolve/main/run_gpu_full.py
9.61 kB
| #!/usr/bin/env python3 | |
| """GPU runner: full-scale AIP-AD experiments on Real-IAD v1 and Real-IAD D3. | |
| Intended for the arc-3 box (RTX Pro 6000 Blackwell 96GB, offline) or a Kaggle | |
| GPU kernel. Downloads the official gated datasets from Hugging Face with | |
| HF_TOKEN, then runs the same protocol as the local pipeline but at full scale: | |
| all categories, all test samples, wide_resnet50_2 backbone. | |
| Examples | |
| HF_TOKEN=hf_xxx python3 run_gpu_full.py --dataset realiad --backbone wide_resnet50_2 | |
| HF_TOKEN=hf_xxx python3 run_gpu_full.py --dataset d3 --categories power_jack fuse_holder | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import os | |
| import sys | |
| import zipfile | |
| import numpy as np | |
| HERE = os.path.dirname(os.path.abspath(__file__)) | |
| sys.path.insert(0, HERE) | |
| DATA = os.environ.get("AIPAD_DATA", "/tmp/aipad_data") | |
| RESULTS = os.path.join(HERE, "results") | |
| REALIAD_REPO = "Real-IAD/Real-IAD" | |
| D3_REPO = "Real-IAD/Real-IAD_D3" | |
| REALIAD_JSON_FALLBACK = "https://huggingface.co/datasets/thuong1504/real-iad/resolve/main/realiad_jsons" # not used; official repo has realiad_jsons.zip | |
| def bootstrap_realiad(categories, resolution="realiad_256"): | |
| from huggingface_hub import hf_hub_download | |
| root = os.path.join(DATA, "realiad") | |
| os.makedirs(root, exist_ok=True) | |
| # annotations | |
| jz = os.path.join(root, "realiad_jsons.zip") | |
| if not os.path.isdir(os.path.join(root, "realiad_jsons")): | |
| p = hf_hub_download(REALIAD_REPO, "realiad_jsons.zip", repo_type="dataset", | |
| token=os.environ.get("HF_TOKEN"), local_dir=root) | |
| zipfile.ZipFile(p).extractall(root) | |
| json_root = os.path.join(root, "realiad_jsons") | |
| img_root = os.path.join(root, resolution) | |
| for cat in categories: | |
| zname = f"{resolution}/{cat}.zip" | |
| if not os.path.isdir(os.path.join(img_root, cat)): | |
| p = hf_hub_download(REALIAD_REPO, zname, repo_type="dataset", | |
| token=os.environ.get("HF_TOKEN"), local_dir=root) | |
| zipfile.ZipFile(p).extractall(os.path.join(root, resolution)) | |
| return img_root, json_root | |
| def bootstrap_d3(categories): | |
| from huggingface_hub import hf_hub_download | |
| root = os.path.join(DATA, "d3") | |
| os.makedirs(root, exist_ok=True) | |
| jz = os.path.join(root, "realiad_d3_jsons.zip") | |
| if not os.path.isdir(os.path.join(root, "realiad_d3_jsons")): | |
| p = hf_hub_download(D3_REPO, "realiad_d3_jsons.zip", repo_type="dataset", | |
| token=os.environ.get("HF_TOKEN"), local_dir=root) | |
| zipfile.ZipFile(p).extractall(root) | |
| json_root = os.path.join(root, "realiad_d3_jsons") | |
| img_root = os.path.join(root, "realiad_d3_raw") | |
| for cat in categories: | |
| if not os.path.isdir(os.path.join(img_root, cat)): | |
| p = hf_hub_download(D3_REPO, f"realiad_d3_raw/{cat}.zip", repo_type="dataset", | |
| token=os.environ.get("HF_TOKEN"), local_dir=root) | |
| zipfile.ZipFile(p).extractall(img_root) | |
| return img_root, json_root | |
| # ---------------- Real-IAD v1 ---------------- | |
| def run_realiad(categories, backbone, device, n_ok, n_ng, n_train, seed): | |
| import torch | |
| torch.set_num_threads(os.cpu_count() or 8) | |
| from src.data import load_samples, subsample_samples, split_calibration | |
| from src.scorer import PatchScorer | |
| from run_experiments import stage_score | |
| # reuse the local stage_score by pointing module constants at the bootstrapped copy | |
| import run_experiments as RE | |
| img_root, json_root = bootstrap_realiad(categories) | |
| RE.IMG_ROOT, RE.JSON_ROOT = img_root, json_root | |
| RE.RESULTS = RESULTS | |
| os.makedirs(os.path.join(RESULTS, "scores"), exist_ok=True) | |
| stage_score(categories, n_ok, n_ng, n_train, seed, force=False) | |
| # ---------------- Real-IAD D3 ---------------- | |
| def _load_modality_array(path, mod): | |
| """Return HxWx3 uint8 for rgb/ps/xyz/pcd.""" | |
| if mod in ("rgb", "ps"): | |
| from PIL import Image | |
| return np.asarray(Image.open(path).convert("RGB").resize((256, 256))) | |
| if mod == "xyz": | |
| import tifffile | |
| a = tifffile.imread(path).astype(np.float32) | |
| a = a[..., :3] if a.ndim == 3 else a[..., None] | |
| lo, hi = np.percentile(a, 1), np.percentile(a, 99) | |
| a = np.clip((a - lo) / (hi - lo + 1e-9), 0, 1) | |
| return (a * 255).astype(np.uint8) | |
| if mod == "pcd": | |
| import open3d as o3d | |
| pcd = o3d.io.read_point_cloud(path) | |
| pts = np.asarray(pcd.points) | |
| xy = pts[:, :2] | |
| z = pts[:, 2] | |
| lo = xy.min(0) | |
| hi = xy.max(0) | |
| G = 256 | |
| grid = np.full((G, G), np.nan, np.float32) | |
| ix = np.clip(((xy[:, 0] - lo[0]) / (hi[0] - lo[0] + 1e-9) * (G - 1)).astype(int), 0, G - 1) | |
| iy = np.clip(((xy[:, 1] - lo[1]) / (hi[1] - lo[1] + 1e-9) * (G - 1)).astype(int), 0, G - 1) | |
| zlo, zhi = np.percentile(z, 1), np.percentile(z, 99) | |
| zn = np.clip((z - zlo) / (zhi - zlo + 1e-9), 0, 1) | |
| # keep highest surface point per pixel (top-view depth) | |
| flat = iy * G + ix | |
| order = np.argsort(zn) | |
| gridflat = grid.reshape(-1) | |
| gridflat[flat[order]] = zn[order] | |
| grid = np.nan_to_num(grid, nan=0.0) | |
| return (np.stack([grid] * 3, -1) * 255).astype(np.uint8) | |
| raise ValueError(mod) | |
| def run_d3(categories, backbone, device, seed): | |
| import torch | |
| from src.data_d3 import load_d3_samples, MODALITIES | |
| from src.scorer import PatchScorer | |
| from src.data import split_calibration | |
| import json | |
| img_root, json_root = bootstrap_d3(categories) | |
| os.makedirs(os.path.join(RESULTS, "scores"), exist_ok=True) | |
| for cat in categories: | |
| out = os.path.join(RESULTS, "scores", f"d3_{cat}.npz") | |
| if os.path.exists(out + ".eval.npz"): | |
| print(f"[skip] d3 {cat}") | |
| continue | |
| train = load_d3_samples(cat, img_root, json_root, "train", modalities=MODALITIES) | |
| test = load_d3_samples(cat, img_root, json_root, "test", modalities=MODALITIES) | |
| val = load_d3_samples(cat, img_root, json_root, "validation", modalities=MODALITIES) | |
| # per-modality scorers = per-modality agents | |
| scorers = {} | |
| for m in MODALITIES: | |
| sc = PatchScorer(device=device, seed=seed, backbone=backbone) | |
| paths = [s.paths[m] for s in train if m in s.paths] | |
| sc.build_bank(paths, patches_per_image=120, bank_size=4096) | |
| scorers[m] = sc | |
| calib, ev = split_calibration(test, calib_frac=0.25, seed=seed) | |
| rows = [] | |
| for split_name, samples in (("calib", calib), ("eval", ev)): | |
| n = len(samples) | |
| vs = np.full((n, len(MODALITIES)), np.nan, np.float32) | |
| vl = np.zeros((n, len(MODALITIES)), np.int8) | |
| for si, s in enumerate(samples): | |
| for mi, m in enumerate(MODALITIES): | |
| if m not in s.paths: | |
| continue | |
| arr = _load_modality_array(s.paths[m], m) | |
| sc_arr, _ = scorers[m].score_arrays([arr]) | |
| vs[si, mi] = sc_arr[0] | |
| vl[si, mi] = s.label | |
| np.savez_compressed(out + f".{split_name}.npz", view_scores=vs, view_labels=vl) | |
| for s in samples: | |
| rows.append(dict(split=split_name, category=cat, sample_id=s.sample_id, | |
| label=int(s.label), anomaly_class=s.anomaly_class)) | |
| with open(out + ".meta.json", "w") as f: | |
| json.dump({"rows": rows, "modalities": list(MODALITIES), "seed": seed}, f) | |
| print(f"[done] d3 {cat}") | |
| if __name__ == "__main__": | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--dataset", choices=["realiad", "d3"], default="realiad") | |
| ap.add_argument("--categories", nargs="+", default=None, | |
| help="default: all 30 realiad / first N d3") | |
| ap.add_argument("--backbone", default="wide_resnet50_2", | |
| choices=["resnet18", "wide_resnet50_2"]) | |
| ap.add_argument("--device", default="cuda" if os.environ.get("CUDA_VISIBLE_DEVICES", "0") else "cpu") | |
| ap.add_argument("--resolution", default="realiad_256", choices=["realiad_256", "realiad_1024"]) | |
| ap.add_argument("--n-ok", type=int, default=10 ** 7, help="samples per category (realiad)") | |
| ap.add_argument("--n-ng", type=int, default=10 ** 7) | |
| ap.add_argument("--n-train", type=int, default=10 ** 7) | |
| ap.add_argument("--seed", type=int, default=0) | |
| args = ap.parse_args() | |
| if args.dataset == "realiad": | |
| cats = args.categories or ["audiojack", "bottle_cap", "button_battery", "end_cap", "eraser", | |
| "fire_hood", "mint", "mounts", "pcb", "phone_battery", "plastic_nut", | |
| "plastic_plug", "porcelain_doll", "regulator", "rolled_strip_base", | |
| "sim_card_set", "switch", "tape", "terminalblock", "toothbrush", "toy", | |
| "toy_brick", "transistor1", "u_block", "usb", "usb_adaptor", "vcpill", | |
| "wooden_beads", "woodstick", "zipper"] | |
| run_realiad(cats, args.backbone, args.device, args.n_ok, args.n_ng, args.n_train, args.seed) | |
| from run_experiments import stage_analyze | |
| stage_analyze(cats, os.path.join(RESULTS, "results_full.json")) | |
| else: | |
| cats = args.categories or ["power_jack", "fuse_holder", "knob_cap"] | |
| run_d3(cats, args.backbone, args.device, args.seed) | |
| from run_experiments import stage_analyze | |
| stage_analyze([f"d3_{c}" for c in cats], os.path.join(RESULTS, "results_d3.json")) | |