Delete raven/manifest.py

raven/manifest.py

@@ -1,176 +0,0 @@
-from __future__ import annotations
-from collections import Counter
-from pathlib import Path
-import numpy as np
-from PIL import Image, ImageOps
-try:
-    import imagehash
-except Exception:
-    imagehash = None
-from tqdm import tqdm
-from .utils import iter_images, stable_unit, write_jsonl
-from .parquet_io import discover_parquet_files, materialize_parquet
-
-
-def _phash(path: Path, hash_size: int = 12) -> str | None:
-    try:
-        with Image.open(path) as im:
-            im = ImageOps.exif_transpose(im).convert("RGB")
-            if imagehash is not None:
-                return str(imagehash.phash(im, hash_size=hash_size))
-            g = im.convert("L").resize((32, 32), Image.Resampling.LANCZOS)
-            a = np.asarray(g, dtype=np.float32) / 255.0
-            low = np.abs(np.fft.fft2(a))[:hash_size, :hash_size]
-            flat = low.ravel(); med = float(np.median(flat[1:])) if flat.size > 1 else float(flat[0])
-            bits = (flat > med).astype(np.uint8)
-            value = 0
-            for bit in bits: value = (value << 1) | int(bit)
-            width = (len(bits) + 3) // 4
-            return f"{value:0{width}x}"
-    except Exception:
-        return None
-
-
-def _hamming_hex(a: str, b: str) -> int:
-    try: return (int(a, 16) ^ int(b, 16)).bit_count()
-    except Exception: return 9999
-
-
-class _HammingIndex:
-
-    def __init__(self, max_distance:int):
-        self.d=max(0,int(max_distance)); self.values=[]; self.buckets={}
-
-    def _segments(self, value:int, bits:int):
-        parts=max(1,self.d+1); base=bits//parts; rem=bits%parts; shift=0
-        for i in range(parts):
-            width=base+(1 if i<rem else 0); mask=(1<<width)-1
-            yield i,(value>>shift)&mask
-            shift+=width
-
-    def duplicate_or_add(self, ph:str|None)->bool:
-        if not ph or self.d<0: return False
-        try: value=int(ph,16)
-        except Exception: return False
-        bits=max(4,len(ph)*4); cand=set(); segs=list(self._segments(value,bits))
-        for key in segs:
-            cand.update(self.buckets.get(key,()))
-        for idx in cand:
-            if (value^self.values[idx]).bit_count()<=self.d:
-                return True
-        idx=len(self.values); self.values.append(value)
-        for key in segs:
-            self.buckets.setdefault(key,[]).append(idx)
-        return False
-
-def _stats(path: Path) -> dict:
-    try:
-        with Image.open(path) as im:
-            fmt = (im.format or path.suffix.lstrip(".")).upper(); exif = im.getexif(); has_exif = bool(exif)
-            im = ImageOps.exif_transpose(im).convert("RGB"); w, h = im.size
-            tiny = im.copy(); tiny.thumbnail((192, 192)); a = np.asarray(tiny, dtype=np.float32) / 255.0
-            y = a[..., 0] * 0.2126 + a[..., 1] * 0.7152 + a[..., 2] * 0.0722
-            mean = float(y.mean()); std = float(y.std()); dark_fraction = float((y < 0.12).mean())
-            if mean < 0.08: bucket = "extreme-dark"
-            elif mean < 0.20: bucket = "dark"
-            elif mean < 0.38: bucket = "dim"
-            else: bucket = "normal"
-            return {"width":int(w),"height":int(h),"megapixels":float(w*h/1e6),"aspect_ratio":float(w/max(1,h)),"format":fmt,"has_exif":has_exif,"mean_luma":mean,"std_luma":std,"dark_fraction":dark_fraction,"light_bucket":bucket}
-    except Exception:
-        return {"width":None,"height":None,"megapixels":None,"aspect_ratio":None,"format":path.suffix.lstrip(".").upper(),"has_exif":False,"mean_luma":None,"std_luma":None,"dark_fraction":None,"light_bucket":"unknown"}
-
-
-def _normal_descriptors(root_spec:str|Path,label:int) -> list[dict]:
-    root=Path(root_spec).resolve(); out=[]
-    if not root.exists(): return out
-
-    if root.is_file(): return out
-    for p in iter_images(root):
-        try: rel=p.relative_to(root)
-        except Exception: continue
-        source_leaf=rel.parts[0] if len(rel.parts)>1 else ("unknown-camera" if label==0 else "unknown-generator")
-        out.append({"path":str(p.resolve()),"label":int(label),"source_leaf":source_leaf,"group":str(rel.with_suffix("")),"origin_kind":"image","origin_parquet":None,"origin_row":None})
-    return out
-
-
-def _parquet_descriptors(root_spec:str|Path,label:int,cache_root:Path,image_column:str|None,source_column:str|None) -> list[dict]:
-    out=[]
-    for pq in discover_parquet_files(root_spec):
-        default_source=pq.stem
-        rows=materialize_parquet(pq,cache_root,label,default_source,image_column,source_column)
-        out.extend(rows)
-    return out
-
-
-def build_manifest(
-    real_dir: str,
-    ai_dir: str,
-    out: str,
-    holdout_sources: list[str] | None = None,
-    val_fraction: float = 0.15,
-    real_test_fraction: float = 0.15,
-    dedupe_hamming_distance: int = 3,
-    parquet_cache_dir: str | None = None,
-    real_parquet_image_column: str | None = None,
-    real_parquet_source_column: str | None = None,
-    ai_parquet_image_column: str | None = None,
-    ai_parquet_source_column: str | None = None,
-    force_real_source: str | None = None,
-    force_ai_source: str | None = None,
-) -> list[dict]:
-    holdout = {s.strip().lower() for s in (holdout_sources or []) if s.strip()}
-    rows=[]; dedupe_index=_HammingIndex(dedupe_hamming_distance)
-    manifest_parent=Path(out).resolve().parent
-    pq_cache=Path(parquet_cache_dir).resolve() if parquet_cache_dir else (manifest_parent/"cache"/"parquet_materialized")
-    pq_cache.mkdir(parents=True,exist_ok=True)
-
-    def duplicate(ph):
-        if dedupe_hamming_distance < 0:
-            return False
-        return dedupe_index.duplicate_or_add(ph)
-
-    specs=[
-        (0,real_dir,"real",real_parquet_image_column,real_parquet_source_column),
-        (1,ai_dir,"ai",ai_parquet_image_column,ai_parquet_source_column),
-    ]
-    for label,root_spec,kind,img_col,src_col in specs:
-        descriptors=_normal_descriptors(root_spec,label)
-        pq_desc=_parquet_descriptors(root_spec,label,pq_cache/kind,img_col,src_col)
-        descriptors.extend(pq_desc)
-        print(f"[{kind}] normal_images={len(descriptors)-len(pq_desc)} parquet_images={len(pq_desc)}")
-        for d in tqdm(descriptors,desc=f"manifest-{kind}"):
-            p=Path(d["path"]); raw_source_leaf=str(d.get("source_leaf") or ("unknown-camera" if label==0 else "unknown-generator"))
-            forced = force_real_source if label==0 else force_ai_source
-            source_leaf = str(forced).strip() if forced and str(forced).strip() else raw_source_leaf
-            ph=_phash(p)
-            if duplicate(ph): continue
-            group=str(d.get("group") or p.stem)
-            if label==1 and source_leaf.lower() in holdout: split="test"
-            else:
-                r=stable_unit(f"{kind}:{group}")
-                if label==0:
-                    if r<real_test_fraction: split="test"
-                    elif r<real_test_fraction+val_fraction: split="val"
-                    else: split="train"
-                else: split="val" if r<val_fraction else "train"
-            rows.append({
-                "path":str(p.resolve()),"label":int(label),
-                "source":(f"real:{source_leaf}" if label==0 else source_leaf),
-                "raw_source":raw_source_leaf,
-                "generator":source_leaf if label==1 else None,"camera_bucket":source_leaf if label==0 else None,
-                "group":group,"phash":ph,"split":split,
-                "origin_kind":d.get("origin_kind","image"),"origin_parquet":d.get("origin_parquet"),"origin_row":d.get("origin_row"),
-                **_stats(p),
-            })
-    write_jsonl(out,rows); return rows
-
-
-def manifest_summary(rows:list[dict])->str:
-    c=Counter((r.get("split"),int(r.get("label",-1))) for r in rows); src=Counter((r.get("split"),str(r.get("source","unknown"))) for r in rows); origin=Counter((r.get("split"),str(r.get("origin_kind","image"))) for r in rows)
-    lines=["Counts by split / label (0=REAL, 1=AI):"]
-    for k,v in sorted(c.items(),key=lambda kv:str(kv[0])): lines.append(f"  {k}: {v}")
-    lines.append("\nCounts by split / source:")
-    for k,v in sorted(src.items(),key=lambda kv:str(kv[0])): lines.append(f"  {k}: {v}")
-    lines.append("\nCounts by split / storage origin:")
-    for k,v in sorted(origin.items(),key=lambda kv:str(kv[0])): lines.append(f"  {k}: {v}")
-    return "\n".join(lines)