Spaces:
Running on Zero
Running on Zero
Download batch.py from seromy/laya-document-batch: direct link, hf CLI and curl.
- Browser
- Download file 5.88 kB
-
https://huggingface.co/spaces/seromy/laya-document-batch/resolve/main/batch.py
- Command line
-
hf download hf://spaces/seromy/laya-document-batch/batch.py
-
curl -L -o batch.py https://huggingface.co/spaces/seromy/laya-document-batch/resolve/main/batch.py
5.88 kB
| """Resumable local folder classification. Originals are never moved or modified.""" | |
| import argparse | |
| import csv | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import time | |
| from classifier import EXTENSIONS, MODEL_ID, POLICY_VERSION, REVISION, classify, load_agent | |
| from PIL import Image | |
| FIELDS = ["filename", "label", "document_probability", "selected_probability", "needs_review", "elapsed_seconds", "error"] | |
| def export_csv(rows, destination): | |
| """Atomically publish an Excel-friendly CSV snapshot.""" | |
| destination = Path(destination) | |
| temporary = destination.with_suffix(".tmp") | |
| with temporary.open("w", encoding="utf-8-sig", newline="") as f: | |
| writer = csv.DictWriter(f, fieldnames=FIELDS, extrasaction="ignore") | |
| writer.writeheader() | |
| for row in rows: | |
| safe = dict(row) | |
| for field in ("filename", "error"): | |
| value = str(safe.get(field, "")) | |
| safe[field] = "'" + value if value.startswith(("=", "+", "-", "@")) else value | |
| writer.writerow(safe) | |
| temporary.replace(destination) | |
| def run(folder, output, threshold=0.5, review_confidence=0.8, limit=None, device=None): | |
| folder, output = Path(folder).resolve(), Path(output).resolve() | |
| output.mkdir(parents=True, exist_ok=True) | |
| source = Path(__file__).resolve().parent | |
| skip_source = not folder.is_relative_to(source) | |
| files = sorted(p for p in folder.rglob("*") if p.is_file() and p.suffix.lower() in EXTENSIONS | |
| and output not in p.parents and (not skip_source or source not in p.parents) | |
| and not any(x.startswith('.') for x in p.relative_to(folder).parts)) | |
| if limit: | |
| files = files[:limit] | |
| settings = {"model": MODEL_ID, "revision": REVISION, "policy": POLICY_VERSION, | |
| "threshold": threshold, "review_confidence": review_confidence, "folder": str(folder)} | |
| config_path = output / "settings.json" | |
| if config_path.exists() and json.loads(config_path.read_text()) != settings: | |
| raise ValueError("Output folder belongs to different settings. Choose a new --output folder.") | |
| config_path.write_text(json.dumps(settings, indent=2)) | |
| journal = output / "predictions.jsonl" | |
| previous = {} | |
| if journal.exists(): | |
| for line in journal.read_text().splitlines(): | |
| try: | |
| row = json.loads(line) | |
| previous[row["filename"]] = row | |
| except (json.JSONDecodeError, KeyError): | |
| continue # A power loss can leave the last journal line incomplete. | |
| model = None | |
| started, processed = time.monotonic(), 0 | |
| rows = [] | |
| # Repair a partial final line before appending any new durable records. | |
| if journal.exists() and journal.stat().st_size: | |
| with journal.open("rb+") as f: | |
| data = f.read() | |
| if not data.endswith(b"\n"): | |
| f.truncate(data.rfind(b"\n") + 1) | |
| with journal.open("a", encoding="utf-8") as log: | |
| for index, path in enumerate(files, 1): | |
| name = path.relative_to(folder).as_posix() | |
| fingerprint = hashlib.sha256(path.read_bytes()).hexdigest() | |
| old = previous.get(name) | |
| if old and old.get("sha256") == fingerprint and not old.get("error"): | |
| rows.append(old) | |
| continue | |
| if model is None: | |
| model = load_agent(device) | |
| try: | |
| with Image.open(path) as image: | |
| result = classify(image, threshold, review_confidence, model) | |
| row = {"filename": name, "sha256": fingerprint, **result, "error": ""} | |
| except Exception as exc: | |
| row = {"filename": name, "sha256": fingerprint, "label": "error", "needs_review": True, | |
| "error": f"{type(exc).__name__}: {exc}"} | |
| log.write(json.dumps(row, ensure_ascii=False) + "\n") | |
| log.flush() | |
| os.fsync(log.fileno()) | |
| rows.append(row) | |
| processed += 1 | |
| if processed % 25 == 0 or index == len(files) or row.get("error"): | |
| export_csv(rows, output / "results.csv") | |
| speed = (time.monotonic() - started) / processed | |
| remaining = len(files) - index | |
| print(f"{index}/{len(files)} | {speed:.2f}s/image | ETA {remaining*speed/60:.1f}min | {name}: {row['label']}", flush=True) | |
| export_csv(rows, output / "results.csv") | |
| summary = {"total": len(rows), "documents": sum(r["label"] == "document" for r in rows), | |
| "non_documents": sum(r["label"] == "non_document" for r in rows), | |
| "errors": sum(r["label"] == "error" for r in rows), | |
| "needs_review": sum(bool(r.get("needs_review")) for r in rows), | |
| "newly_processed": processed, "wall_seconds": round(time.monotonic()-started, 2)} | |
| (output / "summary.json").write_text(json.dumps(summary, indent=2)) | |
| print(json.dumps(summary), flush=True) | |
| return summary | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("folder", type=Path) | |
| parser.add_argument("--output", type=Path, default=Path("document-results")) | |
| parser.add_argument("--threshold", type=float, default=0.5) | |
| parser.add_argument("--review-confidence", type=float, default=0.8) | |
| parser.add_argument("--limit", type=int) | |
| parser.add_argument("--device", choices=["cpu", "mps", "cuda"]) | |
| args = parser.parse_args() | |
| if not 0 < args.threshold < 1 or not 0.5 <= args.review_confidence <= 1: | |
| parser.error("threshold must be between 0 and 1; review confidence between 0.5 and 1") | |
| if args.limit is not None and args.limit < 1: | |
| parser.error("limit must be positive") | |
| run(args.folder, args.output, args.threshold, args.review_confidence, args.limit, args.device) | |