"""Every import of the postkitty engine and every call into it lives in this module. The engine is pinned by commit (POSTKITTY_REF in the Dockerfile) and used through its public Python API, ``postkitty.engine.pipeline``: plain functions, a dict in and a dict out, errors raised. """ from __future__ import annotations import json import os import shutil import zipfile from collections.abc import Callable from dataclasses import asdict from functools import lru_cache from pathlib import Path from postkitty.engine import __version__ as ENGINE_VERSION from postkitty.engine import pipeline as _pipeline from postkitty.engine import platforms as _platforms from postkitty.engine import probe as _probe from postkitty.engine.detect import models as _models from postkitty.engine.text import fit as _text from postkitty.engine.video import captions as _captions __all__ = ["ALIASES", "CAPTIONS", "ENGINE_VERSION", "LAYOUTS", "EngineError"] ALIASES = _pipeline.ALIASES LAYOUTS = _pipeline.LAYOUTS # True once the pinned engine takes a ``captions`` override for a video target. CAPTIONS = "captions" in _pipeline.OVERRIDE_KEYS JOB = "post" # What a job may hand out. HTML is absent on purpose: the review board carries the user's text. _SERVED = {".jpg": "image", ".jpeg": "image", ".png": "image", ".gif": "image", ".webp": "image", ".mp4": "video", ".mov": "video", ".webm": "video", ".txt": "text"} class EngineError(Exception): """The engine refused the input. The message is written for the person who sent it.""" def _call(step: Callable[[dict], dict], args: dict) -> dict: """Run one pipeline step and turn the errors it raises for bad input into ``EngineError``.""" try: return step(args) except (ValueError, KeyError, FileNotFoundError, _probe.ProbeError, _pipeline.Unavailable) as exc: # str() of a KeyError wraps the message in quotes. raise EngineError(str(exc).strip("'\"")) from exc def fetch_model() -> Path: """Download the face model into the engine's cache (the engine pins its checksum).""" return _models.get_yunet_model() @lru_cache(maxsize=1) def video_features() -> dict[str, bool]: """What this image's ffmpeg build can do: burn captions (libass) and tone-map HDR video (zimg).""" from postkitty.engine.video import render # loads OpenCV, so it is imported only when asked return {"captions": render.has_filter("ass"), "hdr": render.has_filter("zscale")} def read_captions(path: Path) -> int: """Parse a caption file with the reader the render uses and return how many captions it holds.""" try: return len(_captions.load_captions(path).cues) except (ValueError, FileNotFoundError) as exc: raise EngineError(str(exc)) from exc @lru_cache(maxsize=1) def platform_names() -> tuple[str, ...]: return tuple(_platforms.platform_names()) @lru_cache(maxsize=1) def target_kinds() -> dict[str, str]: """Target id -> ``image``, ``video`` or ``image_or_video``.""" return {target.id: target.kind for platform in _platforms.load_platforms().values() for target in platform.targets.values()} def layouts_for(kind: str) -> tuple[str, ...]: """The layouts that work on a photo or on a video: speaker needs people talking, seam carves a still.""" return tuple(name for name in LAYOUTS if name != ("speaker" if kind == "image" else "seam")) def planned_targets(kind: str, wanted: list[str]) -> list[str]: """The target ids a scan would plan for one kind of input and the requested names.""" # A target named by its id is selected whatever its kind, and the scan then drops it for lack of a source. return [target.id for target in _platforms.targets_for({kind}, wanted) if target.kind in (kind, "image_or_video")] @lru_cache(maxsize=1) def platforms_doc() -> list[dict]: """Every platform with its text limits and its targets, as plain data.""" def spec(item) -> dict: return {key: value for key, value in asdict(item).items() if value is not None} return [ {"name": platform.name, "label": platform.label, "text": {key: spec(value) for key, value in platform.text.items()}, "targets": [{"id": target.id, "label": target.label, "kind": target.kind, "width": target.width, "height": target.height, "limits": {key: spec(value) for key, value in target.limits.items()}} for target in platform.targets.values()]} for platform in _platforms.load_platforms().values() ] def to_platforms(names: list[str]) -> list[str]: """Platform names for a list of platform names and target ids, in order and without repeats.""" known = platform_names() out: list[str] = [] for name in names: platform = name if name in known else _platforms.platform_for_target(name) if platform not in out: out.append(platform) return out def text_report(text: str, names: list[str], split_threads: bool = True) -> dict[str, dict]: """Count ``text`` with each platform's own rule and split it into a thread where that is allowed.""" labels = {platform["name"]: platform["label"] for platform in platforms_doc()} try: report = _text.check_text(text, names, split_threads) except (ValueError, KeyError) as exc: raise EngineError(str(exc).strip("'\"")) from exc return {name: { "label": labels[name], "ok": item["ok"], "enforcement": "hard" if item["hard"] else "recommendation", **{key: item[key] for key in ("unit", "limit", "count", "overflow", "split", "parts", "hashtags", "mentions", "urls")}, # X shows a whole post, so it has no fold. "fold": None if name == "x" else {"limit": item["fold_limit"], "shown": len(item["fold"]), "text": item["fold"]}, "notes": item["notes"], } for name, item in report.items()} def check_file(path: Path, target: str) -> dict: """Check a file postkitty did not make against one target. Returns pass or fail and the failing checks.""" out = _call(_pipeline.check, {"path": str(path), "target": target}) # The engine answers with a sentence when everything passes, and lists checks it had to skip. found = out["checks"] if isinstance(out["checks"], list) else [] keep = ("id", "status", "hard", "measured", "expected", "message", "source_url", "checked_on") return {"target": out["target"], "pass": bool(out["pass"]), "checks": [{key: check.get(key) for key in keep} for check in found if check["status"] != "skipped"]} def run_job(root: Path, request: dict, progress: Callable[[str, str], None]) -> dict: """Scan the uploads, write the text, render and check every target, then pack the results.""" root = root.resolve() # The engine caches detections by content hash. Inside the job folder they are deleted with the job. os.environ["POSTKITTY_FACES_CACHE"] = str(root / "faces") with _pipeline.scoped(output_dir=str(root)): return _run(root, request, progress) def _run(root: Path, request: dict, progress: Callable[[str, str], None]) -> dict: labels = {item["stored"]: item["name"] for item in request["inputs"]} captions = request.get("captions") def plain(message: str) -> str: """Engine messages name files by their stored path; show the uploader's file names instead.""" message = message.replace(f"{root}/in/", "").replace(f"{root}/", "") for stored, name in (*labels.items(), *([(captions["stored"], captions["name"])] if captions else [])): message = message.replace(stored, name) return message def call(step: Callable, args: dict) -> dict: try: return _call(step, args) except EngineError as exc: raise EngineError(plain(str(exc))) from exc text = request["text"] progress("scan", "Reading the files and finding the people in them.") scan = call(_pipeline.scan, {"inputs": [str(root / "in" / stored) for stored in labels], "text": text, "platforms": request["platforms"], "job": JOB}) planned = [item["id"] for item in scan["plan"]] report: dict[str, dict] = {} if text.strip(): progress("text", "Counting the text for each platform.") names = to_platforms(request["platforms"] or (planned if labels else list(platform_names()))) written = call(_pipeline.write, {"job": JOB, "drafts": dict.fromkeys(names, text)}) report = text_report(text, names) for name, item in report.items(): item["files"] = [_ready_path(path) for path in written["platforms"][name]["files"]] if planned: progress("render", f"Rendering and checking {len(planned)} file{'s' if len(planned) > 1 else ''}.") override = {key: request[key] for key in ("layout", "focus") if request.get(key)} if captions: override["captions"] = str(root / "in" / captions["stored"]) call(_pipeline.make, {"job": JOB, "overrides": dict.fromkeys(planned, override) if override else {}}) progress("pack", "Packing the results.") job_dir = root / JOB manifest = json.loads((job_dir / "manifest.json").read_text(encoding="utf-8")) result = { "ready_dir": f"{JOB}/ready", "inputs": [{"name": item["name"], "kind": entry["kind"], "width": entry["width"], "height": entry["height"], "duration": entry.get("duration"), "people": entry.get("people")} for item, entry in zip(request["inputs"], scan["inputs"], strict=True)], "targets": [_target_row(entry, job_dir) for entry in manifest.get("targets", [])], "text": report, "files": _ready_files(job_dir / "ready"), "notes": [plain(note) for note in scan.get("notes") or []], } _pack(job_dir, root / "download.zip") for leftover in (root / "in", root / "faces", *(path for path in job_dir.iterdir() if path.name != "ready")): if leftover.is_dir(): shutil.rmtree(leftover) else: leftover.unlink(missing_ok=True) return result def _ready_path(job_relative: str | None) -> str | None: """``ready/x/image.jpg`` (relative to the job folder) -> ``x/image.jpg`` (relative to ``ready/``).""" return job_relative.removeprefix("ready/") if job_relative else None def _target_row(entry: dict, job_dir: Path) -> dict: checks = entry.get("checks") or [] layout = entry.get("layout") or {} file = entry.get("file") return { "id": entry["id"], "platform": entry["platform"], "kind": entry["kind"], "status": entry["status"], "file": _ready_path(file), "cover": _ready_path(entry.get("cover")), "bytes": (job_dir / file).stat().st_size if file else None, "width": entry.get("width"), "height": entry.get("height"), "duration": entry.get("duration"), "layout": layout.get("mode"), "note": layout.get("note") or None, "problems": [check["message"] for check in checks if check["status"] == "fail" and check["hard"]], "advisories": [check["message"] for check in checks if check["status"] == "advisory"], "repairs": [f"{repair['check']}: {repair['action']}" for repair in entry.get("repairs") or [] if repair["ok"]], "reason": entry.get("reason"), "captions_shown": (entry.get("captions") or {}).get("shown"), } def _ready_files(ready: Path) -> list[dict]: if not ready.is_dir(): return [] return [{"path": path.relative_to(ready).as_posix(), "platform": path.relative_to(ready).parts[0], "kind": _SERVED[path.suffix.lower()], "bytes": path.stat().st_size} for path in sorted(ready.rglob("*")) if path.is_file() and path.suffix.lower() in _SERVED] def _pack(job_dir: Path, out: Path) -> None: """Zip ``ready/`` with the review board, whose links point into ``ready/``.""" ready = job_dir / "ready" members = sorted(path for path in ready.rglob("*") if path.is_file()) if ready.is_dir() else [] members += [job_dir / name for name in ("board.html", "contact.png") if (job_dir / name).is_file()] with zipfile.ZipFile(out, "w") as archive: for path in members: method = zipfile.ZIP_DEFLATED if path.suffix in {".txt", ".html"} else zipfile.ZIP_STORED archive.write(path, path.relative_to(job_dir).as_posix(), compress_type=method)