ZeroGPU video: auto-purge corrupted HF cache snapshots on tokenizer parse errors

The LTX-Video and CogVideoX-5B pipelines were failing with
'Error parsing line in spiece.model' because the Space's persistent
HF cache held truncated tokenizer downloads. On such errors we now
purge the broken models--org--repo snapshot, evict the in-memory
pipeline cache, and retry the generation with a fresh download.

Refs: Albina Bondar video generation batch.

app.py

@@ -1233,6 +1233,52 @@ def generate_zerogpu_video(
     repo_id = VIDEO_MODELS.get(model_choice, "cerspense/zeroscope_v2_576w")
     out_video_path = f"/tmp/zerogpu_video_{int(time.time())}_{abs(hash(prompt)) % 10000}.mp4"
 
+    def _purge_corrupt_hf_cache(err_text: str):
+        """Nuke the broken model snapshot from the HF cache when a tokenizer/blob parse fails."""
+        import re as _re
+        m = _re.search(r"models--([\w.\-]+)--([\w.\-]+)", err_text)
+        if m:
+            broken = os.path.join("/root/.cache/huggingface/hub", f"models--{m.group(1)}--{m.group(2)}")
+            if os.path.exists(broken):
+                print(f"[ZeroGPU Video] Purging corrupted HF cache: {broken}", flush=True)
+                shutil.rmtree(broken, ignore_errors=True)
+                _video_pipeline_cache.pop(repo_id, None)
+                return True
+        return False
+
+    def _retry_video_generation(prompt, negative_prompt, model_choice, num_frames, fps, guidance_scale, seed):
+        """Fresh re-run of the pipeline body after a cache purge (recursive safe: 1 level)."""
+        import torch
+        from diffusers import DiffusionPipeline, DPMSolverMultistepScheduler
+        from diffusers.utils import export_to_video
+
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        dtype = torch.float16 if device == "cuda" else torch.float32
+        pipe = DiffusionPipeline.from_pretrained(repo_id, torch_dtype=dtype)
+        if hasattr(pipe, "scheduler"):
+            try:
+                pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
+            except Exception:
+                pass
+        if hasattr(pipe, "enable_model_cpu_offload") and device == "cuda":
+            pipe.enable_model_cpu_offload()
+        else:
+            pipe = pipe.to(device)
+        _video_pipeline_cache[repo_id] = pipe
+
+        actual_seed = seed if (seed and int(seed) >= 0) else random.randint(0, 2**31 - 1)
+        generator = torch.Generator(device=device).manual_seed(actual_seed)
+        video_frames = pipe(
+            prompt=prompt.strip(),
+            negative_prompt=negative_prompt.strip() if negative_prompt else None,
+            num_inference_steps=24,
+            guidance_scale=float(guidance_scale),
+            num_frames=int(num_frames),
+            generator=generator
+        ).frames[0]
+        export_to_video(video_frames, out_video_path, fps=int(fps))
+        return out_video_path
+
     try:
         import torch
         from diffusers import DiffusionPipeline, DPMSolverMultistepScheduler
@@ -1271,6 +1317,19 @@ def generate_zerogpu_video(
         export_to_video(video_frames, out_video_path, fps=int(fps))
         return out_video_path
     except Exception as exc:
+        # Auto-recovery: corrupted HF cache snapshots (e.g. truncated spiece.model)
+        # are purged once, then the whole pipeline load is retried fresh.
+        if "Error parsing" in str(exc) or "spiece.model" in str(exc) or "tokenizer" in str(exc).lower():
+            if _purge_corrupt_hf_cache(str(exc)):
+                try:
+                    print(f"[ZeroGPU Video] Retrying {repo_id} with fresh download...", flush=True)
+                    return generate_zerogpu_video.__wrapped__(
+                        prompt, negative_prompt, model_choice, num_frames, fps, guidance_scale, seed
+                    ) if hasattr(generate_zerogpu_video, "__wrapped__") else _retry_video_generation(
+                        prompt, negative_prompt, model_choice, num_frames, fps, guidance_scale, seed
+                    )
+                except Exception as retry_exc:
+                    print(f"[ZeroGPU Video] Retry also failed: {retry_exc}", flush=True)
         print(f"[ZeroGPU Video] Direct pipeline exception: {exc}, running ffmpeg dynamic visualizer fallback...", flush=True)
         try:
             from engine.model_dispatcher import ModelDispatcher