Spaces:
Running on Zero
Running on Zero
Download app.py from ymo2017/seedvr2-upscale: direct link, hf CLI and curl.
- Browser
- Download file 25.9 kB
-
https://huggingface.co/spaces/ymo2017/seedvr2-upscale/resolve/main/app.py
- Command line
-
hf download hf://spaces/ymo2017/seedvr2-upscale/app.py
-
curl -L -o app.py https://huggingface.co/spaces/ymo2017/seedvr2-upscale/resolve/main/app.py
25.9 kB
| # SeedVR2-3B video restoration / upscaling + image upscaling. | |
| # | |
| # 2026-10-26: an IMAGE endpoint (upscale_image, fn_index 1) was merged in | |
| # from the Upsampler v3 image recipe (~/workspace/tmp/seedvr2-src/upsampler). | |
| # Both endpoints share this tree, the config, the weights and the memoized | |
| # runner (the image path decodes t=1, where the runner's disabled causal | |
| # slicing is inactive anyway). The owner asked to raise the image working | |
| # area to 2560*1440 (3.7MP) to test detail retention; it was deployed and | |
| # probed twice that day and the 3.7MP decode died ~10 s in both times | |
| # (ZeroGPU RuntimeError, both allocator configs) -- same ceiling the v3 | |
| # author measured. IMAGE_WORK_AREA therefore defaults to the working | |
| # 1920*1080 and stays env-overridable for future re-probes. | |
| # expandable_segments stays OFF process-wide (see the note below); the | |
| # image function applies the v3 recipe's runtime-only set instead. | |
| # | |
| # Built on the Upsampler seedvr2-3b packaging of ByteDance's SeedVR2 code: | |
| # same trimmed tree, same pinned requirements, same patched config (pure | |
| # PyTorch rms/layer norms instead of the apex fused paths that cannot | |
| # install on ZeroGPU's Python 3.12, SDPA fallback instead of a hard | |
| # flash-attn dependency, torch 2.11 instead of upstream's 2.4.0 which | |
| # ZeroGPU does not support). What this app adds back is the VIDEO path | |
| # (the Upsampler port is image-only), restructured for ZeroGPU: | |
| # | |
| # - ONE @spaces.GPU entry with a dynamic duration (upstream booked three | |
| # separate 100 s GPU allocations per request). | |
| # - The runner is built lazily inside the first GPU call and memoized. | |
| # - Frames are processed in temporal chunks of 41 ((t - 1) % 4 == 0 for | |
| # the VAE's 4x temporal stride), last frame repeated to pad the tail | |
| # chunk, padded frames trimmed from the output. Chunking bounds the | |
| # VAE decode peak, which is what trips the ZeroGPU NVML assert at | |
| # large working areas. | |
| # - Modes: "restore" re-renders at the input's own area (deblur / | |
| # denoise / detail rebuild, no resolution change -- this is what the | |
| # upstream demo does by default); "1.5x" / "2x" scale the working | |
| # area by 2.25x / 4x, capped at 2560x1440 area. | |
| # - cond_noise_scale 0.0 (upstream's own inference script value; the | |
| # demo's 0.1 was measured to scatter bright blemishes). | |
| # - Optional "sliding" window mode (restore only): windows of 5 frames | |
| # advance by 4; every window after the first opens with the previous | |
| # window's finalized restored last frame as a causal anchor at | |
| # position 0 (the causal VAE encodes frame 0 independently, so it | |
| # anchors what follows), and that frame's re-render is averaged | |
| # 0.5/0.5 with its finalized version. Same 5-frame cap (latent t=2); | |
| # ~25% more windows than chunked. | |
| # - Original audio is stream-copied onto the result. | |
| import os | |
| # NOTE: no expandable_segments here. The image port uses it, but on this | |
| # runtime the video VAE decoder (list-based causal inflation ops) trips | |
| # the CUDACachingAllocator NVML assert with it enabled -- measured | |
| # 2026-10-18, two runs, CHUNK 41 and CHUNK 9 alike, always inside the | |
| # decoder's up blocks. Trying the native allocator for the video path. | |
| import gc | |
| import subprocess | |
| import tempfile | |
| from pathlib import Path | |
| import cv2 | |
| import gradio as gr | |
| import imageio | |
| import imageio_ffmpeg | |
| import numpy as np | |
| import spaces | |
| import torch | |
| from einops import rearrange | |
| from huggingface_hub import hf_hub_download | |
| from omegaconf import OmegaConf | |
| from PIL import Image | |
| from torchvision.transforms import Compose, Lambda, Normalize | |
| import torchvision.transforms as T | |
| from data.image.transforms.divisible_crop import DivisibleCrop | |
| from data.image.transforms.na_resize import NaResize | |
| from data.video.transforms.rearrange import Rearrange | |
| from common.config import load_config | |
| from common.distributed import init_torch | |
| from common.seed import set_seed | |
| from projects.video_diffusion_sr.infer import VideoDiffusionInfer | |
| try: | |
| from projects.video_diffusion_sr.color_fix import wavelet_reconstruction | |
| USE_COLOR_FIX = True | |
| except ImportError: | |
| USE_COLOR_FIX = False | |
| print("color fix unavailable") | |
| WEIGHTS_REPO = "ByteDance-Seed/SeedVR2-3B" | |
| CKPT_DIR = Path("./ckpts") | |
| CKPT_DIR.mkdir(exist_ok=True) | |
| def _fetch(filename: str, target: Path) -> str: | |
| if target.exists(): | |
| return str(target) | |
| path = hf_hub_download(repo_id=WEIGHTS_REPO, filename=filename) | |
| target.symlink_to(path) | |
| return str(target) | |
| DIT_CKPT = _fetch("seedvr2_ema_3b.pth", CKPT_DIR / "seedvr2_ema_3b.pth") | |
| VAE_CKPT = _fetch("ema_vae.pth", CKPT_DIR / "ema_vae.pth") | |
| POS_EMB = _fetch("pos_emb.pt", Path("./pos_emb.pt")) | |
| NEG_EMB = _fetch("neg_emb.pt", Path("./neg_emb.pt")) | |
| INPUT_NOISE = 0.0 | |
| # Image endpoint working area (see header note). 2560*1440 = upstream's own | |
| # image value; the public Upsampler Space caps at 1920*1080 because at | |
| # 3.7MP their VAE decode tripped the ZeroGPU NVML assert. Re-probed on | |
| # this Space 2026-10-26 at the owner's request: two runs at 2560*1440 | |
| # (with and without runtime expandable_segments) both died ~10 s in with | |
| # the same opaque ZeroGPU RuntimeError as video '2x' -- the 3.7MP decode | |
| # does not survive on this runtime in either allocator config, matching | |
| # the v3 author's measurement. Default therefore stays at the measured | |
| # ceiling 1920*1080; raise SEEDVR2_WORK_AREA to re-probe. | |
| IMAGE_WORK_AREA = int(os.environ.get("SEEDVR2_WORK_AREA", 1920 * 1080)) | |
| IMAGE_DEFAULT_SEED = int(os.environ.get("SEEDVR2_SEED", 0)) | |
| OUTPUT_WEBP_QUALITY = 95 | |
| WEBP_MAX_SIDE = 16383 # libwebp hard limit; larger results fall back to PNG | |
| # Measured threshold on this ZeroGPU runtime (2026-10-18, card119): | |
| # a chunk of 5 frames (latent t=2) decodes fine; 9 frames (latent t=3) | |
| # and up die in the VAE decoder with the CUDACachingAllocator NVML | |
| # assert, regardless of chunk slicing, causal slicing on/off, or | |
| # allocator settings. So chunks are 5 frames: each is decoded as an | |
| # independent causal tensor (no cross-chunk cache), which is also why | |
| # causal slicing is disabled in _ensure_runner. Per-chunk cost measured | |
| # ~4.2 s at 720x1280 restore on a cold worker. | |
| CHUNK = 5 | |
| SLIDE_STEP = CHUNK - 1 # sliding windows overlap by one frame | |
| MAX_FRAMES = 481 | |
| MAX_TARGET_AREA = 2560 * 1440 | |
| MODE_AREA_SCALE = {"restore": 1.0, "1.5x": 2.25, "2x": 4.0} | |
| os.environ.setdefault("MASTER_ADDR", "127.0.0.1") | |
| os.environ.setdefault("MASTER_PORT", "12355") | |
| os.environ.setdefault("RANK", "0") | |
| os.environ.setdefault("WORLD_SIZE", "1") | |
| os.environ.setdefault("LOCAL_RANK", "0") | |
| _runner = None | |
| def _ensure_runner(): | |
| global _runner | |
| if _runner is not None: | |
| return _runner | |
| if not torch.distributed.is_initialized(): | |
| init_torch(cudnn_benchmark=False) | |
| runner = VideoDiffusionInfer(load_config(os.path.join("./configs_3b", "main.yaml"))) | |
| OmegaConf.set_readonly(runner.config, False) | |
| runner.configure_dit_model(device="cuda", checkpoint=DIT_CKPT) | |
| runner.configure_vae_model() | |
| # Disable the VAE's causal slicing for decode: every multi-frame run | |
| # so far died inside the sliced decode path (memory-state cache | |
| # machinery in the decoder up blocks), while the image port -- which | |
| # always decodes with slicing inactive (t=1) -- runs fine on this | |
| # runtime. With slicing off, each chunk decodes as one causal tensor, | |
| # the same code path the image port exercises. Conv-level memory | |
| # limits (0.15 GiB) still apply. | |
| try: | |
| runner.vae.set_causal_slicing(split_size=None, memory_device="same") | |
| print("[vae] causal slicing disabled", flush=True) | |
| except Exception as exc: | |
| print(f"[vae] could not disable slicing: {exc}", flush=True) | |
| if hasattr(runner.vae, "set_memory_limit"): | |
| runner.vae.set_memory_limit(**runner.config.vae.memory_limit) | |
| _runner = runner | |
| return _runner | |
| def _probe(video_path): | |
| cap = cv2.VideoCapture(video_path) | |
| # CAP_PROP_FRAME_COUNT trusts container duration metadata, which is | |
| # wrong for stream-copied excerpts (it reported ~120 for a 48-frame | |
| # clip and inflated the ZeroGPU duration request). Count by grabbing. | |
| frames = 0 | |
| while cap.grab(): | |
| frames += 1 | |
| if frames == 0: | |
| frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) | |
| w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) | |
| h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) | |
| fps = cap.get(cv2.CAP_PROP_FPS) or 24.0 | |
| cap.release() | |
| return frames, w, h, fps | |
| def _target_area(w, h, mode): | |
| return int(min(w * h * MODE_AREA_SCALE[mode], MAX_TARGET_AREA)) | |
| def _duration(video, mode="restore", steps=1, seed=666, window="chunked", progress=None) -> int: | |
| # ZeroGPU's per-request maximum tracks the account's remaining GPU | |
| # quota; after a heavy session it drops below what an honest estimate | |
| # for even a short clip requests, and every job is rejected up front. | |
| # The dynamic estimate also never reliably saw the real frame count | |
| # at duration-eval time. Request the floor instead: overruns only | |
| # produce a warning (observed on the 384-frame run), not a kill. | |
| return 60 | |
| def _transform(target_area): | |
| return Compose( | |
| [ | |
| NaResize( | |
| resolution=target_area**0.5, mode="area", downsample_only=False | |
| ), | |
| Lambda(lambda x: torch.clamp(x, 0.0, 1.0)), | |
| DivisibleCrop((16, 16)), | |
| Normalize(0.5, 0.5), | |
| Rearrange("t c h w -> c t h w"), | |
| ] | |
| ) | |
| def _read_frames(video_path, max_frames): | |
| cap = cv2.VideoCapture(video_path) | |
| fps = cap.get(cv2.CAP_PROP_FPS) or 24.0 | |
| frames = [] | |
| while len(frames) < max_frames: | |
| ok, frame = cap.read() | |
| if not ok: | |
| break | |
| frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) | |
| cap.release() | |
| if not frames: | |
| raise gr.Error("Could not read any frames from the uploaded video.") | |
| return np.stack(frames), fps # (t, h, w, c) uint8 | |
| def _pad_chunk(chunk): | |
| """Pad a (t, c, h, w) tensor by repeating the last frame until | |
| (t - 1) % 4 == 0. Returns (padded, original_t).""" | |
| t = chunk.size(0) | |
| if t <= 4: | |
| need = 5 - t | |
| elif (t - 1) % 4 == 0: | |
| return chunk, t | |
| else: | |
| need = 4 - ((t - 1) % 4) | |
| pad = chunk[-1:].repeat(need, 1, 1, 1) | |
| return torch.cat([chunk, pad], dim=0), t | |
| def _process_window(runner, raw_frames, transform, text_embeds): | |
| """Run one window of uint8 frames (t, h, w, c) through | |
| encode -> DiT -> decode; return restored uint8 frames | |
| (t, h_out, w_out, c) with the same frame count as the input.""" | |
| tensor = torch.from_numpy(np.ascontiguousarray(raw_frames)).float().div_(255.0) | |
| tensor = tensor.permute(0, 3, 1, 2) | |
| tensor, ori_t = _pad_chunk(tensor) | |
| cond = transform(tensor.to("cuda")) | |
| reference = rearrange(cond, "c t h w -> t c h w") | |
| latents = runner.vae_encode([cond]) | |
| noise = [torch.randn_like(latent) for latent in latents] | |
| aug_noise = [torch.randn_like(latent) for latent in latents] | |
| def _add_noise(x, aug): | |
| t = torch.tensor([1000.0], device="cuda") * INPUT_NOISE | |
| shape = torch.tensor(x.shape[1:], device="cuda")[None] | |
| return runner.schedule.forward(x, aug, runner.timestep_transform(t, shape)) | |
| conditions = [ | |
| runner.get_condition(n, task="sr", latent_blur=_add_noise(latent, a)) | |
| for n, a, latent in zip(noise, aug_noise, latents) | |
| ] | |
| with torch.autocast("cuda", torch.bfloat16, enabled=True): | |
| videos = runner.inference( | |
| noises=noise, conditions=conditions, dit_offload=False, **text_embeds | |
| ) | |
| sample = videos[0] | |
| sample = ( | |
| rearrange(sample[:, None], "c t h w -> t c h w") | |
| if sample.ndim == 3 | |
| else rearrange(sample, "c t h w -> t c h w") | |
| ) | |
| if USE_COLOR_FIX: | |
| sample = wavelet_reconstruction( | |
| sample.to("cpu"), reference[: sample.size(0)].to("cpu") | |
| ) | |
| else: | |
| sample = sample.to("cpu") | |
| sample = sample[:ori_t] | |
| sample = rearrange(sample, "t c h w -> t h w c") | |
| sample = sample.clip(-1, 1).mul_(0.5).add_(0.5).mul_(255).round().to(torch.uint8).numpy() | |
| del tensor, cond, reference, latents, noise, aug_noise, conditions, videos | |
| return sample | |
| def _run_sliding(runner, frames_np, transform, text_embeds): | |
| """Sliding-window pass (restore mode only). Windows of CHUNK frames | |
| advance by SLIDE_STEP. Every window after the first opens with the | |
| previous window's last restored frame -- restored pixels, resized | |
| back to the input's frame size so the window runs through the exact | |
| same transform/inference path as chunked mode -- at position 0, | |
| where the causal VAE encodes it independently as the anchor for the | |
| frames that follow. That frame's re-render is averaged 0.5/0.5 with | |
| its already-finalized version; each window's new last frame is held | |
| back as the next window's anchor.""" | |
| total = frames_np.shape[0] | |
| h_in, w_in = frames_np.shape[1], frames_np.shape[2] | |
| results = [] | |
| pending = None # last restored frame (uint8, output size), unfinalized | |
| next_src = 0 | |
| win_no = 0 | |
| while next_src < total: | |
| if pending is None: | |
| take = frames_np[next_src : next_src + CHUNK] | |
| valid_new = take.shape[0] | |
| win = list(take) | |
| else: | |
| ctx = cv2.resize(pending, (w_in, h_in), interpolation=cv2.INTER_LINEAR) | |
| take = frames_np[next_src : next_src + SLIDE_STEP] | |
| valid_new = take.shape[0] | |
| win = [ctx] + list(take) | |
| while len(win) < CHUNK: | |
| win.append(win[-1]) | |
| out = _process_window(runner, np.stack(win), transform, text_embeds) | |
| last = next_src + valid_new >= total | |
| if pending is None: | |
| if last: | |
| results.extend(out[:valid_new]) | |
| else: | |
| results.extend(out[: valid_new - 1]) | |
| pending = out[valid_new - 1] | |
| else: | |
| averaged = np.clip( | |
| np.rint( | |
| 0.5 * out[0].astype(np.float32) | |
| + 0.5 * pending.astype(np.float32) | |
| ), | |
| 0, | |
| 255, | |
| ).astype(np.uint8) | |
| results.append(averaged) | |
| if last: | |
| results.extend(out[1 : 1 + valid_new]) | |
| pending = None | |
| else: | |
| results.extend(out[1:valid_new]) | |
| pending = out[valid_new] | |
| next_src += valid_new | |
| win_no += 1 | |
| print(f"[job] sliding window {win_no} done ({len(results)} frames finalized)", flush=True) | |
| gc.collect() | |
| torch.cuda.empty_cache() | |
| assert len(results) == total, f"sliding produced {len(results)} frames for {total} input frames" | |
| return np.stack(results) | |
| def restore_video(video, mode="restore", steps=1, seed=666, window="chunked", progress=gr.Progress(track_tqdm=True)): | |
| if video is None: | |
| raise gr.Error("Upload a video first.") | |
| in_path = video["video"]["path"] if isinstance(video, dict) else video | |
| print(f"[alloc] PYTORCH_CUDA_ALLOC_CONF={os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!r}", flush=True) | |
| frames_np, fps = _read_frames(in_path, MAX_FRAMES) | |
| total = frames_np.shape[0] | |
| h, w = frames_np.shape[1], frames_np.shape[2] | |
| target_area = _target_area(w, h, mode) | |
| window_mode = window | |
| if window_mode == "sliding" and mode != "restore": | |
| print(f"[job] sliding applies to restore mode only; falling back to chunked for mode={mode}", flush=True) | |
| window_mode = "chunked" | |
| print(f"[job] {total} frames {w}x{h}@{fps:.3f} mode={mode} window={window_mode} target_area={target_area}", flush=True) | |
| runner = _ensure_runner() | |
| runner.config.diffusion.cfg.scale = 1.0 | |
| runner.config.diffusion.cfg.rescale = 0.0 | |
| runner.config.diffusion.timesteps.sampling.steps = int(steps) | |
| runner.configure_diffusion() | |
| set_seed(int(seed) % (2**32), same_across_ranks=True) | |
| text_embeds = { | |
| "texts_pos": [torch.load(POS_EMB).to("cuda")], | |
| "texts_neg": [torch.load(NEG_EMB).to("cuda")], | |
| } | |
| transform = _transform(target_area) | |
| if window_mode == "sliding": | |
| out_frames = _run_sliding(runner, frames_np, transform, text_embeds) | |
| else: | |
| out_chunks = [] | |
| for start in range(0, total, CHUNK): | |
| raw = frames_np[start : start + CHUNK] | |
| sample = _process_window(runner, raw, transform, text_embeds) | |
| out_chunks.append(sample) | |
| print(f"[job] chunk {start}-{start + sample.shape[0]} done", flush=True) | |
| gc.collect() | |
| torch.cuda.empty_cache() | |
| out_frames = np.concatenate(out_chunks, axis=0) | |
| workdir = tempfile.mkdtemp(prefix="seedvr2-out-") | |
| silent_path = os.path.join(workdir, "video.mp4") | |
| writer = imageio.get_writer( | |
| silent_path, | |
| fps=fps, | |
| codec="libx264", | |
| quality=8, | |
| macro_block_size=None, | |
| ffmpeg_params=["-pix_fmt", "yuv420p", "-movflags", "+faststart"], | |
| ) | |
| for frame in out_frames: | |
| writer.append_data(frame) | |
| writer.close() | |
| # Stream-copy the original audio track if there is one (no -shortest: | |
| # it truncates video tails against audio duration). | |
| final_path = os.path.join(workdir, "seedvr2-out.mp4") | |
| ffmpeg = imageio_ffmpeg.get_ffmpeg_exe() | |
| r = subprocess.run( | |
| [ffmpeg, "-y", "-i", silent_path, "-i", in_path, | |
| "-map", "0:v:0", "-map", "1:a?", "-c", "copy", final_path], | |
| capture_output=True, text=True, | |
| ) | |
| if r.returncode != 0 or not os.path.exists(final_path): | |
| print(f"[job] audio mux failed, returning silent: {r.stderr[-300:]}", flush=True) | |
| final_path = silent_path | |
| print(f"[job] done -> {final_path} ({out_frames.shape})", flush=True) | |
| return final_path | |
| # --------------------------------------------------------------------------- | |
| # Image endpoint (merged from the Upsampler v3 image recipe, 2026-10-26). | |
| # Differences from the v3 original, all deliberate: it shares this app's | |
| # _ensure_runner (which disables VAE causal slicing -- inactive at t=1 | |
| # anyway) and it does NOT set expandable_segments, neither via env at | |
| # import nor via torch.cuda.memory._set_allocator_settings at runtime: | |
| # on this runtime that setting trips the NVML assert in the video VAE | |
| # decoder, and the v3 author measured it made no difference there either. | |
| # --------------------------------------------------------------------------- | |
| def _image_transform(): | |
| return Compose( | |
| [ | |
| NaResize(resolution=IMAGE_WORK_AREA**0.5, mode="area", downsample_only=False), | |
| Lambda(lambda x: torch.clamp(x, 0.0, 1.0)), | |
| DivisibleCrop((16, 16)), | |
| Normalize(0.5, 0.5), | |
| Rearrange("t c h w -> c t h w"), | |
| ] | |
| ) | |
| def _image_duration(image, steps=1, seed=0, progress=None) -> int: | |
| # Every request restores at the same working resolution, so cost tracks | |
| # the step count. v3 formula: 12 s/step measured at 2MP, scaled by the | |
| # working area; at 3.7MP and 1 step this requests 35 s. | |
| area_scale = IMAGE_WORK_AREA / (1920 * 1080) | |
| return int(min(90, 14 + 12 * int(steps) * area_scale)) | |
| def _save_result(image): | |
| # Gradio would save a PIL/array result as WebP at quality 80, throwing | |
| # away the fine detail the model just produced. Write it ourselves at | |
| # quality 95 and return the path; Gradio serves a path as is. | |
| if not isinstance(image, Image.Image): | |
| image = Image.fromarray(image) | |
| out_dir = tempfile.mkdtemp(prefix="seedvr2-img-") | |
| if max(image.size) > WEBP_MAX_SIDE: | |
| path = os.path.join(out_dir, "upscaled.png") | |
| image.save(path, "PNG") | |
| else: | |
| path = os.path.join(out_dir, "upscaled.webp") | |
| image.save(path, "WEBP", quality=OUTPUT_WEBP_QUALITY, method=4) | |
| return path | |
| def upscale_image(image, steps=1, seed=0, progress=gr.Progress(track_tqdm=True)): | |
| if image is None: | |
| raise gr.Error("Upload an image first.") | |
| print(f"[alloc] PYTORCH_CUDA_ALLOC_CONF={os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!r}", flush=True) | |
| # The v3 recipe's runtime allocator set, kept in its original form | |
| # (image function only, never at import -- the env form breaks the | |
| # video decoder). The 2026-10-26 3.7MP probe showed it does not change | |
| # the decode ceiling either way; at the 2MP default it matches the | |
| # public Space's known-good configuration. | |
| try: | |
| torch.cuda.memory._set_allocator_settings("expandable_segments:True") | |
| print("[alloc] expandable_segments set at runtime", flush=True) | |
| except Exception as exc: | |
| print(f"[alloc] runtime allocator setting unavailable: {exc}", flush=True) | |
| print(f"[img] work area {IMAGE_WORK_AREA} px", flush=True) | |
| runner = _ensure_runner() | |
| runner.config.diffusion.cfg.scale = 1.0 | |
| runner.config.diffusion.cfg.rescale = 0.0 | |
| runner.config.diffusion.timesteps.sampling.steps = int(steps) | |
| runner.configure_diffusion() | |
| # Explicit API seed wins; 0 falls back to the pinned default or random. | |
| import random | |
| chosen = int(seed) or IMAGE_DEFAULT_SEED or random.randrange(1, 2**31) | |
| print(f"[img] seed {chosen}", flush=True) | |
| set_seed(chosen, same_across_ranks=True) | |
| img = Image.open(image).convert("RGB") if isinstance(image, str) else image.convert("RGB") | |
| tensor = T.ToTensor()(img).unsqueeze(0) # (t=1, c, h, w) | |
| cond = _image_transform()(tensor.to("cuda")) | |
| original = cond | |
| latents = runner.vae_encode([cond]) | |
| text_embeds = { | |
| "texts_pos": [torch.load(POS_EMB).to("cuda")], | |
| "texts_neg": [torch.load(NEG_EMB).to("cuda")], | |
| } | |
| noise = [torch.randn_like(latent) for latent in latents] | |
| aug_noise = [torch.randn_like(latent) for latent in latents] | |
| def _add_noise(x, aug): | |
| t = torch.tensor([1000.0], device="cuda") * INPUT_NOISE | |
| shape = torch.tensor(x.shape[1:], device="cuda")[None] | |
| return runner.schedule.forward(x, aug, runner.timestep_transform(t, shape)) | |
| conditions = [ | |
| runner.get_condition(n, task="sr", latent_blur=_add_noise(latent, a)) | |
| for n, a, latent in zip(noise, aug_noise, latents) | |
| ] | |
| with torch.autocast("cuda", torch.bfloat16, enabled=True): | |
| videos = runner.inference( | |
| noises=noise, conditions=conditions, dit_offload=False, **text_embeds | |
| ) | |
| sample = videos[0] | |
| sample = ( | |
| rearrange(sample[:, None], "c t h w -> t c h w") | |
| if sample.ndim == 3 | |
| else rearrange(sample, "c t h w -> t c h w") | |
| ) | |
| reference = ( | |
| rearrange(original[:, None], "c t h w -> t c h w") | |
| if original.ndim == 3 | |
| else rearrange(original, "c t h w -> t c h w") | |
| ) | |
| if USE_COLOR_FIX: | |
| sample = wavelet_reconstruction(sample.to("cpu"), reference[: sample.size(0)].to("cpu")) | |
| else: | |
| sample = sample.to("cpu") | |
| sample = rearrange(sample, "t c h w -> t h w c") | |
| sample = sample.clip(-1, 1).mul_(0.5).add_(0.5).mul_(255).round().to(torch.uint8).numpy() | |
| del latents, conditions, videos | |
| gc.collect() | |
| torch.cuda.empty_cache() | |
| return _save_result(sample[0]) | |
| with gr.Blocks(title="SeedVR2 Video + Image") as demo: | |
| gr.Markdown( | |
| "## SeedVR2-3B Restoration\n" | |
| "**Video**: one-step diffusion restoration. **Restore** re-renders at the " | |
| "input's own resolution (deblur / detail rebuild); **1.5x / 2x** also " | |
| "enlarge. Up to 481 frames per job; longer clips are trimmed. " | |
| "**Window**: *chunked* processes independent 5-frame windows; " | |
| "*sliding* (restore only) carries each window's last restored frame " | |
| "into the next window as context and averages the overlap, for " | |
| "better temporal continuity at ~25% more compute.\n\n" | |
| "**Image**: single-image restoration at a 1920x1080 working area." | |
| ) | |
| with gr.Tab("Video"): | |
| with gr.Row(): | |
| with gr.Column(): | |
| video_in = gr.Video(label="Video", sources=["upload"]) | |
| mode = gr.Dropdown( | |
| choices=["restore", "1.5x", "2x"], value="restore", label="Mode" | |
| ) | |
| window = gr.Radio( | |
| choices=["chunked", "sliding"], | |
| value="chunked", | |
| label="Window", | |
| info="chunked: independent 5-frame windows. sliding (restore only): each window carries the previous restored frame as context; the overlap frame is averaged.", | |
| ) | |
| steps = gr.Slider(1, 4, value=1, step=1, label="Steps") | |
| seed = gr.Number(value=666, precision=0, label="Seed") | |
| run = gr.Button("Restore Video", variant="primary") | |
| with gr.Column(): | |
| video_out = gr.Video(label="Result") | |
| # Registered first so /restore stays fn_index 0. | |
| run.click( | |
| restore_video, | |
| inputs=[video_in, mode, steps, seed, window], | |
| outputs=[video_out], | |
| api_name="restore", | |
| ) | |
| with gr.Tab("Image"): | |
| with gr.Row(): | |
| with gr.Column(): | |
| image_in = gr.Image(label="Image", type="filepath") | |
| img_steps = gr.Slider(1, 4, value=1, step=1, label="Steps") | |
| img_seed = gr.Number(value=0, precision=0, visible=False) | |
| img_run = gr.Button("Upscale Image", variant="primary") | |
| with gr.Column(): | |
| image_out = gr.Image(label="Result", type="filepath") | |
| # No leading slash: gradio prefixes it. | |
| img_run.click( | |
| upscale_image, | |
| inputs=[image_in, img_steps, img_seed], | |
| outputs=[image_out], | |
| api_name="upscale_image", | |
| ) | |
| demo.launch(ssr_mode=False, show_error=True) | |