relayvsr / app.py
luca115's picture
Upload folder using huggingface_hub
1596cde verified
Raw History Blame Contribute Delete
10.1 kB
"""RelayVSR video upscaler on ZeroGPU.
RelayVSR (kopperx/RelayVSR, Apache-2.0) computes 4x only: a Wan transformer
renders a keyframe every KEYFRAME_WINDOW - 1 frames and a small Flash decoder
relays the frames between them. The vendored `refvsr_inference` package is the
upstream inference code at commit e856773, unchanged.
Same policy as the RunPod worker (workflows/relayvsr_video_upscale.py): the
model always sees the input at its own size, shrunk only when 4x of it would
pass MAX_MODEL_PIXELS, and the 4x frame is resized to the requested output.
"""
import math
import os
import subprocess
import tempfile
import time
from fractions import Fraction
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import av
import gradio as gr
import imageio_ffmpeg
import spaces
import torch
import torch.nn.functional as F
from huggingface_hub import hf_hub_download
from refvsr_inference.checkpoints import load_checkpoints
from refvsr_inference.models.wan.sampling import WanVSRPipeline
from refvsr_inference.runtime.streaming import CollaborativeFrameStream, InputFrame
from upsampler_theme import UPSAMPLER_CSS, UPSAMPLER_THEME, footer_html, header_html
WEIGHTS_REPO = "kopper/RelayVSR"
WEIGHTS_REVISION = "d43dc5545f9a3bd1c7730b68ebff4ab1f9199bd1"
MODEL_SCALE = 4
KEYFRAME_WINDOW = 16
SEED = 42
DTYPE = torch.bfloat16
# 4x of 960x540: the size the 2026-11-06 bench and the worker run.
MAX_MODEL_PIXELS = 3840 * 2160
# Requested output cap per frame (4K UHD).
MAX_OUTPUT_PIXELS = 3840 * 2160
MAX_FPS = 30
MAX_SECONDS = 10.5
# GPU-duration model. ZeroGPU bills the DECLARED duration (x1.5 on this size,
# no refund), and under-declaring gets jobs aborted, so this is 1.3x measured.
# Measured 2026-11-07 ("TIMING" log lines): ~8 s fixed (setup + flex-attention
# compile on a cold worker) plus 0.025 s per frame per megapixel of the 4x model
# output. 121 frames at 3072x1728: 23.7 s cold / 16.4 s warm; 300 frames at
# 3840x2160: 66.4 s.
STARTUP_SECONDS = 10
SECONDS_PER_MODEL_MEGAFRAME = 0.033
MAX_GPU_SECONDS = 180
print("loading RelayVSR weights")
_wan_path = hf_hub_download(WEIGHTS_REPO, "wan.safetensors", revision=WEIGHTS_REVISION)
_flash_path = hf_hub_download(WEIGHTS_REPO, "flash.safetensors", revision=WEIGHTS_REVISION)
WAN, FLASH = load_checkpoints(_wan_path, _flash_path, device="cpu", dtype=DTYPE)
WAN.to("cuda")
FLASH.to("cuda")
def _even(v):
return max(2, int(v) // 2 * 2)
def _probe(video_path):
"""(fps, frame_count, width, height) of the first video stream."""
with av.open(video_path) as container:
if not container.streams.video:
raise gr.Error("This file has no video stream.")
stream = container.streams.video[0]
fps = stream.average_rate or stream.guessed_rate or Fraction(30)
width, height = stream.codec_context.width, stream.codec_context.height
frames = stream.frames
if not frames:
duration = float(stream.duration * stream.time_base) if stream.duration else (
container.duration / 1_000_000 if container.duration else 0
)
frames = int(round(duration * float(fps)))
return Fraction(fps), int(frames), int(width), int(height)
def _plan(video_path, scale):
fps, frames, width, height = _probe(video_path)
out_fps = min(fps, Fraction(MAX_FPS))
out_frames = max(1, int(round(frames * out_fps / fps))) if frames else 1
area = MODEL_SCALE * MODEL_SCALE * width * height
shrink = min(1.0, math.sqrt(MAX_MODEL_PIXELS / area))
model_w, model_h = _even(width * shrink), _even(height * shrink)
out_w, out_h = width * scale, height * scale
if out_w * out_h > MAX_OUTPUT_PIXELS:
s = math.sqrt(MAX_OUTPUT_PIXELS / (out_w * out_h))
out_w, out_h = out_w * s, out_h * s
return {
"fps": fps, "out_fps": out_fps, "frames": frames, "out_frames": out_frames,
"src": (width, height), "model": (model_w, model_h), "out": (_even(out_w), _even(out_h)),
}
def get_duration(video_path, scale="2x"):
if not video_path:
return 30
p = _plan(video_path, int(scale.rstrip("x")))
model_mp = MODEL_SCALE * MODEL_SCALE * p["model"][0] * p["model"][1] / 1_000_000
estimate = STARTUP_SECONDS + SECONDS_PER_MODEL_MEGAFRAME * model_mp * p["out_frames"]
return min(MAX_GPU_SECONDS, math.ceil(estimate))
def _frames(video_path, step, model_w, model_h, limit):
"""Yield uint8 [3,H,W] frames at the model size, dropping to <= MAX_FPS."""
with av.open(video_path) as container:
stream = container.streams.video[0]
stream.thread_type = "AUTO"
next_capture, emitted = 0.0, 0
for i, frame in enumerate(container.decode(stream)):
if i < next_capture:
continue
next_capture += step
img = frame.to_ndarray(format="rgb24", width=model_w, height=model_h,
interpolation="BICUBIC")
yield torch.from_numpy(img).permute(2, 0, 1).contiguous()
emitted += 1
if emitted >= limit:
return
def _mux_audio(silent_path, source_path):
out = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name
cmd = [
imageio_ffmpeg.get_ffmpeg_exe(), "-y", "-i", silent_path, "-i", source_path,
"-map", "0:v:0", "-map", "1:a:0?", "-c:v", "copy", "-c:a", "aac", "-shortest",
"-movflags", "+faststart", out,
]
if subprocess.run(cmd, capture_output=True).returncode != 0:
return silent_path
return out
@spaces.GPU(duration=get_duration)
@torch.no_grad()
def upscale_video(video_path, scale="2x"):
if not video_path:
raise gr.Error("Please upload a video first.")
started = time.perf_counter()
p = _plan(video_path, int(scale.rstrip("x")))
if p["frames"] and p["frames"] / float(p["fps"]) > MAX_SECONDS:
raise gr.Error(f"Please use a clip of at most {int(MAX_SECONDS)} seconds.")
model_w, model_h = p["model"]
out_w, out_h = p["out"]
step = float(p["fps"] / p["out_fps"])
limit = int(MAX_SECONDS * float(p["out_fps"])) + 1
device = torch.device("cuda")
pipeline = WanVSRPipeline(
scheduler_config={"num_train_timesteps": 1000, "shift": 5.0},
sampling={
"execution_mode": "streaming",
"num_inference_steps": 1,
"timesteps": [1000],
"guidance_scale": 1.0,
"seed": SEED,
"kv_cache_window_size": WAN.temporal_context_frames,
},
)
pipeline.prepare_streaming(model=WAN, prompt_embeds=WAN.prompt_embeds, device=device,
compute_dtype=DTYPE)
stream = CollaborativeFrameStream(
wan_model=WAN, flash_model=FLASH, wan_pipeline=pipeline,
prompt_embeds=WAN.prompt_embeds, device=device, compute_dtype=DTYPE,
window_size=KEYFRAME_WINDOW, seed=SEED,
)
silent = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name
container = av.open(silent, mode="w")
vstream = container.add_stream("libx264", rate=p["out_fps"])
vstream.width, vstream.height, vstream.pix_fmt = out_w, out_h, "yuv420p"
vstream.options = {"crf": "18", "preset": "veryfast"}
packets = (InputFrame(i, rgb) for i, rgb in
enumerate(_frames(video_path, step, model_w, model_h, limit)))
written = 0
first_frame_at = None
try:
for result in stream.frames(packets):
if first_frame_at is None:
first_frame_at = time.perf_counter()
frame = result.rgb.float().add_(1).mul_(0.5)
if tuple(frame.shape[-2:]) != (out_h, out_w):
frame = F.interpolate(frame, size=(out_h, out_w), mode="bicubic",
antialias=True, align_corners=False)
rgb = frame.clamp_(0, 1).mul_(255).round_().to(torch.uint8)[0]
rgb = rgb.permute(1, 2, 0).contiguous().cpu().numpy()
for packet in vstream.encode(av.VideoFrame.from_ndarray(rgb, format="rgb24")):
container.mux(packet)
written += 1
for packet in vstream.encode():
container.mux(packet)
finally:
packets.close()
container.close()
done = time.perf_counter()
print(
f"TIMING src={p['src'][0]}x{p['src'][1]}@{float(p['fps']):.2f} "
f"model={model_w}x{model_h}x4 out={out_w}x{out_h}@{float(p['out_fps']):.2f} "
f"frames={written} declared={get_duration(video_path, scale)} "
f"first_frame={(first_frame_at or done) - started:.1f}s total={done - started:.1f}s"
)
return _mux_audio(silent, video_path)
def clear_result():
return gr.update(value=None)
with gr.Blocks(theme=UPSAMPLER_THEME, css=UPSAMPLER_CSS) as demo:
gr.HTML(header_html(
"RelayVSR Video Upscaler",
"Diffusion video super-resolution up to 4x, temporally stable, in seconds.",
))
with gr.Row():
with gr.Column():
input_video = gr.Video(label="Input Video")
scale = gr.Radio(["2x", "4x"], value="2x", label="Upscale factor")
gr.Markdown(f"Clips up to {int(MAX_SECONDS)} seconds, output up to 4K.")
run_button = gr.Button("Upscale Video", variant="primary")
with gr.Column():
result = gr.Video(interactive=False, label="Upscaled Video")
run_button.click(fn=clear_result, inputs=None, outputs=result).then(
fn=upscale_video, inputs=[input_video, scale], outputs=result,
api_name="upscale_video",
)
gr.HTML(footer_html(
"Free online RelayVSR video upscaler. RelayVSR is a one-step diffusion video "
"super-resolution model: a Wan transformer restores sparse keyframes and a light "
"decoder carries that detail to every frame in between, so textures stay sharp and "
"stable without the flicker of frame-by-frame upscalers.",
"https://upsampler.com/free-video-upscaler-no-signup",
"free AI video upscaler",
))
demo.queue().launch(ssr_mode=False)