Spaces:
Running on Zero
Running on Zero
Download app.py from LoloSemper/relayvsr: direct link, hf CLI and curl.
- Browser
- Download file 49 kB
-
https://huggingface.co/spaces/LoloSemper/relayvsr/resolve/main/app.py
- Command line
-
hf download hf://spaces/LoloSemper/relayvsr/app.py
-
curl -L -o app.py https://huggingface.co/spaces/LoloSemper/relayvsr/resolve/main/app.py
49 kB
"""RelayVSR / ZeroGPU: original intact + long videos + adjustable restoration.
Based on the user's original RelayVSR app.py (Upsampler fork, Apache-2.0).
The unmodified original inference entry point is exposed in its own tab.
Long videos use independent ZeroGPU leases for successive frame windows.
Requires the existing refvsr_inference and upsampler_theme packages.
For optional automatic small-object protection, also install `ultralytics`.
The global neural strength mixes RelayVSR with genuine original-resolution
source frames. It changes neither the model weights nor the number of steps.
IMPORTANT: The model weights are deliberately still downloaded from the
original revision of kopper/RelayVSR. The Google Drive backup is independent.
"""
import bisect
import json
import math
import os
import subprocess
import tempfile
import threading
import time
import traceback
import uuid
from fractions import Fraction
from pathlib import Path
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import av
import gradio as gr
import imageio_ffmpeg
import numpy as np
import spaces
import torch
import torch.nn.functional as F
from huggingface_hub import hf_hub_download
from PIL import Image, ImageDraw, ImageFilter
from refvsr_inference.checkpoints import load_checkpoints
from refvsr_inference.models.wan.sampling import WanVSRPipeline
from refvsr_inference.runtime.streaming import CollaborativeFrameStream, InputFrame
from upsampler_theme import UPSAMPLER_CSS, UPSAMPLER_THEME, footer_html, header_html
WEIGHTS_REPO = "kopper/RelayVSR"
WEIGHTS_REVISION = "d43dc5545f9a3bd1c7730b68ebff4ab1f9199bd1"
MODEL_SCALE = 4
KEYFRAME_WINDOW = 16
SEED = 42
DTYPE = torch.bfloat16
MAX_MODEL_PIXELS = 3840 * 2160
MAX_OUTPUT_PIXELS = 3840 * 2160
MAX_FPS = 30
MAX_SECONDS = 10.5
STARTUP_SECONDS = 10
SECONDS_PER_MODEL_MEGAFRAME = 0.033
MAX_GPU_SECONDS = 180
# New workflow. 8 seconds is a good baseline for 720p clips at <=30 fps.
# A more demanding model size/codec can automatically select shorter chunks.
AUTO_CHUNK_SECONDS = 8
AUTO_MIN_CHUNK_SECONDS = 2
CONTEXT_FRAMES = KEYFRAME_WINDOW - 1 # warm up decoder across each boundary
TARGET_GPU_SECONDS = 155 # leave headroom below ZeroGPU's 180 s budget
AUTO_MAX_SOURCE_SECONDS = 600 # guard against enormous jobs / unbounded disk
DETAIL_MODE_ORIGINAL = "Original (máxima compatibilidad)"
DETAIL_MODE_EXTENDED = "Más detalle neuronal (experimental)"
EXPORT_H264 = "H.264 (MP4)"
EXPORT_PRORES = "ProRes 422 HQ (MOV)"
# Small CPU segmentation model. Kept separate from the large RelayVSR weights
# and loaded ONLY when the user explicitly enables object protection.
PROTECTION_REPO = "Ultralytics/YOLO11"
PROTECTION_WEIGHTS = "yolo11n-seg.pt"
PROTECTION_CLASSES = [0, 2, 3, 5, 7, 17] # person, road vehicles, horse (COCO)
PROTECTION_SAMPLE_EVERY = 3 # CPU detection at 8 fps for a 24 fps input
PROTECTION_CUT_THRESHOLD = 0.22
_PROTECTION_MODEL = None
_PROTECTION_MODEL_LOCK = threading.Lock()
print("loading RelayVSR weights")
_wan_path = hf_hub_download(WEIGHTS_REPO, "wan.safetensors", revision=WEIGHTS_REVISION)
_flash_path = hf_hub_download(WEIGHTS_REPO, "flash.safetensors", revision=WEIGHTS_REVISION)
WAN, FLASH = load_checkpoints(_wan_path, _flash_path, device="cpu", dtype=DTYPE)
WAN.to("cuda")
FLASH.to("cuda")
def _even(v):
return max(2, int(v) // 2 * 2)
def _probe(video_path):
"""(fps, frame_count, width, height) of the first video stream."""
with av.open(video_path) as container:
if not container.streams.video:
raise gr.Error("This file has no video stream.")
stream = container.streams.video[0]
fps = stream.average_rate or stream.guessed_rate or Fraction(30)
width, height = stream.codec_context.width, stream.codec_context.height
frames = stream.frames
if not frames:
duration = float(stream.duration * stream.time_base) if stream.duration else (
container.duration / 1_000_000 if container.duration else 0
)
frames = int(round(duration * float(fps)))
return Fraction(fps), int(frames), int(width), int(height)
def _plan(video_path, scale):
fps, frames, width, height = _probe(video_path)
out_fps = min(fps, Fraction(MAX_FPS))
out_frames = max(1, int(round(frames * out_fps / fps))) if frames else 1
area = MODEL_SCALE * MODEL_SCALE * width * height
shrink = min(1.0, math.sqrt(MAX_MODEL_PIXELS / area))
model_w, model_h = _even(width * shrink), _even(height * shrink)
out_w, out_h = width * scale, height * scale
if out_w * out_h > MAX_OUTPUT_PIXELS:
s = math.sqrt(MAX_OUTPUT_PIXELS / (out_w * out_h))
out_w, out_h = out_w * s, out_h * s
return {
"fps": fps, "out_fps": out_fps, "frames": frames, "out_frames": out_frames,
"src": (width, height), "model": (model_w, model_h),
"out": (_even(out_w), _even(out_h)),
}
def get_duration(video_path, scale="2x"):
if not video_path:
return 30
p = _plan(video_path, int(scale.rstrip("x")))
model_mp = MODEL_SCALE * MODEL_SCALE * p["model"][0] * p["model"][1] / 1_000_000
estimate = STARTUP_SECONDS + SECONDS_PER_MODEL_MEGAFRAME * model_mp * p["out_frames"]
return min(MAX_GPU_SECONDS, math.ceil(estimate))
def _frames(video_path, step, model_w, model_h, limit):
"""ORIGINAL sampling algorithm: kept unchanged."""
with av.open(video_path) as container:
stream = container.streams.video[0]
stream.thread_type = "AUTO"
next_capture, emitted = 0.0, 0
for i, frame in enumerate(container.decode(stream)):
if i < next_capture:
continue
next_capture += step
img = frame.to_ndarray(format="rgb24", width=model_w, height=model_h,
interpolation="BICUBIC")
yield torch.from_numpy(img).permute(2, 0, 1).contiguous()
emitted += 1
if emitted >= limit:
return
def _mux_audio(silent_path, source_path):
"""ORIGINAL muxer: kept unchanged."""
out = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name
cmd = [
imageio_ffmpeg.get_ffmpeg_exe(), "-y", "-i", silent_path, "-i", source_path,
"-map", "0:v:0", "-map", "1:a:0?", "-c:v", "copy", "-c:a", "aac", "-shortest",
"-movflags", "+faststart", out,
]
if subprocess.run(cmd, capture_output=True).returncode != 0:
return silent_path
return out
# ---- ORIGINAL PROCESSOR: model, stream, sampling and export unchanged ----
@spaces.GPU(duration=get_duration)
@torch.no_grad()
def upscale_video(video_path, scale="2x"):
if not video_path:
raise gr.Error("Please upload a video first.")
started = time.perf_counter()
p = _plan(video_path, int(scale.rstrip("x")))
if p["frames"] and p["frames"] / float(p["fps"]) > MAX_SECONDS:
raise gr.Error(f"Please use a clip of at most {int(MAX_SECONDS)} seconds.")
model_w, model_h = p["model"]
out_w, out_h = p["out"]
step = float(p["fps"] / p["out_fps"])
limit = int(MAX_SECONDS * float(p["out_fps"])) + 1
device = torch.device("cuda")
pipeline = WanVSRPipeline(
scheduler_config={"num_train_timesteps": 1000, "shift": 5.0},
sampling={
"execution_mode": "streaming", "num_inference_steps": 1,
"timesteps": [1000], "guidance_scale": 1.0, "seed": SEED,
"kv_cache_window_size": WAN.temporal_context_frames,
},
)
pipeline.prepare_streaming(model=WAN, prompt_embeds=WAN.prompt_embeds, device=device,
compute_dtype=DTYPE)
stream = CollaborativeFrameStream(
wan_model=WAN, flash_model=FLASH, wan_pipeline=pipeline,
prompt_embeds=WAN.prompt_embeds, device=device, compute_dtype=DTYPE,
window_size=KEYFRAME_WINDOW, seed=SEED,
)
silent = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name
container = av.open(silent, mode="w")
vstream = container.add_stream("libx264", rate=p["out_fps"])
vstream.width, vstream.height, vstream.pix_fmt = out_w, out_h, "yuv420p"
vstream.options = {"crf": "18", "preset": "veryfast"}
packets = (InputFrame(i, rgb) for i, rgb in
enumerate(_frames(video_path, step, model_w, model_h, limit)))
written = 0
first_frame_at = None
try:
for result in stream.frames(packets):
if first_frame_at is None:
first_frame_at = time.perf_counter()
frame = result.rgb.float().add_(1).mul_(0.5)
if tuple(frame.shape[-2:]) != (out_h, out_w):
frame = F.interpolate(frame, size=(out_h, out_w), mode="bicubic",
antialias=True, align_corners=False)
rgb = frame.clamp_(0, 1).mul_(255).round_().to(torch.uint8)[0]
rgb = rgb.permute(1, 2, 0).contiguous().cpu().numpy()
for packet in vstream.encode(av.VideoFrame.from_ndarray(rgb, format="rgb24")):
container.mux(packet)
written += 1
for packet in vstream.encode():
container.mux(packet)
finally:
packets.close()
container.close()
done = time.perf_counter()
print(
f"TIMING src={p['src'][0]}x{p['src'][1]}@{float(p['fps']):.2f} "
f"model={model_w}x{model_h}x4 out={out_w}x{out_h}@{float(p['out_fps']):.2f} "
f"frames={written} declared={get_duration(video_path, scale)} "
f"first_frame={(first_frame_at or done) - started:.1f}s total={done - started:.1f}s"
)
return _mux_audio(silent, video_path)
# ---- NEW: FULL-LENGTH PROCESSING IN AUTONOMOUS 8-SECOND GPU LEASES ----
def _auto_plan(video_path, scale, detail_mode):
"""Original sizes by default; optionally allow the true 720p neural input.
The latter raises the INTERNAL model output ceiling, not the delivered
resolution ceiling, and may exceed the GPU RAM of a particular worker.
"""
p = _plan(video_path, int(scale.rstrip("x")))
if detail_mode == DETAIL_MODE_EXTENDED:
w, h = p["src"]
max_internal_pixels = 5120 * 2880 # 4x of 1280x720
shrink = min(1.0, math.sqrt(max_internal_pixels / (16 * w * h)))
p["model"] = (_even(w * shrink), _even(h * shrink))
return p
def _gpu_time_estimate(p, frames, export_mode, blend_original=False):
mp = 16 * p["model"][0] * p["model"][1] / 1_000_000
# Allow additional time for encoding ProRes and cold starts.
overhead = 18 if export_mode == EXPORT_PRORES else 10
if blend_original:
overhead += 12 # source decode + spatial mask compositing
return STARTUP_SECONDS + SECONDS_PER_MODEL_MEGAFRAME * mp * frames + overhead
def _chunk_layout(p, export_mode, blend_original=False):
"""Exact output-frame ranges, each with up to 15 warm-up input frames."""
fps = float(p["out_fps"])
maximum = max(1, int(round(AUTO_CHUNK_SECONDS * fps)))
minimum = max(1, int(round(AUTO_MIN_CHUNK_SECONDS * fps)))
layout = []
cursor = 0
total = p["out_frames"]
while cursor < total:
count = min(maximum, total - cursor)
while count > minimum:
warmup = min(CONTEXT_FRAMES, cursor)
if _gpu_time_estimate(p, count + warmup, export_mode,
blend_original) <= TARGET_GPU_SECONDS:
break
count = max(minimum, count - max(1, int(round(fps))))
if _gpu_time_estimate(p, count + min(CONTEXT_FRAMES, cursor),
export_mode, blend_original) > MAX_GPU_SECONDS:
raise gr.Error(
"Esta combinación de resolución/formato necesita más tiempo de GPU del "
"permitido por ZeroGPU. Prueba el modo de detalle original."
)
layout.append((cursor, cursor + count))
cursor += count
return layout
def _sampled_source_index(output_index, p):
"""Global source position using exact rational arithmetic (ceil cadence)."""
step = Fraction(p["fps"], p["out_fps"])
return (output_index * step.numerator + step.denominator - 1) // step.denominator
def _source_rgb_window(video_path, p, first_output, last_output):
"""Yield ORIGINAL-resolution frames at the very same indices as RelayVSR.
A separate decoder avoids a costly in-memory copy of all source frames.
Crucially, the conservative reconstruction does not inherit the reduced
960x540 neural input of the default mode.
"""
wanted = (_sampled_source_index(j, p) for j in range(first_output, last_output))
target = next(wanted, None)
if target is None:
return
with av.open(video_path) as container:
stream = container.streams.video[0]
stream.thread_type = "AUTO"
for i, frame in enumerate(container.decode(stream)):
if i < target:
continue
if i > target:
raise RuntimeError(f"Fotograma original {target} no encontrado")
yield frame.to_ndarray(format="rgb24")
target = next(wanted, None)
if target is None:
return
def _neural_fraction(global_strength, protection_strength=0.0, mask=None):
"""Fraction of RelayVSR output, optionally reduced at detected objects.
Background: global_strength. Fully protected pixel: global_strength *
(1 - protection_strength). This is a POST-inference blend, not a change
to the diffusion guidance/weights; it cannot recover missing anatomy.
"""
strength = float(global_strength)
if mask is None:
return strength
return strength * (1.0 - float(protection_strength) * mask)
def _get_protection_detector():
"""Lazy CPU-only segmentation; no extra GPU VRAM or CUDA lease required."""
global _PROTECTION_MODEL
with _PROTECTION_MODEL_LOCK:
if _PROTECTION_MODEL is None:
try:
from ultralytics import YOLO
except ImportError as error:
raise gr.Error(
"Para activar Protección de objetos pequeños, añade "
"'ultralytics>=8.3,<9' a requirements.txt del Space "
"y deja que se reconstruya. Los otros modos no la necesitan."
) from error
weights = hf_hub_download(PROTECTION_REPO, PROTECTION_WEIGHTS,
repo_type="model")
_PROTECTION_MODEL = YOLO(weights, task="segment")
return _PROTECTION_MODEL
def _detect_protection_masks(video_path, p, first_output, last_output, dest,
detection_size=640, max_size_pct=10):
"""Run sparse segmentation on CPU BEFORE requesting the ZeroGPU lease.
The result is a small JSON of source-coordinate polygons, not a giant stack
of bitmaps. The GPU pass uses exactly the same global frame positions.
"""
detector = _get_protection_detector()
sw, sh = p["src"]
sample_every = max(1, round(PROTECTION_SAMPLE_EVERY * float(p["out_fps"]) / 24))
keyframes = []
cuts = []
scene = 0
prev_thumbnail = None
detections = 0
# User-configurable upper limit: 10 % of 1248 px = 125 px, not the
# previous very broad 20 % threshold that caught foreground animals.
max_side = max(1.0, float(max_size_pct) / 100.0 * max(sw, sh))
max_box_area = 0.035 * sw * sh
# The model cannot reliably identify every historical horse-drawn cart.
# If no object is found, the Space explicitly reports zero detections.
for index, rgb in enumerate(_source_rgb_window(
video_path, p, first_output, last_output), start=first_output):
thumb = np.asarray(Image.fromarray(rgb).convert("L").resize(
(96, 54), Image.Resampling.BILINEAR), dtype=np.float32)
if prev_thumbnail is not None:
delta = float(np.mean(np.abs(thumb - prev_thumbnail))) / 255.0
if delta >= PROTECTION_CUT_THRESHOLD:
scene += 1
cuts.append(index)
prev_thumbnail = thumb
if (index != first_output and index != last_output - 1
and index % sample_every != 0 and index not in cuts):
continue
# Ultralytics interprets NumPy inputs as BGR, whereas PyAV gave RGB.
bgr = np.ascontiguousarray(rgb[:, :, ::-1])
# Ultralytics' prediction state is not guaranteed to be thread-safe.
with _PROTECTION_MODEL_LOCK:
result = detector.predict(
source=bgr, device="cpu", imgsz=int(detection_size),
conf=0.16, iou=0.5,
classes=PROTECTION_CLASSES, max_det=60, verbose=False,
)[0]
objects = []
if result.boxes is not None and len(result.boxes):
boxes = result.boxes.xyxy.cpu().numpy()
categories = result.boxes.cls.int().cpu().numpy()
polygons = result.masks.xy if result.masks is not None else []
for k, (box, category) in enumerate(zip(boxes, categories)):
x1, y1, x2, y2 = (float(v) for v in box)
bw, bh = x2 - x1, y2 - y1
if bw <= 0 or bh <= 0 or max(bw, bh) > max_side:
continue
if bw * bh > max_box_area:
continue
if k < len(polygons) and len(polygons[k]) >= 3:
polygon = polygons[k]
else:
# Fallback if instance segmentation omits a tiny mask.
polygon = [(x1, y1), (x2, y1), (x2, y2), (x1, y2)]
points = [[int(round(float(x))), int(round(float(y)))]
for x, y in polygon]
if len(points) >= 3:
objects.append({"class": int(category), "poly": points})
detections += len(objects)
keyframes.append({"index": index, "scene": scene, "objects": objects})
payload = {
"src": [sw, sh], "first": first_output, "last": last_output,
"cuts": cuts, "keyframes": keyframes, "detections": detections,
"sample_every": sample_every, "max_size_pct": float(max_size_pct),
}
Path(dest).write_text(json.dumps(payload, separators=(",", ":")), encoding="utf-8")
return detections
class _ProtectionMasks:
"""Soft temporally coherent masks; never blend across a detected cut."""
def __init__(self, path):
self.data = json.loads(Path(path).read_text(encoding="utf-8"))
sw, sh = self.data["src"]
self.width = min(1024, max(256, sw // 2))
self.height = max(2, round(self.width * sh / sw))
self.indexes = [k["index"] for k in self.data["keyframes"]]
self.cuts = self.data["cuts"]
self.cache = {}
@property
def has_objects(self):
return self.data["detections"] > 0
def _keyframe_mask(self, keyframe_index):
if keyframe_index in self.cache:
return self.cache[keyframe_index]
keyframe = self.data["keyframes"][keyframe_index]
sw, sh = self.data["src"]
mask = Image.new("L", (self.width, self.height), 0)
painter = ImageDraw.Draw(mask)
for obj in keyframe["objects"]:
points = [(int(round(x * self.width / sw)),
int(round(y * self.height / sh))) for x, y in obj["poly"]]
if len(points) >= 3:
painter.polygon(points, fill=255)
if keyframe["objects"]:
# Smaller dilation/feathering: foreground details should not acquire
# unnecessarily broad green protection halos.
mask = mask.filter(ImageFilter.MaxFilter(5))
mask = mask.filter(ImageFilter.GaussianBlur(2))
self.cache[keyframe_index] = mask
if len(self.cache) > 6:
self.cache.pop(next(iter(self.cache)))
return mask
def at(self, output_index):
if not self.has_objects or not self.indexes:
return None
current_scene = bisect.bisect_right(self.cuts, output_index)
pos = bisect.bisect_right(self.indexes, output_index)
neighbors = (pos - 1, pos)
relevant = []
for k in neighbors:
if 0 <= k < len(self.indexes):
if self.data["keyframes"][k]["scene"] == current_scene:
relevant.append(k)
if not relevant:
return None
alpha = self._keyframe_mask(relevant[0])
if len(relevant) == 2:
# Temporal interpolation instead of the old MAX/union. Union
# unnecessarily enlarged masks around moving horses and people.
i0, i1 = (self.indexes[k] for k in relevant)
portion = min(1.0, max(0.0,
(output_index - i0) / max(1, i1 - i0)))
alpha = Image.blend(alpha, self._keyframe_mask(relevant[1]), portion)
return np.asarray(alpha, dtype=np.float32) / 255.0
def _frames_window(video_path, p, first_output, last_output):
"""Yield deterministic GLOBAL frame indices for [first_output,last_output).
Sequential decoding costs more CPU on later chunks, but avoids inaccurate
seeking and frame loss at H.264 keyframes, VFR timestamps and video cuts.
No temporary input recompression is introduced.
"""
model_w, model_h = p["model"]
wanted = (_sampled_source_index(j, p) for j in range(first_output, last_output))
target = next(wanted, None)
if target is None:
return
with av.open(video_path) as container:
stream = container.streams.video[0]
stream.thread_type = "AUTO"
for i, frame in enumerate(container.decode(stream)):
if i < target:
continue
if i > target:
raise RuntimeError(f"Frame source {target} no encontrado")
img = frame.to_ndarray(format="rgb24", width=model_w, height=model_h,
interpolation="BICUBIC")
yield torch.from_numpy(img).permute(2, 0, 1).contiguous()
target = next(wanted, None)
if target is None:
return
def _chunk_gpu_duration(video_path, scale, detail_mode, export_mode,
crf, first_output, last_output, part_path,
protection_path=None, protection_strength=0.7,
neural_strength=1.0):
p = _auto_plan(video_path, scale, detail_mode)
start = max(0, first_output - CONTEXT_FRAMES)
estimated = _gpu_time_estimate(p, last_output - start, export_mode,
protection_path is not None or
float(neural_strength) < 0.999)
return min(MAX_GPU_SECONDS, max(35, int(math.ceil(estimated + 8))))
@spaces.GPU(duration=_chunk_gpu_duration)
@torch.no_grad()
def _render_gpu_chunk(video_path, scale, detail_mode, export_mode,
crf, first_output, last_output, part_path,
protection_path=None, protection_strength=0.7,
neural_strength=1.0):
"""One new ZeroGPU lease per invocation. Return video without source audio."""
p = _auto_plan(video_path, scale, detail_mode)
start = max(0, first_output - CONTEXT_FRAMES)
discard = first_output - start
expected = last_output - first_output
model_w, model_h = p["model"]
out_w, out_h = p["out"]
device = torch.device("cuda")
pipeline = WanVSRPipeline(
scheduler_config={"num_train_timesteps": 1000, "shift": 5.0},
sampling={
"execution_mode": "streaming", "num_inference_steps": 1,
"timesteps": [1000], "guidance_scale": 1.0, "seed": SEED,
"kv_cache_window_size": WAN.temporal_context_frames,
},
)
pipeline.prepare_streaming(model=WAN, prompt_embeds=WAN.prompt_embeds,
device=device, compute_dtype=DTYPE)
stream = CollaborativeFrameStream(
wan_model=WAN, flash_model=FLASH, wan_pipeline=pipeline,
prompt_embeds=WAN.prompt_embeds, device=device, compute_dtype=DTYPE,
window_size=KEYFRAME_WINDOW, seed=SEED,
)
is_prores = export_mode == EXPORT_PRORES
output = av.open(part_path, mode="w", format="mov" if is_prores else "mp4")
vstream = output.add_stream("prores_ks" if is_prores else "libx264", rate=p["out_fps"])
vstream.width = out_w
vstream.height = out_h
vstream.pix_fmt = "yuv422p10le" if is_prores else "yuv420p"
vstream.options = ({"profile": "3", "vendor": "apl0"} if is_prores
else {"crf": str(crf), "preset": "fast"})
protection = _ProtectionMasks(protection_path) if protection_path else None
use_protection = bool(protection and protection.has_objects)
use_original = use_protection or float(neural_strength) < 0.999
originals = (iter(_source_rgb_window(video_path, p, first_output, last_output))
if use_original else None)
frames = _frames_window(video_path, p, start, last_output)
packets = (InputFrame(i, rgb) for i, rgb in enumerate(frames))
written = 0
produced = 0
try:
for result in stream.frames(packets):
produced += 1
if produced <= discard:
continue
frame = result.rgb.float().add_(1).mul_(0.5)
if tuple(frame.shape[-2:]) != (out_h, out_w):
frame = F.interpolate(frame, size=(out_h, out_w), mode="bicubic",
antialias=True, align_corners=False)
if use_original:
# Always consume the corresponding ORIGINAL-resolution frame:
# if the current mask is empty, subsequent indices still align.
original = next(originals)
mask_np = (protection.at(first_output + written)
if use_protection else None)
if mask_np is not None and np.any(mask_np):
mask = torch.from_numpy(mask_np.copy()).to(
device=device, dtype=torch.float32)[None, None]
mask = F.interpolate(mask, size=(out_h, out_w),
mode="bilinear", align_corners=False)
mask.clamp_(0.0, 1.0)
neural_weight = _neural_fraction(
neural_strength, protection_strength, mask)
else:
neural_weight = _neural_fraction(neural_strength)
# At 100 % strength with no mask, the original RelayVSR frame
# is passed through unchanged and no bicubic work is required.
if not (isinstance(neural_weight, float) and neural_weight >= 0.999):
base = torch.from_numpy(np.ascontiguousarray(original)).to(
device=device).permute(2, 0, 1)[None].float().div_(255)
base = F.interpolate(base, size=(out_h, out_w),
mode="bicubic", antialias=True,
align_corners=False).clamp_(0.0, 1.0)
frame = torch.lerp(base, frame, neural_weight)
rgb = frame.clamp_(0, 1).mul_(255).round_().to(torch.uint8)[0]
rgb = rgb.permute(1, 2, 0).contiguous().cpu().numpy()
for packet in vstream.encode(av.VideoFrame.from_ndarray(rgb, format="rgb24")):
output.mux(packet)
written += 1
for packet in vstream.encode():
output.mux(packet)
finally:
packets.close()
if originals is not None:
originals.close()
output.close()
if written != expected:
# Never silently deliver a clip with missing or duplicated frames.
raise RuntimeError(f"Fragmento incompleto: esperado {expected}, obtenido {written}")
print(f"CHUNK {first_output}:{last_output} warmup={discard} frames={written} "
f"neural={model_w}x{model_h} output={out_w}x{out_h} "
f"protection={use_protection} neural_strength={neural_strength}")
return part_path
def _ffmpeg(args):
result = subprocess.run([imageio_ffmpeg.get_ffmpeg_exe(), "-y", "-nostdin", *args],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if result.returncode:
raise RuntimeError(result.stderr.decode(errors="replace")[-2800:])
def _join_losslessly(job):
"""Concat video packets with -c:v copy (no further generation loss)."""
folder = Path(job["folder"])
ext = ".mov" if job["export"] == EXPORT_PRORES else ".mp4"
file_list = folder / "parts.txt"
with file_list.open("w", encoding="utf-8") as handle:
for part in job["parts"]:
# All fragment paths are created by us, inside our temporary folder.
handle.write(f"file '{Path(part).as_posix()}'\n")
joined = folder / ("video_joined" + ext)
_ffmpeg(["-f", "concat", "-safe", "0", "-i", str(file_list),
"-map", "0:v:0", "-c:v", "copy", "-an", str(joined)])
final = folder / ("RelayVSR_Final_ProResHQ.mov" if ext == ".mov"
else "RelayVSR_Final_H264.mp4")
# First try copying the original sound bit-for-bit. If the input codec
# cannot be muxed into MOV/MP4, encode only the audio as a fallback.
base = ["-i", str(joined), "-i", job["video"], "-map", "0:v:0",
"-map", "1:a:0?", "-c:v", "copy", "-c:a", "copy"]
if ext == ".mp4":
base.extend(["-movflags", "+faststart"])
try:
_ffmpeg([*base, str(final)])
except RuntimeError:
audio_codec = "pcm_s24le" if ext == ".mov" else "aac"
args = ["-i", str(joined), "-i", job["video"],
"-map", "0:v:0", "-map", "1:a:0?", "-c:v", "copy",
"-c:a", audio_codec]
if ext == ".mp4":
args.extend(["-b:a", "320k", "-movflags", "+faststart"])
_ffmpeg([*args, str(final)])
# Validation: the number of output frames must match the global plan.
# PyAV's stream.frames may be zero for MP4, so decode to count if needed.
count = 0
with av.open(str(final)) as container:
stream = container.streams.video[0]
for _ in container.decode(stream):
count += 1
if count != job["plan"]["out_frames"]:
raise RuntimeError(f"Error al unir: {count} frames finales frente a "
f"{job['plan']['out_frames']} previstos")
for part in job["parts"]:
try:
Path(part).unlink()
except OSError:
pass
return str(final)
def _nth_rgb(video_path, index):
"""Read a frame at a global frame index for a genuine before/after."""
with av.open(video_path) as container:
for i, frame in enumerate(container.decode(video=0)):
if i == index:
return Image.fromarray(frame.to_ndarray(format="rgb24"))
return None
def _previews(job, output_path, sample_seconds=3):
p = job["plan"]
index_out = min(p["out_frames"] - 1,
max(0, round(float(sample_seconds) * float(p["out_fps"]))))
index_in = _sampled_source_index(index_out, p)
before = _nth_rgb(job["video"], index_in)
after = _nth_rgb(output_path, index_out)
if before is None or after is None:
return None, None, None, None, None
# Align both to exact output dimensions. The source is intentionally only
# enlarged bicubically: it does not contain AI-restored detail.
before = before.resize(after.size, Image.Resampling.BICUBIC)
cw, ch = min(640, after.width), min(640, after.height)
x = (after.width - cw) // 2
y = (after.height - ch) // 2
bbox = (x, y, x + cw, y + ch)
protection_preview = _preview_protection(job, index_out)
return before, after, before.crop(bbox), after.crop(bbox), protection_preview
def _preview_protection(job, index_out):
"""Overlay only actually detected protection polygons, without GPU."""
if not job.get("protect_small"):
return None
for (first, last), path in zip(job["layout"], job.get("mask_paths", [])):
if first <= index_out < last and path:
plan = _ProtectionMasks(path)
alpha = plan.at(index_out)
if alpha is None or not np.any(alpha):
return None # Do not claim to have detected invisible objects.
source_index = _sampled_source_index(index_out, job["plan"])
source = _nth_rgb(job["video"], source_index)
if source is None:
return None
view_w = min(960, source.width)
view_h = round(view_w * source.height / source.width)
source = source.resize((view_w, view_h), Image.Resampling.BICUBIC)
mask = Image.fromarray((alpha * 255).astype(np.uint8), "L")
mask = mask.resize(source.size, Image.Resampling.BILINEAR)
mask = mask.point(lambda value: int(value *
min(1.0, job["protection_strength"]) * 0.75))
return Image.composite(Image.new("RGB", source.size, (0, 230, 105)),
source, mask)
return None
def compare_at_time(job, seconds):
"""Refresh the preview without using GPU or re-running the restoration."""
if not job or not job.get("final"):
raise gr.Error("Primero termina de procesar el vídeo.")
return _previews(job, job["final"], seconds)
_CLAIMS_LOCK = threading.Lock()
_CLAIMED_NEXT = {} # run ID -> next chunk number; prevents duplicate events
def _claim(job):
with _CLAIMS_LOCK:
run = job["run"]
current = len(job["parts"])
known = _CLAIMED_NEXT.get(run, 0)
if current != known:
return False
_CLAIMED_NEXT[run] = current + 1
return True
def prepare_auto_job(video_path, scale, detail_mode, export_mode, crf,
protect_small=False, protection_strength=0.7,
detection_size=640, neural_strength=1.0,
protection_max_pct=10):
if not video_path:
raise gr.Error("Sube primero un vídeo.")
p = _auto_plan(video_path, scale, detail_mode)
if p["frames"] <= 0:
raise gr.Error("No se ha podido determinar la duración del vídeo.")
seconds = p["frames"] / float(p["fps"])
if seconds > AUTO_MAX_SOURCE_SECONDS:
raise gr.Error(f"Por seguridad, máximo {AUTO_MAX_SOURCE_SECONDS // 60} min por trabajo.")
if seconds < 0.25:
raise gr.Error("El vídeo es demasiado corto.")
if not 0.3 <= float(neural_strength) <= 1.0:
raise gr.Error("La fuerza neuronal debe estar entre 30 % y 100 %.")
if not 4 <= float(protection_max_pct) <= 20:
raise gr.Error("El tamaño máximo debe estar entre 4 % y 20 %.")
blend_original = bool(protect_small) or float(neural_strength) < 0.999
layout = _chunk_layout(p, export_mode, blend_original)
folder = tempfile.mkdtemp(prefix="relayvsr_long_")
job = {
"run": uuid.uuid4().hex, "video": str(video_path), "scale": scale,
"detail": detail_mode, "export": export_mode, "crf": int(crf),
"plan": p, "layout": layout, "parts": [], "folder": folder,
"protect_small": bool(protect_small),
"protection_strength": float(protection_strength),
"detection_size": int(detection_size),
"neural_strength": float(neural_strength),
"protection_max_pct": float(protection_max_pct),
"mask_paths": [], "detections": [],
}
estimates = sum(_gpu_time_estimate(
p, b - max(0, a - CONTEXT_FRAMES), export_mode,
blend_original) for a, b in layout)
status = (
f"⏳ **Preparado:** {seconds:.1f} s; {len(layout)} fragmentos "
f"(máximo {AUTO_CHUNK_SECONDS} s cada uno). "
f"Tiempo de GPU estimado: {estimates / 60:.1f} min, más colas y exportación. "
"El límite diario de ZeroGPU puede detener el proceso. "
+ f"Fuerza neuronal global: {int(round(float(neural_strength) * 100))} %. "
+ ("Protección activada: detección CPU previa a cada fragmento. "
if protect_small else "")
)
return job, None, None, status, None, None, None, None, None
def advance_auto_job(job):
"""This event automatically retriggers on the next gr.State change.
Each invocation is a fresh browser request and therefore a fresh ZeroGPU
authorization: essential for long videos (proxy token expiry otherwise).
"""
if not job or len(job["parts"]) >= len(job["layout"]):
return (gr.skip(),) * 9
if not _claim(job):
return (gr.skip(),) * 9
n = len(job["parts"])
a, b = job["layout"][n]
ext = ".mov" if job["export"] == EXPORT_PRORES else ".mp4"
part_path = str(Path(job["folder"]) / f"segment_{n:04d}{ext}")
try:
mask_path = None
object_count = 0
if job.get("protect_small"):
mask_path = str(Path(job["folder"]) / f"protection_{n:04d}.json")
object_count = _detect_protection_masks(
job["video"], job["plan"], a, b, mask_path,
job.get("detection_size", 640),
job.get("protection_max_pct", 10))
processed = _render_gpu_chunk(
job["video"], job["scale"], job["detail"], job["export"],
job["crf"], a, b, part_path, mask_path,
job.get("protection_strength", 0.7),
job.get("neural_strength", 1.0),
)
updated = {
**job,
"parts": [*job["parts"], processed],
"mask_paths": [*job["mask_paths"], mask_path],
"detections": [*job["detections"], object_count],
}
done = len(updated["parts"])
total = len(updated["layout"])
if done < total:
return (updated, gr.skip(), gr.skip(),
f"⏳ **{done}/{total} fragmentos procesados** "
f"({b / float(job['plan']['out_fps']):.1f} s del vídeo). "
+ (f"{sum(updated['detections'])} detecciones pequeñas. "
if job.get("protect_small") else "")
+ "Continuando automáticamente…",
gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip())
output_path = _join_losslessly(updated)
previews = _previews(updated, output_path)
updated = {**updated, "final": output_path}
p = updated["plan"]
status = (f"✅ **Completado:** {done} fragmentos unidos sin recomprimir vídeo. "
f"{p['out_frames']} fotogramas, "
f"{p['out'][0]}×{p['out'][1]} a {float(p['out_fps']):.3f} fps. "
f"Fuerza neuronal global: {int(round(job.get('neural_strength', 1) * 100))} %. "
"El recorte comparativo está al 100 % de la resolución de salida. "
+ (f"Protección: {sum(updated['detections'])} detecciones "
"en fotogramas analizados (no objetos únicos). "
"Revisa las máscaras: los carruajes históricos pueden "
"pasar inadvertidos. " if job.get("protect_small") else ""))
# MOV/ProRes is intended for downloading/editing. Browsers and
# Gradio's video player often cannot decode it directly.
browser_video = output_path if job["export"] == EXPORT_H264 else None
return (updated, browser_video, output_path, status, *previews)
except Exception as error:
traceback.print_exc()
# Stop without resetting the already finished parts. A new execution
# from the button starts a new clean job; the previous original remains.
status = (f"❌ **Detenido en fragmento {n + 1}/{len(job['layout'])}:** "
f"{type(error).__name__}: {str(error)[:600]}. "
"Los fragmentos anteriores se conservan temporalmente.")
return (gr.skip(), gr.skip(), gr.skip(), status,
gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip())
def clear_result():
return gr.update(value=None)
with gr.Blocks(theme=UPSAMPLER_THEME, css=UPSAMPLER_CSS) as demo:
gr.HTML(header_html(
"RelayVSR Video Upscaler — ampliado",
"Original intacto, vídeos largos, fuerza neuronal y protección de objetos pequeños.",
))
with gr.Tabs():
with gr.Tab("Original · clips cortos"):
with gr.Row():
with gr.Column():
input_video = gr.Video(label="Input Video")
scale = gr.Radio(["2x", "4x"], value="2x", label="Upscale factor")
gr.Markdown(f"Clips hasta {MAX_SECONDS} s. Mismo modelo y ajustes originales.")
run_button = gr.Button("Upscale Video — original", variant="primary")
with gr.Column():
result = gr.Video(interactive=False, label="Upscaled Video")
run_button.click(fn=clear_result, inputs=None, outputs=result).then(
fn=upscale_video, inputs=[input_video, scale], outputs=result,
api_name="upscale_video",
)
with gr.Tab("Vídeo largo · automático"):
gr.Markdown(
"**Un solo clic:** trocea automáticamente en fragmentos de hasta "
"8 segundos, añade 15 fotogramas de contexto para reducir saltos, "
"procesa cada fragmento con RelayVSR y los vuelve a unir **sin "
"segunda compresión del vídeo**. El audio original se reincorpora "
"al final. La cuota de ZeroGPU sigue aplicándose a cada fragmento."
)
with gr.Row():
with gr.Column():
long_video = gr.Video(label="Vídeo completo")
long_scale = gr.Radio(["2x", "4x"], value="2x",
label="Resolución solicitada (tope 4K UHD)")
detail_mode = gr.Radio(
[DETAIL_MODE_ORIGINAL, DETAIL_MODE_EXTENDED],
value=DETAIL_MODE_ORIGINAL,
label="Resolución de inferencia (NO cambia los pesos del modelo)",
)
gr.Markdown(
"El modo **original** usa hasta 960×540 antes del 4× neuronal, "
"como el Space actual. **Más detalle** permite hasta 1280×720 "
"antes del 4×: más lento, mayor consumo de GPU y riesgo de "
"agotamiento de memoria. No garantiza mejores caras o letras."
)
neural_strength = gr.Slider(
0.30, 1.0, value=1.0, step=0.05,
label="Fuerza neuronal global: 1 = RelayVSR completo, "
"0.7 = 70 % RelayVSR + 30 % ampliación original",
)
gr.Markdown(
"**Nuevo:** controla cuánto de la imagen reconstruida "
"se conserva en **todo el fotograma**. Para planos aéreos "
"prueba entre **0,65 y 0,75**; para arquitectura cercana "
"entre **0,80 y 0,90**. Al 1,0 se conserva el comportamiento "
"anterior. No modifica los pesos ni los pasos de RelayVSR. "
"Si una secuencia mezcla planos muy distintos, conviene "
"procesarlos por separado."
)
with gr.Accordion("Protección de objetos pequeños (opcional)", open=False):
protect_small = gr.Checkbox(
value=False,
label="Detectar y proteger automáticamente caballos, "
"personas y vehículos pequeños",
)
protection_strength = gr.Slider(
0.25, 1.0, value=0.70, step=0.05,
label="Protección adicional: 0.7 reduce un 70 % "
"la fuerza neuronal restante dentro de la máscara",
)
protection_max_pct = gr.Slider(
4, 20, value=10, step=1,
label="Tamaño máximo protegido (% del lado mayor "
"del vídeo original; 10 = objetos realmente pequeños)",
)
detection_size = gr.Radio(
[640, 960], value=640,
label="Resolución del detector CPU "
"(960: mejor para objetos pequeños, más lento)",
)
gr.Markdown(
"Un detector ligero analiza fotogramas en **CPU** "
"antes de la GPU y mezcla sus máscaras con la ampliación "
"normal. No corrige errores de anatomía del vídeo original. "
"Los **carruajes históricos** y objetos demasiado pequeños "
"pueden no detectarse. Las máscaras ahora son más ajustadas "
"y se interpolan entre fotogramas: el verde permite "
"comprobar qué se está protegiendo. "
"Requiere añadir `ultralytics>=8.3,<9` a `requirements.txt`."
)
export_mode = gr.Radio(
[EXPORT_H264, EXPORT_PRORES], value=EXPORT_H264,
label="Archivo final",
)
crf = gr.Slider(8, 20, value=12, step=1,
label="Calidad H.264 (CRF: menor = mejor)")
long_button = gr.Button("Procesar vídeo completo", variant="primary")
status = gr.Markdown("Listo para comenzar.")
with gr.Column():
long_result = gr.Video(label="Vídeo final", interactive=False)
download_result = gr.File(label="Descargar vídeo final (MP4 / MOV)")
gr.Markdown("Si eliges ProRes, usa **Descargar vídeo final**: el navegador puede no reproducir MOV.")
gr.Markdown("### Comparación real del mismo fotograma")
with gr.Row():
sample_seconds = gr.Number(value=3, minimum=0,
label="Segundo para comparar")
preview_button = gr.Button("Actualizar comparación (sin GPU)")
with gr.Row():
before_img = gr.Image(label="Antes (re-escalado bicúbico)", interactive=False)
after_img = gr.Image(label="Después (RelayVSR)", interactive=False)
gr.Markdown("### Centro al 100 % (píxeles originales de salida)")
with gr.Row():
before_crop = gr.Image(label="Antes · recorte 1:1", interactive=False)
after_crop = gr.Image(label="Después · recorte 1:1", interactive=False)
protection_preview = gr.Image(
label="Objetos protegidos: verde = máscara detectada",
interactive=False,
)
job_state = gr.State(None)
auto_outputs = [job_state, long_result, download_result, status,
before_img, after_img, before_crop, after_crop,
protection_preview]
long_button.click(
fn=prepare_auto_job,
inputs=[long_video, long_scale, detail_mode, export_mode, crf,
protect_small, protection_strength, detection_size,
neural_strength, protection_max_pct],
outputs=auto_outputs,
api_name=False,
)
# Crucial: one freshly authorized GPU request per State change.
job_state.change(
fn=advance_auto_job,
inputs=job_state,
outputs=auto_outputs,
trigger_mode="multiple",
api_name=False,
)
preview_button.click(
fn=compare_at_time,
inputs=[job_state, sample_seconds],
outputs=[before_img, after_img, before_crop, after_crop,
protection_preview],
api_name=False,
)
gr.HTML(footer_html(
"RelayVSR restoration by one-step diffusion. Original weights are fetched "
"from the pinned upstream revision; this Space does not need Google Drive.",
"https://upsampler.com/free-video-upscaler-no-signup",
"free AI video upscaler",
))
demo.queue().launch(ssr_mode=False)
"""RelayVSR / ZeroGPU: original intact + long videos + adjustable restoration.
Based on the user's original RelayVSR app.py (Upsampler fork, Apache-2.0).
The unmodified original inference entry point is exposed in its own tab.
Long videos use independent ZeroGPU leases for successive frame windows.
Requires the existing refvsr_inference and upsampler_theme packages.
For optional automatic small-object protection, also install `ultralytics`.
The global neural strength mixes RelayVSR with genuine original-resolution
source frames. It changes neither the model weights nor the number of steps.
IMPORTANT: The model weights are deliberately still downloaded from the
original revision of kopper/RelayVSR. The Google Drive backup is independent.
"""
import bisect
import json
import math
import os
import subprocess
import tempfile
import threading
import time
import traceback
import uuid
from fractions import Fraction
from pathlib import Path
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import av
import gradio as gr
import imageio_ffmpeg
import numpy as np
import spaces
import torch
import torch.nn.functional as F
from huggingface_hub import hf_hub_download
from PIL import Image, ImageDraw, ImageFilter
from refvsr_inference.checkpoints import load_checkpoints
from refvsr_inference.models.wan.sampling import WanVSRPipeline
from refvsr_inference.runtime.streaming import CollaborativeFrameStream, InputFrame
from upsampler_theme import UPSAMPLER_CSS, UPSAMPLER_THEME, footer_html, header_html
WEIGHTS_REPO = "kopper/RelayVSR"
WEIGHTS_REVISION = "d43dc5545f9a3bd1c7730b68ebff4ab1f9199bd1"
MODEL_SCALE = 4
KEYFRAME_WINDOW = 16
SEED = 42
DTYPE = torch.bfloat16
MAX_MODEL_PIXELS = 3840 * 2160
MAX_OUTPUT_PIXELS = 3840 * 2160
MAX_FPS = 30
MAX_SECONDS = 10.5
STARTUP_SECONDS = 10
SECONDS_PER_MODEL_MEGAFRAME = 0.033
MAX_GPU_SECONDS = 180
# New workflow. 8 seconds is a good baseline for 720p clips at <=30 fps.
# A more demanding model size/codec can automatically select shorter chunks.
AUTO_CHUNK_SECONDS = 8
AUTO_MIN_CHUNK_SECONDS = 2
CONTEXT_FRAMES = KEYFRAME_WINDOW - 1 # warm up decoder across each boundary
TARGET_GPU_SECONDS = 155 # leave headroom below ZeroGPU's 180 s budget
AUTO_MAX_SOURCE_SECONDS = 600 # guard against enormous jobs / unbounded disk
DETAIL_MODE_ORIGINAL = "Original (máxima compatibilidad)"
DETAIL_MODE_EXTENDED = "Más detalle neuronal (experimental)"
EXPORT_H264 = "H.264 (MP4)"
EXPORT_PRORES = "ProRes 422 HQ (MOV)"
# Small CPU segmentation model. Kept separate from the large RelayVSR weights
# and loaded ONLY when the user explicitly enables object protection.
PROTECTION_REPO = "Ultralytics/YOLO11"
PROTECTION_WEIGHTS = "yolo11n-seg.pt"
PROTECTION_CLASSES = [0, 2, 3, 5, 7, 17] # person, road vehicles, horse (COCO)
PROTECTION_SAMPLE_EVERY = 3 # CPU detection at 8 fps for a 24 fps input
PROTECTION_CUT_THRESHOLD = 0.22
_PROTECTION_MODEL = None
_PROTECTION_MODEL_LOCK = threading.Lock()
print("loading RelayVSR weights")
_wan_path = hf_hub_download(WEIGHTS_REPO, "wan.safetensors", revision=WEIGHTS_REVISION)
_flash_path = hf_hub_download(WEIGHTS_REPO, "flash.safetensors", revision=WEIGHTS_REVISION)
WAN, FLASH = load_checkpoints(_wan_path, _flash_path, device="cpu", dtype=DTYPE)
WAN.to("cuda")
FLASH.to("cuda")
def _even(v):
return max(2, int(v) // 2 * 2)
def _probe(video_path):
"""(fps, frame_count, width, height) of the first video stream."""
with av.open(video_path) as container:
if not container.streams.video:
raise gr.Error("This file has no video stream.")
stream = container.streams.video[0]