Spaces:
Paused
Paused
Commit ·
da72558
0
Parent(s):
RIFE 4.9 frame interpolation on ZeroGPU
Browse files- .gitattributes +37 -0
- .gitignore +4 -0
- README.md +151 -0
- app.py +1301 -0
- cli.py +52 -0
- examples/sample.mp4 +3 -0
- models/rife49.onnx +3 -0
- requirements.txt +8 -0
- rife_core.py +999 -0
.gitattributes
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
*.mp4 filter=lfs diff=lfs merge=lfs -text
|
.gitignore
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.trt/
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.pyc
|
| 4 |
+
.gradio/
|
README.md
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: Flowframes
|
| 3 |
+
emoji: 🎞️
|
| 4 |
+
colorFrom: pink
|
| 5 |
+
colorTo: green
|
| 6 |
+
sdk: gradio
|
| 7 |
+
sdk_version: 6.26.0
|
| 8 |
+
python_version: "3.10"
|
| 9 |
+
app_file: app.py
|
| 10 |
+
pinned: false
|
| 11 |
+
fullWidth: true
|
| 12 |
+
license: mit
|
| 13 |
+
short_description: RIFE frame interpolation, streams a clip of any length
|
| 14 |
+
startup_duration_timeout: 1h
|
| 15 |
+
---
|
| 16 |
+
|
| 17 |
+
# Flowframes
|
| 18 |
+
|
| 19 |
+
RIFE frame interpolation as a Space. Upload a clip, get it back at two or four
|
| 20 |
+
times the frame rate.
|
| 21 |
+
|
| 22 |
+
[Flowframes](https://github.com/n00mkrad/flowframes) does this job on Windows.
|
| 23 |
+
That project is a C# GUI wrapping ffmpeg and a set of interpolation backends, so
|
| 24 |
+
none of it ports directly. What I reimplemented is the orchestration, in one
|
| 25 |
+
`app.py` over one ONNX model.
|
| 26 |
+
|
| 27 |
+
## Length is free, pixels are not
|
| 28 |
+
|
| 29 |
+
Nothing accumulates in memory. ffmpeg decodes raw frames into a pipe, two frames
|
| 30 |
+
are held at a time, each pair is interpolated, and the results stream straight
|
| 31 |
+
into a second ffmpeg. A 4K frame is about 100 MB as float32, so a five minute 4K
|
| 32 |
+
source costs the same memory as a five second one. Only time scales.
|
| 33 |
+
|
| 34 |
+
Time scales with pixels rather than seconds, which is why "Max height" is the
|
| 35 |
+
control that matters most. Halving the height is roughly four times faster.
|
| 36 |
+
|
| 37 |
+
## Segments
|
| 38 |
+
|
| 39 |
+
Work is cut into segments sized from a per-frame rate measured on the hardware
|
| 40 |
+
that is actually running, never from a table. On ZeroGPU each segment is its own
|
| 41 |
+
GPU call, so the 120 s limit bounds a segment rather than the whole job and clip
|
| 42 |
+
length stops being a constraint. On CPU the same seams act as checkpoints, so
|
| 43 |
+
pressing Stop keeps everything already rendered.
|
| 44 |
+
|
| 45 |
+
The Interpolate button carries the estimate and updates as you change the clip,
|
| 46 |
+
the multiplier or Max height. It counts what you actually wait for, which is not
|
| 47 |
+
the same as inference time: the per-segment overhead the plan under-charges, and
|
| 48 |
+
on ZeroGPU the allocation queue, which for a short clip is the largest term of
|
| 49 |
+
the three. A job needing more than six hours is refused up front with a suggested
|
| 50 |
+
resolution, rather than discovering the problem hours in.
|
| 51 |
+
|
| 52 |
+
If a segment overruns its plan badly, the remaining segments are re-cut from what
|
| 53 |
+
that segment actually cost. A segment reports progress only when it finishes, so
|
| 54 |
+
an oversized one freezes the log and adds its whole duration to how long Stop
|
| 55 |
+
takes to respond.
|
| 56 |
+
|
| 57 |
+
## What it costs, measured
|
| 58 |
+
|
| 59 |
+
Every number here came off this Space.
|
| 60 |
+
|
| 61 |
+
| clip | where | result |
|
| 62 |
+
| --- | --- | --- |
|
| 63 |
+
| 6 s, 1280x720, 2x | ZeroGPU | 180 -> 359 frames, 15 s end to end |
|
| 64 |
+
| 2 min, 426x240, 2x | CPU, 12 threads | 3600 -> 7199 frames, 589 s, 47 segments |
|
| 65 |
+
|
| 66 |
+
Per frame at 720p: 0.021 s on the GPU, 0.83 s on a Space CPU. The header quotes
|
| 67 |
+
that number, and it is why the header says seconds on GPU hardware and minutes on
|
| 68 |
+
CPU.
|
| 69 |
+
|
| 70 |
+
That 0.021 was 0.130 until the model was converted from opset 19 to opset 18.
|
| 71 |
+
Nothing else changed and the two graphs are bit identical, but at opset 19 the
|
| 72 |
+
CUDA provider declined all 22 `Resize` nodes and ran them on the CPU, which
|
| 73 |
+
forced a host round trip at each one. Profiling inside the GPU call showed it:
|
| 74 |
+
|
| 75 |
+
```
|
| 76 |
+
opset 19 MemcpyFromHost 34%, Resize 27%, Conv 19% memcpy 44%, on CPU EP 28%
|
| 77 |
+
opset 18 Conv 38%, Add 16%, LeakyRelu 8% memcpy 0%, on CPU EP 0%
|
| 78 |
+
```
|
| 79 |
+
|
| 80 |
+
The copies were never input transfer. They were one op bouncing the graph between
|
| 81 |
+
devices twenty two times per inference, and the fix was a version number.
|
| 82 |
+
|
| 83 |
+
Frame counts are exact. A 47-segment run reassembles to 7199 frames, which is
|
| 84 |
+
`3600 * 2 - 1` to the frame. Segment bounds carry a one-frame overlap and only
|
| 85 |
+
the last segment writes its tail, so no seam duplicates or drops a frame. At 4x
|
| 86 |
+
the same holds across ten segments, `(180 - 1) * 4 + 1 = 717`. Audio carries over
|
| 87 |
+
from the source and survives the segment concat. Odd source dimensions round down
|
| 88 |
+
to even, because yuv420p subsamples chroma 2x2 and x264 refuses an odd side.
|
| 89 |
+
|
| 90 |
+
## Execution providers
|
| 91 |
+
|
| 92 |
+
One model file, best available provider, in order.
|
| 93 |
+
|
| 94 |
+
| provider | status |
|
| 95 |
+
| --- | --- |
|
| 96 |
+
| TensorRT | Registered, never selected, and deliberately not pursued. See below. |
|
| 97 |
+
| CUDA, fp32 | what runs on GPU |
|
| 98 |
+
| CPU, fp32 | no GPU |
|
| 99 |
+
|
| 100 |
+
The TensorRT EP ships inside the ONNX Runtime wheel, so ORT always reports it as
|
| 101 |
+
available, but it needs `libnvinfer.so.10`, TensorRT's own runtime, which is not
|
| 102 |
+
in the image. Adding it is one pinned dependency. The reason I have not: a
|
| 103 |
+
TensorRT engine compiles on first inference, which on ZeroGPU means inside the
|
| 104 |
+
120 s window, and the engine cache lives on a filesystem that dies with the
|
| 105 |
+
container. Every cold start would recompile and might time out. That trades a
|
| 106 |
+
working 0.021 s/frame for a maybe-faster path that intermittently fails to start.
|
| 107 |
+
The code probes for the library before attempting the provider, so a missing
|
| 108 |
+
runtime costs nothing rather than a failed session construction on every GPU
|
| 109 |
+
call, and the path lights up by itself if the image ever ships it.
|
| 110 |
+
|
| 111 |
+
Providers are tried one at a time rather than as a single list, because ORT's
|
| 112 |
+
list is all or nothing: a missing TensorRT takes CUDA down with it and lands on
|
| 113 |
+
CPU. ORT also does not raise when a provider fails. It prints `EP Error`, falls
|
| 114 |
+
back to CPU inside the constructor and hands back a valid session, so the code
|
| 115 |
+
reads the selected provider back from the session and prints it in the header. A
|
| 116 |
+
Space running fifty times slower than it should looks exactly like one that is
|
| 117 |
+
not.
|
| 118 |
+
|
| 119 |
+
fp16 is not used on the GPU either, and that was a measurement rather than an
|
| 120 |
+
oversight. I converted the model to fp16 and A/B'd it against fp32 on the CUDA
|
| 121 |
+
provider inside a single GPU allocation on one container: fp16 0.1096 s/frame
|
| 122 |
+
against fp32 0.0306. Three and a half times slower. Cosine agreement was 0.99994,
|
| 123 |
+
so a quality gate would have passed it straight through.
|
| 124 |
+
|
| 125 |
+
On the CPU path fp16 is wrong for a different reason. Those cores have no half
|
| 126 |
+
precision ALU, so every operation promotes to fp32 and you pay the conversion on
|
| 127 |
+
top.
|
| 128 |
+
|
| 129 |
+
Thread count comes from the cgroup, not `os.cpu_count()`, which reports the host.
|
| 130 |
+
The ZeroGPU container's `cpu.max` reads `1600000 100000`, so sixteen CPUs, and
|
| 131 |
+
the cap takes twelve of them: measured at 720p this graph stops scaling around
|
| 132 |
+
twelve threads and thirty two run 4.3x slower. Sixteen is worth knowing because
|
| 133 |
+
it is not the two vCPUs a free CPU Space gets, and sizing anything against two
|
| 134 |
+
here would be wrong by a factor of six.
|
| 135 |
+
|
| 136 |
+
## Model
|
| 137 |
+
|
| 138 |
+
RIFE 4.9, ONNX, 21 MB, committed to this repo rather than downloaded at startup,
|
| 139 |
+
so a wake costs nothing and no upstream repo can disappear from under it.
|
| 140 |
+
|
| 141 |
+
Stored at opset 18, converted from the opset 19 export. The conversion is bit
|
| 142 |
+
identical on this graph, verified across three resolutions and three timesteps,
|
| 143 |
+
and it is what keeps `Resize` on the CUDA provider. Re-exporting at opset 19
|
| 144 |
+
would silently cost a factor of five.
|
| 145 |
+
|
| 146 |
+
## Credit
|
| 147 |
+
|
| 148 |
+
[RIFE](https://github.com/hzwer/Practical-RIFE) by Huang et al. ONNX export from
|
| 149 |
+
[yuvraj108c/rife-onnx](https://huggingface.co/yuvraj108c/rife-onnx).
|
| 150 |
+
Orchestration modelled on [Flowframes](https://github.com/n00mkrad/flowframes) by
|
| 151 |
+
n00mkrad.
|
app.py
ADDED
|
@@ -0,0 +1,1301 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Flowframes on Spaces: RIFE frame interpolation that runs on CPU or GPU.
|
| 2 |
+
|
| 3 |
+
One code path, one model file. The execution provider is chosen at startup
|
| 4 |
+
(TensorRT, else CUDA, else CPU) and whatever was actually selected is printed in
|
| 5 |
+
the header, because a missing CUDA library drops ORT to CPU silently and a Space
|
| 6 |
+
running 50x slower looks identical to one that is not.
|
| 7 |
+
|
| 8 |
+
Work is cut into segments sized from a measured per-frame rate. On ZeroGPU each
|
| 9 |
+
segment is its own GPU call, so the 120 s cap bounds a segment rather than the
|
| 10 |
+
job and clip length stops being a limit. On CPU the same seams are checkpoints,
|
| 11 |
+
so cancelling keeps the minutes already spent.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
import os
|
| 15 |
+
import pathlib
|
| 16 |
+
import re
|
| 17 |
+
import shutil
|
| 18 |
+
import sys
|
| 19 |
+
import tempfile
|
| 20 |
+
import threading
|
| 21 |
+
import time
|
| 22 |
+
import uuid
|
| 23 |
+
|
| 24 |
+
# Gradio copies every file it serves into GRADIO_TEMP_DIR and never prunes it,
|
| 25 |
+
# so each delivered output existed twice and only one copy was ever cleaned.
|
| 26 |
+
# Point that cache somewhere sweepable. Must happen before gradio is imported.
|
| 27 |
+
SCRATCH = pathlib.Path(tempfile.gettempdir()) / "flowframes"
|
| 28 |
+
OUTDIR = SCRATCH / "out"
|
| 29 |
+
CACHEDIR = SCRATCH / "gradio"
|
| 30 |
+
for _d in (OUTDIR, CACHEDIR):
|
| 31 |
+
_d.mkdir(parents=True, exist_ok=True)
|
| 32 |
+
os.environ.setdefault("GRADIO_TEMP_DIR", str(CACHEDIR))
|
| 33 |
+
CACHEDIR = pathlib.Path(os.environ["GRADIO_TEMP_DIR"]) # setdefault may not have won
|
| 34 |
+
|
| 35 |
+
import gradio as gr
|
| 36 |
+
|
| 37 |
+
import rife_core as R
|
| 38 |
+
|
| 39 |
+
try:
|
| 40 |
+
import spaces
|
| 41 |
+
GPU = spaces.GPU
|
| 42 |
+
except Exception: # noqa: BLE001
|
| 43 |
+
def GPU(*_a, **_k):
|
| 44 |
+
return lambda f: f
|
| 45 |
+
|
| 46 |
+
# The import above is NOT the test. `spaces` is in requirements.txt and installs
|
| 47 |
+
# anywhere; off ZeroGPU its decorator simply no-ops, so `import spaces` succeeded
|
| 48 |
+
# on cpu-basic too and every ZeroGPU branch was taken there: BUDGET_S 90 s
|
| 49 |
+
# instead of 900, segments ten times smaller than intended, and a header
|
| 50 |
+
# announcing a GPU. Only the environment knows. Read from the live container:
|
| 51 |
+
# SPACES_ZERO_GPU='1' ZEROGPU_V2='true' SPACES_ZERO_DEVICE_API_URL=...
|
| 52 |
+
ZEROGPU = bool(os.environ.get("SPACES_ZERO_GPU"))
|
| 53 |
+
|
| 54 |
+
HERE = pathlib.Path(__file__).parent
|
| 55 |
+
LOG_LINES = 14
|
| 56 |
+
CANCELLED: set[str] = set() # session hashes that asked to stop
|
| 57 |
+
LOCK = threading.Lock()
|
| 58 |
+
OWNER: str | None = None # session hash of the job currently holding LOCK
|
| 59 |
+
|
| 60 |
+
# Built on CPU at import: on ZeroGPU there is no CUDA device until a @spaces.GPU
|
| 61 |
+
# call is running, so a session created here could never see one.
|
| 62 |
+
BACKEND = R.OnnxBackend(prefer_gpu=False)
|
| 63 |
+
_GPU_BE = None
|
| 64 |
+
# (height, width, device) -> (seconds per frame, backend name, build seconds).
|
| 65 |
+
# Survives between jobs because it lives in the parent, not in a @spaces.GPU fork.
|
| 66 |
+
_RATE_CACHE: dict[tuple[int, int, str], tuple[float, str, float]] = {}
|
| 67 |
+
# Set when a GPU call fails on quota, cleared when one next succeeds. Module
|
| 68 |
+
# level because the header is rendered per page load by visitors who are not
|
| 69 |
+
# running a job, and a Space whose daily quota is gone was still announcing
|
| 70 |
+
# "GPU (ZeroGPU), about 39s" while every job crawled along on CPU.
|
| 71 |
+
GPU_DEAD = False
|
| 72 |
+
# Whether NVENC actually encodes in a GPU fork. False until the first fork
|
| 73 |
+
# reports back, because the parent process has no GPU and cannot find out.
|
| 74 |
+
NVENC_OK = False
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def worker() -> R.Backend:
|
| 78 |
+
"""The backend to actually infer with. Inside a GPU call this upgrades to
|
| 79 |
+
CUDA or TensorRT once, then stays."""
|
| 80 |
+
global _GPU_BE
|
| 81 |
+
if not ZEROGPU:
|
| 82 |
+
return BACKEND
|
| 83 |
+
if _GPU_BE is None:
|
| 84 |
+
t0 = time.time()
|
| 85 |
+
_GPU_BE = R.OnnxBackend(prefer_gpu=True, trt_cache=str(HERE / ".trt"))
|
| 86 |
+
print(f"[ff] BUILT gpu worker {_GPU_BE.name} in {time.time()-t0:.0f}s "
|
| 87 |
+
f"pid={os.getpid()}", flush=True)
|
| 88 |
+
else:
|
| 89 |
+
print(f"[ff] REUSED gpu worker pid={os.getpid()}", flush=True)
|
| 90 |
+
return _GPU_BE
|
| 91 |
+
# Reference rate at 720p so the header can quote a real number rather than a table.
|
| 92 |
+
REF_RATE = R.measure_rate(BACKEND, 720, 1280)
|
| 93 |
+
# One GPU call must finish inside its ZeroGPU budget; CPU has no such cap but
|
| 94 |
+
# still benefits from the checkpoint seams.
|
| 95 |
+
#
|
| 96 |
+
# 60, not 90, and not the 75 I first reasoned my way to. plan_gaps charges only
|
| 97 |
+
# the inference term, so a segment costs more than it plans for. Measured on this
|
| 98 |
+
# Space against a hard 120 s @spaces.GPU window:
|
| 99 |
+
#
|
| 100 |
+
# 852x480 f2 1.416x, 1.456x 360x640 f2 1.393x, 1.496x
|
| 101 |
+
# 852x480 f4 1.566x <-- worst
|
| 102 |
+
#
|
| 103 |
+
# I predicted f4 would be BELOW f2, on the theory that a fixed per-gap overhead
|
| 104 |
+
# is diluted by three inferences instead of one. It came in highest. The overhead
|
| 105 |
+
# is 0.0235 s/gap at f2 and 0.0957 at f4 - four times larger, not fixed - and per
|
| 106 |
+
# generated frame it is superlinear too, so there is no model here, only a worst
|
| 107 |
+
# case. At the worst observed 1.566x: 90 plans 141 s (killed), 75 plans 117 s
|
| 108 |
+
# (2% of headroom, which is not headroom), 60 plans 94 s.
|
| 109 |
+
#
|
| 110 |
+
# The cost of being conservative is refunded by the re-cut below, which grows the
|
| 111 |
+
# plan back when a segment proves cheaper than budgeted. CPU stays 900: no window
|
| 112 |
+
# to overrun, and bigger segments mean fewer ffmpeg restarts.
|
| 113 |
+
BUDGET_S = 60.0 if ZEROGPU else 900.0
|
| 114 |
+
print(f"[ff] provider={BACKEND.provider} rate720p={REF_RATE:.3f}s/frame "
|
| 115 |
+
f"budget={BUDGET_S:.0f}s zerogpu={ZEROGPU}", flush=True)
|
| 116 |
+
# Kept: if HF ever renames the marker, this line is how it gets noticed.
|
| 117 |
+
print("[ff] env " + " ".join(sorted(k for k in os.environ if "ZERO" in k.upper())),
|
| 118 |
+
flush=True)
|
| 119 |
+
# On GPU the plan-check ratio says 40-66% of a segment is decode, emit and x264
|
| 120 |
+
# on two vCPUs rather than the model, so whether ffmpeg can offload any of that
|
| 121 |
+
# is the largest open speed question. It depends on the image's ffmpeg build,
|
| 122 |
+
# which nothing else here can see.
|
| 123 |
+
R.report_ffmpeg_hw()
|
| 124 |
+
# Whether the thread count came from the cgroup or from a failed read matters:
|
| 125 |
+
# one is the container's real allowance, the other is the host's count and means
|
| 126 |
+
# ORT is oversubscribing the two vCPUs a Space is supposed to have.
|
| 127 |
+
print(f"[rife] threads={R.CPUS} from: {R.cgroup_report()}", flush=True)
|
| 128 |
+
# One-shot: is the 12-thread cap right for THIS container? The sweep behind it
|
| 129 |
+
# measured 12 against 32 on a dev box, and the ZeroGPU container allows 16, which
|
| 130 |
+
# was never tested. Both measured in the same boot so the comparison is not
|
| 131 |
+
# across two containers with different noise. Costs a few seconds once, against a
|
| 132 |
+
# 1h startup budget, and the CPU path is what users hit when quota runs out.
|
| 133 |
+
if ZEROGPU and R.container_cpus() > R.CPUS:
|
| 134 |
+
try:
|
| 135 |
+
_alt = R.OnnxBackend(prefer_gpu=False, threads=R.container_cpus())
|
| 136 |
+
_r_alt = R.measure_rate(_alt, 720, 1280)
|
| 137 |
+
del _alt
|
| 138 |
+
print(f"[rife] thread sweep at 720p: {R.CPUS} threads {REF_RATE:.3f}s/frame, "
|
| 139 |
+
f"{R.container_cpus()} threads {_r_alt:.3f}s/frame -> "
|
| 140 |
+
f"{'raise the cap' if _r_alt < REF_RATE * 0.95 else 'cap is right'}",
|
| 141 |
+
flush=True)
|
| 142 |
+
except Exception as _e: # noqa: BLE001
|
| 143 |
+
print(f"[rife] thread sweep failed ({type(_e).__name__})", flush=True)
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def hms(s: float) -> str:
|
| 147 |
+
s = int(s)
|
| 148 |
+
return f"{s//3600}:{(s%3600)//60:02d}:{s%60:02d}" if s >= 3600 else f"{s//60}:{s%60:02d}"
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def plain(seconds: float) -> str:
|
| 152 |
+
"""Wall clock the way a person says it, not seconds-per-frame."""
|
| 153 |
+
if seconds < 90:
|
| 154 |
+
return f"{int(seconds)}s"
|
| 155 |
+
if seconds < 5400:
|
| 156 |
+
return f"{seconds/60:.0f} min"
|
| 157 |
+
return f"{seconds/3600:.1f} h"
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
# Measured at 1280x720 on the ZeroGPU device (RTX PRO 6000 Blackwell MIG 2g.48gb),
|
| 161 |
+
# fp32 CUDA, by running a real job through this Space. Used only as the a-priori
|
| 162 |
+
# number in the header before any job has run here, because there is no GPU at
|
| 163 |
+
# import time to measure on. Quoted at 720p directly rather than scaled up from a
|
| 164 |
+
# smaller measurement, so the header carries a number and not an extrapolation.
|
| 165 |
+
GPU_REF_RATE_720P = 0.130
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def measured() -> tuple[float, str, int, int] | None:
|
| 169 |
+
"""The rate this Space actually runs at, with the device that produced it.
|
| 170 |
+
|
| 171 |
+
One snapshot and one preference order, so the name and the number in the
|
| 172 |
+
header always describe the same device. Two independent lookups over this
|
| 173 |
+
dict used to disagree: the estimate preferred a GPU entry while the name took
|
| 174 |
+
whatever was inserted first, which on a Space that ran on CPU before the
|
| 175 |
+
quota reset printed a CPU backend beside a GPU rate.
|
| 176 |
+
|
| 177 |
+
The snapshot also matters: demo.load runs outside the queue on every page
|
| 178 |
+
load, so a job in another session can insert here mid-iteration.
|
| 179 |
+
"""
|
| 180 |
+
snap = list(_RATE_CACHE.items())
|
| 181 |
+
devices = ("cpu",) if GPU_DEAD else ("gpu", "cpu")
|
| 182 |
+
# 1280x720 first, because "10s 720p clip" is the phrase a person can act on
|
| 183 |
+
# and the default Max height produces exactly that key on any 16:9 source,
|
| 184 |
+
# which is most video. Failing that, quote whatever WAS measured and name its
|
| 185 |
+
# size: a number that is true about an odd resolution beats a familiar one
|
| 186 |
+
# restated from a measurement taken somewhere else.
|
| 187 |
+
for exact in (True, False):
|
| 188 |
+
for want in devices:
|
| 189 |
+
for (h, w, dev), (rate, name, _b) in snap:
|
| 190 |
+
if dev == want and (not exact or (h, w) == (720, 1280)):
|
| 191 |
+
return rate, name, h, w
|
| 192 |
+
return None
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def header() -> str:
|
| 196 |
+
"""A 10 s 30fps clip at 2x is 300 generated frames.
|
| 197 |
+
|
| 198 |
+
The measured rate is quoted at the resolution it was measured at, never
|
| 199 |
+
restated at 720p. Scaling by pixel count looked exact from 640x360 (0.032
|
| 200 |
+
measured, 0.033 predicted) and was 15% out from 852x480 (0.067 measured
|
| 201 |
+
predicts 45 s; the directly measured 720p answer is 39 s), because a small
|
| 202 |
+
frame leaves the GPU idle. Naming the resolution costs nothing and means the
|
| 203 |
+
number is always something that was actually observed.
|
| 204 |
+
"""
|
| 205 |
+
hit = measured()
|
| 206 |
+
if hit:
|
| 207 |
+
rate, name, h, w = hit
|
| 208 |
+
size = "720p" if (h, w) == (720, 1280) else f"{w}x{h}"
|
| 209 |
+
cost = f"about {plain(rate * 300)} for a 10s {size} clip at 2x"
|
| 210 |
+
elif ZEROGPU:
|
| 211 |
+
name = "GPU (ZeroGPU)"
|
| 212 |
+
cost = f"about {plain(GPU_REF_RATE_720P * 300)} for a 10s 720p clip at 2x"
|
| 213 |
+
else:
|
| 214 |
+
name = BACKEND.name
|
| 215 |
+
cost = f"about {plain(REF_RATE * 300)} for a 10s 720p clip at 2x"
|
| 216 |
+
return (f"[flowframes](https://github.com/n00mkrad/flowframes): "
|
| 217 |
+
f"RIFE 4.9 frame interpolation · **{name}** · **{cost}** · "
|
| 218 |
+
f"streams two frames at a time, so length costs time and not memory")
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
class Log:
|
| 222 |
+
"""Ring buffer with elapsed-time prefixes.
|
| 223 |
+
|
| 224 |
+
Every yield re-sends the whole joined string over SSE, so an unbounded log
|
| 225 |
+
grows each frame. Elapsed rather than wall clock because on a long job the
|
| 226 |
+
delta between two lines is the only ETA instrument the viewer has.
|
| 227 |
+
"""
|
| 228 |
+
|
| 229 |
+
def __init__(self, cap: int = 400):
|
| 230 |
+
self.t0 = time.time()
|
| 231 |
+
self.lines: list[str] = []
|
| 232 |
+
self.cap = cap
|
| 233 |
+
|
| 234 |
+
def __call__(self, msg: str) -> str:
|
| 235 |
+
self.lines.append(f"[+{hms(time.time() - self.t0)}] {msg}")
|
| 236 |
+
if len(self.lines) > self.cap:
|
| 237 |
+
del self.lines[: self.cap // 2]
|
| 238 |
+
return "\n".join(self.lines[-LOG_LINES:])
|
| 239 |
+
|
| 240 |
+
@property
|
| 241 |
+
def text(self) -> str:
|
| 242 |
+
return "\n".join(self.lines[-LOG_LINES:])
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
def stop(request: gr.Request):
|
| 246 |
+
"""Cancel only the caller's own job.
|
| 247 |
+
|
| 248 |
+
This handler runs outside the queue, so a second visitor waiting behind the
|
| 249 |
+
first used to be able to kill the first visitor's run by pressing Stop, and
|
| 250 |
+
anyone could POST the endpoint directly. Cancellation is keyed on the session
|
| 251 |
+
that owns the running job.
|
| 252 |
+
"""
|
| 253 |
+
me = getattr(request, "session_hash", None) or f"anon-{uuid.uuid4()}"
|
| 254 |
+
if OWNER is not None and me != OWNER:
|
| 255 |
+
return "That job belongs to another visitor. Yours has not started yet."
|
| 256 |
+
CANCELLED.add(me)
|
| 257 |
+
return "Stopping at the next frame boundary. The finished part is still saved."
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def _segment_duration(src, dst, start, gaps, factor, clip, crf, write_tail,
|
| 261 |
+
me, tw, th) -> int:
|
| 262 |
+
"""Ask for what this particular segment needs, not for the ceiling.
|
| 263 |
+
|
| 264 |
+
Takes the same arguments as the call it sizes, which is how a `duration`
|
| 265 |
+
callable is invoked. Reuses the planner's own arithmetic and the measured
|
| 266 |
+
`_OVERHEAD`, so a segment cut for a 60s budget asks for roughly 60s rather
|
| 267 |
+
than the 120s cap, and a one-gap segment asks for seconds.
|
| 268 |
+
|
| 269 |
+
Under-asking is not free: ZeroGPU kills a call that outlives its declaration.
|
| 270 |
+
That lands in the existing `is_timeout_error` path, which halves the segment
|
| 271 |
+
and retries up to four times, so it degrades into a slower job rather than a
|
| 272 |
+
failed one. Hence the 1.4x on top of an overhead figure that already carries
|
| 273 |
+
the measured 1.40-1.66 correction.
|
| 274 |
+
"""
|
| 275 |
+
# A measurement of THIS shape beats the pooled mean. _PER_GAP holds the wall
|
| 276 |
+
# cost per gap of the last full segment at this (h, w, factor), which is the
|
| 277 |
+
# same quantity the re-cut sizes `gaps` from - so the declaration and the
|
| 278 |
+
# plan agree by construction instead of drifting apart with the mean.
|
| 279 |
+
seen = _PER_GAP.get((th, tw, int(factor), "gpu"))
|
| 280 |
+
if seen:
|
| 281 |
+
# 1.35, not 1.4: it gives every segment the same headroom the re-cut
|
| 282 |
+
# already assumes is enough. REAL_BUDGET_S is 0.75 of the cap, so a 25%
|
| 283 |
+
# swing - the deadband's own definition of ordinary - still lands under
|
| 284 |
+
# 119. A short tail segment then gets the same margin as a full one,
|
| 285 |
+
# instead of whatever the pooled mean happened to imply for it.
|
| 286 |
+
return max(15, min(GPU_MAX_S, int(gaps * seen * 1.35) + 5))
|
| 287 |
+
hit = _RATE_CACHE.get((th, tw, "gpu"))
|
| 288 |
+
if not hit:
|
| 289 |
+
return GPU_MAX_S # nothing measured yet, keep the ceiling
|
| 290 |
+
est = gaps * hit[0] * max(1, int(factor) - 1) * _OVERHEAD["gpu"]
|
| 291 |
+
return max(15, min(GPU_MAX_S, int(est * 1.4) + 5))
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
@GPU(duration=_segment_duration)
|
| 295 |
+
def _segment_gpu(src, dst, start, gaps, factor, clip, crf, write_tail, me, tw, th):
|
| 296 |
+
"""One segment, start to finish, inside a single GPU allocation.
|
| 297 |
+
|
| 298 |
+
Returns (frames, seconds_inside). The caller times this call from the parent,
|
| 299 |
+
which means its clock also covers the ZeroGPU allocation queue - 45s on a
|
| 300 |
+
session's first call. That number is not the cost of the work, and using it
|
| 301 |
+
as one shrank the next segment ~2x and permanently skewed the overhead mean.
|
| 302 |
+
"""
|
| 303 |
+
t0 = time.perf_counter()
|
| 304 |
+
n = R.interpolate_segment(src, dst, start, gaps, factor, worker(), clip,
|
| 305 |
+
crf=crf, cancel=lambda: me in CANCELLED,
|
| 306 |
+
write_tail=write_tail, out_w=tw, out_h=th,
|
| 307 |
+
hw_encode=NVENC_OK)
|
| 308 |
+
return n, time.perf_counter() - t0
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
def _segment_cpu(src, dst, start, gaps, factor, clip, crf, write_tail, me, tw, th):
|
| 312 |
+
"""The same work without a GPU allocation, for when ZeroGPU quota is gone.
|
| 313 |
+
|
| 314 |
+
Same (frames, seconds_inside) shape as the GPU worker so the caller does not
|
| 315 |
+
branch on which one ran.
|
| 316 |
+
"""
|
| 317 |
+
t0 = time.perf_counter()
|
| 318 |
+
n = R.interpolate_segment(src, dst, start, gaps, factor, BACKEND, clip,
|
| 319 |
+
crf=crf, cancel=lambda: me in CANCELLED,
|
| 320 |
+
write_tail=write_tail, out_w=tw, out_h=th)
|
| 321 |
+
return n, time.perf_counter() - t0
|
| 322 |
+
|
| 323 |
+
|
| 324 |
+
# Two different failures used to share one bucket. "GPU task aborted" is what a
|
| 325 |
+
# segment that overran its 120 s window looks like, and calling that quota
|
| 326 |
+
# exhaustion moved the whole rest of the job to CPU, told the user the day's
|
| 327 |
+
# budget was gone, and never mentioned the real fault. A timeout means the
|
| 328 |
+
# segment was too big; only the quota messages mean there is no GPU left today.
|
| 329 |
+
QUOTA_HINTS = ("exceeded your zerogpu", "zerogpu runs limit", "proxy token",
|
| 330 |
+
"no gpu available", "gpu quota")
|
| 331 |
+
TIMEOUT_HINTS = ("gpu task aborted", "illegal duration", "exceeded the maximum",
|
| 332 |
+
"timeout")
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
def is_quota_error(e: BaseException) -> bool:
|
| 336 |
+
return any(h in str(e).lower() for h in QUOTA_HINTS)
|
| 337 |
+
|
| 338 |
+
|
| 339 |
+
def is_timeout_error(e: BaseException) -> bool:
|
| 340 |
+
return any(h in str(e).lower() for h in TIMEOUT_HINTS)
|
| 341 |
+
|
| 342 |
+
|
| 343 |
+
def cpu_rate(h: int, w: int) -> tuple[float, str, float]:
|
| 344 |
+
"""Same caching for the CPU path. Measuring costs a few seconds of the two
|
| 345 |
+
vCPUs a job is about to need."""
|
| 346 |
+
key = (h, w, "cpu")
|
| 347 |
+
if key not in _RATE_CACHE:
|
| 348 |
+
_RATE_CACHE[key] = (R.measure_rate(BACKEND, h, w), BACKEND.name, 0.0)
|
| 349 |
+
return _RATE_CACHE[key]
|
| 350 |
+
|
| 351 |
+
|
| 352 |
+
def gpu_rate(h: int, w: int) -> tuple[float, str, float]:
|
| 353 |
+
"""Cached in the parent process, which survives between jobs even though each
|
| 354 |
+
@spaces.GPU call runs in a fork. ZeroGPU meters runs rather than seconds, so a
|
| 355 |
+
90 s measurement costs the same as a 90 s segment; paying it on every job was
|
| 356 |
+
spending a fifth of a short job's budget to re-learn a constant."""
|
| 357 |
+
global NVENC_OK
|
| 358 |
+
key = (h, w, "gpu")
|
| 359 |
+
if key not in _RATE_CACHE:
|
| 360 |
+
rate, name, build, nvenc = _gpu_rate(h, w)
|
| 361 |
+
# The cache stays a 3-tuple. measured() unpacks it as (rate, name, _b),
|
| 362 |
+
# and an element-count mismatch here has already cost one debugging
|
| 363 |
+
# session, so the new field rides beside the cache rather than inside it.
|
| 364 |
+
_RATE_CACHE[key] = (rate, name, build)
|
| 365 |
+
NVENC_OK = nvenc
|
| 366 |
+
print(f"[ff] nvenc usable in the GPU fork: {nvenc}", flush=True)
|
| 367 |
+
return _RATE_CACHE[key]
|
| 368 |
+
|
| 369 |
+
|
| 370 |
+
# measure_rate times three samples. At keep-source 4K that is 3 x ~1.1s, plus a
|
| 371 |
+
# session build the logs show landing in under a second once the CUDA libs are
|
| 372 |
+
# preloaded. 90s was reserving a minute and a half of quota to spend four
|
| 373 |
+
# seconds, on the one call whose queue wait the user actually sits through.
|
| 374 |
+
@GPU(duration=30)
|
| 375 |
+
def _gpu_rate(h: int, w: int) -> tuple[float, str, float]:
|
| 376 |
+
"""Measure the per-frame rate where the work will actually happen.
|
| 377 |
+
|
| 378 |
+
Measuring outside a GPU call times the CPU session, and the segment planner
|
| 379 |
+
would then size GPU segments from a CPU number, making them far too small.
|
| 380 |
+
"""
|
| 381 |
+
t0 = time.time()
|
| 382 |
+
be = worker() # first call in this worker builds the session
|
| 383 |
+
build = time.time() - t0
|
| 384 |
+
rate = R.measure_rate(be, h, w)
|
| 385 |
+
# Probed here because this fork HAS a GPU, and returned rather than cached,
|
| 386 |
+
# because a global set inside a fork dies with it. Riding the rate call keeps
|
| 387 |
+
# it free: a separate @spaces.GPU probe would burn a metered run on one bool.
|
| 388 |
+
# Printed, not returned: this goes to the container log, and adding a fifth
|
| 389 |
+
# element to the tuple is how the 3-vs-2 unpack bug happened once already.
|
| 390 |
+
# FIXED 720p, not the job's resolution. This runs inside _gpu_rate, which
|
| 391 |
+
# declares duration=30; profiling four extra inferences at a keep-source 4K
|
| 392 |
+
# would add ~5-8s plus a second session build to a probe that already spends
|
| 393 |
+
# ~4s measuring, and a probe that outlives its declared duration is KILLED -
|
| 394 |
+
# which would drop the whole job to CPU. The op MIX is what decides between
|
| 395 |
+
# CUDA graphs and IO binding, and that does not need the job's own size.
|
| 396 |
+
print(f"[ff] profile 1280x720: {R.profile_ops(720, 1280)}", flush=True)
|
| 397 |
+
return rate, be.name, build, R.nvenc_works()
|
| 398 |
+
|
| 399 |
+
|
| 400 |
+
def example_run(video, factor, max_h, crf):
|
| 401 |
+
"""Examples cache one value per output, so drain the generator and hand back
|
| 402 |
+
only the video and the final log. Routed through run() so a cached example is
|
| 403 |
+
the same code path the button takes."""
|
| 404 |
+
class _Anon:
|
| 405 |
+
session_hash = "example"
|
| 406 |
+
vid_out, log_out = None, ""
|
| 407 |
+
for v, lg, *_ in run(video, factor, max_h, crf, _Anon()):
|
| 408 |
+
vid_out, log_out = v or vid_out, lg
|
| 409 |
+
if vid_out is None:
|
| 410 |
+
# Raising keeps Gradio from writing the failure into the cache. Returning
|
| 411 |
+
# None here cached a null video permanently, so the example row stayed
|
| 412 |
+
# broken for every later visitor with nothing to explain why.
|
| 413 |
+
raise gr.Error(f"Example produced no video. {log_out.splitlines()[-1:]}")
|
| 414 |
+
return vid_out, log_out
|
| 415 |
+
|
| 416 |
+
|
| 417 |
+
# 50 GB of non-persistent container disk, shared with the model, the upload and
|
| 418 |
+
# every segment. Four is room for several 4K jobs and nowhere near the ceiling.
|
| 419 |
+
DISK_BUDGET = 4 * 1024 ** 3
|
| 420 |
+
SWEEP_MIN_AGE = 3600 # never touch anything younger than an hour
|
| 421 |
+
SWEEP_KEEP = 3 # ...nor the newest few, whatever the budget says
|
| 422 |
+
ABANDONED_AGE = 6 * 3600 # work dirs left by a kill, not by the finally
|
| 423 |
+
|
| 424 |
+
|
| 425 |
+
VIDEO_NAME = re.compile(r"[\w.\-]+\.(?:mp4|webm|mkv|mov)")
|
| 426 |
+
|
| 427 |
+
|
| 428 |
+
def pinned_names() -> set[str]:
|
| 429 |
+
"""Filenames the lazily cached example row points at.
|
| 430 |
+
|
| 431 |
+
Gradio postprocesses the example's output through save_file_to_cache, so the
|
| 432 |
+
video is copied into GRADIO_TEMP_DIR and the cached_examples CSV stores that
|
| 433 |
+
path. The row is written once and never rewritten, so its mtime never moves
|
| 434 |
+
and it becomes the OLDEST file in the cache: precisely the first thing an
|
| 435 |
+
oldest-first sweep takes. Losing it breaks the example permanently, for
|
| 436 |
+
everyone, with nothing in the log to say why.
|
| 437 |
+
|
| 438 |
+
Matching on the bare filename over-protects at worst, which is the right way
|
| 439 |
+
to be wrong here.
|
| 440 |
+
"""
|
| 441 |
+
names: set[str] = set()
|
| 442 |
+
try:
|
| 443 |
+
for csv in pathlib.Path(".gradio").rglob("*.csv"):
|
| 444 |
+
names.update(VIDEO_NAME.findall(
|
| 445 |
+
csv.read_text(encoding="utf-8", errors="replace")))
|
| 446 |
+
except OSError:
|
| 447 |
+
pass
|
| 448 |
+
return names
|
| 449 |
+
|
| 450 |
+
|
| 451 |
+
def sweep_outputs(budget: int = DISK_BUDGET, protect: str | None = None) -> None:
|
| 452 |
+
"""Hold finished outputs and Gradio's copies of them under a byte budget.
|
| 453 |
+
|
| 454 |
+
Counting files is the wrong unit: three 4K outputs are far more disk than
|
| 455 |
+
thirty 480p ones. Nothing pruned either directory before, so disk-full was
|
| 456 |
+
the expected end state of a busy day, and the first thing that fails when the
|
| 457 |
+
disk is full is the mkdtemp at the top of the next job.
|
| 458 |
+
|
| 459 |
+
Four things must survive regardless of the budget. `protect` is the source
|
| 460 |
+
video of the job doing the sweeping, pinned by resolved path: GRADIO_TEMP_DIR
|
| 461 |
+
is also Gradio's UPLOAD folder, and the age floor does NOT cover it. That
|
| 462 |
+
claim used to sit here and was wrong - SWEEP_MIN_AGE is one hour while this
|
| 463 |
+
code permits a six hour job, so an upload that sat for ninety minutes before
|
| 464 |
+
the user pressed Interpolate could be unlinked two lines before
|
| 465 |
+
`R.probe(video)` read it, and the failure surfaced as "ffmpeg cannot open
|
| 466 |
+
this file as a video", blaming the user's file. The cached example is pinned
|
| 467 |
+
by name. And the newest few are always kept, because a budget smaller than
|
| 468 |
+
one job's output would otherwise delete its own input.
|
| 469 |
+
"""
|
| 470 |
+
now = time.time()
|
| 471 |
+
pinned = pinned_names()
|
| 472 |
+
safe = None
|
| 473 |
+
if protect:
|
| 474 |
+
try:
|
| 475 |
+
safe = os.path.realpath(protect)
|
| 476 |
+
except OSError:
|
| 477 |
+
safe = None
|
| 478 |
+
files = []
|
| 479 |
+
for root in (OUTDIR, CACHEDIR):
|
| 480 |
+
for f in root.rglob("*"):
|
| 481 |
+
try:
|
| 482 |
+
if f.is_file():
|
| 483 |
+
st = f.stat()
|
| 484 |
+
files.append((st.st_mtime, st.st_size, f))
|
| 485 |
+
except OSError:
|
| 486 |
+
pass
|
| 487 |
+
total = sum(sz for _, sz, _ in files)
|
| 488 |
+
files.sort() # oldest first
|
| 489 |
+
keep = {f for _, _, f in files[-SWEEP_KEEP:]}
|
| 490 |
+
for mt, sz, f in files:
|
| 491 |
+
if total <= budget:
|
| 492 |
+
break
|
| 493 |
+
if f in keep or f.name in pinned or now - mt < SWEEP_MIN_AGE:
|
| 494 |
+
continue
|
| 495 |
+
if safe:
|
| 496 |
+
try:
|
| 497 |
+
if os.path.realpath(f) == safe:
|
| 498 |
+
continue
|
| 499 |
+
except OSError:
|
| 500 |
+
pass
|
| 501 |
+
try:
|
| 502 |
+
f.unlink()
|
| 503 |
+
total -= sz
|
| 504 |
+
except OSError:
|
| 505 |
+
pass # an open handle on Windows; still counted, so the next
|
| 506 |
+
# candidate is tried rather than the entry being forgotten
|
| 507 |
+
|
| 508 |
+
# Work directories are removed by run()'s finally on every normal path, but
|
| 509 |
+
# not after a hard kill: an OOM, a container restart, a ZeroGPU teardown.
|
| 510 |
+
# Those hold a whole segment set, gigabytes for a 4K job, and were invisible
|
| 511 |
+
# to the accounting above, so the budget could report itself satisfied while
|
| 512 |
+
# the disk filled anyway.
|
| 513 |
+
for d in SCRATCH.glob("ff_*"):
|
| 514 |
+
try:
|
| 515 |
+
if d.is_dir() and now - d.stat().st_mtime > ABANDONED_AGE:
|
| 516 |
+
shutil.rmtree(d, ignore_errors=True)
|
| 517 |
+
except OSError:
|
| 518 |
+
pass
|
| 519 |
+
|
| 520 |
+
|
| 521 |
+
def preflight(video):
|
| 522 |
+
"""Reject before the buttons swap, never after."""
|
| 523 |
+
if video is None:
|
| 524 |
+
raise gr.Error("Upload a video first, or click the example row below.")
|
| 525 |
+
return BUSY
|
| 526 |
+
|
| 527 |
+
|
| 528 |
+
# The heights the Max height control actually offers, descending. Advice about
|
| 529 |
+
# that control reads the same list, so a suggestion can never name a value the
|
| 530 |
+
# dropdown does not have - which it did: at 480p it proposed 240, and at 360p it
|
| 531 |
+
# claimed 480 was "the smallest step" when 480 IS the smallest step.
|
| 532 |
+
HEIGHTS = (1080, 720, 480)
|
| 533 |
+
|
| 534 |
+
|
| 535 |
+
def next_lower_height(th: int) -> int | None:
|
| 536 |
+
"""The largest offered height below the current one, or None at the floor."""
|
| 537 |
+
return next((h for h in HEIGHTS if h < th), None)
|
| 538 |
+
|
| 539 |
+
|
| 540 |
+
# What the plan does NOT charge, seeded from this Space's own `plan-check` lines
|
| 541 |
+
# and then re-fitted from every full segment that runs. plan_gaps budgets only
|
| 542 |
+
# inference; a real segment also decodes, emits factor-1 frames per gap and runs
|
| 543 |
+
# x264. Seeds are the medians of what was actually logged:
|
| 544 |
+
# gpu 1.400 1.417 1.456 1.566 1.663
|
| 545 |
+
# cpu 1.050 1.187 1.192 1.204 1.230
|
| 546 |
+
# Not a model: a constant would be wrong, because the ratio moves with device and
|
| 547 |
+
# resolution - an earlier additive attempt fitted three points and was falsified
|
| 548 |
+
# by a fourth. This is a running mean of measured ratios and converges on
|
| 549 |
+
# whatever this container actually does.
|
| 550 |
+
# The gpu seed moved 1.46 -> 2.10 when the model went to opset 18 and Resize
|
| 551 |
+
# stopped falling back to the CPU provider. Inference went from 0.105 to 0.021
|
| 552 |
+
# s/frame, so ffmpeg - decode, emit, x264, none of which got faster - became the
|
| 553 |
+
# dominant share of a segment and the ratio rose accordingly. Measured 2.120 and
|
| 554 |
+
# 2.111 on two consecutive 179-gap 720p segments.
|
| 555 |
+
#
|
| 556 |
+
# Leaving it at 1.46 was not merely inaccurate: _segment_duration falls back to
|
| 557 |
+
# `gaps * rate * overhead * 1.4` when no _PER_GAP measurement exists, so the
|
| 558 |
+
# FIRST segment of a long clip would have declared ~85s for ~89s of work and been
|
| 559 |
+
# killed by ZeroGPU. Making the model faster broke the duration declaration.
|
| 560 |
+
_OVERHEAD = {"gpu": 2.10, "cpu": 1.19}
|
| 561 |
+
_OVERHEAD_N = {"gpu": 5, "cpu": 5}
|
| 562 |
+
|
| 563 |
+
# Wall seconds per gap of the last FULL segment at each (h, w, factor, device).
|
| 564 |
+
# Set in the parent after a segment returns, read in the parent by the duration
|
| 565 |
+
# callable, so no fork boundary is crossed. Empty until a segment completes,
|
| 566 |
+
# which is why the reconstruction below it still has to exist.
|
| 567 |
+
#
|
| 568 |
+
# The DEVICE belongs in the key. The ratio moves with resolution and factor - the
|
| 569 |
+
# note in plan_gaps is explicit about that - but the per-gap cost itself moves
|
| 570 |
+
# with the hardware by roughly 4x, so a CPU-measured value would size a GPU
|
| 571 |
+
# segment at four times its true length and hand it a duration to match.
|
| 572 |
+
_PER_GAP: dict[tuple[int, int, int, str], float] = {}
|
| 573 |
+
|
| 574 |
+
# The wait the estimate always ignored. A ZeroGPU allocation queues before it
|
| 575 |
+
# forks: measured 45-46s for the first call of a session, then 1-2s per later
|
| 576 |
+
# call. On a short clip that first wait is the LARGEST term - a real job logged
|
| 577 |
+
# "estimate 0:26", did 31s of work and took 1:17 wall clock, and the 46s nobody
|
| 578 |
+
# was accounting for is the whole difference.
|
| 579 |
+
# Seeded from a COLD container, where the first allocation of a session queued
|
| 580 |
+
# 45-46s. That is the worst case, not the normal one: a warm Space allocates in
|
| 581 |
+
# ~1s, and a browser run whose button promised 59s finished in 22 - 39 phantom
|
| 582 |
+
# seconds, all of them this constant. Now that `took - spent` measures the queue
|
| 583 |
+
# directly, it is a running mean like _OVERHEAD rather than a guess.
|
| 584 |
+
GPU_QUEUE_S = 45.0
|
| 585 |
+
_QUEUE_N = 1
|
| 586 |
+
GPU_ACQUIRE_S = 1.5
|
| 587 |
+
|
| 588 |
+
# ZeroGPU's per-call ceiling. Declaring above a tier's cap does not run slower,
|
| 589 |
+
# it is refused outright with "ZeroGPU illegal duration", so this is a hard limit
|
| 590 |
+
# rather than a target and every duration is clamped to it.
|
| 591 |
+
GPU_MAX_S = 119
|
| 592 |
+
|
| 593 |
+
# What the RE-CUT aims a segment at, in wall-clock seconds.
|
| 594 |
+
#
|
| 595 |
+
# BUDGET_S is not this. plan_gaps charges it against measure_rate, which times
|
| 596 |
+
# one bare inference, so BUDGET_S = 60 means 60s of INFERENCE - about 88s of wall
|
| 597 |
+
# time once decode, emit and x264 are added. The re-cut divides a wall-clock
|
| 598 |
+
# measurement, so it needs a wall-clock budget or it silently re-plans every job
|
| 599 |
+
# down by the overhead factor: want = gaps_per / 1.46, which trips the 0.8
|
| 600 |
+
# deadband on the first segment of every GPU job and then settles there. That put
|
| 601 |
+
# segments at 60s inside a 119s window and cost 1.46x more GPU runs than
|
| 602 |
+
# intended, reported in the log as ordinary adaptive behaviour.
|
| 603 |
+
#
|
| 604 |
+
# 0.75 of the cap restores the planner's own intent (~88s) with 30s of headroom.
|
| 605 |
+
# Safe because real_per_gap is a measurement of the segment that just ran, not
|
| 606 |
+
# the 1.566x worst case that forced BUDGET_S to half the cap - and an overrun
|
| 607 |
+
# still lands in the halve-and-retry path at the cost of one run.
|
| 608 |
+
REAL_BUDGET_S = 0.75 * GPU_MAX_S if ZEROGPU else 900.0
|
| 609 |
+
|
| 610 |
+
|
| 611 |
+
def note_queue(seconds: float) -> None:
|
| 612 |
+
"""Fold one observed allocation wait into the running mean.
|
| 613 |
+
|
| 614 |
+
Bounded below at 0 because a clock skew or a cancelled segment can make the
|
| 615 |
+
subtraction negative, and above at 120 because anything larger is not a queue
|
| 616 |
+
- it is a stall the estimate should not learn from.
|
| 617 |
+
"""
|
| 618 |
+
global GPU_QUEUE_S, _QUEUE_N
|
| 619 |
+
if not 0.0 <= seconds <= 120.0:
|
| 620 |
+
return
|
| 621 |
+
_QUEUE_N = min(_QUEUE_N + 1, 20)
|
| 622 |
+
GPU_QUEUE_S += (seconds - GPU_QUEUE_S) / _QUEUE_N
|
| 623 |
+
|
| 624 |
+
|
| 625 |
+
def note_overhead(dev: str, ratio: float) -> None:
|
| 626 |
+
"""Fold one full segment's measured/assumed ratio into the running mean."""
|
| 627 |
+
if not 0.2 < ratio < 20: # a cancelled or starved segment
|
| 628 |
+
return
|
| 629 |
+
n = _OVERHEAD_N[dev] = min(_OVERHEAD_N[dev] + 1, 40)
|
| 630 |
+
_OVERHEAD[dev] += (ratio - _OVERHEAD[dev]) / n
|
| 631 |
+
|
| 632 |
+
|
| 633 |
+
# How far pixel-linear scaling may be trusted before the estimate stops being a
|
| 634 |
+
# single number. It was 15% out at 852x480, which is fine, and 31x out at
|
| 635 |
+
# 3840x2160, which is not. Two is deliberately conservative: past it, say a
|
| 636 |
+
# range rather than a number nobody can stand behind.
|
| 637 |
+
TRUST_RATIO = 2.0
|
| 638 |
+
|
| 639 |
+
|
| 640 |
+
def _rate_for(h: int, w: int, dev: str | None = None
|
| 641 |
+
) -> tuple[float, bool, float] | None:
|
| 642 |
+
"""Per-frame rate at this size, whether it was measured AT this size, and how
|
| 643 |
+
far the answer had to be extrapolated (target pixels / measured pixels)."""
|
| 644 |
+
if dev is None:
|
| 645 |
+
dev = "gpu" if (ZEROGPU and not GPU_DEAD) else "cpu"
|
| 646 |
+
hit = _RATE_CACHE.get((h, w, dev))
|
| 647 |
+
if hit:
|
| 648 |
+
return hit[0], True, 1.0
|
| 649 |
+
# Fall back to pixel-scaling from the nearest measurement on the same device.
|
| 650 |
+
# That model was measured 15% out at 852x480, which is why it only ever feeds
|
| 651 |
+
# a rounded label carrying a "~", and never the segment planner.
|
| 652 |
+
same = [(hh, ww, v[0]) for (hh, ww, d), v in list(_RATE_CACHE.items())
|
| 653 |
+
if d == dev]
|
| 654 |
+
if not same:
|
| 655 |
+
# Cold container: _RATE_CACHE is empty until a job runs, and seeding it
|
| 656 |
+
# here would be a bug - gpu_rate/cpu_rate treat a present key as measured
|
| 657 |
+
# and would plan the whole job off an a-priori number. So fall back the
|
| 658 |
+
# way header() already does, without writing anything back. REF_RATE was
|
| 659 |
+
# measured at import on THIS container; GPU_REF_RATE_720P is the only
|
| 660 |
+
# a-priori constant in the estimate, and it is replaced the moment a real
|
| 661 |
+
# GPU rate lands.
|
| 662 |
+
same = [(720, 1280, GPU_REF_RATE_720P if dev == "gpu" else REF_RATE)]
|
| 663 |
+
hh, ww, r = min(same, key=lambda t: abs(t[0] * t[1] - h * w))
|
| 664 |
+
ratio = (h * w) / (hh * ww)
|
| 665 |
+
return r * ratio, False, ratio
|
| 666 |
+
|
| 667 |
+
|
| 668 |
+
def eta(clip: R.Clip, factor: int, max_h: int) -> tuple[float, float] | None:
|
| 669 |
+
"""Wall clock from click to result, as (likely, worst) seconds.
|
| 670 |
+
|
| 671 |
+
The log's `estimate` line has always been rate x frames, so it promised less
|
| 672 |
+
than the user waits: it left out the per-segment overhead the plan
|
| 673 |
+
under-charges, and the ZeroGPU allocation queue entirely.
|
| 674 |
+
|
| 675 |
+
The two numbers differ only where the rate had to be extrapolated well past
|
| 676 |
+
anything measured on this container. A 3840x2160 job was quoted 13 min and
|
| 677 |
+
ran 3.5 h: the label scaled the measured 720p GPU rate up by pixel count
|
| 678 |
+
while the job itself fell back to the CPU, whose 4K cost the planner had
|
| 679 |
+
right all along. Past TRUST_RATIO the CPU price is therefore the honest
|
| 680 |
+
upper bound.
|
| 681 |
+
"""
|
| 682 |
+
tw, th = scale_for(clip, int(max_h))
|
| 683 |
+
dev = "gpu" if (ZEROGPU and not GPU_DEAD) else "cpu"
|
| 684 |
+
hit = _rate_for(th, tw, dev)
|
| 685 |
+
if hit is None:
|
| 686 |
+
return None
|
| 687 |
+
rate, _exact, ratio = hit
|
| 688 |
+
|
| 689 |
+
def total_for(r: float, d: str) -> float:
|
| 690 |
+
t = r * clip.frames * (int(factor) - 1) * _OVERHEAD[d]
|
| 691 |
+
if d == "gpu":
|
| 692 |
+
gaps_per = max(1, R.plan_gaps(clip, int(factor), r, BUDGET_S))
|
| 693 |
+
segs = max(1, -(-(clip.frames - 1) // gaps_per))
|
| 694 |
+
t += segs * GPU_ACQUIRE_S
|
| 695 |
+
if (th, tw, "gpu") not in _RATE_CACHE:
|
| 696 |
+
t += GPU_QUEUE_S # the cold probe the user waits through
|
| 697 |
+
return t
|
| 698 |
+
|
| 699 |
+
likely = total_for(rate, dev)
|
| 700 |
+
if ratio <= TRUST_RATIO or dev != "gpu":
|
| 701 |
+
return likely, likely
|
| 702 |
+
slow = _rate_for(th, tw, "cpu")
|
| 703 |
+
worst = total_for(slow[0], "cpu") if slow else likely
|
| 704 |
+
return likely, max(likely, worst)
|
| 705 |
+
|
| 706 |
+
|
| 707 |
+
_ETA_PROBE: dict[str, R.Clip] = {}
|
| 708 |
+
|
| 709 |
+
|
| 710 |
+
def button_label(video, factor, max_h):
|
| 711 |
+
"""Put the cost in the control that starts it, recomputed as inputs change.
|
| 712 |
+
|
| 713 |
+
Never raises into the UI and never blocks it: a probe failure or an unknown
|
| 714 |
+
rate just leaves the plain label, because a button that cannot be clicked is
|
| 715 |
+
worse than a button with no number on it.
|
| 716 |
+
"""
|
| 717 |
+
if not video:
|
| 718 |
+
return gr.update(value="Interpolate")
|
| 719 |
+
try:
|
| 720 |
+
clip = _ETA_PROBE.get(video)
|
| 721 |
+
if clip is None:
|
| 722 |
+
if len(_ETA_PROBE) > 32:
|
| 723 |
+
_ETA_PROBE.clear()
|
| 724 |
+
clip = _ETA_PROBE[video] = R.probe(video)
|
| 725 |
+
pair = eta(clip, factor, max_h)
|
| 726 |
+
except Exception: # noqa: BLE001
|
| 727 |
+
return gr.update(value="Interpolate")
|
| 728 |
+
if not pair or not pair[0]:
|
| 729 |
+
return gr.update(value="Interpolate")
|
| 730 |
+
likely, worst = pair
|
| 731 |
+
if worst > likely * 1.5:
|
| 732 |
+
# Never a lone "~13 min" for a size nothing was measured at. The spread
|
| 733 |
+
# IS the message: it says the height cap is the lever worth moving.
|
| 734 |
+
return gr.update(value=f"Interpolate ({plain(likely)} - {plain(worst)})")
|
| 735 |
+
return gr.update(value=f"Interpolate (~{plain(likely)})")
|
| 736 |
+
|
| 737 |
+
|
| 738 |
+
def scale_for(clip: R.Clip, max_h: int) -> tuple[int, int]:
|
| 739 |
+
"""Both dimensions are forced even unconditionally: yuv420p subsamples chroma
|
| 740 |
+
2x2 and x264 refuses an odd side, so a 1920x1038 or phone-cropped source used
|
| 741 |
+
to die on the first frame written even though nothing else about it was hard."""
|
| 742 |
+
if max_h <= 0 or clip.height <= max_h:
|
| 743 |
+
return clip.width // 2 * 2, clip.height // 2 * 2
|
| 744 |
+
w = int(round(clip.width * max_h / clip.height)) // 2 * 2
|
| 745 |
+
return w, max_h // 2 * 2
|
| 746 |
+
|
| 747 |
+
|
| 748 |
+
def run(video: str, factor: int, max_h: int, crf: float,
|
| 749 |
+
request: gr.Request, progress=gr.Progress()):
|
| 750 |
+
"""Interpolate a video to a higher frame rate with RIFE.
|
| 751 |
+
|
| 752 |
+
Args:
|
| 753 |
+
video: Path to the source clip. Any container ffmpeg can read.
|
| 754 |
+
factor: Frame rate multiplier, 2 doubles the frame rate and 4 quadruples it.
|
| 755 |
+
max_h: Cap the output height, 0 keeps the source resolution. Lowering this is
|
| 756 |
+
the single biggest lever on runtime, since cost scales with pixels.
|
| 757 |
+
crf: x264 quality, lower is better and larger. 18 is visually lossless.
|
| 758 |
+
"""
|
| 759 |
+
global OWNER, GPU_DEAD
|
| 760 |
+
log = Log()
|
| 761 |
+
me = getattr(request, "session_hash", None) or f"anon-{uuid.uuid4()}"
|
| 762 |
+
if not LOCK.acquire(blocking=False):
|
| 763 |
+
raise gr.Error("A job is already running. Two vCPUs, one job. "
|
| 764 |
+
"Press Stop to cancel it.")
|
| 765 |
+
CANCELLED.discard(me)
|
| 766 |
+
OWNER = me
|
| 767 |
+
closing = False
|
| 768 |
+
stopped = "" # why the run ended early, if it did
|
| 769 |
+
out_path = None
|
| 770 |
+
parts: list[str] = []
|
| 771 |
+
work = None
|
| 772 |
+
try:
|
| 773 |
+
# Before the mkdtemp, because mkdtemp is the first thing that fails on a
|
| 774 |
+
# full disk. Inside the try, not before it: failing between acquire() and
|
| 775 |
+
# try: held the lock with nothing to release it, wedging every later job.
|
| 776 |
+
sweep_outputs(protect=video)
|
| 777 |
+
work = tempfile.mkdtemp(prefix="ff_", dir=str(SCRATCH))
|
| 778 |
+
clip = R.probe(video)
|
| 779 |
+
tw, th = scale_for(clip, int(max_h))
|
| 780 |
+
src = video
|
| 781 |
+
|
| 782 |
+
yield None, log(f"source {clip.width}x{clip.height} {clip.fps:.2f}fps "
|
| 783 |
+
f"{clip.frames} frames {clip.duration:.1f}s"), *BUSY
|
| 784 |
+
|
| 785 |
+
if (tw, th) != (clip.width, clip.height):
|
| 786 |
+
yield None, log(f"scaling to {tw}x{th} inside each segment decoder, "
|
| 787 |
+
f"no pre-transcode"), *BUSY
|
| 788 |
+
|
| 789 |
+
# Estimate from a measured rate at this resolution, before committing.
|
| 790 |
+
# Start from what the last job learned: a cached GPU rate never raises,
|
| 791 |
+
# so an optimistic retry costs a failed segment and a re-plan per job.
|
| 792 |
+
gpu_dead = GPU_DEAD
|
| 793 |
+
if ZEROGPU:
|
| 794 |
+
try:
|
| 795 |
+
rate, be_name, build_s = gpu_rate(th, tw)
|
| 796 |
+
# Clear what the LAST job learned. Without this, GPU_DEAD from an
|
| 797 |
+
# earlier quota-out job survived a successful probe: gpu_rate
|
| 798 |
+
# returned a GPU rate and name, the header and the estimate
|
| 799 |
+
# quoted them, and then every segment ran on _segment_cpu because
|
| 800 |
+
# gpu_dead was still True. Seen live on 2026-09-06 - "GPU CUDA:
|
| 801 |
+
# 0.053s/frame ... estimate 0:09" against a measured 57.4s, with
|
| 802 |
+
# the plan-check line labelling its own segment "cpu".
|
| 803 |
+
#
|
| 804 |
+
# The comment above this block already described the intended
|
| 805 |
+
# behaviour ("an optimistic retry costs a failed segment and a
|
| 806 |
+
# re-plan per job"); the retry just never happened. A cached rate
|
| 807 |
+
# does not re-prove the quota, so this is deliberately
|
| 808 |
+
# optimistic: the segment loop catches the quota error, sets the
|
| 809 |
+
# flag and re-cuts the rest for CPU. One failed segment is the
|
| 810 |
+
# price of not pinning the Space to CPU until it restarts, and it
|
| 811 |
+
# keeps the one invariant that matters - the rate that plans the
|
| 812 |
+
# job is the rate of the device that runs it.
|
| 813 |
+
gpu_dead = GPU_DEAD = False
|
| 814 |
+
except Exception as e: # noqa: BLE001
|
| 815 |
+
if not (is_quota_error(e) or is_timeout_error(e)):
|
| 816 |
+
raise
|
| 817 |
+
gpu_dead = GPU_DEAD = True
|
| 818 |
+
# str(e), not type(e).__name__: ZeroGPU raises gradio's own
|
| 819 |
+
# Error, so the class name rendered as the literal "no usable
|
| 820 |
+
# GPU (Error)" - true, and carrying none of the quota message
|
| 821 |
+
# that tells the user when the GPU comes back.
|
| 822 |
+
why = " ".join(str(e).split())[:110] or type(e).__name__
|
| 823 |
+
yield None, log(f"no usable GPU ({why}), "
|
| 824 |
+
f"running on CPU"), *BUSY
|
| 825 |
+
rate, be_name, build_s = cpu_rate(th, tw)
|
| 826 |
+
else:
|
| 827 |
+
rate, be_name, build_s = cpu_rate(th, tw)
|
| 828 |
+
new_frames = clip.frames * (int(factor) - 1)
|
| 829 |
+
est = rate * new_frames
|
| 830 |
+
if build_s > 1:
|
| 831 |
+
yield None, log(f"{be_name} session built in {build_s:.0f}s"), *BUSY
|
| 832 |
+
yield None, log(f"{be_name}: {rate:.3f}s/frame here, "
|
| 833 |
+
f"{new_frames} frames to generate, estimate {hms(est)}"), *BUSY
|
| 834 |
+
|
| 835 |
+
if est > 6 * 3600:
|
| 836 |
+
# BACKEND is always the CPU session, built at import with
|
| 837 |
+
# prefer_gpu=False, so testing BACKEND.on_gpu here named the CPU
|
| 838 |
+
# ceiling on GPU hardware every time. Ask the flag that knows.
|
| 839 |
+
fits = int(clip.duration * (6 * 3600) / est)
|
| 840 |
+
raise gr.Error(
|
| 841 |
+
f"Estimated {hms(est)} at {tw}x{th}, past the 6 hour ceiling"
|
| 842 |
+
f"{' and well past a ZeroGPU day' if ZEROGPU else ''}. "
|
| 843 |
+
f"Lower 'Max height' to {min(720, max(240, th // 2))}, which is "
|
| 844 |
+
f"about 4x faster, or trim the clip to under "
|
| 845 |
+
f"{hms(max(1, fits))} at this size.")
|
| 846 |
+
|
| 847 |
+
# Plan segment 0 from a MEASUREMENT when this container has already run
|
| 848 |
+
# this shape on this device. plan_gaps charges BUDGET_S against a bare
|
| 849 |
+
# inference rate, so it deliberately mis-sizes the first segment and the
|
| 850 |
+
# re-cut then fires once to correct it - on every repeat job, forever.
|
| 851 |
+
#
|
| 852 |
+
# It also mis-states the plan LENGTH, and `len(plan) > 30` below is what
|
| 853 |
+
# gates the "you will run out of ZeroGPU quota, lower your resolution"
|
| 854 |
+
# warning. At a clean overhead of 1.26 a job that really needs 30 calls
|
| 855 |
+
# was announced as 35, so the user was told to halve their height for a
|
| 856 |
+
# limit they were not going to hit.
|
| 857 |
+
dev0 = "cpu" if gpu_dead else "gpu"
|
| 858 |
+
seen0 = _PER_GAP.get((th, tw, int(factor), dev0))
|
| 859 |
+
if seen0:
|
| 860 |
+
gaps_per = max(1, int(REAL_BUDGET_S / seen0))
|
| 861 |
+
budget_note = f"{REAL_BUDGET_S:.0f}s measured budget"
|
| 862 |
+
else:
|
| 863 |
+
# Charge the overhead here too. plan_gaps bills against measure_rate,
|
| 864 |
+
# which times bare inference, so BUDGET_S seconds of inference is
|
| 865 |
+
# BUDGET_S * overhead of WALL time - and the cap is on wall time.
|
| 866 |
+
#
|
| 867 |
+
# That gap widened when the model got five times faster: ffmpeg did
|
| 868 |
+
# not, so the ratio went 1.46 -> 1.85 and the same BUDGET_S = 60 went
|
| 869 |
+
# from 87.6s (74% of the 119s cap, the intent) to 111.0s (93%). The
|
| 870 |
+
# first segment of every job was being planned with 7% headroom, and
|
| 871 |
+
# the re-cut that would fix it only runs AFTER that segment lands.
|
| 872 |
+
#
|
| 873 |
+
# Billing rate * overhead against REAL_BUDGET_S puts the first
|
| 874 |
+
# segment on the same 75%-of-cap target the re-cut aims for, and it
|
| 875 |
+
# stays correct automatically as the ratio moves.
|
| 876 |
+
gaps_per = R.plan_gaps(clip, int(factor), rate * _OVERHEAD[dev0],
|
| 877 |
+
REAL_BUDGET_S)
|
| 878 |
+
budget_note = f"{REAL_BUDGET_S:.0f}s wall budget"
|
| 879 |
+
plan = R.segment_bounds(clip, gaps_per)
|
| 880 |
+
total_gaps = max(1, clip.frames - 1)
|
| 881 |
+
yield None, log(f"{len(plan)} segment(s) of up to {gaps_per} gaps "
|
| 882 |
+
f"({gaps_per / clip.fps:.1f}s), inside the "
|
| 883 |
+
f"{budget_note}"), *BUSY
|
| 884 |
+
|
| 885 |
+
# On ZeroGPU each segment is a separate allocation and the daily budget
|
| 886 |
+
# is counted in RUNS, so segment count - not estimated hours - is what
|
| 887 |
+
# decides whether a job can finish on the GPU. The 6 hour ceiling above
|
| 888 |
+
# is checked against the GPU estimate, so a 4K five minute clip at source
|
| 889 |
+
# resolution sails through it at ~2.8h and then needs ~165 allocations:
|
| 890 |
+
# quota dies partway, the CPU fallback takes over at roughly 6 s/frame,
|
| 891 |
+
# and the user who was shown "2:45" is waiting most of a day. Say so
|
| 892 |
+
# before any of it starts, with the number, rather than letting them find
|
| 893 |
+
# out hours in.
|
| 894 |
+
if ZEROGPU and not gpu_dead and len(plan) > 30:
|
| 895 |
+
# The remedy has to be an action the user can actually take. A
|
| 896 |
+
# suggestion to cap height at the height they are already using is
|
| 897 |
+
# not advice, and 240 is the floor of the dropdown - below that the
|
| 898 |
+
# only lever left is a shorter clip. Cost scales with pixels, so
|
| 899 |
+
# halving the height is ~4x fewer calls; say the number either way.
|
| 900 |
+
lower = next_lower_height(th)
|
| 901 |
+
if lower:
|
| 902 |
+
# Cost scales with pixels, so the saving is the area ratio.
|
| 903 |
+
saved = max(1, int(len(plan) * (lower / th) ** 2))
|
| 904 |
+
remedy = (f"Capping 'Max height' at {lower} (from {th}) would "
|
| 905 |
+
f"cut it to about {saved} calls.")
|
| 906 |
+
else:
|
| 907 |
+
fits = clip.duration * 30 / len(plan)
|
| 908 |
+
# Reached when no offered cap is BELOW the current height -
|
| 909 |
+
# either it is already the lowest option, or the source is
|
| 910 |
+
# smaller than every option. Saying "the control's lowest" would
|
| 911 |
+
# be wrong in the second case, so say what is true in both.
|
| 912 |
+
remedy = (f"no lower cap is available for a {th}p source, so the "
|
| 913 |
+
f"lever left is length: about "
|
| 914 |
+
f"{hms(max(1, int(fits)))} of this clip fits in "
|
| 915 |
+
f"30 calls.")
|
| 916 |
+
yield None, log(
|
| 917 |
+
f"heads up: {len(plan)} GPU calls needed, and ZeroGPU meters a "
|
| 918 |
+
f"daily number of them. This will likely run out partway and "
|
| 919 |
+
f"finish on CPU, which is far slower. " + remedy), *BUSY
|
| 920 |
+
|
| 921 |
+
done = 0
|
| 922 |
+
cursor = 0 # gaps already rendered
|
| 923 |
+
i = 0
|
| 924 |
+
shrinks = 0 # GPU-window overruns absorbed so far
|
| 925 |
+
while cursor < total_gaps:
|
| 926 |
+
if me in CANCELLED:
|
| 927 |
+
# Set `stopped` like every other early exit. Without it the
|
| 928 |
+
# summary read "done: 121 frames" on a job that expected 599 -
|
| 929 |
+
# the word done, and a number with nothing to compare it to.
|
| 930 |
+
# Cancel is the most frequently taken of the early exits and was
|
| 931 |
+
# the only one not marking the result partial.
|
| 932 |
+
stopped = "cancelled at your request"
|
| 933 |
+
yield None, log("cancelled, keeping what finished"), *BUSY
|
| 934 |
+
break
|
| 935 |
+
if i >= len(plan):
|
| 936 |
+
# Never silent. cursor advancing without a segment to consume it
|
| 937 |
+
# is a planner bug, and the two guards below cover the ways it
|
| 938 |
+
# has actually happened; this catches any third.
|
| 939 |
+
stopped = (f"the plan ran out at {cursor} of {total_gaps} gaps")
|
| 940 |
+
yield None, log(f"stopped early: {stopped}"), *BUSY
|
| 941 |
+
break
|
| 942 |
+
t0, gaps, is_last = plan[i]
|
| 943 |
+
part = os.path.join(work, f"seg{i:04d}.mp4")
|
| 944 |
+
t_seg = time.perf_counter()
|
| 945 |
+
try:
|
| 946 |
+
worker_fn = _segment_cpu if gpu_dead else _segment_gpu
|
| 947 |
+
n, inside = worker_fn(src, part, t0, gaps, int(factor), clip,
|
| 948 |
+
int(crf), is_last, me, tw, th)
|
| 949 |
+
except Exception as e: # noqa: BLE001
|
| 950 |
+
# ZEROGPU gates this: only there is there a window to overrun,
|
| 951 |
+
# and TIMEOUT_HINTS is broad enough to match ordinary ffmpeg
|
| 952 |
+
# errors that retrying cannot help.
|
| 953 |
+
overran = ZEROGPU and not gpu_dead and is_timeout_error(e)
|
| 954 |
+
if overran and shrinks < 4 and gaps_per > 1:
|
| 955 |
+
# The segment was too big for its GPU window, not the day's
|
| 956 |
+
# budget. measure_rate times one bare inference while a real
|
| 957 |
+
# segment also runs factor-1 encodes and emits per gap, and
|
| 958 |
+
# at 4x that overhead is where a 90 s plan meets a 120 s cap.
|
| 959 |
+
# Halve and retry the same range rather than abandoning the
|
| 960 |
+
# GPU for the rest of the job.
|
| 961 |
+
shrinks += 1
|
| 962 |
+
gaps_per = max(1, gaps_per // 2)
|
| 963 |
+
plan = plan[:i] + R.segment_bounds(clip, gaps_per, cursor)
|
| 964 |
+
yield None, log(f"segment overran its GPU window, re-cut to "
|
| 965 |
+
f"{gaps_per} gaps and retrying "
|
| 966 |
+
f"({len(plan) - i} segments left)"), *BUSY
|
| 967 |
+
continue
|
| 968 |
+
if gpu_dead or not (overran or is_quota_error(e)):
|
| 969 |
+
raise
|
| 970 |
+
# Two ways to get here. Quota exhausted, or a single gap that
|
| 971 |
+
# cannot fit the GPU window even after shrinking - which used to
|
| 972 |
+
# kill the job outright, the one case where the GPU truly cannot
|
| 973 |
+
# proceed being the one case that refused the existing fallback.
|
| 974 |
+
why = ("one gap will not fit the GPU window" if overran
|
| 975 |
+
else "ZeroGPU quota exhausted")
|
| 976 |
+
#
|
| 977 |
+
# Re-cutting the remainder is the part that matters. These
|
| 978 |
+
# segments were sized from a GPU rate, so one of them is roughly
|
| 979 |
+
# six times more work than the CPU budget allows: a segment
|
| 980 |
+
# yields once, at the end, so an over-sized one freezes progress,
|
| 981 |
+
# gives Stop that much latency, and stops the seams being
|
| 982 |
+
# checkpoints at all. Caching the GPU rate made this the common
|
| 983 |
+
# path, because a cached rate never raises the quota error that
|
| 984 |
+
# used to be caught before any planning happened.
|
| 985 |
+
gpu_dead = GPU_DEAD = True
|
| 986 |
+
rate, be_name, _ = cpu_rate(th, tw)
|
| 987 |
+
gaps_per = R.plan_gaps(clip, int(factor), rate, BUDGET_S)
|
| 988 |
+
plan = plan[:i] + R.segment_bounds(clip, gaps_per, cursor)
|
| 989 |
+
left = rate * (total_gaps - cursor) * max(1, int(factor) - 1)
|
| 990 |
+
yield None, log(f"{why}. Finishing on {be_name} at "
|
| 991 |
+
f"{rate:.3f}s/frame: re-cut the rest into "
|
| 992 |
+
f"{len(plan) - i} segments, about {hms(left)} to "
|
| 993 |
+
f"go. Frames so far are kept."), *BUSY
|
| 994 |
+
continue
|
| 995 |
+
# plan_gaps budgets rate*(factor-1) per gap from measure_rate, which
|
| 996 |
+
# times one bare inference. A real segment also runs `factor` emit
|
| 997 |
+
# passes and `factor` x264 frames per gap. That overhead is roughly
|
| 998 |
+
# fixed per gap while the inference term scales with the device, so
|
| 999 |
+
# the plan under-charges least on CPU and most on a fast GPU - which
|
| 1000 |
+
# is exactly where the 120 s cap lives. Logged rather than guessed,
|
| 1001 |
+
# so any real job on any device reports its own correction factor.
|
| 1002 |
+
took = time.perf_counter() - t_seg # wall, includes the queue
|
| 1003 |
+
spent = inside # what the work actually cost
|
| 1004 |
+
assumed = gaps * rate * max(1, int(factor) - 1)
|
| 1005 |
+
if assumed > 0:
|
| 1006 |
+
# Only a segment that rendered its whole assignment describes the
|
| 1007 |
+
# overhead. A cancelled or starved one returns early and logs
|
| 1008 |
+
# ratios like 0.399x, which would drag the mean toward nonsense.
|
| 1009 |
+
if n // max(1, int(factor)) >= gaps:
|
| 1010 |
+
note_overhead("gpu" if (ZEROGPU and not gpu_dead) else "cpu",
|
| 1011 |
+
spent / assumed)
|
| 1012 |
+
if ZEROGPU and not gpu_dead:
|
| 1013 |
+
note_queue(took - spent)
|
| 1014 |
+
if gaps > 0:
|
| 1015 |
+
_PER_GAP[(th, tw, int(factor),
|
| 1016 |
+
"cpu" if gpu_dead else "gpu")] = spent / gaps
|
| 1017 |
+
print(f"[ff] plan-check seg{i} {gaps} gaps f{factor} {tw}x{th} "
|
| 1018 |
+
f"{'gpu' if (ZEROGPU and not gpu_dead) else 'cpu'}: "
|
| 1019 |
+
f"assumed {assumed:.2f}s "
|
| 1020 |
+
f"measured {spent:.2f}s ratio {spent / assumed:.3f}x "
|
| 1021 |
+
f"(wall {took:.2f}s, queue {took - spent:.2f}s)",
|
| 1022 |
+
flush=True)
|
| 1023 |
+
if not gpu_dead:
|
| 1024 |
+
GPU_DEAD = False # a GPU segment landed, quota is back
|
| 1025 |
+
if n: # a cancelled segment writes no file
|
| 1026 |
+
parts.append(part)
|
| 1027 |
+
done += n
|
| 1028 |
+
# A segment cancelled part-way returns fewer frames than its gap
|
| 1029 |
+
# assignment. Crediting the whole assignment made the last line of a
|
| 1030 |
+
# cancelled job overstate progress: 60 of 179 gaps reported against
|
| 1031 |
+
# 156 frames, which at 4x is 39 gaps of real work. Each gap writes
|
| 1032 |
+
# `factor` frames, so the frames are the honest count.
|
| 1033 |
+
got_gaps = min(gaps, n // max(1, int(factor)))
|
| 1034 |
+
cursor += got_gaps
|
| 1035 |
+
i += 1
|
| 1036 |
+
if got_gaps < gaps and me not in CANCELLED:
|
| 1037 |
+
# The decoder handed back fewer frames than this segment covers,
|
| 1038 |
+
# and nobody cancelled: the source is shorter than its header
|
| 1039 |
+
# claims. On CPU nothing re-cuts the plan, so cursor stopped
|
| 1040 |
+
# short of total_gaps while i walked past the last entry, and the
|
| 1041 |
+
# next pass raised "IndexError: list index out of range" - which
|
| 1042 |
+
# is what the live Space showed for a truncated upload while this
|
| 1043 |
+
# machine's ffmpeg raised the clean starved-decoder error before
|
| 1044 |
+
# ever reaching here. Same file, two decoders, one path untested.
|
| 1045 |
+
stopped = (f"source ended early: this segment covers {gaps} gaps "
|
| 1046 |
+
f"but the decoder supplied {got_gaps}. The file is "
|
| 1047 |
+
f"probably truncated or corrupt - its header claims "
|
| 1048 |
+
f"{clip.frames} frames.")
|
| 1049 |
+
yield None, log(f"stopped early: {stopped}"), *BUSY
|
| 1050 |
+
break
|
| 1051 |
+
|
| 1052 |
+
# Re-cut from what this segment ACTUALLY cost, not from a constant.
|
| 1053 |
+
# The plan charges only inference, so it is optimistic by a fixed
|
| 1054 |
+
# per-gap amount that is a bigger fraction the faster the device is -
|
| 1055 |
+
# measured 1.456x at 852x480 on the GPU against 1.03x on CPU. That
|
| 1056 |
+
# ratio cannot be bounded ahead of time: a smaller frame makes
|
| 1057 |
+
# inference cheaper without making ffmpeg cheaper, so it climbs as
|
| 1058 |
+
# resolution drops, and four measurements were neither constant nor
|
| 1059 |
+
# pixel-linear. So do not model it - measure it here and shrink the
|
| 1060 |
+
# rest of the plan if the segment that just ran says the next one
|
| 1061 |
+
# would not fit the window. Costs nothing when the plan is right.
|
| 1062 |
+
# Every device, not just the GPU. The gate used to read
|
| 1063 |
+
# `ZEROGPU and not gpu_dead`, on the reasoning that only a GPU has a
|
| 1064 |
+
# window to overrun - but a segment 5x its plan is a problem on any
|
| 1065 |
+
# device, because a segment yields once at the END. A 2290x1708 job
|
| 1066 |
+
# planned at 59s a segment took 318s and showed the user a frozen log
|
| 1067 |
+
# for 5m18s with no progress line, no ETA and 5 minutes of Stop
|
| 1068 |
+
# latency (2026-09-06). The 120 s cap is why the GPU MUST re-cut; the
|
| 1069 |
+
# seams being checkpoints at all is why everything else should.
|
| 1070 |
+
#
|
| 1071 |
+
# Cheap when the plan is right: the measured CPU under-charge is
|
| 1072 |
+
# 1.05-1.23x, which lands inside the 20% deadband and re-cuts nothing.
|
| 1073 |
+
if gaps > 0 and cursor < total_gaps:
|
| 1074 |
+
real_per_gap = spent / gaps
|
| 1075 |
+
# Both directions. Shrinking keeps a segment inside the window;
|
| 1076 |
+
# growing refunds a conservative BUDGET_S when the plan turns out
|
| 1077 |
+
# pessimistic, so starting safe does not cost a GPU run per
|
| 1078 |
+
# segment for the whole job. real_per_gap already contains the
|
| 1079 |
+
# under-charge, so this is a measurement and not a correction
|
| 1080 |
+
# factor. The 20% deadband stops one noisy segment churning the
|
| 1081 |
+
# plan, and BUDGET_S sits at half the hard cap, so even a segment
|
| 1082 |
+
# twice as slow as the one just measured still lands inside it.
|
| 1083 |
+
want = max(1, int(REAL_BUDGET_S / real_per_gap))
|
| 1084 |
+
if want < gaps_per * 0.8 or want > gaps_per * 1.25:
|
| 1085 |
+
grew = want > gaps_per
|
| 1086 |
+
gaps_per = want
|
| 1087 |
+
plan = plan[:i] + R.segment_bounds(clip, gaps_per, cursor)
|
| 1088 |
+
yield None, log(f"segment cost {real_per_gap:.3f}s/gap against "
|
| 1089 |
+
f"{rate * max(1, int(factor) - 1):.3f}s planned; "
|
| 1090 |
+
f"{'widened' if grew else 're-cut'} the rest into "
|
| 1091 |
+
f"{len(plan) - i} segments"), *BUSY
|
| 1092 |
+
progress(cursor / total_gaps, desc=f"{cursor}/{total_gaps} gaps")
|
| 1093 |
+
yield None, log(f"[{cursor}/{total_gaps} gaps] {n} frames, "
|
| 1094 |
+
f"{done} total"), *BUSY
|
| 1095 |
+
except GeneratorExit:
|
| 1096 |
+
# Tab closed or SSE torn down. Yielding from finally here would raise
|
| 1097 |
+
# "generator ignored GeneratorExit" and block teardown while ffmpeg muxes.
|
| 1098 |
+
closing = True
|
| 1099 |
+
CANCELLED.add(me)
|
| 1100 |
+
raise
|
| 1101 |
+
except gr.Error as e:
|
| 1102 |
+
# Mirror the generic handler below. This clause exists so a validation
|
| 1103 |
+
# error raised BEFORE any work reaches the user as a clean toast, and
|
| 1104 |
+
# with no parts that is still exactly what happens. But ZeroGPU itself
|
| 1105 |
+
# raises gr.Error, so a mid-job GPU failure whose text matches neither
|
| 1106 |
+
# QUOTA_HINTS nor TIMEOUT_HINTS lands here with segments already on disk:
|
| 1107 |
+
# the finally delivered them labelled "done", for a job that produced a
|
| 1108 |
+
# fraction of its frames, with the log replaced by an Error badge.
|
| 1109 |
+
if not parts:
|
| 1110 |
+
raise
|
| 1111 |
+
stopped = f"{type(e).__name__}: {e}"
|
| 1112 |
+
yield None, log(f"stopped early: {stopped}"), *IDLE
|
| 1113 |
+
except Exception as e: # noqa: BLE001
|
| 1114 |
+
if parts:
|
| 1115 |
+
# Segments landed, so there IS a result. Raising here would hand
|
| 1116 |
+
# Gradio an errored generator, and verified in Chrome on 2026-09-06
|
| 1117 |
+
# against truncated.mp4 that means: the video the finally yields
|
| 1118 |
+
# survives and stays downloadable, but the log component is replaced
|
| 1119 |
+
# by a bare "Error" badge. The user gets a silently truncated clip
|
| 1120 |
+
# with the one line explaining it wiped off the screen. Record the
|
| 1121 |
+
# reason and let the finally deliver both.
|
| 1122 |
+
stopped = f"{type(e).__name__}: {e}"
|
| 1123 |
+
yield None, log(f"stopped early: {stopped}"), *IDLE
|
| 1124 |
+
else:
|
| 1125 |
+
yield None, log(f"failed: {type(e).__name__}: {e}"), *IDLE
|
| 1126 |
+
raise
|
| 1127 |
+
finally:
|
| 1128 |
+
# Package from finally so cancel, error and success all hand back the
|
| 1129 |
+
# frames that were actually produced. The flag is re-derived here because
|
| 1130 |
+
# a GeneratorExit raised at the yield inside `except Exception` is a
|
| 1131 |
+
# sibling of the GeneratorExit handler and never reaches it.
|
| 1132 |
+
closing = closing or sys.exc_info()[0] is GeneratorExit
|
| 1133 |
+
try:
|
| 1134 |
+
if closing:
|
| 1135 |
+
# Do not YIELD - that raises "generator ignored GeneratorExit"
|
| 1136 |
+
# and blocks teardown. But keeping is a separate decision from
|
| 1137 |
+
# yielding, and the inner finally is about to rmtree `work` with
|
| 1138 |
+
# every finished segment inside it. Concat into OUTDIR so the
|
| 1139 |
+
# result survives the cleanup and is recoverable from the log,
|
| 1140 |
+
# instead of a dropped SSE costing a 40 minute job outright.
|
| 1141 |
+
if parts:
|
| 1142 |
+
try:
|
| 1143 |
+
fd, sv = tempfile.mkstemp(suffix=".mp4", dir=str(OUTDIR))
|
| 1144 |
+
os.close(fd)
|
| 1145 |
+
R.concat(parts, sv)
|
| 1146 |
+
print(f"[ff] stream closed mid-job; kept {len(parts)} "
|
| 1147 |
+
f"segments at {sv}", flush=True)
|
| 1148 |
+
except Exception as e: # noqa: BLE001
|
| 1149 |
+
# Best effort on a path that is already unwinding.
|
| 1150 |
+
print(f"[ff] stream closed mid-job and the salvage "
|
| 1151 |
+
f"failed: {type(e).__name__}: {e}", flush=True)
|
| 1152 |
+
elif parts:
|
| 1153 |
+
# Guarded for the same reason the probe below is: describing OR
|
| 1154 |
+
# packaging the result must never be able to discard it. A raise
|
| 1155 |
+
# from concat, mkstemp or the copy escapes this finally so
|
| 1156 |
+
# nothing is yielded, REPLACES the exception already in flight -
|
| 1157 |
+
# overwriting a segment failure already recorded in `stopped` -
|
| 1158 |
+
# and still reaches the inner finally that deletes `work`. The
|
| 1159 |
+
# parts would be destroyed by cleanup for a delivery that never
|
| 1160 |
+
# happened. Falling back to the largest single segment keeps most
|
| 1161 |
+
# of the job rather than none of it.
|
| 1162 |
+
out_path = None
|
| 1163 |
+
try:
|
| 1164 |
+
joined = os.path.join(work, "joined.mp4")
|
| 1165 |
+
R.concat(parts, joined)
|
| 1166 |
+
fd, out_path = tempfile.mkstemp(suffix=".mp4", dir=str(OUTDIR))
|
| 1167 |
+
os.close(fd)
|
| 1168 |
+
if not R.mux_audio(joined, video, out_path):
|
| 1169 |
+
shutil.copy(joined, out_path)
|
| 1170 |
+
except Exception as e: # noqa: BLE001
|
| 1171 |
+
stopped = stopped or f"{type(e).__name__}: {e}"
|
| 1172 |
+
big = max(parts, key=lambda p: os.path.getsize(p)
|
| 1173 |
+
if os.path.exists(p) else 0)
|
| 1174 |
+
try:
|
| 1175 |
+
fd, out_path = tempfile.mkstemp(suffix=".mp4",
|
| 1176 |
+
dir=str(OUTDIR))
|
| 1177 |
+
os.close(fd)
|
| 1178 |
+
shutil.copy(big, out_path)
|
| 1179 |
+
log(f"could not join the segments ({e}); handing back the "
|
| 1180 |
+
f"longest one instead")
|
| 1181 |
+
except Exception: # noqa: BLE001
|
| 1182 |
+
out_path = None
|
| 1183 |
+
if out_path is None:
|
| 1184 |
+
# No `return` here: a return inside a finally DISCARDS an
|
| 1185 |
+
# exception that is still propagating. An else instead, so
|
| 1186 |
+
# the summary below cannot run on a None path and probe it.
|
| 1187 |
+
yield None, log("the frames were rendered but could not be "
|
| 1188 |
+
"packaged; nothing to hand back"), *IDLE
|
| 1189 |
+
else:
|
| 1190 |
+
# Describing the result must never be able to discard it.
|
| 1191 |
+
# probe raises when it cannot establish a frame count, and
|
| 1192 |
+
# this call happens after the work is finished and written.
|
| 1193 |
+
head = "partial" if stopped else "done"
|
| 1194 |
+
try:
|
| 1195 |
+
got = R.probe(out_path)
|
| 1196 |
+
summary = (f"{head}: {got.width}x{got.height} "
|
| 1197 |
+
f"{got.fps:.2f}fps {got.frames} frames")
|
| 1198 |
+
if stopped:
|
| 1199 |
+
summary += (f" of the "
|
| 1200 |
+
f"{(clip.frames - 1) * int(factor) + 1} "
|
| 1201 |
+
f"expected - the download below is the "
|
| 1202 |
+
f"part that finished")
|
| 1203 |
+
except Exception as e: # noqa: BLE001
|
| 1204 |
+
summary = (f"{head}, {done} frames written "
|
| 1205 |
+
f"({type(e).__name__})")
|
| 1206 |
+
yield out_path, log(summary), *IDLE
|
| 1207 |
+
else:
|
| 1208 |
+
yield None, log("no frames produced"), *IDLE
|
| 1209 |
+
finally:
|
| 1210 |
+
if work:
|
| 1211 |
+
shutil.rmtree(work, ignore_errors=True)
|
| 1212 |
+
OWNER = None
|
| 1213 |
+
LOCK.release()
|
| 1214 |
+
|
| 1215 |
+
|
| 1216 |
+
BUSY = (gr.update(visible=False), gr.update(visible=True))
|
| 1217 |
+
IDLE = (gr.update(visible=True), gr.update(visible=False))
|
| 1218 |
+
|
| 1219 |
+
CSS = """
|
| 1220 |
+
footer{display:none}
|
| 1221 |
+
.gradio-container{padding:4px !important;max-width:100% !important}
|
| 1222 |
+
.form{gap:2px !important}
|
| 1223 |
+
#hdr{max-height:34px}
|
| 1224 |
+
#hdr p{margin:0 !important;font-size:.82em;opacity:.75}
|
| 1225 |
+
#log textarea{font-family:ui-monospace,monospace;font-size:.78em}
|
| 1226 |
+
#vid video{max-height:190px}
|
| 1227 |
+
:where(button,input,textarea,select,[role="button"],[tabindex]):focus-visible{
|
| 1228 |
+
outline:2px solid var(--color-accent) !important;outline-offset:2px}
|
| 1229 |
+
@media (prefers-reduced-motion:reduce){*,*::before,*::after{
|
| 1230 |
+
animation-duration:.01ms !important;transition-duration:.01ms !important}}
|
| 1231 |
+
"""
|
| 1232 |
+
|
| 1233 |
+
with gr.Blocks(title="Flowframes", fill_width=True) as demo:
|
| 1234 |
+
hdr = gr.Markdown(header(), elem_id="hdr")
|
| 1235 |
+
with gr.Row():
|
| 1236 |
+
with gr.Column(scale=4):
|
| 1237 |
+
vid = gr.Video(label="Source clip", elem_id="vid", height=190)
|
| 1238 |
+
go = gr.Button("Interpolate", variant="primary")
|
| 1239 |
+
halt = gr.Button("Stop", variant="stop", visible=False)
|
| 1240 |
+
# label with show_label=False: hidden visually, still the control's
|
| 1241 |
+
# accessible name. Without it this is the one element on the page a
|
| 1242 |
+
# screen reader announces as just "Textbox", and it is the element
|
| 1243 |
+
# carrying every progress line, the ETA and every error message.
|
| 1244 |
+
status = gr.Textbox(lines=8, max_lines=LOG_LINES + 2, autoscroll=True,
|
| 1245 |
+
label="Progress log", show_label=False,
|
| 1246 |
+
container=False, elem_id="log")
|
| 1247 |
+
with gr.Column(scale=2):
|
| 1248 |
+
factor = gr.Radio([2, 4], value=2, label="Frame rate multiplier",
|
| 1249 |
+
info="4x generates three frames per gap, so it costs "
|
| 1250 |
+
"three times 2x.")
|
| 1251 |
+
max_h = gr.Dropdown(
|
| 1252 |
+
[("Keep source resolution", 0)]
|
| 1253 |
+
+ [(f"Cap at {h}p" + (" (fastest)" if h == 720 else ""), h)
|
| 1254 |
+
for h in HEIGHTS],
|
| 1255 |
+
value=720, label="Max height",
|
| 1256 |
+
info="Cost scales with pixels. Halving height is roughly 4x faster.")
|
| 1257 |
+
with gr.Accordion("Advanced", open=False):
|
| 1258 |
+
crf = gr.Slider(12, 28, value=18, step=1, label="x264 CRF",
|
| 1259 |
+
info="Lower is better quality and a larger file.")
|
| 1260 |
+
with gr.Column(scale=4):
|
| 1261 |
+
out = gr.Video(label="Result", height=190, interactive=False)
|
| 1262 |
+
dl = gr.DownloadButton("Download result", size="sm", interactive=False)
|
| 1263 |
+
|
| 1264 |
+
# Registered before the work handler and outside the queue, so the swap lands
|
| 1265 |
+
# immediately rather than waiting behind a job that may run for hours.
|
| 1266 |
+
# Validation lives in the swap handler, so a rejected click never swaps the
|
| 1267 |
+
# buttons. Raising after the swap stranded the UI showing Stop forever.
|
| 1268 |
+
# The trailing .then restores the buttons on paths that never reach run()'s
|
| 1269 |
+
# final yield: the lock-busy gr.Error is raised before the try, so the swap
|
| 1270 |
+
# that preflight already made would otherwise leave the page showing Stop
|
| 1271 |
+
# forever. .then runs on failure too, which .success would not.
|
| 1272 |
+
go.click(preflight, inputs=[vid], outputs=[go, halt], queue=False).then(
|
| 1273 |
+
run, [vid, factor, max_h, crf], [out, status, go, halt]).then(
|
| 1274 |
+
lambda: IDLE, outputs=[go, halt], queue=False)
|
| 1275 |
+
out.change(lambda v: gr.update(value=v, interactive=v is not None),
|
| 1276 |
+
inputs=out, outputs=dl, queue=False)
|
| 1277 |
+
# queue=False so the label never waits behind a running job, which under
|
| 1278 |
+
# concurrency_limit=1 would mean the number arrives after the work finishes.
|
| 1279 |
+
for _c in (vid, factor, max_h):
|
| 1280 |
+
_c.change(button_label, [vid, factor, max_h], [go], queue=False)
|
| 1281 |
+
halt.click(stop, outputs=status, queue=False)
|
| 1282 |
+
# Recomputed per page load, so the first real job replaces the reference
|
| 1283 |
+
# estimate with the rate measured on the hardware actually serving this Space.
|
| 1284 |
+
demo.load(header, outputs=[hdr], queue=False)
|
| 1285 |
+
|
| 1286 |
+
ex = HERE / "examples" / "sample.mp4"
|
| 1287 |
+
if ex.exists():
|
| 1288 |
+
# Cached so the first visitor is not asked to supply a video before they
|
| 1289 |
+
# can see anything work. Lazy, not eager: eager runs the whole pipeline
|
| 1290 |
+
# before the Space serves its first request, and on ZeroGPU that would
|
| 1291 |
+
# also spend quota on every rebuild. Cached after the first click, then
|
| 1292 |
+
# instant for everyone.
|
| 1293 |
+
gr.Examples(examples=[[str(ex), 2, 480, 20]],
|
| 1294 |
+
inputs=[vid, factor, max_h, crf],
|
| 1295 |
+
outputs=[out, status],
|
| 1296 |
+
fn=example_run,
|
| 1297 |
+
cache_examples=True, cache_mode="lazy",
|
| 1298 |
+
label="Example (cached after first run)")
|
| 1299 |
+
|
| 1300 |
+
if __name__ == "__main__":
|
| 1301 |
+
demo.queue(max_size=4, default_concurrency_limit=1).launch(css=CSS, show_error=True)
|
cli.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Headless end-to-end check. Runs before app.py exists, and keeps working after."""
|
| 2 |
+
import argparse, pathlib, sys, time
|
| 3 |
+
sys.path.insert(0, str(pathlib.Path(__file__).parent))
|
| 4 |
+
import rife_core as R
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def main() -> int:
|
| 8 |
+
ap = argparse.ArgumentParser()
|
| 9 |
+
ap.add_argument("src")
|
| 10 |
+
ap.add_argument("-o", "--out", default="out.mp4")
|
| 11 |
+
ap.add_argument("-x", "--factor", type=int, default=2)
|
| 12 |
+
ap.add_argument("--budget", type=float, default=100.0, help="seconds per segment")
|
| 13 |
+
a = ap.parse_args()
|
| 14 |
+
|
| 15 |
+
clip = R.probe(a.src)
|
| 16 |
+
print(f"[probe] {clip.width}x{clip.height} {clip.fps:.3f}fps "
|
| 17 |
+
f"{clip.frames} frames {clip.duration:.2f}s")
|
| 18 |
+
|
| 19 |
+
be = R.OnnxBackend()
|
| 20 |
+
print(f"[backend] {be.name}, {R.CPUS} threads")
|
| 21 |
+
rate = R.measure_rate(be, clip.height, clip.width)
|
| 22 |
+
new_frames = clip.frames * (a.factor - 1)
|
| 23 |
+
print(f"[rate] {rate:.3f} s/frame -> est {rate*new_frames:.1f}s "
|
| 24 |
+
f"for {new_frames} generated frames")
|
| 25 |
+
|
| 26 |
+
seg_s = R.plan_segments(clip, a.factor, rate, a.budget)
|
| 27 |
+
nseg = max(1, int(clip.duration / seg_s + 0.999))
|
| 28 |
+
print(f"[plan] {nseg} segment(s) of {seg_s:.2f}s")
|
| 29 |
+
|
| 30 |
+
parts, t0 = [], time.time()
|
| 31 |
+
tmpdir = pathlib.Path(a.out).parent / "_segs"
|
| 32 |
+
tmpdir.mkdir(exist_ok=True)
|
| 33 |
+
total = 0
|
| 34 |
+
for i in range(nseg):
|
| 35 |
+
p = str(tmpdir / f"seg{i:04d}.mp4")
|
| 36 |
+
n = R.interpolate_segment(a.src, p, i * seg_s, seg_s, a.factor, be, clip)
|
| 37 |
+
total += n
|
| 38 |
+
parts.append(p)
|
| 39 |
+
print(f"[+{time.time()-t0:6.1f}s] [{i+1}/{nseg}] {n} frames -> {pathlib.Path(p).name}")
|
| 40 |
+
R.concat(parts, a.out)
|
| 41 |
+
print(f"[done] {total} frames in {time.time()-t0:.1f}s -> {a.out}")
|
| 42 |
+
|
| 43 |
+
got = R.probe(a.out)
|
| 44 |
+
print(f"[verify] {got.width}x{got.height} {got.fps:.3f}fps {got.frames} frames "
|
| 45 |
+
f"{got.duration:.2f}s")
|
| 46 |
+
ok = abs(got.fps - clip.fps * a.factor) < 0.5 and got.frames >= clip.frames * a.factor * 0.8
|
| 47 |
+
print(f"[verify] fps x{a.factor} and frame count plausible: {ok}")
|
| 48 |
+
return 0 if ok else 1
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
if __name__ == "__main__":
|
| 52 |
+
raise SystemExit(main())
|
examples/sample.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4d612ca50fbac3481c7b03bddc5b9789e9458923f748787cef509a0ee0bd04c7
|
| 3 |
+
size 2971056
|
models/rife49.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d42a41188d5c23291f18295da726671423d245b674ef086661c9ab2b0921e5dd
|
| 3 |
+
size 21459539
|
requirements.txt
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
gradio
|
| 2 |
+
numpy
|
| 3 |
+
spaces
|
| 4 |
+
# The cu12 nvidia pins that used to live here were the bug, not the fix: they
|
| 5 |
+
# forced pip to resolve an older CUDA-12 build of onnxruntime-gpu, which then
|
| 6 |
+
# reported "Require cuDNN 9.* and CUDA 12.*" against an image where torch 2.11
|
| 7 |
+
# ships cu13. The extras pull the matching CUDA 13 runtime instead.
|
| 8 |
+
onnxruntime-gpu[cuda,cudnn]
|
rife_core.py
ADDED
|
@@ -0,0 +1,999 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Streaming RIFE frame interpolation.
|
| 2 |
+
|
| 3 |
+
Two frames are resident at a time, never the clip. ffmpeg decodes raw RGB into a
|
| 4 |
+
pipe, each consecutive pair is interpolated, and the results go straight out to a
|
| 5 |
+
second ffmpeg encoding a segment. Memory is therefore O(1) in clip length, so a
|
| 6 |
+
5 minute 4K source costs time and nothing else.
|
| 7 |
+
|
| 8 |
+
Work is always cut into segments. On ZeroGPU that keeps each GPU call inside its
|
| 9 |
+
duration budget; on CPU the same seams act as checkpoints, so a cancel keeps the
|
| 10 |
+
minutes already spent. Segment length is derived from a measured per-frame rate
|
| 11 |
+
rather than a guess.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import ctypes
|
| 17 |
+
import json
|
| 18 |
+
import os
|
| 19 |
+
import pathlib
|
| 20 |
+
import queue
|
| 21 |
+
import shutil
|
| 22 |
+
import statistics
|
| 23 |
+
import subprocess
|
| 24 |
+
import threading
|
| 25 |
+
import tempfile
|
| 26 |
+
import time
|
| 27 |
+
from dataclasses import dataclass
|
| 28 |
+
from typing import Callable
|
| 29 |
+
|
| 30 |
+
import numpy as np
|
| 31 |
+
|
| 32 |
+
HERE = pathlib.Path(__file__).parent
|
| 33 |
+
MODEL = HERE / "models" / "rife49.onnx"
|
| 34 |
+
# fp16 IS NOT USED, ON EITHER PATH, AND THE SELECTION CODE IS GONE ON PURPOSE.
|
| 35 |
+
# Converting to fp16 and running it on the CUDA EP was predicted at ~1.8x faster
|
| 36 |
+
# and came out 3.6x SLOWER, A/B'd inside a single GPU allocation on the same
|
| 37 |
+
# container: fp16 0.1096 s/frame vs fp32 0.0306 s/frame. keep_io_types inserts
|
| 38 |
+
# Cast nodes throughout, and GridSample/Resize appear to have no fp16 CUDA
|
| 39 |
+
# kernel, so the graph thrashes between precisions. The cosine gate PASSED at
|
| 40 |
+
# 0.99994549, so quality checks alone would have shipped the regression.
|
| 41 |
+
# On CPU it is a trap for a different reason: those cores have no half ALU, so
|
| 42 |
+
# every op promotes back to fp32 and you pay the conversion on top.
|
| 43 |
+
# This used to be a path guarded by a filename that deliberately did not exist,
|
| 44 |
+
# which is a disable that a later tidy-up "fixes" by accident. Re-adding it means
|
| 45 |
+
# re-running that A/B first.
|
| 46 |
+
ALIGN = 32 # RIFE downsamples 5 times; both dims must be a multiple of 32
|
| 47 |
+
FFMPEG = shutil.which("ffmpeg") or "ffmpeg"
|
| 48 |
+
FFPROBE = shutil.which("ffprobe") or "ffprobe"
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
# Set False by container_cpus() when every cgroup read fails, meaning the count
|
| 52 |
+
# is the HOST's and only a guess at this container's allowance.
|
| 53 |
+
CPUS_TRUSTED = True
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def cgroup_report() -> str:
|
| 57 |
+
"""What the cgroup actually says, for the startup line.
|
| 58 |
+
|
| 59 |
+
`container_cpus` returning 12 is ambiguous on its own: it means either the
|
| 60 |
+
container really may use 12, or every cgroup read failed and it fell back to
|
| 61 |
+
the host's count clamped by THREAD_CAP. Those are opposite situations and
|
| 62 |
+
only one of them is a bug, so print the raw values.
|
| 63 |
+
"""
|
| 64 |
+
bits = []
|
| 65 |
+
for path in ("/sys/fs/cgroup/cpu.max",
|
| 66 |
+
"/sys/fs/cgroup/cpu/cpu.cfs_quota_us",
|
| 67 |
+
"/sys/fs/cgroup/cpu/cpu.cfs_period_us",
|
| 68 |
+
os.environ.get("ZEROGPU_PROC_SELF_CGROUP_PATH", "")):
|
| 69 |
+
if not path:
|
| 70 |
+
continue
|
| 71 |
+
try:
|
| 72 |
+
bits.append(f"{path}={pathlib.Path(path).read_text().strip()[:40]}")
|
| 73 |
+
except Exception as e: # noqa: BLE001
|
| 74 |
+
bits.append(f"{path}=<{type(e).__name__}>")
|
| 75 |
+
bits.append(f"os.cpu_count()={os.cpu_count()}")
|
| 76 |
+
return " ".join(bits)
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def container_cpus(default: int = 2) -> int:
|
| 80 |
+
"""CPUs this container may actually use. os.cpu_count() reports the host, so
|
| 81 |
+
on a Spaces box it over-reports badly and ORT would spawn threads that only
|
| 82 |
+
fight each other for the two vCPUs the container is allowed."""
|
| 83 |
+
for quota_p, period_p in (("/sys/fs/cgroup/cpu.max", None),
|
| 84 |
+
("/sys/fs/cgroup/cpu/cpu.cfs_quota_us",
|
| 85 |
+
"/sys/fs/cgroup/cpu/cpu.cfs_period_us")):
|
| 86 |
+
try:
|
| 87 |
+
raw = pathlib.Path(quota_p).read_text().split()
|
| 88 |
+
quota = raw[0]
|
| 89 |
+
period = raw[1] if period_p is None else pathlib.Path(period_p).read_text().strip()
|
| 90 |
+
if quota != "max" and int(quota) > 0:
|
| 91 |
+
return max(1, int(quota) // int(period))
|
| 92 |
+
except Exception:
|
| 93 |
+
continue
|
| 94 |
+
# Reached only when no cgroup file answered. That is the case this function
|
| 95 |
+
# exists to catch, so say so rather than silently returning the host count:
|
| 96 |
+
# ZeroGPU sets ZEROGPU_PROC_SELF_CGROUP_PATH, which hints the cgroup is not
|
| 97 |
+
# where this looks, and an over-reported count means ORT spawns threads that
|
| 98 |
+
# only fight each other.
|
| 99 |
+
global CPUS_TRUSTED
|
| 100 |
+
CPUS_TRUSTED = False
|
| 101 |
+
n = max(1, os.cpu_count() or default)
|
| 102 |
+
print(f"[rife] cgroup cpu quota unreadable, falling back to os.cpu_count()"
|
| 103 |
+
f"={n}; if this container is smaller than that, threads are "
|
| 104 |
+
f"oversubscribed", flush=True)
|
| 105 |
+
return n
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
# Measured sweep at 720p: this graph scales to about 12 threads and then falls
|
| 109 |
+
# apart, 32 threads being 4.3x slower than 12. A Space gets 2 vCPUs so the cgroup
|
| 110 |
+
# read decides it there; the cap only stops a fat host oversubscribing.
|
| 111 |
+
# Two different jobs, previously conflated into one cap of 12.
|
| 112 |
+
#
|
| 113 |
+
# When the cgroup answers, it states this container's real allowance and should
|
| 114 |
+
# be believed: ZeroGPU reports 16, and measured in one boot at 720p, 16 threads
|
| 115 |
+
# run 0.637 s/frame against 12 threads' 0.790 - 19% faster. The old cap of 12
|
| 116 |
+
# came from a sweep on a dev box that compared 12 with 32 and never tried 16.
|
| 117 |
+
#
|
| 118 |
+
# When no cgroup answers we are reading the HOST's core count, which on a fat
|
| 119 |
+
# machine over-reports badly, and ORT would spawn threads that only fight each
|
| 120 |
+
# other. That is the case the 12 was really protecting against, and it still
|
| 121 |
+
# does, because the same sweep showed 32 threads running 4.3x slower than 12.
|
| 122 |
+
_raw_cpus = container_cpus()
|
| 123 |
+
CPUS = _raw_cpus if CPUS_TRUSTED else min(_raw_cpus, 12)
|
| 124 |
+
|
| 125 |
+
# intra_op_num_threads governs ORT's pool only. OpenMP and the BLAS backends size
|
| 126 |
+
# themselves from the host core count, which on a Spaces box is dozens of cores
|
| 127 |
+
# the container will never get. Must be set before onnxruntime is imported.
|
| 128 |
+
for _v in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS"):
|
| 129 |
+
os.environ.setdefault(_v, str(CPUS))
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
@dataclass
|
| 133 |
+
class Clip:
|
| 134 |
+
width: int
|
| 135 |
+
height: int
|
| 136 |
+
fps: float
|
| 137 |
+
frames: int
|
| 138 |
+
duration: float
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def probe(path: str) -> Clip:
|
| 142 |
+
# avg_frame_rate must be REQUESTED. Reading st.get("avg_frame_rate") without
|
| 143 |
+
# naming it here returned None on every call, so the whole VFR fix silently
|
| 144 |
+
# fell through to r_frame_rate and did nothing. Nothing raised.
|
| 145 |
+
#
|
| 146 |
+
# Bounded, too: this runs on every job before the first yield, with the lock
|
| 147 |
+
# held and no Stop button in existence yet, so a malformed container that
|
| 148 |
+
# sends ffprobe into a long analyzeduration scan would otherwise block the
|
| 149 |
+
# whole Space with nothing on screen.
|
| 150 |
+
try:
|
| 151 |
+
out = subprocess.run(
|
| 152 |
+
[FFPROBE, "-v", "error", "-select_streams", "v:0", "-show_entries",
|
| 153 |
+
"stream=width,height,r_frame_rate,avg_frame_rate,nb_frames"
|
| 154 |
+
":format=duration",
|
| 155 |
+
"-of", "json", path],
|
| 156 |
+
capture_output=True, text=True, check=True, timeout=30).stdout
|
| 157 |
+
except subprocess.TimeoutExpired:
|
| 158 |
+
# Not "Ffprobe ...": the tool's name is lowercase, so capitalising it to
|
| 159 |
+
# open the sentence looks like a typo. Reword instead.
|
| 160 |
+
raise RuntimeError("Could not read this file within 30s (ffprobe timed "
|
| 161 |
+
"out). It may be malformed; try re-muxing it, for "
|
| 162 |
+
"example ffmpeg -i in -c copy out.mp4") from None
|
| 163 |
+
except subprocess.CalledProcessError as e:
|
| 164 |
+
# An upload ffprobe rejects outright used to escape as CalledProcessError,
|
| 165 |
+
# whose str() is the full argv, so the user was shown an absolute path to
|
| 166 |
+
# ffprobe.EXE on someone else's machine and no hint that their FILE is the
|
| 167 |
+
# problem. Keep ffprobe's own last line, drop the command.
|
| 168 |
+
why = " ".join((e.stderr or "").split())[-160:]
|
| 169 |
+
raise RuntimeError("ffmpeg cannot open this file as a video"
|
| 170 |
+
+ (f" ({why})" if why else "")
|
| 171 |
+
+ ". Upload a normal video container such as mp4, "
|
| 172 |
+
"mkv or webm.") from None
|
| 173 |
+
d = json.loads(out)
|
| 174 |
+
if not d.get("streams"):
|
| 175 |
+
raise RuntimeError("This file has no video stream that ffmpeg will read "
|
| 176 |
+
"(is it audio only?)")
|
| 177 |
+
st = d["streams"][0]
|
| 178 |
+
# avg_frame_rate is the real cadence; r_frame_rate is the smallest tick that
|
| 179 |
+
# times every frame, which on a VFR source is the MAXIMUM rate. Segment start
|
| 180 |
+
# times are seconds computed from this while frame counts come from
|
| 181 |
+
# -frames:v, so a wrong fps slides the seek a frame earlier every ~1000
|
| 182 |
+
# frames and each segment then re-renders content the previous one covered.
|
| 183 |
+
num, _, den = (st.get("avg_frame_rate") or "0/0").partition("/")
|
| 184 |
+
fps = (float(num) / float(den)) if float(den or 0) and float(num) else 0.0
|
| 185 |
+
if fps <= 0:
|
| 186 |
+
num, _, den = st["r_frame_rate"].partition("/")
|
| 187 |
+
den_f = float(den) if den else 1.0
|
| 188 |
+
fps = float(num) / den_f if den_f else 0.0
|
| 189 |
+
if fps <= 0:
|
| 190 |
+
# Every consumer of fps divides by it, so failing here names the file
|
| 191 |
+
# instead of surfacing an opaque ZeroDivisionError inside the planner.
|
| 192 |
+
raise RuntimeError("Could not determine a frame rate for this file "
|
| 193 |
+
"(ffprobe reports 0/0). Re-mux it, for example "
|
| 194 |
+
"ffmpeg -i in -c copy out.mp4")
|
| 195 |
+
dur = float(d.get("format", {}).get("duration") or 0.0)
|
| 196 |
+
n = int(st.get("nb_frames") or 0) or int(round(fps * dur))
|
| 197 |
+
if n == 1:
|
| 198 |
+
raise RuntimeError("This file has a single frame, so there is no gap to "
|
| 199 |
+
"interpolate. Use a clip with at least two frames.")
|
| 200 |
+
if n < 2:
|
| 201 |
+
# nb_frames and format.duration are both absent on raw h264 and some
|
| 202 |
+
# mkv/webm. n then fell to 0, the planner made one segment of one gap,
|
| 203 |
+
# and a two frame file came back reported as a success. Counting is slow,
|
| 204 |
+
# so it is the fallback and not the default.
|
| 205 |
+
try:
|
| 206 |
+
cnt = subprocess.run(
|
| 207 |
+
[FFPROBE, "-v", "error", "-select_streams", "v:0", "-count_frames",
|
| 208 |
+
"-show_entries", "stream=nb_read_frames", "-of", "json", path],
|
| 209 |
+
capture_output=True, text=True, timeout=120).stdout
|
| 210 |
+
except subprocess.TimeoutExpired:
|
| 211 |
+
cnt = ""
|
| 212 |
+
try:
|
| 213 |
+
n = int(json.loads(cnt)["streams"][0]["nb_read_frames"])
|
| 214 |
+
except Exception: # noqa: BLE001
|
| 215 |
+
n = 0
|
| 216 |
+
if n < 2:
|
| 217 |
+
raise RuntimeError(
|
| 218 |
+
f"Could not determine a frame count for this file "
|
| 219 |
+
f"(container reports {n} frames). Re-mux it, for example "
|
| 220 |
+
f"ffmpeg -i in -c copy out.mp4")
|
| 221 |
+
if dur <= 0 and fps > 0:
|
| 222 |
+
dur = n / fps
|
| 223 |
+
if dur > 0 and n > 0 and abs(n / dur - fps) / fps > 0.02:
|
| 224 |
+
print(f"[rife] {path}: declared {fps:.3f}fps but {n} frames over "
|
| 225 |
+
f"{dur:.2f}s averages {n / dur:.3f}fps. Variable rate source: the "
|
| 226 |
+
f"output is stamped on a uniform grid, so it is retimed.",
|
| 227 |
+
flush=True)
|
| 228 |
+
return Clip(int(st["width"]), int(st["height"]), fps, n, dur)
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
# ---------------------------------------------------------------- backends
|
| 232 |
+
|
| 233 |
+
class Backend:
|
| 234 |
+
"""Common shape: infer(img0, img1, t) on NCHW float32 in 0..1."""
|
| 235 |
+
name = "none"
|
| 236 |
+
def infer(self, a: np.ndarray, b: np.ndarray, t: float) -> np.ndarray: # pragma: no cover
|
| 237 |
+
raise NotImplementedError
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def preload_cuda_libs() -> int:
|
| 241 |
+
"""dlopen the CUDA runtime libraries so ONNX Runtime can find them.
|
| 242 |
+
|
| 243 |
+
ORT reports "Require cuDNN 9.* and CUDA 12.*" whenever any of them is missing
|
| 244 |
+
from the loader path, and then silently runs on CPU. The libraries do exist in
|
| 245 |
+
the image (torch ships them under site-packages/nvidia) but nothing puts that
|
| 246 |
+
directory on the path, and LD_LIBRARY_PATH cannot help because glibc reads it
|
| 247 |
+
once at process start. dlopening them RTLD_GLOBAL into this process does.
|
| 248 |
+
|
| 249 |
+
cuDNN 9 is split across several sub-libraries and libcudnn.so.9 depends on
|
| 250 |
+
them, so ordering matters: dependencies first, aggregate last.
|
| 251 |
+
"""
|
| 252 |
+
import ctypes, glob, sysconfig
|
| 253 |
+
site = sysconfig.get_paths().get("purelib", "")
|
| 254 |
+
root = os.path.join(site, "nvidia")
|
| 255 |
+
found = sorted(glob.glob(os.path.join(root, "*", "lib", "*.so*")))
|
| 256 |
+
print(f"[rife] nvidia .so files present: {len(found)}", flush=True)
|
| 257 |
+
if found:
|
| 258 |
+
names = sorted({os.path.basename(f).split(".so")[0] for f in found})
|
| 259 |
+
print(f"[rife] libs: {', '.join(names)}", flush=True)
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
# Dependencies before the aggregates that link them.
|
| 263 |
+
order = ["cudart", "cublasLt", "cublas", "cufft", "curand", "cusparse",
|
| 264 |
+
"cusolver", "nvrtc", "nvJitLink",
|
| 265 |
+
"cudnn_graph", "cudnn_engines_precompiled", "cudnn_engines_runtime_compiled",
|
| 266 |
+
"cudnn_heuristic", "cudnn_ops", "cudnn_adv", "cudnn_cnn", "cudnn"]
|
| 267 |
+
loaded, failed = [], []
|
| 268 |
+
for stem in order:
|
| 269 |
+
for so in sorted(glob.glob(os.path.join(root, "*", "lib", f"lib{stem}.so*"))):
|
| 270 |
+
try:
|
| 271 |
+
ctypes.CDLL(so, mode=ctypes.RTLD_GLOBAL)
|
| 272 |
+
loaded.append(os.path.basename(so))
|
| 273 |
+
except OSError as e:
|
| 274 |
+
failed.append(f"{os.path.basename(so)}: {str(e)[:60]}")
|
| 275 |
+
print(f"[rife] preloaded {len(loaded)} cuda libs, {len(failed)} failed", flush=True)
|
| 276 |
+
for f in failed[:5]:
|
| 277 |
+
print(f"[rife] FAILED {f}", flush=True)
|
| 278 |
+
return len(loaded)
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
def report_ffmpeg_hw() -> None:
|
| 282 |
+
"""What this image's ffmpeg can offload, printed once at startup.
|
| 283 |
+
|
| 284 |
+
The pip nvidia listing above cannot answer this: NVENC and NVDEC come from
|
| 285 |
+
the driver (libnvidia-encode, libnvcuvid) plus ffmpeg's own build flags, not
|
| 286 |
+
from site-packages. Bounded and swallowed, because a diagnostic must never be
|
| 287 |
+
able to stop the Space from starting.
|
| 288 |
+
"""
|
| 289 |
+
try:
|
| 290 |
+
enc = subprocess.run([FFMPEG, "-hide_banner", "-encoders"],
|
| 291 |
+
capture_output=True, text=True, timeout=20).stdout
|
| 292 |
+
dec = subprocess.run([FFMPEG, "-hide_banner", "-decoders"],
|
| 293 |
+
capture_output=True, text=True, timeout=20).stdout
|
| 294 |
+
acc = subprocess.run([FFMPEG, "-hide_banner", "-hwaccels"],
|
| 295 |
+
capture_output=True, text=True, timeout=20).stdout
|
| 296 |
+
except Exception as e: # noqa: BLE001
|
| 297 |
+
print(f"[rife] ffmpeg hw probe failed ({type(e).__name__})", flush=True)
|
| 298 |
+
return
|
| 299 |
+
nvenc = sorted({ln.split()[1] for ln in enc.splitlines()
|
| 300 |
+
if "nvenc" in ln and len(ln.split()) > 1})
|
| 301 |
+
nvdec = sorted({ln.split()[1] for ln in dec.splitlines()
|
| 302 |
+
if ("cuvid" in ln or "_qsv" in ln) and len(ln.split()) > 1})
|
| 303 |
+
hw = [w for w in acc.split() if w not in ("Hardware", "acceleration",
|
| 304 |
+
"methods:")]
|
| 305 |
+
print(f"[rife] ffmpeg hwaccels: {hw or 'none'}", flush=True)
|
| 306 |
+
print(f"[rife] ffmpeg nvenc encoders: {nvenc or 'none'}", flush=True)
|
| 307 |
+
print(f"[rife] ffmpeg hw decoders: {nvdec or 'none'}", flush=True)
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
def have_lib(soname: str) -> bool:
|
| 311 |
+
"""Can the dynamic loader find this shared object right now?
|
| 312 |
+
|
| 313 |
+
ctypes.CDLL does the real search - loader cache, LD_LIBRARY_PATH as it was at
|
| 314 |
+
process start, and anything already dlopened RTLD_GLOBAL - which is exactly
|
| 315 |
+
what ORT will do a moment later.
|
| 316 |
+
"""
|
| 317 |
+
try:
|
| 318 |
+
ctypes.CDLL(soname)
|
| 319 |
+
return True
|
| 320 |
+
except OSError:
|
| 321 |
+
return False
|
| 322 |
+
|
| 323 |
+
|
| 324 |
+
class OnnxBackend(Backend):
|
| 325 |
+
"""One model, best available execution provider.
|
| 326 |
+
|
| 327 |
+
Providers are tried one at a time. ORT's own provider list is all or nothing:
|
| 328 |
+
if the first entry fails to load it falls back to CPU alone and discards the
|
| 329 |
+
rest, so asking for TensorRT and CUDA together means a missing TensorRT takes
|
| 330 |
+
CUDA down with it. Whatever is finally selected is reported rather than
|
| 331 |
+
assumed, because a Space that quietly runs on CPU looks exactly like one that
|
| 332 |
+
does not.
|
| 333 |
+
|
| 334 |
+
fp32 on both paths; see the fp16 note at the top of this file.
|
| 335 |
+
"""
|
| 336 |
+
|
| 337 |
+
def __init__(self, threads: int = CPUS, prefer_gpu: bool = False,
|
| 338 |
+
trt_cache: str | None = None):
|
| 339 |
+
import onnxruntime as ort
|
| 340 |
+
o = ort.SessionOptions()
|
| 341 |
+
o.intra_op_num_threads = threads
|
| 342 |
+
o.inter_op_num_threads = 1
|
| 343 |
+
o.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
|
| 344 |
+
# The other three session knobs were measured at 736x1280, best of 12,
|
| 345 |
+
# and every alternative to the defaults lost: ORT_PARALLEL +61%,
|
| 346 |
+
# enable_mem_pattern=False +59%, enable_cpu_mem_arena=False +78%.
|
| 347 |
+
# ORT_PARALLEL losing badly is the useful one - it says this graph is a
|
| 348 |
+
# deep sequential chain with nothing to overlap, so an inter-op pool only
|
| 349 |
+
# buys synchronisation. Same story as the spin-wait measurement below.
|
| 350 |
+
# Spin-wait stays ON. Disabling it looks right on a 2 vCPU box (one core
|
| 351 |
+
# burned between ops) and measured the opposite: 1.171s vs 2.797s at two
|
| 352 |
+
# threads, 58% worse. RIFE is many small ops and the context-switch
|
| 353 |
+
# latency between them costs far more than the spin.
|
| 354 |
+
|
| 355 |
+
chain: list = []
|
| 356 |
+
if prefer_gpu:
|
| 357 |
+
preload_cuda_libs()
|
| 358 |
+
# No importlib.reload here. onnxruntime is a C extension and the CPU
|
| 359 |
+
# session built at import already holds bindings into it; reloading
|
| 360 |
+
# leaves that session pointing at a stale module. ORT resolves its
|
| 361 |
+
# provider shared objects lazily when a session is created, so the
|
| 362 |
+
# RTLD_GLOBAL preload above is already visible to it.
|
| 363 |
+
avail = set(ort.get_available_providers())
|
| 364 |
+
# Registered is not the same as usable. ORT ships the TensorRT EP
|
| 365 |
+
# inside the wheel, so it is always "available", but it dlopens
|
| 366 |
+
# libnvinfer.so.10 - TensorRT's own runtime, which this image does
|
| 367 |
+
# not have. Attempting it anyway cost a failed session construction
|
| 368 |
+
# on every fork, and ZeroGPU forks per GPU call. Look for the library
|
| 369 |
+
# first: still free if the image ever ships it, free if it does not.
|
| 370 |
+
if "TensorrtExecutionProvider" in avail and not have_lib("libnvinfer.so.10"):
|
| 371 |
+
print("[rife] TensorRT EP registered but libnvinfer.so.10 is "
|
| 372 |
+
"absent, skipping it", flush=True)
|
| 373 |
+
avail.discard("TensorrtExecutionProvider")
|
| 374 |
+
if "TensorrtExecutionProvider" in avail:
|
| 375 |
+
trt = {"trt_fp16_enable": True, "trt_engine_cache_enable": True}
|
| 376 |
+
if trt_cache:
|
| 377 |
+
os.makedirs(trt_cache, exist_ok=True)
|
| 378 |
+
trt["trt_engine_cache_path"] = trt_cache
|
| 379 |
+
chain.append([("TensorrtExecutionProvider", trt),
|
| 380 |
+
("CUDAExecutionProvider", {})])
|
| 381 |
+
if "CUDAExecutionProvider" in avail:
|
| 382 |
+
chain.append([("CUDAExecutionProvider", {})])
|
| 383 |
+
print(f"[rife] ORT sees providers: {sorted(avail)}", flush=True)
|
| 384 |
+
chain.append([("CPUExecutionProvider", {})])
|
| 385 |
+
|
| 386 |
+
model = str(MODEL)
|
| 387 |
+
|
| 388 |
+
self.s = None
|
| 389 |
+
for attempt in chain:
|
| 390 |
+
want = attempt[0][0]
|
| 391 |
+
try:
|
| 392 |
+
use = model if want != "CPUExecutionProvider" else str(MODEL)
|
| 393 |
+
sess = ort.InferenceSession(use, o, providers=attempt)
|
| 394 |
+
except Exception as e: # noqa: BLE001
|
| 395 |
+
print(f"[rife] {want} raised: {str(e)[:110]}", flush=True)
|
| 396 |
+
continue
|
| 397 |
+
# ORT does not raise when a provider fails to initialise. It prints
|
| 398 |
+
# "EP Error", quietly falls back to CPU inside the constructor and
|
| 399 |
+
# hands back a perfectly valid session. Constructing without error is
|
| 400 |
+
# therefore no evidence the provider was used: ask what it picked.
|
| 401 |
+
got = sess.get_providers()[0]
|
| 402 |
+
if got != want and want != "CPUExecutionProvider":
|
| 403 |
+
print(f"[rife] asked {want}, got {got}, trying next", flush=True)
|
| 404 |
+
del sess
|
| 405 |
+
continue
|
| 406 |
+
self.s = sess
|
| 407 |
+
break
|
| 408 |
+
if self.s is None:
|
| 409 |
+
raise RuntimeError("No usable execution provider")
|
| 410 |
+
|
| 411 |
+
self.provider = self.s.get_providers()[0]
|
| 412 |
+
self.name = {
|
| 413 |
+
"TensorrtExecutionProvider": "GPU TensorRT",
|
| 414 |
+
"CUDAExecutionProvider": "GPU CUDA",
|
| 415 |
+
"CPUExecutionProvider": f"CPU ONNX fp32 x{threads}",
|
| 416 |
+
}.get(self.provider, self.provider)
|
| 417 |
+
self.on_gpu = self.provider != "CPUExecutionProvider"
|
| 418 |
+
# Names are converter artifacts. Interrogate, never hardcode.
|
| 419 |
+
self.inames = [i.name for i in self.s.get_inputs()]
|
| 420 |
+
|
| 421 |
+
def infer(self, a, b, t):
|
| 422 |
+
return self.s.run(None, {self.inames[0]: a, self.inames[1]: b,
|
| 423 |
+
self.inames[2]: np.array([t], dtype=np.float32)})[0]
|
| 424 |
+
|
| 425 |
+
|
| 426 |
+
|
| 427 |
+
|
| 428 |
+
|
| 429 |
+
class Buffers:
|
| 430 |
+
"""Preallocated padded frame buffers for the streaming loop.
|
| 431 |
+
|
| 432 |
+
The naive path costs five full-size passes per frame: frombuffer, transpose,
|
| 433 |
+
astype, divide, ascontiguousarray, then np.pad again because 480, 720 and
|
| 434 |
+
1080 are none of them multiples of 32. At 1080p that is well over 100 MB of
|
| 435 |
+
memory traffic per decoded frame on a box with two vCPUs, and it is pure
|
| 436 |
+
serial overhead that the model rate measurement never sees.
|
| 437 |
+
|
| 438 |
+
Here the padded buffers are allocated once per segment and reused. Decoded
|
| 439 |
+
bytes are scaled straight into the interior view in a single pass, and only
|
| 440 |
+
the edge strips are re-replicated rather than the whole frame.
|
| 441 |
+
"""
|
| 442 |
+
|
| 443 |
+
def __init__(self, h: int, w: int):
|
| 444 |
+
self.h, self.w = h, w
|
| 445 |
+
self.ph = (h + ALIGN - 1) // ALIGN * ALIGN
|
| 446 |
+
self.pw = (w + ALIGN - 1) // ALIGN * ALIGN
|
| 447 |
+
# uint8 NHWC: exactly what the decoder emits and what the graph now
|
| 448 |
+
# takes, so a frame is copied once and never converted on this CPU.
|
| 449 |
+
self.a = np.zeros((1, self.ph, self.pw, 3), dtype=np.uint8)
|
| 450 |
+
self.b = np.zeros((1, self.ph, self.pw, 3), dtype=np.uint8)
|
| 451 |
+
self._u8 = np.empty((h, w, 3), dtype=np.uint8)
|
| 452 |
+
|
| 453 |
+
def load(self, dst: np.ndarray, raw: bytes) -> np.ndarray:
|
| 454 |
+
"""Copy decoded RGB straight into dst's interior, then fix the edges.
|
| 455 |
+
|
| 456 |
+
No scaling and no transpose any more: the graph takes uint8 NHWC and does
|
| 457 |
+
both on the device. This was 0.42 s of a 3.44 s segment at 852x480.
|
| 458 |
+
|
| 459 |
+
The decoder stays on packed rgb24 deliberately. Asking it for gbrp would
|
| 460 |
+
remove this transpose, but `yuv420p -> gbrp` and `yuv420p -> rgb24` are
|
| 461 |
+
different chroma upsampling paths in swscale and disagree by up to 154
|
| 462 |
+
on a colour-bar source, mean 1.87, with no dither or rounding flag
|
| 463 |
+
closing the gap. That is a change to what the user gets, for 0.74 s a
|
| 464 |
+
segment. The encoder half IS free, but it cannot be taken alone: the
|
| 465 |
+
source-frame pass-through writes decoder bytes straight to the encoder,
|
| 466 |
+
so the two pipes must agree on a format. See emit() for the full result.
|
| 467 |
+
"""
|
| 468 |
+
src = np.frombuffer(raw, dtype=np.uint8).reshape(self.h, self.w, 3)
|
| 469 |
+
dst[0, :self.h, :self.w, :] = src # one contiguous copy
|
| 470 |
+
if self.ph > self.h: # replicate bottom edge
|
| 471 |
+
dst[0, self.h:, :self.w, :] = dst[0, self.h - 1:self.h, :self.w, :]
|
| 472 |
+
if self.pw > self.w: # replicate right edge
|
| 473 |
+
dst[0, :, self.w:, :] = dst[0, :, self.w - 1:self.w, :]
|
| 474 |
+
return dst
|
| 475 |
+
|
| 476 |
+
def emit(self, x: np.ndarray) -> bytes:
|
| 477 |
+
"""Pack the model's uint8 NCHW output into the rgb24 the encoder wants.
|
| 478 |
+
|
| 479 |
+
The scale, round and clip used to happen here: three float passes over
|
| 480 |
+
2.76 M elements per frame, on two vCPUs, while the GPU finished the whole
|
| 481 |
+
frame in 21 ms. Measured on the Space that was 1.13-1.20 s of the 2.3 s
|
| 482 |
+
of non-model time in a 179-gap segment, the largest term after inference.
|
| 483 |
+
Mul/Add/Clip/Cast now live in the graph and run on the device, verified
|
| 484 |
+
bit identical against this function's old output on real model results.
|
| 485 |
+
What is left is the NCHW to HWC pack the encoder's rgb24 requires.
|
| 486 |
+
|
| 487 |
+
Planar (gbrp) on both pipes would remove this and the matching transpose
|
| 488 |
+
in load(), worth ~1.5 s of the 1.94 s of non-model time in a segment. It
|
| 489 |
+
was measured and declined: the ENCODER side is byte identical with gbrp
|
| 490 |
+
(max diff 0 feeding the same pixels both ways), but the DECODER is not -
|
| 491 |
+
`yuv420p -> gbrp` and `yuv420p -> rgb24` take different chroma upsampling
|
| 492 |
+
paths in swscale and disagree by up to 154 on a colour-bar source, mean
|
| 493 |
+
1.87, with no dither or rounding flag closing it. And the two pipes
|
| 494 |
+
cannot disagree: the source-frame pass-through writes decoder bytes
|
| 495 |
+
straight to the encoder, so a packed decoder with a planar encoder feeds
|
| 496 |
+
packed pixels into a planar pipe. Both formats or neither.
|
| 497 |
+
"""
|
| 498 |
+
# The model emits uint8 now. A float32 array here would truncate to
|
| 499 |
+
# near-black without raising, which is the kind of failure that reaches
|
| 500 |
+
# a user as a corrupt video and no error at all.
|
| 501 |
+
assert x.dtype == np.uint8, f"emit expects uint8 from the model, got {x.dtype}"
|
| 502 |
+
self._u8[...] = x[0, :, :self.h, :self.w].transpose(1, 2, 0)
|
| 503 |
+
return self._u8.tobytes()
|
| 504 |
+
|
| 505 |
+
|
| 506 |
+
def measure_rate(be: Backend, h: int, w: int, samples: int = 3) -> float:
|
| 507 |
+
"""Seconds per interpolated frame at this resolution, measured on this box.
|
| 508 |
+
|
| 509 |
+
Everything downstream is estimated from this rather than from a table, so a
|
| 510 |
+
slower vCPU or a faster GPU changes the plan instead of surprising the user.
|
| 511 |
+
"""
|
| 512 |
+
ph, pw = (h + ALIGN - 1) // ALIGN * ALIGN, (w + ALIGN - 1) // ALIGN * ALIGN
|
| 513 |
+
# NOT np.zeros. A freshly calloc'd array can sit on the shared zero page, and
|
| 514 |
+
# reusing one across samples reads it straight out of cache: measured at
|
| 515 |
+
# 640x360, reused zeros time at 0.1415 s against 0.2130 s for real decoded
|
| 516 |
+
# frames, 33% low. The single-sample version escaped that only by allocating
|
| 517 |
+
# fresh arrays each call. Sampling over one buffer would have walked into it
|
| 518 |
+
# and made every estimate optimistic. Random data times like real data
|
| 519 |
+
# (0.2130 vs 0.2130) and is reproducible.
|
| 520 |
+
rng = np.random.default_rng(0)
|
| 521 |
+
a = rng.integers(0, 256, (1, ph, pw, 3), dtype=np.uint8)
|
| 522 |
+
b = rng.integers(0, 256, (1, ph, pw, 3), dtype=np.uint8)
|
| 523 |
+
be.infer(a, b, 0.5) # warm
|
| 524 |
+
# Median of a few, not one. plan_gaps sizes every segment from this number,
|
| 525 |
+
# so its variance is a correctness property: a single sample at 640x360 was
|
| 526 |
+
# observed from 0.157 to 0.232 s across runs, a 1.5x spread, and a plan built
|
| 527 |
+
# on the low end makes segments 1.5x larger than intended - which on ZeroGPU
|
| 528 |
+
# is how a 90 s budget overruns a 120 s cap. At 1280x720 the spread is ~6%,
|
| 529 |
+
# so the error is worst where frames are small, but this is the foundation of
|
| 530 |
+
# every estimate the user is shown.
|
| 531 |
+
ts = []
|
| 532 |
+
for _ in range(max(1, samples)):
|
| 533 |
+
t0 = time.perf_counter()
|
| 534 |
+
be.infer(a, b, 0.5)
|
| 535 |
+
ts.append(time.perf_counter() - t0)
|
| 536 |
+
return statistics.median(ts)
|
| 537 |
+
|
| 538 |
+
|
| 539 |
+
def profile_ops(h: int, w: int, samples: int = 4) -> str:
|
| 540 |
+
"""One-line breakdown of where an inference goes, from ORT's own profiler.
|
| 541 |
+
|
| 542 |
+
Builds a SECOND session with profiling on rather than turning it on for the
|
| 543 |
+
live one: the profiler adds per-op bookkeeping, and measuring a session while
|
| 544 |
+
slowing it down is how you get numbers that describe the measurement.
|
| 545 |
+
"""
|
| 546 |
+
import glob
|
| 547 |
+
import onnxruntime as ort
|
| 548 |
+
try:
|
| 549 |
+
o = ort.SessionOptions()
|
| 550 |
+
o.enable_profiling = True
|
| 551 |
+
o.profile_file_prefix = os.path.join(tempfile.gettempdir(), "ffprof")
|
| 552 |
+
o.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
|
| 553 |
+
sess = ort.InferenceSession(str(MODEL), o, providers=[("CUDAExecutionProvider", {})])
|
| 554 |
+
if sess.get_providers()[0] != "CUDAExecutionProvider":
|
| 555 |
+
return "profile skipped: session did not land on CUDA"
|
| 556 |
+
rng = np.random.default_rng(0)
|
| 557 |
+
a = rng.integers(0, 256, (1, h, w, 3), dtype=np.uint8)
|
| 558 |
+
b = rng.integers(0, 256, (1, h, w, 3), dtype=np.uint8)
|
| 559 |
+
t = np.array([0.5], dtype=np.float32)
|
| 560 |
+
for _ in range(samples):
|
| 561 |
+
# The only hardcoded input names in the file; everything else binds
|
| 562 |
+
# off session.get_inputs(). They move with the graph.
|
| 563 |
+
sess.run(None, {"img0_u8": a, "img1_u8": b, "timestep": t})
|
| 564 |
+
path = sess.end_profiling()
|
| 565 |
+
del sess
|
| 566 |
+
with open(path, encoding="utf-8") as f:
|
| 567 |
+
ev = json.load(f)
|
| 568 |
+
tot, by, on_cpu, cpu_by = 0, {}, 0, {}
|
| 569 |
+
for e in ev:
|
| 570 |
+
if e.get("cat") != "Node" or "dur" not in e:
|
| 571 |
+
continue
|
| 572 |
+
args = e.get("args", {}) or {}
|
| 573 |
+
kind = args.get("op_name", "?")
|
| 574 |
+
by[kind] = by.get(kind, 0) + e["dur"]
|
| 575 |
+
# Which EP actually ran it. 38% memcpy has two possible causes and
|
| 576 |
+
# they need opposite fixes: inputs arriving on the host (fix by
|
| 577 |
+
# shrinking what crosses, or by IO binding), or an op with no CUDA
|
| 578 |
+
# kernel splitting the graph and forcing a round trip mid-inference
|
| 579 |
+
# (fix by replacing that op). Bucketing by provider tells them apart,
|
| 580 |
+
# and it is also how a uint8 input graph would be caught failing -
|
| 581 |
+
# ORT can place the leading Cast on the CPU EP, in which case the
|
| 582 |
+
# host casts to float32 and ships the same bytes as before, plus
|
| 583 |
+
# extra nodes.
|
| 584 |
+
if "CPU" in args.get("provider", ""):
|
| 585 |
+
on_cpu += e["dur"]
|
| 586 |
+
cpu_by[kind] = cpu_by.get(kind, 0) + e["dur"]
|
| 587 |
+
tot += e["dur"]
|
| 588 |
+
for f in glob.glob(o.profile_file_prefix + "*"):
|
| 589 |
+
try:
|
| 590 |
+
os.remove(f)
|
| 591 |
+
except OSError:
|
| 592 |
+
pass
|
| 593 |
+
if not tot:
|
| 594 |
+
return "profile produced no node events"
|
| 595 |
+
top = sorted(by.items(), key=lambda kv: -kv[1])[:6]
|
| 596 |
+
copies = sum(v for k, v in by.items() if "Memcpy" in k)
|
| 597 |
+
return ("ops " + ", ".join(f"{k} {v * 100 // tot}%" for k, v in top)
|
| 598 |
+
+ f" | memcpy {copies * 100 // tot}%"
|
| 599 |
+
+ f" | on CPU EP {on_cpu * 100 // tot}%"
|
| 600 |
+
+ (" [" + ", ".join(
|
| 601 |
+
f"{k} {v * 100 // tot}%" for k, v in
|
| 602 |
+
sorted(cpu_by.items(), key=lambda kv: -kv[1])[:5]) + "]"
|
| 603 |
+
if cpu_by else "")
|
| 604 |
+
+ f" | {tot / 1000 / samples:.1f} ms/run")
|
| 605 |
+
except Exception as e: # noqa: BLE001
|
| 606 |
+
return f"profile failed ({type(e).__name__}: {str(e)[:80]})"
|
| 607 |
+
|
| 608 |
+
|
| 609 |
+
def nvenc_works() -> bool:
|
| 610 |
+
"""Can this process actually ENCODE with NVENC right now?
|
| 611 |
+
|
| 612 |
+
`ffmpeg -encoders` listing h264_nvenc proves only that ffmpeg was built with
|
| 613 |
+
it. The runtime needs libnvidia-encode and a real device, and on ZeroGPU the
|
| 614 |
+
parent process has neither - a GPU exists only inside a @spaces.GPU fork. So
|
| 615 |
+
this has to run in the fork, and it has to encode something rather than ask.
|
| 616 |
+
"""
|
| 617 |
+
try:
|
| 618 |
+
r = subprocess.run(
|
| 619 |
+
[FFMPEG, "-v", "error", "-f", "lavfi", "-i", "nullsrc=s=256x256:d=0.1",
|
| 620 |
+
"-c:v", "h264_nvenc", "-f", "null", "-"],
|
| 621 |
+
capture_output=True, timeout=30)
|
| 622 |
+
return r.returncode == 0
|
| 623 |
+
except Exception: # noqa: BLE001
|
| 624 |
+
return False
|
| 625 |
+
|
| 626 |
+
|
| 627 |
+
# NVENC only pays above roughly 2 MP. Measured against x264 veryfast pinned to
|
| 628 |
+
# 2 threads, which is what the Space actually has, at MATCHED SSIM rather than
|
| 629 |
+
# matched knob number:
|
| 630 |
+
# 720p 0.9 MP 0.86x speed 1.02x size <- slower, so do not
|
| 631 |
+
# 1080p 2.1 MP 1.14x speed 0.92x size
|
| 632 |
+
# 1440p 3.7 MP 1.27x speed 0.85x size
|
| 633 |
+
# 2160p 8.3 MP 1.51x speed 0.82x size
|
| 634 |
+
# The threshold sits between the first two rows.
|
| 635 |
+
NVENC_MIN_PIXELS = 1_400_000
|
| 636 |
+
|
| 637 |
+
# cq is NOT crf. Passing crf straight through produced files 2.5-2.8x larger at
|
| 638 |
+
# the same number. Matching SSIM instead put the offset at +2: nvenc cq22 scores
|
| 639 |
+
# at or above x264 crf20 everywhere measured. Picking the offset by FILE SIZE
|
| 640 |
+
# would have said +6, and cq26 is a real quality drop (SSIM 0.941 against 0.967)
|
| 641 |
+
# that looks like a win on the size column alone.
|
| 642 |
+
NVENC_CQ_OFFSET = 2
|
| 643 |
+
|
| 644 |
+
|
| 645 |
+
def encoder_args(crf: int, hw: bool) -> list[str]:
|
| 646 |
+
"""x264, or NVENC when it is actually the faster of the two.
|
| 647 |
+
|
| 648 |
+
`-b:v 0` is required or vbr silently targets a bitrate and ignores -cq.
|
| 649 |
+
"""
|
| 650 |
+
if not hw:
|
| 651 |
+
return ["-c:v", "libx264", "-preset", "veryfast", "-crf", str(crf)]
|
| 652 |
+
cq = max(0, min(51, crf + NVENC_CQ_OFFSET))
|
| 653 |
+
return ["-c:v", "h264_nvenc", "-preset", "p4", "-tune", "hq",
|
| 654 |
+
"-rc", "vbr", "-cq", str(cq), "-b:v", "0"]
|
| 655 |
+
|
| 656 |
+
|
| 657 |
+
def interpolate_segment(src: str, dst: str, start: float, gaps: int, factor: int,
|
| 658 |
+
be: Backend, clip: Clip, crf: int = 18,
|
| 659 |
+
cancel: Callable[[], bool] | None = None,
|
| 660 |
+
on_frame: Callable[[int], None] | None = None,
|
| 661 |
+
write_tail: bool = True,
|
| 662 |
+
out_w: int = 0, out_h: int = 0,
|
| 663 |
+
hw_encode: bool = False) -> int:
|
| 664 |
+
"""Interpolate one time slice straight from decoder pipe to encoder pipe.
|
| 665 |
+
|
| 666 |
+
Segments overlap by exactly one source frame. A segment owns the GAPS that
|
| 667 |
+
start inside it, so it emits `prev` plus the generated frames for each gap and
|
| 668 |
+
leaves the trailing frame to whoever owns the next gap. Without that, every
|
| 669 |
+
seam would both duplicate a source frame and lose the frame belonging in the
|
| 670 |
+
gap across it; only the final segment writes its tail.
|
| 671 |
+
|
| 672 |
+
Two decoded frames are resident at a time, plus at most four more in the
|
| 673 |
+
encoder queue and the writer - about 100 MB at 4K rather than 50. Still O(1)
|
| 674 |
+
in clip length at any resolution, which is the property that matters; only
|
| 675 |
+
the constant grew when the writer thread arrived.
|
| 676 |
+
"""
|
| 677 |
+
w = out_w or clip.width
|
| 678 |
+
h = out_h or clip.height
|
| 679 |
+
fsize = w * h * 3
|
| 680 |
+
need = gaps + 1 # frames required to close them
|
| 681 |
+
|
| 682 |
+
# Scaling here rather than in a pre-pass: a whole-clip transcode before the
|
| 683 |
+
# first segment costs an extra encode AND decode over the entire source, is
|
| 684 |
+
# serial time before any progress shows, and adds a generation-loss hop.
|
| 685 |
+
vf = ([] if (w, h) == (clip.width, clip.height)
|
| 686 |
+
else ["-vf", f"scale={w}:{h}:flags=bicubic"])
|
| 687 |
+
# stderr goes to temp FILES, not pipes. A pipe nobody drains fills at ~64 KB,
|
| 688 |
+
# at which point ffmpeg blocks writing stderr, stops producing on stdout, and
|
| 689 |
+
# the unbounded dec.stdout.read() below waits forever. "-v error" hides
|
| 690 |
+
# warnings but not per-frame decode errors, and a slightly corrupt upload is
|
| 691 |
+
# routine input here, so that pipe filling is a question of when.
|
| 692 |
+
dec_err = tempfile.TemporaryFile()
|
| 693 |
+
enc_err = tempfile.TemporaryFile()
|
| 694 |
+
dec = subprocess.Popen(
|
| 695 |
+
[FFMPEG, "-v", "error", "-ss", f"{start:.6f}", "-i", src,
|
| 696 |
+
"-frames:v", str(need), *vf, "-f", "rawvideo", "-pix_fmt", "rgb24", "-"],
|
| 697 |
+
stdout=subprocess.PIPE, stderr=dec_err, bufsize=fsize * 2)
|
| 698 |
+
# -preset veryfast is MEASURED, not picked. Stage profiling puts the encoder
|
| 699 |
+
# at 1-2% of a CPU job and ~11% of a GPU one (ffmpeg does not get faster when
|
| 700 |
+
# the model does), so it is only worth anything on GPU, and only then if the
|
| 701 |
+
# trade is good. It is not. Benchmarked on real frames at fixed crf 20:
|
| 702 |
+
# 720p ultrafast -6.5% time +195% size superfast +20% time +100% size
|
| 703 |
+
# 2160p ultrafast -13% time +209% size superfast -14% time +102% size
|
| 704 |
+
# Three times the download to shave 13% off a stage that is a ninth of the
|
| 705 |
+
# job, about 1.5% overall. The slower presets cost 26-89% more time and did
|
| 706 |
+
# not come back smaller either. veryfast is the right point on this curve.
|
| 707 |
+
enc = subprocess.Popen(
|
| 708 |
+
[FFMPEG, "-v", "error", "-y", "-f", "rawvideo", "-pix_fmt", "rgb24",
|
| 709 |
+
"-s", f"{w}x{h}", "-r", f"{clip.fps * factor:.6f}", "-i", "-",
|
| 710 |
+
*encoder_args(crf, hw_encode and w * h >= NVENC_MIN_PIXELS),
|
| 711 |
+
"-pix_fmt", "yuv420p", dst],
|
| 712 |
+
stdin=subprocess.PIPE, stderr=enc_err)
|
| 713 |
+
|
| 714 |
+
written = 0
|
| 715 |
+
starved = False
|
| 716 |
+
# Stage accumulators. perf_counter calls are ~100 ns, so at a few per frame
|
| 717 |
+
# this is far below the millisecond effects being measured.
|
| 718 |
+
t_read = t_load = t_inf = t_emit = t_put = 0.0
|
| 719 |
+
bufs = Buffers(h, w)
|
| 720 |
+
prev_p, cur_p = bufs.a, bufs.b # ping-pong, no per-frame allocation
|
| 721 |
+
|
| 722 |
+
# The encoder gets its own thread so x264 overlaps inference instead of
|
| 723 |
+
# stalling it. A 4K frame is ~25 MB against a 64 KB default pipe buffer, so
|
| 724 |
+
# without this every write waits for ffmpeg to drain the last one.
|
| 725 |
+
wq: queue.Queue = queue.Queue(maxsize=3)
|
| 726 |
+
werr: list[BaseException] = []
|
| 727 |
+
|
| 728 |
+
def _writer():
|
| 729 |
+
while True:
|
| 730 |
+
item = wq.get()
|
| 731 |
+
if item is None:
|
| 732 |
+
return
|
| 733 |
+
try:
|
| 734 |
+
enc.stdin.write(item)
|
| 735 |
+
except BaseException as e: # noqa: BLE001
|
| 736 |
+
werr.append(e)
|
| 737 |
+
return # stop; put() notices below
|
| 738 |
+
|
| 739 |
+
wt = threading.Thread(target=_writer, daemon=True)
|
| 740 |
+
wt.start()
|
| 741 |
+
|
| 742 |
+
def put(b: bytes) -> None:
|
| 743 |
+
"""Hand a frame to the writer, refusing to block forever if it died."""
|
| 744 |
+
while True:
|
| 745 |
+
try:
|
| 746 |
+
wq.put(b, timeout=1.0)
|
| 747 |
+
return
|
| 748 |
+
except queue.Full:
|
| 749 |
+
if not wt.is_alive() or werr:
|
| 750 |
+
raise RuntimeError(
|
| 751 |
+
"the encoder stopped accepting frames"
|
| 752 |
+
+ (f": {werr[0]}" if werr else ""))
|
| 753 |
+
try:
|
| 754 |
+
raw_prev = dec.stdout.read(fsize)
|
| 755 |
+
if len(raw_prev) < fsize:
|
| 756 |
+
# Returning 0 from inside the try skipped the encoder check below and
|
| 757 |
+
# let concat splice straight over the gap: a jump cut whose only
|
| 758 |
+
# trace was one line in a 14-line ring buffer. A segment that decodes
|
| 759 |
+
# nothing is a bug in the bounds or a truncated source, so say so.
|
| 760 |
+
starved = True
|
| 761 |
+
else:
|
| 762 |
+
bufs.load(prev_p, raw_prev)
|
| 763 |
+
while not starved:
|
| 764 |
+
if cancel is not None and cancel():
|
| 765 |
+
break
|
| 766 |
+
_t = time.perf_counter()
|
| 767 |
+
raw_cur = dec.stdout.read(fsize)
|
| 768 |
+
t_read += time.perf_counter() - _t
|
| 769 |
+
if len(raw_cur) < fsize:
|
| 770 |
+
break
|
| 771 |
+
_t = time.perf_counter()
|
| 772 |
+
bufs.load(cur_p, raw_cur)
|
| 773 |
+
t_load += time.perf_counter() - _t
|
| 774 |
+
# A SOURCE frame goes out as the bytes that came in. It used to be
|
| 775 |
+
# scaled to float32 by load(), then scaled back by emit() - multiply,
|
| 776 |
+
# add, clip, transpose, copy - to reproduce what was already in hand.
|
| 777 |
+
# Verified byte-identical for all 256 levels: u/255 in float32, then
|
| 778 |
+
# clip(v*255 + 0.5), returns u exactly, so this changes no output.
|
| 779 |
+
# At factor 2 it is HALF of all emit work, at factor 4 a quarter.
|
| 780 |
+
_t = time.perf_counter()
|
| 781 |
+
put(raw_prev)
|
| 782 |
+
t_put += time.perf_counter() - _t
|
| 783 |
+
written += 1
|
| 784 |
+
for i in range(1, factor):
|
| 785 |
+
_t = time.perf_counter()
|
| 786 |
+
out = be.infer(prev_p, cur_p, i / factor)
|
| 787 |
+
t_inf += time.perf_counter() - _t
|
| 788 |
+
_t = time.perf_counter()
|
| 789 |
+
blob = bufs.emit(out)
|
| 790 |
+
t_emit += time.perf_counter() - _t
|
| 791 |
+
_t = time.perf_counter()
|
| 792 |
+
put(blob)
|
| 793 |
+
t_put += time.perf_counter() - _t
|
| 794 |
+
written += 1
|
| 795 |
+
if on_frame is not None:
|
| 796 |
+
on_frame(written)
|
| 797 |
+
prev_p, cur_p = cur_p, prev_p # swap, never copy
|
| 798 |
+
raw_prev = raw_cur
|
| 799 |
+
if write_tail and not starved:
|
| 800 |
+
put(raw_prev); written += 1
|
| 801 |
+
finally:
|
| 802 |
+
# Drain before closing stdin, or the frames still queued are discarded
|
| 803 |
+
# and the segment silently comes up short. The sentinel is put without a
|
| 804 |
+
# timeout guard on purpose: if the writer is gone the queue has room.
|
| 805 |
+
try:
|
| 806 |
+
wq.put(None, timeout=5.0)
|
| 807 |
+
except Exception as e: # noqa: BLE001
|
| 808 |
+
# Not swallowed. A dropped sentinel means the writer never learns to
|
| 809 |
+
# stop, so frames may still be queued and unwritten - which would
|
| 810 |
+
# otherwise return a short segment as a complete one.
|
| 811 |
+
werr.append(RuntimeError(f"could not signal the writer to finish: {e}"))
|
| 812 |
+
wt.join(timeout=120.0)
|
| 813 |
+
if wt.is_alive():
|
| 814 |
+
# join() returns identically whether the thread finished or the
|
| 815 |
+
# timeout expired, so this has to be asked explicitly.
|
| 816 |
+
werr.append(RuntimeError("the encoder writer did not finish in 120s"))
|
| 817 |
+
if written > 1:
|
| 818 |
+
tot = t_read + t_load + t_inf + t_emit + t_put
|
| 819 |
+
print(f"[rife] stages {w}x{h} {written}f: infer {t_inf:.2f}s "
|
| 820 |
+
f"read {t_read:.2f}s load {t_load:.2f}s emit {t_emit:.2f}s "
|
| 821 |
+
f"put {t_put:.2f}s | non-model {(tot - t_inf) * 1000 / written:.1f} "
|
| 822 |
+
f"ms/frame", flush=True)
|
| 823 |
+
for closer in (lambda: enc.stdin.close(), lambda: dec.stdout.close()):
|
| 824 |
+
try:
|
| 825 |
+
closer()
|
| 826 |
+
except Exception:
|
| 827 |
+
pass
|
| 828 |
+
rc = enc.wait() # safe now: stderr is a file, so it cannot block
|
| 829 |
+
try:
|
| 830 |
+
dec.wait(timeout=10)
|
| 831 |
+
except Exception:
|
| 832 |
+
dec.kill()
|
| 833 |
+
try:
|
| 834 |
+
# Order matters, all three ways.
|
| 835 |
+
# Starvation first: it is decided by the very FIRST read, before any
|
| 836 |
+
# cancel check, so it is a property of the bounds or the source and never
|
| 837 |
+
# of the cancellation. Swallowing it under a cancel would lose the one
|
| 838 |
+
# diagnostic built for bad bounds exactly when a user interrupts a job
|
| 839 |
+
# that was already going wrong. It also makes the encoder exit non-zero
|
| 840 |
+
# for want of input, so checking rc ahead of it reported "encoder failed"
|
| 841 |
+
# for what is really a decode fault.
|
| 842 |
+
if werr:
|
| 843 |
+
# Raised HERE, not in the finally. Raising from inside the finally
|
| 844 |
+
# jumped over enc.stdin.close(), dec.stdout.close(), enc.wait() and
|
| 845 |
+
# dec.wait(), so a failed encoder left the DECODER alive, blocked
|
| 846 |
+
# writing into a pipe with no reader, holding fds on the source and
|
| 847 |
+
# on its stderr temp file - one orphan pair per failed segment, and
|
| 848 |
+
# the shrink-and-retry path can try the same range four times.
|
| 849 |
+
#
|
| 850 |
+
# Ordered after starvation because a starved segment never calls
|
| 851 |
+
# put(), so werr cannot be set there, and before the cancel check
|
| 852 |
+
# because a broken write means the file is incomplete whatever the
|
| 853 |
+
# user pressed.
|
| 854 |
+
raise RuntimeError(f"encoder write failed: {werr[0]}")
|
| 855 |
+
if starved:
|
| 856 |
+
raise RuntimeError(
|
| 857 |
+
f"Decoder produced no frames at {start:.3f}s of a "
|
| 858 |
+
f"{clip.duration:.3f}s source. The file is probably truncated "
|
| 859 |
+
f"or corrupt: its header claims more than it contains. "
|
| 860 |
+
f"ffmpeg said: {tail(dec_err)}")
|
| 861 |
+
# Then the clean stop. A cancel landing between the loop's own check and
|
| 862 |
+
# the first frame leaves the encoder with nothing, and rc below would be
|
| 863 |
+
# reporting that cancellation rather than a fault.
|
| 864 |
+
if written == 0 and cancel is not None and cancel():
|
| 865 |
+
return 0
|
| 866 |
+
# A dead encoder used to read as success: written was returned
|
| 867 |
+
# regardless, so concat later spliced a truncated part.
|
| 868 |
+
if rc != 0:
|
| 869 |
+
raise RuntimeError(f"Encoder failed (exit {rc}). "
|
| 870 |
+
f"ffmpeg said: {tail(enc_err)}")
|
| 871 |
+
finally:
|
| 872 |
+
for f in (dec_err, enc_err):
|
| 873 |
+
try:
|
| 874 |
+
f.close()
|
| 875 |
+
except Exception: # noqa: BLE001
|
| 876 |
+
pass
|
| 877 |
+
return written
|
| 878 |
+
|
| 879 |
+
|
| 880 |
+
def tail(f, n: int = 400) -> str:
|
| 881 |
+
"""The last COMPLETE stderr line or two, for an error message.
|
| 882 |
+
|
| 883 |
+
Slicing the last n bytes lands mid-word: a truncated h264 file produced
|
| 884 |
+
"decoder produced no frames at 10.783s of a 20.000s source ze (2176 > 104)."
|
| 885 |
+
where "ze (2176 > 104)" is the tail of "Invalid NAL unit size (2176 > 104)".
|
| 886 |
+
ffmpeg also repeats the same complaint per frame, so the last couple of
|
| 887 |
+
distinct lines carry everything the earlier thousands do.
|
| 888 |
+
"""
|
| 889 |
+
try:
|
| 890 |
+
f.seek(0)
|
| 891 |
+
raw = f.read()[-4096:].decode("utf-8", "replace")
|
| 892 |
+
except Exception: # noqa: BLE001
|
| 893 |
+
return ""
|
| 894 |
+
lines, seen = [], set()
|
| 895 |
+
for ln in reversed([l.strip() for l in raw.splitlines() if l.strip()]):
|
| 896 |
+
if ln in seen:
|
| 897 |
+
continue
|
| 898 |
+
seen.add(ln)
|
| 899 |
+
lines.append(ln)
|
| 900 |
+
if len(lines) == 2 or sum(map(len, lines)) > n:
|
| 901 |
+
break
|
| 902 |
+
return " | ".join(reversed(lines))[:n]
|
| 903 |
+
|
| 904 |
+
|
| 905 |
+
# NOT corrected by a per-gap constant, and the measurements are why.
|
| 906 |
+
#
|
| 907 |
+
# plan_gaps charges `rate * (factor - 1)` per gap. A real segment also does a
|
| 908 |
+
# decode read, two buffer passes and `factor` x264 frames, so it under-charges.
|
| 909 |
+
# Measured, as measured/assumed on real segments:
|
| 910 |
+
#
|
| 911 |
+
# CPU 1280x720 f2 inference 0.9103 s/frame 1.028x -> 0.0258 s/gap missing
|
| 912 |
+
# CPU 1280x720 f4 inference 0.9103 s/frame 1.008x -> 0.0217 s/gap missing
|
| 913 |
+
# GPU 852x480 f2 inference 0.0579 s/frame 1.456x -> 0.0264 s/gap missing
|
| 914 |
+
# CPU 426x240 f2 inference 0.1023 s/frame 1.042x -> 0.0043 s/gap missing
|
| 915 |
+
#
|
| 916 |
+
# A single additive 0.026 s/gap fits the first three to within 0.4%. It is 6x too
|
| 917 |
+
# large for the fourth, where it would over-charge by 21% - and on ZeroGPU an
|
| 918 |
+
# over-charge is not free either, since 21% more segments is 21% more GPU
|
| 919 |
+
# allocations against a daily run budget. Four points that are neither constant
|
| 920 |
+
# nor pixel-linear (480p and 720p agree despite 2.25x the pixels) are not a
|
| 921 |
+
# model, so no constant is applied.
|
| 922 |
+
#
|
| 923 |
+
# What IS established: the under-charge is largest on a fast device, exactly
|
| 924 |
+
# where the 120 s ZeroGPU cap lives. That is handled by sizing the budget for it
|
| 925 |
+
# in app.py rather than by inventing a curve through four points. Every segment
|
| 926 |
+
# now logs its own ratio, so the data to build a real model accumulates from
|
| 927 |
+
# production instead of from one afternoon's harness.
|
| 928 |
+
|
| 929 |
+
|
| 930 |
+
def plan_gaps(clip: Clip, factor: int, rate_s: float, budget_s: float) -> int:
|
| 931 |
+
"""How many gaps one segment may own so it fits the compute budget.
|
| 932 |
+
|
| 933 |
+
Counted in gaps, not seconds. Second-based boundaries round against the frame
|
| 934 |
+
rate and the error accumulates across segments, which showed up as a handful
|
| 935 |
+
of extra frames on a seven segment run.
|
| 936 |
+
"""
|
| 937 |
+
per_gap = rate_s * max(1, factor - 1)
|
| 938 |
+
if per_gap <= 0:
|
| 939 |
+
return max(1, clip.frames - 1)
|
| 940 |
+
return max(1, int(budget_s / per_gap))
|
| 941 |
+
|
| 942 |
+
|
| 943 |
+
def segment_bounds(clip: Clip, gaps_per_seg: int,
|
| 944 |
+
start_frame: int = 0) -> list[tuple[float, int, bool]]:
|
| 945 |
+
"""(start_seconds, gaps_owned, is_last) for each segment, frame exact.
|
| 946 |
+
|
| 947 |
+
start_frame re-cuts only the tail of a clip, which is what a mid-job switch
|
| 948 |
+
from GPU to CPU needs: the segments already rendered keep their bounds and
|
| 949 |
+
the rest are re-sized for the slower device.
|
| 950 |
+
"""
|
| 951 |
+
total = max(1, clip.frames - 1)
|
| 952 |
+
out: list[tuple[float, int, bool]] = []
|
| 953 |
+
f = start_frame
|
| 954 |
+
while f < total:
|
| 955 |
+
n = min(gaps_per_seg, total - f)
|
| 956 |
+
# Half a frame early, deliberately. -ss lands on the first frame whose
|
| 957 |
+
# pts >= the target, and the target is a float division, so a target that
|
| 958 |
+
# rounds a hair above frame f's true pts silently starts the segment at
|
| 959 |
+
# f+1. Aiming between f-1 and f cannot reach f+1 and cannot fall short of
|
| 960 |
+
# f, which removes the rounding question entirely.
|
| 961 |
+
out.append((max(0.0, (f - 0.5) / clip.fps), n, f + n >= total))
|
| 962 |
+
f += n
|
| 963 |
+
return out
|
| 964 |
+
|
| 965 |
+
|
| 966 |
+
def concat(parts: list[str], dst: str) -> None:
|
| 967 |
+
lst = tempfile.NamedTemporaryFile("w", suffix=".txt", delete=False)
|
| 968 |
+
for p in parts:
|
| 969 |
+
lst.write(f"file '{pathlib.Path(p).as_posix()}'\n")
|
| 970 |
+
lst.close()
|
| 971 |
+
try:
|
| 972 |
+
r = subprocess.run([FFMPEG, "-v", "error", "-y", "-f", "concat", "-safe",
|
| 973 |
+
"0", "-i", lst.name, "-c", "copy", dst],
|
| 974 |
+
capture_output=True)
|
| 975 |
+
if r.returncode != 0:
|
| 976 |
+
# check=True raises CalledProcessError, whose str() is the whole
|
| 977 |
+
# argv - an absolute path to ffmpeg and no reason. Same shape probe()
|
| 978 |
+
# was fixed to stop producing. Carry ffmpeg's own words instead.
|
| 979 |
+
why = " ".join((r.stderr or b"").decode("utf-8", "replace").split())
|
| 980 |
+
raise RuntimeError(
|
| 981 |
+
"could not join the finished segments"
|
| 982 |
+
+ (f": {why[-200:]}" if why else " and ffmpeg said nothing"))
|
| 983 |
+
finally:
|
| 984 |
+
# In a finally: a failing concat used to leak its list file, and concat
|
| 985 |
+
# fails on exactly the runs where something else already went wrong.
|
| 986 |
+
try:
|
| 987 |
+
os.unlink(lst.name)
|
| 988 |
+
except OSError:
|
| 989 |
+
pass
|
| 990 |
+
|
| 991 |
+
|
| 992 |
+
def mux_audio(video: str, original: str, dst: str) -> bool:
|
| 993 |
+
"""Carry the original audio across. Interpolation changes frame rate, not
|
| 994 |
+
duration, so the track still lines up."""
|
| 995 |
+
r = subprocess.run(
|
| 996 |
+
[FFMPEG, "-v", "error", "-y", "-i", video, "-i", original,
|
| 997 |
+
"-c", "copy", "-map", "0:v:0", "-map", "1:a:0?", "-shortest", dst],
|
| 998 |
+
capture_output=True)
|
| 999 |
+
return r.returncode == 0 and os.path.exists(dst) and os.path.getsize(dst) > 0
|