Bash2X commited on
Commit
3afc363
·
verified ·
1 Parent(s): 3b670c8

Upload inference_video_OpenCV.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. inference_video_OpenCV.py +344 -0
inference_video_OpenCV.py ADDED
@@ -0,0 +1,344 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import cv2
3
+ import torch
4
+ import importlib.util
5
+ import sys
6
+ import argparse
7
+ import numpy as np
8
+ from torch.nn import functional as F
9
+ import warnings
10
+ import _thread
11
+ from queue import Queue, Empty
12
+ from model.pytorch_msssim import ssim_matlab
13
+ import time
14
+
15
+ warnings.filterwarnings("ignore")
16
+ loglevel = os.environ.get("RIFE_LOGLEVEL", "error")
17
+
18
+ os.environ["SDL_AUDIODRIVER"] = "dummy"
19
+ os.environ["ALSA_CONFIG_PATH"] = "/dev/null"
20
+
21
+ # =============== CUSTOM PROGRESS TRACKER ===============
22
+ def format_time(seconds: float) -> str:
23
+ """Format seconds to H:MM:SS or MM:SS."""
24
+ m, s = divmod(int(seconds), 60)
25
+ h, m = divmod(m, 60)
26
+ if h > 0:
27
+ return f"{h}:{m:02}:{s:02}"
28
+ else:
29
+ return f"{m:02}:{s:02}"
30
+
31
+ class VideoProgressTracker:
32
+ def __init__(self, total_frames):
33
+ self.total_frames = total_frames
34
+ self.current_frame = 0
35
+ self.start_time = time.time()
36
+
37
+ def update(self, frame_num=None):
38
+ if frame_num is not None:
39
+ self.current_frame = frame_num
40
+ else:
41
+ self.current_frame += 1
42
+ self.display_progress()
43
+
44
+ def display_progress(self):
45
+ elapsed = time.time() - self.start_time
46
+ progress_fraction = self.current_frame / self.total_frames if self.total_frames > 0 else 0
47
+ fps = self.current_frame / elapsed if elapsed > 0 else 0
48
+ eta = (elapsed / progress_fraction - elapsed) if progress_fraction > 0 else 0
49
+ percent = int(progress_fraction * 100)
50
+
51
+ info = (f"Interpolating: {percent:3}% "
52
+ f"Frame {self.current_frame}/{self.total_frames} | "
53
+ f"Elapsed: {format_time(elapsed)} | ETA: {format_time(eta)} | {fps:.1f} frame/s")
54
+
55
+ sys.stdout.write('\r' + info)
56
+ sys.stdout.flush()
57
+
58
+ def finish(self):
59
+ """Finalize progress display"""
60
+ # Show 100% completion before finishing
61
+ self.current_frame = self.total_frames
62
+ self.display_progress()
63
+ sys.stdout.write('\n')
64
+ sys.stdout.flush()
65
+
66
+ def transferAudio(sourceVideo, targetVideo):
67
+ import shutil
68
+ import moviepy.editor
69
+ tempAudioFileName = "./temp/audio.mkv"
70
+ if True:
71
+ if os.path.isdir("temp"):
72
+ shutil.rmtree("temp")
73
+ os.makedirs("temp")
74
+ os.system('ffmpeg -hide_banner -loglevel {} -y -i "{}" -c:a copy -vn {}'.format(loglevel, sourceVideo, tempAudioFileName))
75
+
76
+ targetNoAudio = os.path.splitext(targetVideo)[0] + "_noaudio" + os.path.splitext(targetVideo)[1]
77
+ os.rename(targetVideo, targetNoAudio)
78
+ os.system('ffmpeg -hide_banner -loglevel {} -y -i "{}" -i {} -c copy "{}"'.format(loglevel, targetNoAudio, tempAudioFileName, targetVideo))
79
+
80
+ if os.path.getsize(targetVideo) == 0:
81
+ tempAudioFileName = "./temp/audio.m4a"
82
+ os.system('ffmpeg -hide_banner -loglevel {} -y -i "{}" -c:a aac -b:a 160k -vn {}'.format(loglevel, sourceVideo, tempAudioFileName))
83
+ os.system('ffmpeg -hide_banner -loglevel {} -y -i "{}" -i {} -c copy "{}"'.format(loglevel, targetNoAudio, tempAudioFileName, targetVideo))
84
+ if (os.path.getsize(targetVideo) == 0):
85
+ os.rename(targetNoAudio, targetVideo)
86
+ print("Audio transfer failed. Interpolated video will have no audio")
87
+ else:
88
+ print("Lossless audio transfer failed. Audio was transcoded to AAC (M4A) instead.")
89
+ os.remove(targetNoAudio)
90
+ else:
91
+ os.remove(targetNoAudio)
92
+ shutil.rmtree("temp")
93
+
94
+ parser = argparse.ArgumentParser(description='Interpolation for a pair of images')
95
+ parser.add_argument('--video', dest='video', type=str, default=None)
96
+ parser.add_argument('--output', dest='output', type=str, default=None)
97
+ parser.add_argument('--img', dest='img', type=str, default=None)
98
+ parser.add_argument('--montage', dest='montage', action='store_true', help='montage origin video')
99
+ parser.add_argument('--model', dest='modelDir', type=str, default='train_log', help='directory with trained model files')
100
+ parser.add_argument('--interpolation_factor', type=int, default=2, help="How many total frames between two input frames")
101
+ parser.add_argument('--mode', type=str, choices=['fast', 'slow'], default='slow', help="Interpolation mode: 'fast uses multi' (simple split) or 'slow uses exp' (recursive)")
102
+ parser.add_argument('--UHD', dest='UHD', action='store_true', help='support 4k video')
103
+ parser.add_argument('--scale', dest='scale', type=float, default=1.0, help='Try scale=0.5 for 4k video')
104
+ parser.add_argument('--skip', dest='skip', action='store_true', help='whether to remove static frames before processing')
105
+ parser.add_argument('--fps', dest='fps', type=int, default=None)
106
+ parser.add_argument('--png', dest='png', action='store_true', help='whether to vid_out png format vid_outs')
107
+ parser.add_argument('--ext', dest='ext', type=str, default='mp4', help='vid_out video extension')
108
+ args = parser.parse_args()
109
+
110
+ args.multi = args.interpolation_factor
111
+ assert (not args.video is None or not args.img is None)
112
+ if args.skip:
113
+ print("skip flag is abandoned, please refer to issue #207.")
114
+ if args.UHD and args.scale==1.0:
115
+ args.scale = 0.5
116
+ assert args.scale in [0.25, 0.5, 1.0, 2.0, 4.0]
117
+ if not args.img is None:
118
+ args.png = True
119
+
120
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
121
+ torch.set_grad_enabled(False)
122
+ if torch.cuda.is_available():
123
+ torch.backends.cudnn.enabled = True
124
+ torch.backends.cudnn.benchmark = True
125
+
126
+ # Set Path to Practical-RIFE
127
+ args.modelDir = os.path.join("/content/Practical-RIFE", args.modelDir)
128
+ args.modelDir = os.path.abspath(args.modelDir)
129
+ sys.path.insert(0, args.modelDir)
130
+ model_path = os.path.join(args.modelDir, 'RIFE_HDv3.py')
131
+ spec = importlib.util.spec_from_file_location("RIFE_HDv3", model_path)
132
+ RIFE_module = importlib.util.module_from_spec(spec)
133
+ spec.loader.exec_module(RIFE_module)
134
+ Model = RIFE_module.Model
135
+ model = Model()
136
+ if not hasattr(model, 'version'):
137
+ model.version = 0
138
+ model.load_model(args.modelDir, -1)
139
+ model.eval()
140
+ model.device()
141
+ model_name = os.path.basename(os.path.abspath(args.modelDir))
142
+
143
+ if not args.video is None:
144
+ videoCapture = cv2.VideoCapture(args.video)
145
+ fps = videoCapture.get(cv2.CAP_PROP_FPS)
146
+ tot_frame = videoCapture.get(cv2.CAP_PROP_FRAME_COUNT)
147
+ videoCapture.release()
148
+ if args.fps is None:
149
+ fpsNotAssigned = True
150
+ args.fps = fps * args.interpolation_factor
151
+ else:
152
+ fpsNotAssigned = False
153
+ videoCapture = cv2.VideoCapture(args.video)
154
+ success, lastframe = videoCapture.read()
155
+ if success:
156
+ lastframe = cv2.cvtColor(lastframe, cv2.COLOR_BGR2RGB)
157
+ videogen = []
158
+ while success:
159
+ videogen.append(lastframe)
160
+ success, lastframe = videoCapture.read()
161
+ if success:
162
+ lastframe = cv2.cvtColor(lastframe,cv2.COLOR_BGR2RGB)
163
+ videoCapture.release()
164
+ lastframe = videogen.pop(0)
165
+ fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v')
166
+ video_path_wo_ext, ext = os.path.splitext(args.video)
167
+ file_name = os.path.basename(args.video)
168
+ print(f"{tot_frame} frames in total, {fps}FPS to {fps * args.interpolation_factor}FPS\n")
169
+ if args.png == False and fpsNotAssigned == True:
170
+ pass
171
+ else:
172
+ pass
173
+ else:
174
+ videogen = []
175
+ for f in os.listdir(args.img):
176
+ if 'png' in f:
177
+ videogen.append(f)
178
+ tot_frame = len(videogen)
179
+ videogen.sort(key= lambda x:int(x[:-4]))
180
+ lastframe = cv2.imread(os.path.join(args.img, videogen[0]), cv2.IMREAD_UNCHANGED)[:, :, ::-1].copy()
181
+ videogen = videogen[1:]
182
+ folder_path = os.path.abspath(args.img)
183
+ print(f"{len(videogen)} PNG frames found.\n")
184
+ h, w, _ = lastframe.shape
185
+ vid_out_name = None
186
+ vid_out = None
187
+ if args.png:
188
+ if not os.path.exists('vid_out'):
189
+ os.mkdir('vid_out')
190
+ else:
191
+ if args.output is not None:
192
+ vid_out_name = args.output
193
+ else:
194
+ vid_out_name = '{}_{}X_{}fps.{}'.format(video_path_wo_ext, args.interpolation_factor, int(np.round(args.fps)), args.ext)
195
+ vid_out = cv2.VideoWriter(vid_out_name, fourcc, args.fps, (w, h))
196
+
197
+ def clear_write_buffer(user_args, write_buffer):
198
+ cnt = 0
199
+ while True:
200
+ item = write_buffer.get()
201
+ if item is None:
202
+ break
203
+ if user_args.png:
204
+ cv2.imwrite('vid_out/{:0>7d}.png'.format(cnt), item[:, :, ::-1])
205
+ cnt += 1
206
+ else:
207
+ vid_out.write(item[:, :, ::-1])
208
+
209
+ def build_read_buffer(user_args, read_buffer, videogen):
210
+ try:
211
+ for frame in videogen:
212
+ if not user_args.img is None:
213
+ frame = cv2.imread(os.path.join(user_args.img, frame), cv2.IMREAD_UNCHANGED)[:, :, ::-1].copy()
214
+ if user_args.montage:
215
+ frame = frame[:, left: left + w]
216
+ read_buffer.put(frame)
217
+ except:
218
+ pass
219
+ read_buffer.put(None)
220
+
221
+ #OriginalLogic
222
+ def make_inference(I0, I1, n):
223
+ global model
224
+ if args.mode == "slow":
225
+ if n == 1:
226
+ middle = model.inference(I0, I1, 0.5, args.scale)
227
+ return [middle]
228
+ middle = model.inference(I0, I1, 0.5, args.scale)
229
+ left_half = make_inference(I0, middle, n // 2)
230
+ right_half = make_inference(middle, I1, n // 2)
231
+ if n % 2:
232
+ return [*left_half, middle, *right_half]
233
+ else:
234
+ return [*left_half, *right_half]
235
+ else:
236
+ outputs = []
237
+ for i in range(n):
238
+ timestep = (i + 1) / (n + 1)
239
+ middle = model.inference(I0, I1, timestep, args.scale)
240
+ outputs.append(middle)
241
+ return outputs
242
+
243
+ def pad_image(img, padding):
244
+ if any(padding):
245
+ img = F.pad(img, padding)
246
+ return img
247
+
248
+ if args.montage:
249
+ left = w // 4
250
+ w = w // 2
251
+ tmp = max(128, int(128 / args.scale))
252
+ ph = ((h - 1) // tmp + 1) * tmp
253
+ pw = ((w - 1) // tmp + 1) * tmp
254
+ padding = (0, pw - w, 0, ph - h)
255
+
256
+ # Initialize custom progress tracker
257
+ progress = VideoProgressTracker(int(tot_frame))
258
+
259
+ if args.montage:
260
+ lastframe = lastframe[:, left: left + w]
261
+ write_buffer = Queue(maxsize=500)
262
+ read_buffer = Queue(maxsize=500)
263
+ _thread.start_new_thread(build_read_buffer, (args, read_buffer, videogen))
264
+ _thread.start_new_thread(clear_write_buffer, (args, write_buffer))
265
+
266
+ I1 = torch.from_numpy(np.transpose(lastframe, (2,0,1))).to(device, non_blocking=True).unsqueeze(0).float() / 255.
267
+ I1 = pad_image(I1, padding)
268
+ temp = None
269
+
270
+ while True:
271
+ if temp is not None:
272
+ frame = temp
273
+ temp = None
274
+ else:
275
+ frame = read_buffer.get()
276
+ if frame is None:
277
+ break
278
+ I0 = I1
279
+ I1 = torch.from_numpy(np.transpose(frame, (2,0,1))).to(device, non_blocking=True).unsqueeze(0).float() / 255.
280
+ I1 = pad_image(I1, padding)
281
+ I0_small = F.interpolate(I0, (32, 32), mode='bilinear', align_corners=False)
282
+ I1_small = F.interpolate(I1, (32, 32), mode='bilinear', align_corners=False)
283
+ ssim = ssim_matlab(I0_small[:, :3], I1_small[:, :3])
284
+
285
+ break_flag = False
286
+ if ssim > 0.996:
287
+ frame = read_buffer.get()
288
+ if frame is None:
289
+ break_flag = True
290
+ frame = lastframe
291
+ else:
292
+ temp = frame
293
+ I1 = torch.from_numpy(np.transpose(frame, (2,0,1))).to(device, non_blocking=True).unsqueeze(0).float() / 255.
294
+ I1 = pad_image(I1, padding)
295
+ I1 = model.inference(I0, I1, scale=args.scale)
296
+ I1_small = F.interpolate(I1, (32, 32), mode='bilinear', align_corners=False)
297
+ ssim = ssim_matlab(I0_small[:, :3], I1_small[:, :3])
298
+ frame = (I1[0] * 255).byte().cpu().numpy().transpose(1, 2, 0)[:h, :w]
299
+
300
+ if ssim < 0.2:
301
+ output = []
302
+ for i in range(args.interpolation_factor - 1):
303
+ output.append(I0)
304
+ else:
305
+ output = make_inference(I0, I1, args.interpolation_factor - 1)
306
+
307
+ if args.montage:
308
+ write_buffer.put(np.concatenate((lastframe, lastframe), 1))
309
+ for mid in output:
310
+ mid = (((mid[0] * 255.).byte().cpu().numpy().transpose(1, 2, 0)))
311
+ write_buffer.put(np.concatenate((lastframe, mid[:h, :w]), 1))
312
+ else:
313
+ write_buffer.put(lastframe)
314
+ for mid in output:
315
+ mid = (((mid[0] * 255.).byte().cpu().numpy().transpose(1, 2, 0)))
316
+ write_buffer.put(mid[:h, :w])
317
+
318
+ progress.update()
319
+ lastframe = frame
320
+ if break_flag:
321
+ break
322
+
323
+ if args.montage:
324
+ write_buffer.put(np.concatenate((lastframe, lastframe), 1))
325
+ else:
326
+ write_buffer.put(lastframe)
327
+ progress.update() # Add progress update for final frame
328
+ write_buffer.put(None)
329
+
330
+ while(not write_buffer.empty()):
331
+ time.sleep(0.1)
332
+
333
+ progress.finish()
334
+
335
+ if not vid_out is None:
336
+ vid_out.release()
337
+
338
+ if args.png == False and fpsNotAssigned == True and not args.video is None:
339
+ try:
340
+ transferAudio(args.video, vid_out_name)
341
+ except:
342
+ print("Audio transfer failed. Interpolated video will have no audio")
343
+ targetNoAudio = os.path.splitext(vid_out_name)[0] + "_noaudio" + os.path.splitext(vid_out_name)[1]
344
+ os.rename(targetNoAudio, vid_out_name)