"""Browser version of the AI Video Editor's clip and theme workflow.""" from pathlib import Path import tempfile import gradio as gr import numpy as np import spaces import torch from PIL import Image, ImageDraw, ImageFont from moviepy.editor import ( AudioFileClip, CompositeVideoClip, ImageClip, VideoFileClip, concatenate_videoclips, ) from moviepy.audio.fx.all import audio_loop from transformers import CLIPModel, CLIPProcessor # MoviePy 1.0.3 still uses the Pillow alias removed in Pillow 10. if not hasattr(Image, "ANTIALIAS"): Image.ANTIALIAS = Image.Resampling.LANCZOS MODEL_ID = "openai/clip-vit-base-patch32" THEMES = [ "wedding ceremony", "bride", "groom", "dance", "kiss", "group of people", "party", "ring exchange", "cake cutting", "speeches", "couple portrait", "crowd", "outdoor", "indoor", "dinner", "reception", "romantic", "family", "friends", "music performance", ] MAX_VIDEOS = 5 MAX_DURATION = 120 OUTPUT_HEIGHT = 720 PROCESSOR = CLIPProcessor.from_pretrained(MODEL_ID) MODEL = CLIPModel.from_pretrained(MODEL_ID).to("cuda").eval() @spaces.GPU(duration=120) def detect_theme(video_paths): """Detect the first clip's theme using the public CLIP model.""" if not video_paths: raise gr.Error("Upload at least one video.") clip = VideoFileClip(str(video_paths[0])) try: count = max(1, min(12, int(clip.duration / 1.5) + 1)) times = np.linspace(0, max(0, clip.duration - 0.1), count) scores = np.zeros(len(THEMES), dtype=np.float64) for timestamp in times: frame = Image.fromarray(clip.get_frame(float(timestamp)).astype("uint8")) inputs = PROCESSOR(text=THEMES, images=frame, return_tensors="pt", padding=True).to("cuda") with torch.inference_mode(): scores += MODEL(**inputs).logits_per_image.softmax(dim=1)[0].cpu().numpy() index = int(np.argmax(scores)) return THEMES[index], float(scores[index] / scores.sum()) finally: clip.close() def fit_clip(clip, size): width, height = size scale = min(width / clip.w, height / clip.h) new_size = (max(2, round(clip.w * scale)), max(2, round(clip.h * scale))) resized = clip.fl_image( lambda frame: np.asarray( Image.fromarray(frame).resize(new_size, Image.Resampling.LANCZOS) ) ) return resized.on_color(size=size, color=(0, 0, 0), pos="center") def make_title(theme, confidence, size): image = Image.new("RGB", size, (19, 23, 35)) draw = ImageDraw.Draw(image) font_path = "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf" try: font = ImageFont.truetype(font_path, max(20, min(size) // 17)) except OSError: font = ImageFont.load_default() lines = ["AI Video Editor", f"Theme: {theme}", f"Confidence: {confidence:.0%}"] spacing = max(15, size[1] // 30) heights = [draw.textbbox((0, 0), line, font=font)[3] for line in lines] y = (size[1] - sum(heights) - spacing * (len(lines) - 1)) // 2 for line, line_height in zip(lines, heights): box = draw.textbbox((0, 0), line, font=font) x = (size[0] - (box[2] - box[0])) // 2 draw.text((x, y), line, fill="white", font=font) y += line_height + spacing return ImageClip(np.asarray(image)).set_duration(3) def edit_video(video_paths, music_path, transition, add_title, theme_result): if not video_paths: raise gr.Error("Upload at least one video.") if len(video_paths) > MAX_VIDEOS: raise gr.Error(f"Upload at most {MAX_VIDEOS} videos per edit.") clips = [] created = [] try: for path in video_paths: clip = VideoFileClip(str(path)) clips.append(clip) duration = sum(clip.duration for clip in clips) if duration > MAX_DURATION: raise gr.Error(f"The combined clips must be {MAX_DURATION} seconds or less.") first = clips[0] height = min(OUTPUT_HEIGHT, first.h) width = max(2, round(first.w * height / first.h / 2) * 2) size = (width, max(2, round(height / 2) * 2)) fitted = [fit_clip(clip, size) for clip in clips] created.extend(fitted) theme, confidence = theme_result if transition == "Crossfade" and len(fitted) > 1: overlap = min(0.5, *(clip.duration / 4 for clip in fitted)) layers = [fitted[0]] start = fitted[0].duration for clip in fitted[1:]: start -= overlap layers.append(clip.set_start(start).crossfadein(overlap)) start += clip.duration final = CompositeVideoClip(layers, size=size).set_duration(start) else: final = concatenate_videoclips(fitted, method="compose") created.append(final) if add_title and theme: title = make_title(theme, confidence, size) created.append(title) final = concatenate_videoclips([title, final], method="compose") created.append(final) if music_path: music = AudioFileClip(str(music_path)) created.append(music) if music.duration < final.duration: soundtrack = audio_loop(music, duration=final.duration) else: soundtrack = music.subclip(0, final.duration) created.append(soundtrack) final = final.set_audio(soundtrack) created.append(final) output_dir = Path(tempfile.mkdtemp(prefix="ai-video-editor-")) output_path = output_dir / "edited_video.mp4" final.write_videofile( str(output_path), codec="libx264", audio_codec="aac", fps=min(first.fps or 24, 30), threads=2, preset="ultrafast", logger=None, ) message = f"Detected theme: {theme} ({confidence:.0%})" if theme else "Edit complete." return str(output_path), str(output_path), message except gr.Error: raise except Exception as error: raise gr.Error(f"Video processing failed: {error}") from error finally: for clip in reversed(created + clips): try: clip.close() except Exception: pass def edit_without_ai(video_paths, music_path, transition, add_title): """Render clips without AI theme detection or GPU usage.""" return edit_video(video_paths, music_path, transition, add_title, (None, None)) with gr.Blocks(title="AI Video Editor") as demo: gr.Markdown("# AI Video Editor\nUpload clips in order, optionally add music, and export an MP4. AI theme detection uses the public CLIP model.") videos = gr.File(label="Video clips, in order (up to 5)", file_count="multiple", file_types=["video"], type="filepath") music = gr.File(label="Background music (optional)", file_types=["audio"], type="filepath") with gr.Row(): transition_choice = gr.Dropdown(["Cut", "Crossfade"], value="Cut", label="Between clips") title_choice = gr.Checkbox(value=True, label="Add theme title card") with gr.Row(): run_ai = gr.Button("Edit with AI theme detection", variant="primary") run_simple = gr.Button("Edit without AI") detected = gr.State((None, None)) status = gr.Textbox(label="Result", interactive=False) preview = gr.Video(label="Edited video") download = gr.File(label="Download MP4") gr.Markdown("For this hosted demo, total input duration is limited to 2 minutes. Files are processed in temporary storage.") run_ai.click(detect_theme, inputs=videos, outputs=detected).then( edit_video, inputs=[videos, music, transition_choice, title_choice, detected], outputs=[preview, download, status], ) run_simple.click( edit_without_ai, inputs=[videos, music, transition_choice, title_choice], outputs=[preview, download, status], ) demo.queue(max_size=3).launch()