Spaces:
Running on Zero
Running on Zero
| """Browser version of the AI Video Editor's clip and theme workflow.""" | |
| from pathlib import Path | |
| import tempfile | |
| import gradio as gr | |
| import numpy as np | |
| import spaces | |
| import torch | |
| from PIL import Image, ImageDraw, ImageFont | |
| from moviepy.editor import ( | |
| AudioFileClip, | |
| CompositeVideoClip, | |
| ImageClip, | |
| VideoFileClip, | |
| concatenate_videoclips, | |
| ) | |
| from moviepy.audio.fx.all import audio_loop | |
| from transformers import CLIPModel, CLIPProcessor | |
| # MoviePy 1.0.3 still uses the Pillow alias removed in Pillow 10. | |
| if not hasattr(Image, "ANTIALIAS"): | |
| Image.ANTIALIAS = Image.Resampling.LANCZOS | |
| MODEL_ID = "openai/clip-vit-base-patch32" | |
| THEMES = [ | |
| "wedding ceremony", "bride", "groom", "dance", "kiss", | |
| "group of people", "party", "ring exchange", "cake cutting", | |
| "speeches", "couple portrait", "crowd", "outdoor", "indoor", | |
| "dinner", "reception", "romantic", "family", "friends", "music performance", | |
| ] | |
| MAX_VIDEOS = 5 | |
| MAX_DURATION = 120 | |
| OUTPUT_HEIGHT = 720 | |
| PROCESSOR = CLIPProcessor.from_pretrained(MODEL_ID) | |
| MODEL = CLIPModel.from_pretrained(MODEL_ID).to("cuda").eval() | |
| def detect_theme(video_paths): | |
| """Detect the first clip's theme using the public CLIP model.""" | |
| if not video_paths: | |
| raise gr.Error("Upload at least one video.") | |
| clip = VideoFileClip(str(video_paths[0])) | |
| try: | |
| count = max(1, min(12, int(clip.duration / 1.5) + 1)) | |
| times = np.linspace(0, max(0, clip.duration - 0.1), count) | |
| scores = np.zeros(len(THEMES), dtype=np.float64) | |
| for timestamp in times: | |
| frame = Image.fromarray(clip.get_frame(float(timestamp)).astype("uint8")) | |
| inputs = PROCESSOR(text=THEMES, images=frame, return_tensors="pt", padding=True).to("cuda") | |
| with torch.inference_mode(): | |
| scores += MODEL(**inputs).logits_per_image.softmax(dim=1)[0].cpu().numpy() | |
| index = int(np.argmax(scores)) | |
| return THEMES[index], float(scores[index] / scores.sum()) | |
| finally: | |
| clip.close() | |
| def fit_clip(clip, size): | |
| width, height = size | |
| scale = min(width / clip.w, height / clip.h) | |
| new_size = (max(2, round(clip.w * scale)), max(2, round(clip.h * scale))) | |
| resized = clip.fl_image( | |
| lambda frame: np.asarray( | |
| Image.fromarray(frame).resize(new_size, Image.Resampling.LANCZOS) | |
| ) | |
| ) | |
| return resized.on_color(size=size, color=(0, 0, 0), pos="center") | |
| def make_title(theme, confidence, size): | |
| image = Image.new("RGB", size, (19, 23, 35)) | |
| draw = ImageDraw.Draw(image) | |
| font_path = "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf" | |
| try: | |
| font = ImageFont.truetype(font_path, max(20, min(size) // 17)) | |
| except OSError: | |
| font = ImageFont.load_default() | |
| lines = ["AI Video Editor", f"Theme: {theme}", f"Confidence: {confidence:.0%}"] | |
| spacing = max(15, size[1] // 30) | |
| heights = [draw.textbbox((0, 0), line, font=font)[3] for line in lines] | |
| y = (size[1] - sum(heights) - spacing * (len(lines) - 1)) // 2 | |
| for line, line_height in zip(lines, heights): | |
| box = draw.textbbox((0, 0), line, font=font) | |
| x = (size[0] - (box[2] - box[0])) // 2 | |
| draw.text((x, y), line, fill="white", font=font) | |
| y += line_height + spacing | |
| return ImageClip(np.asarray(image)).set_duration(3) | |
| def edit_video(video_paths, music_path, transition, add_title, theme_result): | |
| if not video_paths: | |
| raise gr.Error("Upload at least one video.") | |
| if len(video_paths) > MAX_VIDEOS: | |
| raise gr.Error(f"Upload at most {MAX_VIDEOS} videos per edit.") | |
| clips = [] | |
| created = [] | |
| try: | |
| for path in video_paths: | |
| clip = VideoFileClip(str(path)) | |
| clips.append(clip) | |
| duration = sum(clip.duration for clip in clips) | |
| if duration > MAX_DURATION: | |
| raise gr.Error(f"The combined clips must be {MAX_DURATION} seconds or less.") | |
| first = clips[0] | |
| height = min(OUTPUT_HEIGHT, first.h) | |
| width = max(2, round(first.w * height / first.h / 2) * 2) | |
| size = (width, max(2, round(height / 2) * 2)) | |
| fitted = [fit_clip(clip, size) for clip in clips] | |
| created.extend(fitted) | |
| theme, confidence = theme_result | |
| if transition == "Crossfade" and len(fitted) > 1: | |
| overlap = min(0.5, *(clip.duration / 4 for clip in fitted)) | |
| layers = [fitted[0]] | |
| start = fitted[0].duration | |
| for clip in fitted[1:]: | |
| start -= overlap | |
| layers.append(clip.set_start(start).crossfadein(overlap)) | |
| start += clip.duration | |
| final = CompositeVideoClip(layers, size=size).set_duration(start) | |
| else: | |
| final = concatenate_videoclips(fitted, method="compose") | |
| created.append(final) | |
| if add_title and theme: | |
| title = make_title(theme, confidence, size) | |
| created.append(title) | |
| final = concatenate_videoclips([title, final], method="compose") | |
| created.append(final) | |
| if music_path: | |
| music = AudioFileClip(str(music_path)) | |
| created.append(music) | |
| if music.duration < final.duration: | |
| soundtrack = audio_loop(music, duration=final.duration) | |
| else: | |
| soundtrack = music.subclip(0, final.duration) | |
| created.append(soundtrack) | |
| final = final.set_audio(soundtrack) | |
| created.append(final) | |
| output_dir = Path(tempfile.mkdtemp(prefix="ai-video-editor-")) | |
| output_path = output_dir / "edited_video.mp4" | |
| final.write_videofile( | |
| str(output_path), codec="libx264", audio_codec="aac", | |
| fps=min(first.fps or 24, 30), threads=2, preset="ultrafast", logger=None, | |
| ) | |
| message = f"Detected theme: {theme} ({confidence:.0%})" if theme else "Edit complete." | |
| return str(output_path), str(output_path), message | |
| except gr.Error: | |
| raise | |
| except Exception as error: | |
| raise gr.Error(f"Video processing failed: {error}") from error | |
| finally: | |
| for clip in reversed(created + clips): | |
| try: | |
| clip.close() | |
| except Exception: | |
| pass | |
| def edit_without_ai(video_paths, music_path, transition, add_title): | |
| """Render clips without AI theme detection or GPU usage.""" | |
| return edit_video(video_paths, music_path, transition, add_title, (None, None)) | |
| with gr.Blocks(title="AI Video Editor") as demo: | |
| gr.Markdown("# AI Video Editor\nUpload clips in order, optionally add music, and export an MP4. AI theme detection uses the public CLIP model.") | |
| videos = gr.File(label="Video clips, in order (up to 5)", file_count="multiple", file_types=["video"], type="filepath") | |
| music = gr.File(label="Background music (optional)", file_types=["audio"], type="filepath") | |
| with gr.Row(): | |
| transition_choice = gr.Dropdown(["Cut", "Crossfade"], value="Cut", label="Between clips") | |
| title_choice = gr.Checkbox(value=True, label="Add theme title card") | |
| with gr.Row(): | |
| run_ai = gr.Button("Edit with AI theme detection", variant="primary") | |
| run_simple = gr.Button("Edit without AI") | |
| detected = gr.State((None, None)) | |
| status = gr.Textbox(label="Result", interactive=False) | |
| preview = gr.Video(label="Edited video") | |
| download = gr.File(label="Download MP4") | |
| gr.Markdown("For this hosted demo, total input duration is limited to 2 minutes. Files are processed in temporary storage.") | |
| run_ai.click(detect_theme, inputs=videos, outputs=detected).then( | |
| edit_video, inputs=[videos, music, transition_choice, title_choice, detected], | |
| outputs=[preview, download, status], | |
| ) | |
| run_simple.click( | |
| edit_without_ai, inputs=[videos, music, transition_choice, title_choice], | |
| outputs=[preview, download, status], | |
| ) | |
| demo.queue(max_size=3).launch() | |