Happy commited on
Commit
4ea23c0
·
verified ·
1 Parent(s): 7b60682

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +105 -278
app.py CHANGED
@@ -1,197 +1,94 @@
1
  import gradio as gr
2
  import os
3
- from huggingface_hub import InferenceClient
4
- from pathlib import Path
5
  import tempfile
 
 
 
 
6
 
7
- # DO NOT create a global client for paid apps
8
- # Each user's client will be created using their OAuth token
9
 
10
- def generate_video_with_auth(image, prompt, token: gr.OAuthToken | None, progress=gr.Progress()):
11
- """
12
- Generate a video from an image using the Ovi model with authentication check.
13
-
14
- Args:
15
- image: Input image (PIL Image or file path)
16
- prompt: Text prompt describing the desired motion/animation
17
- token: OAuth token for authentication
18
- progress: Gradio progress tracker
19
-
20
- Returns:
21
- Path to the generated video file
22
- """
23
- if token is None or not getattr(token, "token", None):
24
- raise gr.Error("Please sign in with Hugging Face to use this paid app")
 
 
 
 
25
 
26
  if image is None:
27
- raise gr.Error("Please upload an image first!")
28
 
29
  if not prompt or prompt.strip() == "":
30
- raise gr.Error("Please enter a prompt describing the desired motion!")
 
 
 
31
 
32
  try:
33
- progress(0.2, desc="Processing image...")
34
-
35
- # Read the image file
36
  if isinstance(image, str):
37
- with open(image, "rb") as image_file:
38
- input_image = image_file.read()
39
  else:
40
- # If image is a PIL Image, save it temporarily
41
- temp_image = tempfile.NamedTemporaryFile(delete=False, suffix=".png")
42
- image.save(temp_image.name)
43
- with open(temp_image.name, "rb") as image_file:
44
- input_image = image_file.read()
45
-
46
- progress(0.4, desc="Generating video with AI...")
47
-
48
- # CRITICAL FOR PAID APPS: Create client with user's OAuth token
49
- # Do NOT use bill_to parameter - this makes the USER pay, not HuggingFace
50
- client = InferenceClient(
51
- provider="fal-ai",
52
- api_key=token.token, # Use token.token
53
- )
54
-
55
- # Generate video using the inference client
56
- video = client.image_to_video(
57
- input_image,
58
- prompt=prompt,
59
- model="chetwinlow1/Ovi",
60
- )
61
-
62
- progress(0.9, desc="Finalizing video...")
63
-
64
- # Save the video to a temporary file
65
- output_path = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4")
66
-
67
- # Check if video is bytes or a file path
68
- if isinstance(video, bytes):
69
- with open(output_path.name, "wb") as f:
70
- f.write(video)
71
- elif isinstance(video, str) and os.path.exists(video):
72
- # If it's a path, copy it
73
- import shutil
74
- shutil.copy(video, output_path.name)
75
- else:
76
- # Try to write it directly
77
- with open(output_path.name, "wb") as f:
78
- f.write(video)
79
 
80
- progress(1.0, desc="Complete!")
81
 
82
- return output_path.name
83
-
84
  except Exception as e:
85
- raise gr.Error(f"Error generating video: {str(e)}")
86
 
87
- # Create the Gradio interface
88
- with gr.Blocks(
89
- theme=gr.themes.Soft(
90
- primary_hue="blue",
91
- secondary_hue="indigo",
92
- ),
93
- css="""
94
- .header-link {
95
- font-size: 0.9em;
96
- color: #666;
97
- text-decoration: none;
98
- margin-bottom: 1em;
99
- display: inline-block;
100
- }
101
- .header-link:hover {
102
- color: #333;
103
- text-decoration: underline;
104
- }
105
- .main-header {
106
- text-align: center;
107
- margin-bottom: 2em;
108
- }
109
- .info-box {
110
- background-color: #f0f7ff;
111
- border-left: 4px solid #4285f4;
112
- padding: 1em;
113
- margin: 1em 0;
114
- border-radius: 4px;
115
- }
116
- .auth-warning {
117
- background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
118
- color: white;
119
- padding: 14px 16px;
120
- border-radius: 12px;
121
- margin: 18px auto 6px;
122
- max-width: 860px;
123
- text-align: center;
124
- font-size: 0.98rem;
125
- font-weight: bold;
126
- }
127
- """,
128
- title="Image to Video Generator with Ovi (Paid)",
129
- ) as demo:
130
-
131
- gr.HTML(
132
- """
133
- <div class="main-header">
134
- <a href="https://huggingface.co/spaces/akhaliq/anycoder" target="_blank" class="header-link">
135
- Built with anycoder ✨
136
- </a>
137
- </div>
138
- """
139
- )
140
-
141
- gr.Markdown(
142
- """
143
- # 🎬 Image to Video Generator with Ovi
144
-
145
- Transform your static images into dynamic videos with synchronized audio using AI! Upload an image and describe the motion you want to see.
146
-
147
- Powered by Ovi: Twin Backbone Cross-Modal Fusion for Audio-Video Generation via [HuggingFace Inference Providers](https://huggingface.co/docs/huggingface_hub/guides/inference).
148
- """
149
- )
150
-
151
- gr.HTML(
152
- """
153
- <div class="auth-warning">
154
- 💳 <b>PAID APP:</b> This app uses <b>YOUR</b> inference provider credits.
155
- Free users get $0.10 in included credits. PRO users get $2 in credits and can continue with billing.
156
- <a href='http://huggingface.co/subscribe/pro?source=ovi' target='_blank' style='color:#fff; text-decoration:underline;'>Subscribe to PRO</a> for more credits.
157
- Please sign in below to continue.
158
- <br><a href='https://huggingface.co/settings/inference-providers/overview' target='_blank' style='color:#fff; text-decoration:underline; font-weight:bold;'>Check your billing usage here</a>
159
- </div>
160
- """
161
- )
162
-
163
- # Add login button - required for OAuth
164
- gr.LoginButton()
165
 
166
- gr.HTML(
167
- """
168
- <div class="info-box">
169
- <strong>💡 Tips for best results:</strong>
170
- <ul>
171
- <li>Use clear, well-lit images with a single main subject</li>
172
- <li>Write specific prompts describing the desired motion or action</li>
173
- <li>Keep prompts concise and focused on movement and audio elements</li>
174
- <li>Processing generates 5-second videos at 24 FPS with synchronized audio</li>
175
- <li>Processing may take 30-60 seconds depending on server load</li>
176
- </ul>
177
- </div>
178
- """
179
- )
180
-
181
- gr.HTML(
182
- """
183
- <div class="info-box">
184
- <strong>✨ Special Tokens for Enhanced Control:</strong>
185
- <ul>
186
- <li><strong>Speech:</strong> <code>&lt;S&gt;Your speech content here&lt;E&gt;</code> - Text enclosed in these tags will be converted to speech</li>
187
- <li><strong>Audio Description:</strong> <code>&lt;AUDCAP&gt;Audio description here&lt;ENDAUDCAP&gt;</code> - Describes the audio or sound effects present in the video</li>
188
- </ul>
189
- <br>
190
- <strong>📝 Example Prompt:</strong><br>
191
- <code>Dogs bark loudly at a man wearing a red shirt. The man says &lt;S&gt;Please stop barking at me!&lt;E&gt;. &lt;AUDCAP&gt;Dogs barking, angry man yelling in stern voice&lt;ENDAUDCAP&gt;.</code>
192
- </div>
193
- """
194
- )
195
 
196
  with gr.Row():
197
  with gr.Column(scale=1):
@@ -199,126 +96,56 @@ with gr.Blocks(
199
  label="📸 Upload Image",
200
  type="filepath",
201
  sources=["upload", "clipboard"],
202
- height=400,
203
  )
204
-
205
  prompt_input = gr.Textbox(
206
- label="✍️ Text Prompt",
207
  lines=3,
208
- placeholder="Describe the motion and audio you want to see..."
209
- )
210
-
211
- generate_btn = gr.Button(
212
- "🎬 Generate Video",
213
- variant="primary",
214
- size="lg",
215
- )
216
-
217
- clear_btn = gr.Button(
218
- "🗑️ Clear",
219
- variant="secondary",
220
  )
221
-
222
- gr.Examples(
223
- examples=[
224
- [
225
- "5.png",
226
- 'A bearded man wearing large dark sunglasses and a blue patterned cardigan sits in a studio, actively speaking into a large, suspended microphone. He has headphones on and gestures with his hands, displaying rings on his fingers. Behind him, a wall is covered with red, textured sound-dampening foam on the left, and a white banner on the right features the "CHOICE FM" logo and various social media handles like "@ilovechoicefm" with "RALEIGH" below it. The man intently addresses the microphone, articulating, <S>is talent. It\'s all about authenticity. You gotta be who you really are, especially if you\'re working<E>. He leans forward slightly as he speaks, maintaining a serious expression behind his sunglasses.. <AUDCAP>Clear male voice speaking into a microphone, a low background hum.<ENDAUDCAP>'
227
- ]
228
- ],
229
- inputs=[image_input, prompt_input],
230
- label="Example",
231
- )
232
-
233
  with gr.Column(scale=1):
234
  video_output = gr.Video(
235
  label="🎥 Generated Video",
236
- height=400,
237
  autoplay=True,
238
  )
239
-
240
- gr.Markdown(
241
- """
242
- ### About Ovi Model
243
-
244
- **Ovi: Twin Backbone Cross-Modal Fusion for Audio-Video Generation**
245
-
246
- Developed by Chetwin Low, Weimin Wang (Character AI) & Calder Katyal (Yale University)
247
-
248
- 🌟 **Key Features:**
249
- - 🎬 **Video+Audio Generation**: Generates synchronized video and audio content simultaneously
250
- - 📝 **Flexible Input**: Supports text-only or text+image conditioning
251
- - ⏱️ **5-second Videos**: Generates 5-second videos at 24 FPS
252
- - 📐 **Multiple Aspect Ratios**: Supports 720×720 area at various ratios (9:16, 16:9, 1:1, etc)
253
-
254
- Ovi is a veo-3 like model that simultaneously generates both video and audio content from text or text+image inputs.
255
-
256
- ---
257
-
258
- ### 💳 Pricing & Credits
259
-
260
- This is a **paid app** that charges your HuggingFace inference provider account:
261
- - **Free users**: $0.10 in included credits
262
- - **PRO users**: $2 in included credits + ability to continue with billing
263
- - Each video generation consumes credits based on processing time
264
-
265
- [Subscribe to PRO](http://huggingface.co/subscribe/pro?source=ovi) for more credits!
266
- """
267
- )
268
-
269
- # Event handlers with authentication
270
- # NOTE: Do NOT pass profile as input - Gradio injects it automatically
271
  generate_btn.click(
272
- fn=generate_video_with_auth,
273
  inputs=[image_input, prompt_input],
274
  outputs=[video_output],
275
- queue=False,
276
- api_name=False,
277
- show_api=False,
278
  )
279
 
280
  clear_btn.click(
281
  fn=lambda: (None, "", None),
282
  inputs=None,
283
  outputs=[image_input, prompt_input, video_output],
284
- queue=False,
285
- )
286
-
287
- gr.Markdown(
288
- """
289
- ---
290
-
291
- ### 🚀 How it works
292
-
293
- 1. **Sign in** with your Hugging Face account (required for paid app)
294
- 2. **Upload** your image - any photo or illustration
295
- 3. **Describe** the motion you want to see in the prompt
296
- 4. **Generate** and watch your image come to life with synchronized audio!
297
- 5. **Credits are deducted** from your HuggingFace inference provider account
298
-
299
- ### ⚠️ Notes
300
-
301
- - **This is a PAID app** - uses your inference provider credits
302
- - Video generation may take 30-60 seconds
303
- - Generates 5-second videos at 24 FPS with synchronized audio
304
- - Supports multiple aspect ratios (9:16, 16:9, 1:1, etc) at 720×720 area
305
- - Best results with clear, high-quality images
306
- - The model works best with realistic subjects and natural motions
307
- - Free accounts have limited credits - upgrade to PRO for more
308
-
309
- ### 🔗 Resources
310
-
311
- - [Ovi Model Card](https://huggingface.co/chetwinlow1/Ovi)
312
- - [Character AI](https://character.ai)
313
- - [Subscribe to PRO](http://huggingface.co/subscribe/pro?source=ovi)
314
- - [Inference API Documentation](https://huggingface.co/docs/huggingface_hub/guides/inference)
315
- """
316
  )
317
 
318
- # Launch the app
319
  if __name__ == "__main__":
320
- demo.launch(
321
- show_api=False,
322
- enable_monitoring=False,
323
- quiet=True,
324
- )
 
1
  import gradio as gr
2
  import os
 
 
3
  import tempfile
4
+ import shutil
5
+ import torch
6
+ import numpy as np
7
+ from PIL import Image
8
 
9
+ # ── Load OVI Model ──────────────────────────────────────────
10
+ print("Loading OVI model...")
11
 
12
+ from transformers import AutoProcessor, AutoModel
13
+
14
+ MODEL_ID = "chetwinlow1/Ovi"
15
+
16
+ try:
17
+ processor = AutoProcessor.from_pretrained(MODEL_ID)
18
+ model = AutoModel.from_pretrained(MODEL_ID, torch_dtype=torch.float16)
19
+ device = "cuda" if torch.cuda.is_available() else "cpu"
20
+ model = model.to(device)
21
+ model.eval()
22
+ print(f"✅ OVI Model loaded on {device}")
23
+ except Exception as e:
24
+ print(f"❌ Model load error: {e}")
25
+ processor = None
26
+ model = None
27
+
28
+
29
+ def generate_video(image, prompt):
30
+ """Generate talking avatar video - FREE, no auth needed."""
31
 
32
  if image is None:
33
+ raise gr.Error("Please upload an image!")
34
 
35
  if not prompt or prompt.strip() == "":
36
+ raise gr.Error("Please enter a text prompt!")
37
+
38
+ if model is None:
39
+ raise gr.Error("Model not loaded. Please restart the Space.")
40
 
41
  try:
42
+ # Process inputs
 
 
43
  if isinstance(image, str):
44
+ pil_image = Image.open(image).convert("RGB")
 
45
  else:
46
+ pil_image = Image.fromarray(image).convert("RGB")
47
+
48
+ # Run model
49
+ inputs = processor(
50
+ text=prompt,
51
+ images=pil_image,
52
+ return_tensors="pt"
53
+ ).to(device)
54
+
55
+ with torch.no_grad():
56
+ outputs = model.generate(**inputs)
57
+
58
+ # Save output video
59
+ output_path = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name
60
+
61
+ # Write video frames
62
+ if hasattr(outputs, 'video'):
63
+ video_data = outputs.video[0].cpu().numpy()
64
+ import cv2
65
+ h, w = video_data.shape[1:3]
66
+ writer = cv2.VideoWriter(
67
+ output_path,
68
+ cv2.VideoWriter_fourcc(*'mp4v'),
69
+ 24, (w, h)
70
+ )
71
+ for frame in video_data:
72
+ frame_bgr = cv2.cvtColor(
73
+ (frame * 255).astype(np.uint8),
74
+ cv2.COLOR_RGB2BGR
75
+ )
76
+ writer.write(frame_bgr)
77
+ writer.release()
 
 
 
 
 
 
 
78
 
79
+ return output_path
80
 
 
 
81
  except Exception as e:
82
+ raise gr.Error(f"Generation failed: {str(e)}")
83
 
84
+
85
+ # ── Gradio Interface ────────────────────────────────────────
86
+ with gr.Blocks(theme=gr.themes.Soft()) as demo:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
87
 
88
+ gr.Markdown("""
89
+ # 🎬 OVI — Talking Avatar Generator
90
+ **Free & Open Source** | No login required | Powered by OVI AI Model
91
+ """)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
92
 
93
  with gr.Row():
94
  with gr.Column(scale=1):
 
96
  label="📸 Upload Image",
97
  type="filepath",
98
  sources=["upload", "clipboard"],
99
+ height=300,
100
  )
 
101
  prompt_input = gr.Textbox(
102
+ label="💬 Text Prompt",
103
  lines=3,
104
+ placeholder=(
105
+ "Describe what you want. Example:\n"
106
+ "A person looks at camera and says "
107
+ "<S>Hello everyone, welcome!<E> "
108
+ "<AUDCAP>Clear friendly voice<ENDAUDCAP>"
109
+ ),
 
 
 
 
 
 
110
  )
111
+ generate_btn = gr.Button("🎬 Generate Video", variant="primary", size="lg")
112
+ clear_btn = gr.Button("🗑️ Clear", variant="secondary")
113
+
 
 
 
 
 
 
 
 
 
114
  with gr.Column(scale=1):
115
  video_output = gr.Video(
116
  label="🎥 Generated Video",
117
+ height=300,
118
  autoplay=True,
119
  )
120
+
121
+ gr.Markdown("""
122
+ ---
123
+ ### 💡 Prompt Tips:
124
+ - `<S>Your speech here<E>` — what avatar says
125
+ - `<AUDCAP>voice description<ENDAUDCAP>` — voice style
126
+ - Example: `A man speaks to camera. <S>Hello world!<E> <AUDCAP>Deep calm voice<ENDAUDCAP>`
127
+ """)
128
+
129
+ gr.Examples(
130
+ examples=[
131
+ [None, "A person smiles at the camera. <S>Hello! Welcome to my channel.<E> <AUDCAP>Friendly energetic voice<ENDAUDCAP>"],
132
+ [None, "A woman speaks confidently. <S>Today I want to share something amazing with you.<E> <AUDCAP>Clear professional voice<ENDAUDCAP>"],
133
+ ],
134
+ inputs=[image_input, prompt_input],
135
+ )
136
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
137
  generate_btn.click(
138
+ fn=generate_video,
139
  inputs=[image_input, prompt_input],
140
  outputs=[video_output],
 
 
 
141
  )
142
 
143
  clear_btn.click(
144
  fn=lambda: (None, "", None),
145
  inputs=None,
146
  outputs=[image_input, prompt_input, video_output],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
147
  )
148
 
 
149
  if __name__ == "__main__":
150
+ demo.launch(show_api=True)
151
+