Nymbo commited on
Commit
f2d6d0a
·
verified ·
1 Parent(s): aceed58

Gradio 6.29; live caption/SFX upstreams; forward X-IP-Token; drop moviepy

Browse files
Files changed (3) hide show
  1. README.md +1 -1
  2. app.py +78 -133
  3. requirements.txt +1 -1
README.md CHANGED
@@ -4,7 +4,7 @@ emoji: 👂
4
  colorFrom: green
5
  colorTo: pink
6
  sdk: gradio
7
- sdk_version: 4.15.0
8
  app_file: app.py
9
  pinned: false
10
  ---
 
4
  colorFrom: green
5
  colorTo: pink
6
  sdk: gradio
7
+ sdk_version: 6.29.0
8
  app_file: app.py
9
  pinned: false
10
  ---
app.py CHANGED
@@ -1,165 +1,110 @@
 
 
1
  import gradio as gr
2
- from gradio_client import Client
3
- import json
4
- import re
5
- from moviepy.editor import VideoFileClip
6
- from moviepy.audio.AudioClip import AudioClip
7
-
8
- def extract_audio(video_in):
9
- input_video = video_in
10
- output_audio = 'audio.wav'
11
-
12
- # Open the video file and extract the audio
13
- video_clip = VideoFileClip(input_video)
14
- audio_clip = video_clip.audio
15
-
16
- # Save the audio as a .wav file
17
- audio_clip.write_audiofile(output_audio, fps=44100) # Use 44100 Hz as the sample rate for .wav files
18
- print("Audio extraction complete.")
19
-
20
- return 'audio.wav'
21
-
22
- def get_caption_from_kosmos(image_in):
23
- kosmos2_client = Client("https://ydshieh-kosmos-2.hf.space/")
24
-
25
- kosmos2_result = kosmos2_client.predict(
26
- image_in, # str (filepath or URL to image) in 'Test Image' Image component
27
- "Detailed", # str in 'Description Type' Radio component
28
- fn_index=4
29
- )
30
 
31
- print(f"KOSMOS2 RETURNS: {kosmos2_result}")
32
-
33
- with open(kosmos2_result[1], 'r') as f:
34
- data = json.load(f)
35
-
36
- reconstructed_sentence = []
37
- for sublist in data:
38
- reconstructed_sentence.append(sublist[0])
39
-
40
- full_sentence = ' '.join(reconstructed_sentence)
41
- #print(full_sentence)
42
-
43
- # Find the pattern matching the expected format ("Describe this image in detail:" followed by optional space and then the rest)...
44
- pattern = r'^Describe this image in detail:\s*(.*)$'
45
- # Apply the regex pattern to extract the description text.
46
- match = re.search(pattern, full_sentence)
47
- if match:
48
- description = match.group(1)
49
- print(description)
50
- else:
51
- print("Unable to locate valid description.")
52
-
53
- # Find the last occurrence of "."
54
- last_period_index = description.rfind('.')
55
-
56
- # Truncate the string up to the last period
57
- truncated_caption = description[:last_period_index + 1]
58
-
59
- # print(truncated_caption)
60
- print(f"\n—\nIMAGE CAPTION: {truncated_caption}")
61
-
62
- return truncated_caption
63
-
64
- def get_caption(image_in):
65
- client = Client("https://vikhyatk-moondream1.hf.space/")
66
- result = client.predict(
67
- image_in, # filepath in 'image' Image component
68
- "Describe precisely the image in one sentence.", # str in 'Question' Textbox component
69
- api_name="/answer_question"
70
- )
71
- print(result)
72
- return result
73
 
74
- def get_magnet(prompt):
75
- amended_prompt = f"{prompt}"
76
- print(amended_prompt)
77
- client = Client("https://fffiloni-magnet.hf.space/")
78
- result = client.predict(
79
- "facebook/audio-magnet-medium", # Literal['facebook/magnet-small-10secs', 'facebook/magnet-medium-10secs', 'facebook/magnet-small-30secs', 'facebook/magnet-medium-30secs', 'facebook/audio-magnet-small', 'facebook/audio-magnet-medium'] in 'Model' Radio component
80
- "", # str in 'Model Path (custom models)' Textbox component
81
- amended_prompt, # str in 'Input Text' Textbox component
82
- 3, # float in 'Temperature' Number component
83
- 0.9, # float in 'Top-p' Number component
84
- 10, # float in 'Max CFG coefficient' Number component
85
- 1, # float in 'Min CFG coefficient' Number component
86
- 20, # float in 'Decoding Steps (stage 1)' Number component
87
- 10, # float in 'Decoding Steps (stage 2)' Number component
88
- 10, # float in 'Decoding Steps (stage 3)' Number component
89
- 10, # float in 'Decoding Steps (stage 4)' Number component
90
- "prod-stride1 (new!)", # Literal['max-nonoverlap', 'prod-stride1 (new!)'] in 'Span Scoring' Radio component
91
- api_name="/predict_full"
92
- )
93
- print(result)
94
- return result[1]
95
 
96
- def get_audioldm(prompt):
97
- client = Client("https://haoheliu-audioldm2-text2audio-text2music.hf.space/")
98
- result = client.predict(
99
- prompt, # str in 'Input text' Textbox component
100
- "Low quality. Music.", # str in 'Negative prompt' Textbox component
101
- 10, # int | float (numeric value between 5 and 15) in 'Duration (seconds)' Slider component
102
- 3.5, # int | float (numeric value between 0 and 7) in 'Guidance scale' Slider component
103
- 45, # int | float in 'Seed' Number component
104
- 3, # int | float (numeric value between 1 and 5) in 'Number waveforms to generate' Slider component
105
- fn_index=1
106
- )
107
- print(result)
108
- audio_result = extract_audio(result)
109
- return audio_result
110
 
111
- def get_audiogen(prompt):
112
- client = Client("https://fffiloni-audiogen.hf.space/")
113
  result = client.predict(
114
- prompt,
115
- 10,
116
- api_name="/infer"
117
  )
118
- return result
119
-
120
- def infer(image_in, chosen_model):
121
- caption = get_caption(image_in)
122
- if chosen_model == "MAGNet" :
123
- magnet_result = get_magnet(caption)
124
- return magnet_result
125
- elif chosen_model == "AudioLDM-2" :
126
- audioldm_result = get_audioldm(caption)
127
- return audioldm_result
128
- elif chosen_model == "AudioGen" :
129
- audiogen_result = get_audiogen(caption)
130
- return audiogen_result
131
-
132
- css="""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
133
  #col-container{
134
  margin: 0 auto;
135
  max-width: 800px;
136
  }
137
  """
138
 
139
- with gr.Blocks(css=css) as demo:
140
  with gr.Column(elem_id="col-container"):
141
  gr.HTML("""
142
  <h2 style="text-align: center;">
143
  Image to SFX
144
  </h2>
145
  <p style="text-align: center;">
146
- Compare MAGNet, AudioLDM2 and AudioGen sound effects generation from image caption.
147
  </p>
148
  """)
149
-
150
  with gr.Column():
151
  image_in = gr.Image(sources=["upload"], type="filepath", label="Image input", value="oiseau.png")
152
  with gr.Row():
153
- chosen_model = gr.Radio(label="Choose a model", choices=["MAGNet", "AudioLDM-2", "AudioGen"], value="AudioLDM-2")
154
  submit_btn = gr.Button("Submit")
155
  with gr.Column():
 
156
  audio_o = gr.Audio(label="Audio output")
157
-
158
  submit_btn.click(
159
  fn=infer,
160
  inputs=[image_in, chosen_model],
161
- outputs=[audio_o],
162
- concurrency_limit = 4
 
163
  )
164
 
165
- demo.queue(max_size=10).launch(debug=True)
 
1
+ import random
2
+
3
  import gradio as gr
4
+ from gradio_client import Client, handle_file
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5
 
6
+ MAX_SEED = 2**31 - 1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8
 
9
+ def _client(space, request):
10
+ # Forward the visitor's ZeroGPU identity so upstream GPU Spaces bill their quota, not this Space's IP
11
+ headers = {}
12
+ if request is not None:
13
+ ip_token = request.headers.get("x-ip-token")
14
+ if ip_token:
15
+ headers["X-IP-Token"] = ip_token
16
+ return Client(space, headers=headers, verbose=False)
17
+
 
 
 
 
 
18
 
19
+ def get_caption(image_in, request=None):
20
+ client = _client("vikhyatk/moondream2", request)
21
  result = client.predict(
22
+ handle_file(image_in),
23
+ "Describe precisely the image in one sentence.",
24
+ api_name="/answer_question",
25
  )
26
+ print(result)
27
+ return str(result).strip()
28
+
29
+
30
+ def get_audioldm(prompt, request=None):
31
+ try:
32
+ client = _client("fffiloni/audioldm2-text2audio-text2music-API", request)
33
+ return client.predict(
34
+ prompt,
35
+ "Low quality. Music.",
36
+ 10,
37
+ 3.5,
38
+ random.randint(0, MAX_SEED),
39
+ 3,
40
+ api_name="/text2audio",
41
+ )
42
+ except Exception as e:
43
+ raise gr.Error(f"AudioLDM-2 Space API is not ready, please try again in a few minutes ({e})")
44
+
45
+
46
+ def get_audiogen(prompt, request=None):
47
+ try:
48
+ client = _client("fffiloni/audiogen", request)
49
+ return client.predict(prompt, 10, api_name="/infer")
50
+ except Exception as e:
51
+ raise gr.Error(f"AudioGen Space API is not ready, please try again in a few minutes ({e})")
52
+
53
+
54
+ def get_ezaudio(prompt, request=None):
55
+ try:
56
+ client = _client("OpenSound/EzAudio", request)
57
+ return client.predict(prompt, 10, 5, 0.75, 50, 1, 0, True, api_name="/generate_audio")
58
+ except Exception as e:
59
+ raise gr.Error(f"EzAudio Space API is not ready, please try again in a few minutes ({e})")
60
+
61
+
62
+ def infer(image_in, chosen_model, request: gr.Request):
63
+ if not image_in:
64
+ raise gr.Error("Please provide an image.")
65
+ caption = get_caption(image_in, request)
66
+ if chosen_model == "AudioLDM-2":
67
+ return caption, get_audioldm(caption, request)
68
+ elif chosen_model == "AudioGen":
69
+ return caption, get_audiogen(caption, request)
70
+ elif chosen_model == "EzAudio":
71
+ return caption, get_ezaudio(caption, request)
72
+ raise gr.Error(f"Unknown model: {chosen_model}")
73
+
74
+
75
+ css = """
76
  #col-container{
77
  margin: 0 auto;
78
  max-width: 800px;
79
  }
80
  """
81
 
82
+ with gr.Blocks() as demo:
83
  with gr.Column(elem_id="col-container"):
84
  gr.HTML("""
85
  <h2 style="text-align: center;">
86
  Image to SFX
87
  </h2>
88
  <p style="text-align: center;">
89
+ Compare AudioLDM2, AudioGen and EzAudio sound effects generation from image caption.
90
  </p>
91
  """)
92
+
93
  with gr.Column():
94
  image_in = gr.Image(sources=["upload"], type="filepath", label="Image input", value="oiseau.png")
95
  with gr.Row():
96
+ chosen_model = gr.Radio(label="Choose a model", choices=["AudioLDM-2", "AudioGen", "EzAudio"], value="AudioLDM-2")
97
  submit_btn = gr.Button("Submit")
98
  with gr.Column():
99
+ caption_o = gr.Textbox(label="Image caption")
100
  audio_o = gr.Audio(label="Audio output")
101
+
102
  submit_btn.click(
103
  fn=infer,
104
  inputs=[image_in, chosen_model],
105
+ outputs=[caption_o, audio_o],
106
+ concurrency_limit=4,
107
+ api_name="infer",
108
  )
109
 
110
+ demo.queue(max_size=10).launch(css=css)
requirements.txt CHANGED
@@ -1 +1 @@
1
- moviepy
 
1
+ gradio_client