Spaces:
Runtime error
Runtime error
| import gradio as gr | |
| from .h3_ref2va_logic import process_inputs, RESOLUTION_PRESETS | |
| from .ui_components import create_lora_settings_ui, create_h3_controlnet_ui, create_h3_guide_ui | |
| from comfy_integration.nodes import SAMPLER_CHOICES, SCHEDULER_CHOICES | |
| from core.generation_logic import generate_h3_video_wrapper | |
| UI_INFO = { | |
| "workflow_recipe": "h3_ref2va_recipe.yaml", | |
| "main_tab": "VideoGen", | |
| "sub_tab": "H3 REF2VA", | |
| "run_button_text": "🎬 Generate H3 Video" | |
| } | |
| MAX_REF_IMAGES = 9 | |
| MAX_REF_VIDEOS = 3 | |
| MAX_REF_AUDIOS = 3 | |
| def create_ui(): | |
| components = {} | |
| with gr.Column(): | |
| gr.Markdown("## MiniMax H3 Reference Video Generation") | |
| gr.Markdown("💡 **Tip:** Supports uploading up to 9 reference images, 3 reference videos with audio, and 3 reference audio files. Under the default ZeroGPU Duration (s) of 120s, tasks with 544p resolution, 20 steps, 5s duration, and 3 reference images pass successfully.") | |
| components['prompt'] = gr.Textbox(label="Prompt", lines=5) | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| with gr.Row(): | |
| components['resolution'] = gr.Radio( | |
| label="Resolution", | |
| choices=["544p", "768p", "1080p"], | |
| value="544p", | |
| interactive=True | |
| ) | |
| with gr.Row(): | |
| components['aspect_ratio'] = gr.Dropdown( | |
| label="Aspect Ratio", | |
| choices=list(RESOLUTION_PRESETS["544p"].keys()), | |
| value="16:9 (Landscape)", | |
| interactive=True | |
| ) | |
| with gr.Row(): | |
| components['width'] = gr.Number(label="Width", value=960, precision=0) | |
| components['height'] = gr.Number(label="Height", value=544, precision=0) | |
| with gr.Row(visible=False): | |
| components['sampler'] = gr.Dropdown( | |
| label="Sampler", | |
| choices=SAMPLER_CHOICES, | |
| value="res_multistep" if "res_multistep" in SAMPLER_CHOICES else (SAMPLER_CHOICES[0] if SAMPLER_CHOICES else "res_multistep"), | |
| interactive=True, | |
| visible=False | |
| ) | |
| components['scheduler'] = gr.Dropdown( | |
| label="Scheduler", | |
| choices=SCHEDULER_CHOICES, | |
| value="simple" if "simple" in SCHEDULER_CHOICES else (SCHEDULER_CHOICES[0] if SCHEDULER_CHOICES else "simple"), | |
| interactive=True, | |
| visible=False | |
| ) | |
| with gr.Row(visible=False): | |
| components['cfg_scale'] = gr.Slider( | |
| label="CFG Scale", | |
| minimum=1.0, | |
| maximum=20.0, | |
| step=0.1, | |
| value=1.0, | |
| interactive=True, | |
| visible=False | |
| ) | |
| with gr.Row(): | |
| components['steps'] = gr.Slider( | |
| label="Steps", | |
| minimum=1, | |
| maximum=50, | |
| step=1, | |
| value=20, | |
| interactive=True | |
| ) | |
| components['duration'] = gr.Slider( | |
| label="Duration (s)", | |
| minimum=0.2, | |
| maximum=15.0, | |
| step=0.1, | |
| value=5.0, | |
| interactive=True | |
| ) | |
| with gr.Row(): | |
| components['seed'] = gr.Number(label="Seed (-1 for random)", value=-1, precision=0) | |
| components['zero_gpu_duration'] = gr.Number( | |
| label="ZeroGPU Duration (s)", | |
| value=120, | |
| precision=0 | |
| ) | |
| with gr.Column(scale=1): | |
| components['output_video'] = gr.Video( | |
| label="Result Video", | |
| interactive=False, | |
| height=492 | |
| ) | |
| components.update(create_lora_settings_ui("h3_ref2va")) | |
| with gr.Accordion("Reference Image Settings", open=False): | |
| ref_image_groups = [] | |
| ref_image_inputs = [] | |
| with gr.Row(): | |
| for i in range(MAX_REF_IMAGES): | |
| with gr.Column(visible=(i < 1), min_width=160) as img_col: | |
| img_comp = gr.Image( | |
| type="pil", | |
| label=f"Ref Image {i+1}", | |
| sources=["upload"], | |
| height=160 | |
| ) | |
| ref_image_groups.append(img_col) | |
| ref_image_inputs.append(img_comp) | |
| components['ref_image_groups'] = ref_image_groups | |
| components['ref_image_inputs'] = ref_image_inputs | |
| components['ref_images'] = ref_image_inputs | |
| with gr.Row(): | |
| components['add_ref_button'] = gr.Button("✚ Add Reference Image") | |
| components['delete_ref_button'] = gr.Button("➖ Delete Reference Image", visible=True) | |
| components['ref_count_state'] = gr.State(1) | |
| with gr.Accordion("Reference Video Settings", open=False): | |
| ref_video_groups = [] | |
| ref_video_inputs = [] | |
| with gr.Row(): | |
| for i in range(MAX_REF_VIDEOS): | |
| with gr.Column(visible=(i < 1), min_width=200) as vid_col: | |
| vid_comp = gr.Video( | |
| label=f"Ref Video {i+1}", | |
| sources=["upload"], | |
| height=200 | |
| ) | |
| ref_video_groups.append(vid_col) | |
| ref_video_inputs.append(vid_comp) | |
| components['ref_video_groups'] = ref_video_groups | |
| components['ref_video_inputs'] = ref_video_inputs | |
| components['ref_videos'] = ref_video_inputs | |
| with gr.Row(): | |
| components['add_ref_video_button'] = gr.Button("✚ Add Reference Video") | |
| components['delete_ref_video_button'] = gr.Button("➖ Delete Reference Video", visible=True) | |
| components['ref_video_count_state'] = gr.State(1) | |
| with gr.Accordion("Reference Audio Settings", open=False): | |
| ref_audio_groups = [] | |
| ref_audio_inputs = [] | |
| with gr.Row(): | |
| for i in range(MAX_REF_AUDIOS): | |
| with gr.Column(visible=(i < 1), min_width=200) as aud_col: | |
| aud_comp = gr.Audio( | |
| label=f"Ref Audio {i+1}", | |
| sources=["upload"], | |
| type="filepath" | |
| ) | |
| ref_audio_groups.append(aud_col) | |
| ref_audio_inputs.append(aud_comp) | |
| components['ref_audio_groups'] = ref_audio_groups | |
| components['ref_audio_inputs'] = ref_audio_inputs | |
| components['ref_audios'] = ref_audio_inputs | |
| with gr.Row(): | |
| components['add_ref_audio_button'] = gr.Button("✚ Add Reference Audio") | |
| components['delete_ref_audio_button'] = gr.Button("➖ Delete Reference Audio", visible=True) | |
| components['ref_audio_count_state'] = gr.State(1) | |
| components.update(create_h3_guide_ui("h3_ref2va")) | |
| components.update(create_h3_controlnet_ui("h3_ref2va")) | |
| components['run_button'] = gr.Button(UI_INFO["run_button_text"], variant="primary", elem_classes=["run-shortcut"]) | |
| return components | |
| def create_event_handlers(components: dict, demo: gr.Blocks): | |
| def update_dimensions(resolution, aspect_ratio): | |
| w, h = RESOLUTION_PRESETS.get(resolution, {}).get(aspect_ratio, (960, 544)) | |
| return w, h | |
| components['resolution'].change( | |
| fn=update_dimensions, | |
| inputs=[components['resolution'], components['aspect_ratio']], | |
| outputs=[components['width'], components['height']], | |
| api_name=False | |
| ) | |
| components['aspect_ratio'].change( | |
| fn=update_dimensions, | |
| inputs=[components['resolution'], components['aspect_ratio']], | |
| outputs=[components['width'], components['height']], | |
| api_name=False | |
| ) | |
| ref_count_state = components['ref_count_state'] | |
| add_ref_btn = components['add_ref_button'] | |
| del_ref_btn = components['delete_ref_button'] | |
| ref_image_groups = components['ref_image_groups'] | |
| ref_image_inputs = components['ref_image_inputs'] | |
| def add_ref_row(count): | |
| count += 1 | |
| visibility_updates = tuple(gr.update(visible=i < count) for i in range(MAX_REF_IMAGES)) | |
| return (count, gr.update(visible=count < MAX_REF_IMAGES), gr.update(visible=count > 0)) + visibility_updates | |
| def delete_ref_row(count): | |
| count -= 1 | |
| image_clear_updates = [gr.update()] * MAX_REF_IMAGES | |
| if count >= 0: | |
| image_clear_updates[count] = None | |
| visibility_updates = tuple(gr.update(visible=i < count) for i in range(MAX_REF_IMAGES)) | |
| return (count, gr.update(visible=count < MAX_REF_IMAGES), gr.update(visible=count > 0)) + visibility_updates + tuple(image_clear_updates) | |
| add_ref_outputs = [ref_count_state, add_ref_btn, del_ref_btn] + ref_image_groups | |
| del_ref_outputs = [ref_count_state, add_ref_btn, del_ref_btn] + ref_image_groups + ref_image_inputs | |
| add_ref_btn.click( | |
| fn=add_ref_row, | |
| inputs=[ref_count_state], | |
| outputs=add_ref_outputs, | |
| show_progress=False, | |
| api_name=False | |
| ) | |
| del_ref_btn.click( | |
| fn=delete_ref_row, | |
| inputs=[ref_count_state], | |
| outputs=del_ref_outputs, | |
| show_progress=False, | |
| api_name=False | |
| ) | |
| ref_video_count_state = components['ref_video_count_state'] | |
| add_ref_vid_btn = components['add_ref_video_button'] | |
| del_ref_vid_btn = components['delete_ref_video_button'] | |
| ref_video_groups = components['ref_video_groups'] | |
| ref_video_inputs = components['ref_video_inputs'] | |
| def add_ref_video_row(count): | |
| count += 1 | |
| visibility_updates = tuple(gr.update(visible=i < count) for i in range(MAX_REF_VIDEOS)) | |
| return (count, gr.update(visible=count < MAX_REF_VIDEOS), gr.update(visible=count > 0)) + visibility_updates | |
| def delete_ref_video_row(count): | |
| count -= 1 | |
| video_clear_updates = [gr.update()] * MAX_REF_VIDEOS | |
| if count >= 0: | |
| video_clear_updates[count] = None | |
| visibility_updates = tuple(gr.update(visible=i < count) for i in range(MAX_REF_VIDEOS)) | |
| return (count, gr.update(visible=count < MAX_REF_VIDEOS), gr.update(visible=count > 0)) + visibility_updates + tuple(video_clear_updates) | |
| add_ref_vid_outputs = [ref_video_count_state, add_ref_vid_btn, del_ref_vid_btn] + ref_video_groups | |
| del_ref_vid_outputs = [ref_video_count_state, add_ref_vid_btn, del_ref_vid_btn] + ref_video_groups + ref_video_inputs | |
| add_ref_vid_btn.click( | |
| fn=add_ref_video_row, | |
| inputs=[ref_video_count_state], | |
| outputs=add_ref_vid_outputs, | |
| show_progress=False, | |
| api_name=False | |
| ) | |
| del_ref_vid_btn.click( | |
| fn=delete_ref_video_row, | |
| inputs=[ref_video_count_state], | |
| outputs=del_ref_vid_outputs, | |
| show_progress=False, | |
| api_name=False | |
| ) | |
| ref_audio_count_state = components['ref_audio_count_state'] | |
| add_ref_aud_btn = components['add_ref_audio_button'] | |
| del_ref_aud_btn = components['delete_ref_audio_button'] | |
| ref_audio_groups = components['ref_audio_groups'] | |
| ref_audio_inputs = components['ref_audio_inputs'] | |
| def add_ref_audio_row(count): | |
| count += 1 | |
| visibility_updates = tuple(gr.update(visible=i < count) for i in range(MAX_REF_AUDIOS)) | |
| return (count, gr.update(visible=count < MAX_REF_AUDIOS), gr.update(visible=count > 0)) + visibility_updates | |
| def delete_ref_audio_row(count): | |
| count -= 1 | |
| audio_clear_updates = [gr.update()] * MAX_REF_AUDIOS | |
| if count >= 0: | |
| audio_clear_updates[count] = None | |
| visibility_updates = tuple(gr.update(visible=i < count) for i in range(MAX_REF_AUDIOS)) | |
| return (count, gr.update(visible=count < MAX_REF_AUDIOS), gr.update(visible=count > 0)) + visibility_updates + tuple(audio_clear_updates) | |
| add_ref_aud_outputs = [ref_audio_count_state, add_ref_aud_btn, del_ref_aud_btn] + ref_audio_groups | |
| del_ref_aud_outputs = [ref_audio_count_state, add_ref_aud_btn, del_ref_aud_btn] + ref_audio_groups + ref_audio_inputs | |
| add_ref_aud_btn.click( | |
| fn=add_ref_audio_row, | |
| inputs=[ref_audio_count_state], | |
| outputs=add_ref_aud_outputs, | |
| show_progress=False, | |
| api_name=False | |
| ) | |
| del_ref_aud_btn.click( | |
| fn=delete_ref_audio_row, | |
| inputs=[ref_audio_count_state], | |
| outputs=del_ref_aud_outputs, | |
| show_progress=False, | |
| api_name=False | |
| ) | |
| lora_sources = components.get('lora_sources_h3_ref2va', []) | |
| lora_ids = components.get('lora_ids_h3_ref2va', []) | |
| lora_scales = components.get('lora_scales_h3_ref2va', []) | |
| cn_videos = components.get('h3_controlnet_videos_h3_ref2va', []) | |
| cn_types = components.get('h3_controlnet_types_h3_ref2va', []) | |
| cn_series = components.get('h3_controlnet_series_h3_ref2va', []) | |
| cn_strengths = components.get('h3_controlnet_strengths_h3_ref2va', []) | |
| cn_start_percents = components.get('h3_controlnet_start_percents_h3_ref2va', []) | |
| cn_end_percents = components.get('h3_controlnet_end_percents_h3_ref2va', []) | |
| cn_filepaths = components.get('h3_controlnet_filepaths_h3_ref2va', []) | |
| guide_images = components.get('h3_guide_images_h3_ref2va', []) | |
| guide_videos = components.get('h3_guide_videos_h3_ref2va', []) | |
| guide_audios = components.get('h3_guide_audios_h3_ref2va', []) | |
| guide_times = components.get('h3_guide_times_h3_ref2va', []) | |
| guide_frames = components.get('h3_guide_frames_h3_ref2va', []) | |
| inputs_list = [ | |
| components['prompt'], | |
| components['resolution'], | |
| components['aspect_ratio'], | |
| components['width'], | |
| components['height'], | |
| components['duration'], | |
| components['sampler'], | |
| components['scheduler'], | |
| components['steps'], | |
| components['cfg_scale'], | |
| components['seed'], | |
| components['zero_gpu_duration'] | |
| ] + ref_image_inputs + ref_video_inputs + ref_audio_inputs + lora_sources + lora_ids + lora_scales + cn_videos + cn_types + cn_series + cn_strengths + cn_start_percents + cn_end_percents + cn_filepaths + guide_images + guide_videos + guide_audios + guide_times + guide_frames | |
| def on_run_click(*args, progress=gr.Progress(track_tqdm=True)): | |
| prompt, resolution, aspect_ratio, width, height, duration, sampler, scheduler, steps, cfg_scale, seed, zero_gpu_duration = args[:12] | |
| idx = 12 | |
| imgs = list(args[idx : idx + len(ref_image_inputs)]) | |
| idx += len(ref_image_inputs) | |
| vids = list(args[idx : idx + len(ref_video_inputs)]) | |
| idx += len(ref_video_inputs) | |
| auds = list(args[idx : idx + len(ref_audio_inputs)]) | |
| idx += len(ref_audio_inputs) | |
| num_loras = len(lora_sources) | |
| l_sources = args[idx : idx + num_loras] | |
| l_ids = args[idx + num_loras : idx + 2*num_loras] | |
| l_scales = args[idx + 2*num_loras : idx + 3*num_loras] | |
| idx += 3 * num_loras | |
| num_cns = len(cn_videos) | |
| c_vids = args[idx : idx + num_cns] | |
| c_types = args[idx + num_cns : idx + 2*num_cns] | |
| c_series = args[idx + 2*num_cns : idx + 3*num_cns] | |
| c_strengths = args[idx + 3*num_cns : idx + 4*num_cns] | |
| c_starts = args[idx + 4*num_cns : idx + 5*num_cns] | |
| c_ends = args[idx + 5*num_cns : idx + 6*num_cns] | |
| c_fps = args[idx + 6*num_cns : idx + 7*num_cns] | |
| idx += 7 * num_cns | |
| num_guides = len(guide_images) | |
| g_imgs = args[idx : idx + num_guides] | |
| g_vids = args[idx + num_guides : idx + 2*num_guides] | |
| g_auds = args[idx + 2*num_guides : idx + 3*num_guides] | |
| g_times = args[idx + 3*num_guides : idx + 4*num_guides] | |
| g_frames = args[idx + 4*num_guides : idx + 5*num_guides] if guide_frames else [] | |
| ui_values = { | |
| 'prompt': prompt, | |
| 'resolution': resolution, | |
| 'aspect_ratio': aspect_ratio, | |
| 'width': width, | |
| 'height': height, | |
| 'duration': duration, | |
| 'sampler_name': sampler, | |
| 'scheduler': scheduler, | |
| 'steps': steps, | |
| 'cfg_scale': cfg_scale, | |
| 'seed': seed, | |
| 'zero_gpu_duration': zero_gpu_duration, | |
| 'batch_count': 1, | |
| 'ref_images': imgs, | |
| 'ref_videos': vids, | |
| 'ref_audios': auds, | |
| 'lora_sources_h3_ref2va': list(l_sources), | |
| 'lora_ids_h3_ref2va': list(l_ids), | |
| 'lora_scales_h3_ref2va': list(l_scales), | |
| 'h3_controlnet_videos_h3_ref2va': list(c_vids), | |
| 'h3_controlnet_types_h3_ref2va': list(c_types), | |
| 'h3_controlnet_series_h3_ref2va': list(c_series), | |
| 'h3_controlnet_strengths_h3_ref2va': list(c_strengths), | |
| 'h3_controlnet_start_percents_h3_ref2va': list(c_starts), | |
| 'h3_controlnet_end_percents_h3_ref2va': list(c_ends), | |
| 'h3_controlnet_filepaths_h3_ref2va': list(c_fps), | |
| 'h3_guide_images_h3_ref2va': list(g_imgs), | |
| 'h3_guide_videos_h3_ref2va': list(g_vids), | |
| 'h3_guide_audios_h3_ref2va': list(g_auds), | |
| 'h3_guide_times_h3_ref2va': list(g_times), | |
| 'h3_guide_frames_h3_ref2va': list(g_frames), | |
| } | |
| return generate_h3_video_wrapper(process_inputs, ui_values, progress=progress) | |
| components['run_button'].click( | |
| fn=on_run_click, | |
| inputs=inputs_list, | |
| outputs=[components['output_video']], | |
| api_name=False | |
| ) | |