import spaces from spaces.config import Config as SpacesConfig from collections import deque import base64 import os from pathlib import Path import sys import tempfile import threading import time import uuid import gradio as gr from huggingface_hub import hf_hub_download import numpy as np import torch ROOT = Path(__file__).resolve().parent sys.path.insert(0, str(ROOT / 'space')) from space.assets_loader import resolve_t5, REVISION, REPO from space.inference import load_model from space.text_encoder import load_text_encoder, load_prompt_features, save_prompt_features from fast_window_runtime import FastWindowRuntime from hml263_runtime import load_hml263_model from window_control import WindowPath as SeedPath from hml263_control import WindowPath as HumanML3DPath from window_viewer import create_viewer from cloud_sessions import SessionStore MODELS = ('SEED', 'HumanML3D') DEFAULTS = [('A person walks forward.', .8), ('A person runs forward.', 2.5), ('A person walks forward in a crouched position.', .45), ('A person is dancing.', .8)] GPU_CONCURRENCY = None if SpacesConfig.zero_gpu else 1 LOCAL_GPU_LOCK = threading.Lock() STORE = SessionStore(Path(tempfile.gettempdir()) / 'flood2_live_window_controls') FEATURES = ROOT / 'space/assets/prompts.npz' DEFAULT_BANK = load_prompt_features(ROOT / 'space/assets/default_prompts.npz') BASE_BANK = load_prompt_features(FEATURES) seed_checkpoint = os.getenv('FLOOD2_SEED_CHECKPOINT') or hf_hub_download( REPO, 'checkpoints/seed_path_fk_300k/model.ckpt', revision=REVISION) hml_checkpoint = os.getenv('FLOOD2_HML_CHECKPOINT') or hf_hub_download( REPO, 'checkpoints/humanml3d_babel_path_200k/model.ckpt', revision=REVISION) hml_config = os.getenv('FLOOD2_HML_CONFIG') or hf_hub_download( REPO, 'checkpoints/humanml3d_babel_path_200k/config.yaml', revision=REVISION) RUNTIMES = { 'SEED': FastWindowRuntime.from_runtime(load_model(ROOT/'space/vendor', seed_checkpoint, FEATURES)), 'HumanML3D': load_hml263_model(ROOT/'space/vendor', hml_checkpoint, hml_config, FEATURES), } # Module-scope loads are virtualized by ZeroGPU. Encoding and graph capture # happen only inside decorated allocations. encoder_path, tokenizer_path = resolve_t5() ENCODER = load_text_encoder(ROOT/'space/vendor', encoder_path, tokenizer_path) def owner(request): value = getattr(request, 'session_hash', None) if not value: raise gr.Error('Reload the page to start a session.') return value def checked_model(value): if value not in MODELS: raise gr.Error('Choose SEED or HumanML3D.') return value def read(session, request): try: return STORE.read(session, owner(request)) except (ValueError, FileNotFoundError) as exc: raise gr.Error(str(exc)) from exc @spaces.GPU(duration=30) def prepare(previous_session, model, p1, p2, p3, p4, v1, v2, v3, v4, request: gr.Request): model = checked_model(model) browser = owner(request) prompts = [str(prompt).strip() for prompt in (p1, p2, p3, p4)] if any(not prompt or len(prompt) > 500 for prompt in prompts): raise gr.Error('Enter 1 to 500 characters in every prompt slot.') try: speeds = [float(value) for value in (v1, v2, v3, v4)] except (TypeError, ValueError): raise gr.Error('Every speed must be between 0 and 5 m/s.') if any(not np.isfinite(value) or not 0 <= value <= 5 for value in speeds): raise gr.Error('Every speed must be between 0 and 5 m/s.') if previous_session: previous = read(previous_session, request) if previous['state'] not in ('paused', 'error') or previous.get('stream_active'): raise gr.Error('Stop before starting a new motion.') bank = {text: DEFAULT_BANK[text] for text in set(prompts) if text in DEFAULT_BANK} missing = [text for text in dict.fromkeys(prompts) if text not in bank] if missing: bank.update(ENCODER.encode(missing)) session = uuid.uuid4().hex save_prompt_features(STORE.path(session).with_suffix('.npz'), bank) STORE.create(session, dict(owner=browser, model=model, slots=prompts, speeds=speeds, slot=0, x=0., z=0., client_seq=-1, keys_updated_at=0., version=0, state='starting', stop=False, stream_active=False, frames=0, runtime={}, error=None)) return session def control(session, seq, x, z, shift=False, slot=0, request: gr.Request=None): if not session: return 'Ready' try: data = STORE.control(session, owner(request), seq, x, z, slot) except (ValueError, FileNotFoundError) as exc: raise gr.Error(str(exc)) from exc return 'Running' if data['state'] == 'running' else 'Ready' def keyboard_control(event: gr.EventData, request: gr.Request): data = event._data if isinstance(data, dict): control(data.get('session_id', ''), data.get('seq', -1), data.get('x', 0.), data.get('z', 0.), False, data.get('slot', 0), request) def stop_session(session, request: gr.Request): if not session: return 'Ready' try: data = STORE.stop(session, owner(request)) except (ValueError, FileNotFoundError) as exc: raise gr.Error(str(exc)) from exc return 'Stopping…' if data['stream_active'] else 'Stopped' def session_status(session, request: gr.Request): if not session: return dict(state='ready', frames=0, stream_active=False) data = read(session, request) return {key: value for key, value in data.items() if key != 'owner'} @spaces.GPU(duration=60) def stream(session, request: gr.Request): browser = owner(request) initial = STORE.claim(session, browser) model = initial['model'] runtime = RUNTIMES[model] planner = SeedPath() if model == 'SEED' else HumanML3DPath() fps = 30 if model == 'SEED' else 20 completed = sequence = 0 started = time.perf_counter() deadline = started + 55. playback_origin = first_frame_seconds = None compute_seconds = 0. frame_seconds = deque(maxlen=60) text_metas = [] latest = initial last_meta = dict(slot=0, frame=-1, prompt=initial['slots'][0]) gpu_name = None acquired = False runtime_started = False def packet(status, frames=None, metas=None, rotations=None, error=None): nonlocal sequence frames, metas = frames or [], metas or [] info = runtime.status() if runtime_started else {} active_slot = last_meta['slot'] if completed else latest['slot'] result = dict(session_id=session, sequence=sequence, run_revision=1, model=model, representation='mesh' if model == 'SEED' else 'skeleton', start_frame=completed-len(frames), frames=frames, frame_meta=metas, control_points=[dict(meta) for meta in metas]+[dict(point) for point in planner.points], control_points_delta=True, control_replace_from_frame=completed-len(frames), spring_point=dict(planner.points[-1]) if planner.points else None, target_point=planner.target_point(active_slot), slots=list(latest['slots']), speeds=list(latest['speeds']), active_slot=active_slot, base_slot=latest['slot'], prompt=latest['slots'][active_slot], action=latest['slots'][active_slot], speed=latest['speeds'][active_slot], completed_frame=completed-1, input_frame=info.get('steps', 0)-1, fps=fps, generated_frames=completed, elapsed_seconds=time.perf_counter()-started, first_frame_seconds=first_frame_seconds, inference_fps=len(frame_seconds)/max(sum(frame_seconds), 1e-9), session_inference_fps=completed/max(compute_seconds, 1e-9), graph_enabled=bool(info.get('graph', {}).get('replays')), graph=info.get('graph'), text_bank=info.get('fixed_text'), control_version=last_meta.get('version', 0), path_mode='window', prompt_history=dict(enabled=True, max_history=30, resets=info.get('history_resets', 0), visible_history=info.get('visible_history', 0), cutoff=info.get('history_cutoff_absolute')), gpu=gpu_name, status=status, error=error) if rotations: result['mesh_pose'] = dict( rotations=base64.b64encode(np.asarray(rotations, dtype='= deadline: STORE.patch(session, browser, state='paused', stop=True, stream_active=False) yield packet('paused') return tick = time.perf_counter() runtime.start(seed=0, history=60, use_graph=True, reset_history_on_prompt_change=True, cfg_scale=4.) runtime_started = True compute_seconds += time.perf_counter()-tick latest = STORE.read(session, browser) if not latest['stop']: latest = STORE.patch(session, browser, state='running') while time.perf_counter() < deadline: latest = STORE.read(session, browser) if latest['stop']: break tick = time.perf_counter() alive = time.time()-latest['keys_updated_at'] < 1.2 x, z = (latest['x'], latest['z']) if alive else (0., 0.) slot = latest['slot'] prompt = latest['slots'][slot] for meta in text_metas: meta.update(slot=slot, base_slot=slot, run=False, prompt=prompt, text_version=latest['version']) text_metas.append(dict(frame=runtime.status()['steps'], slot=slot, base_slot=slot, run=False, prompt=prompt, text_version=latest['version'])) rows, points = planner.plan(text_metas, x=x, z=z, shift=False, slot=slot, speeds=latest['speeds'], version=latest['version']) raw = runtime.step_replanned(prompt, rows, replan_text=True) frames, metas, rotations = [], [], [] if raw is not None: native, meta = planner.commit_first() text_metas.pop(0) if meta['frame'] != completed or not np.allclose(raw[:3], native, rtol=1e-5, atol=1e-6): raise RuntimeError('Motion and path frame alignment was lost.') last_meta = meta frames = [np.round(runtime.recover(raw)[0], 5).tolist()] metas = [dict(meta)] if model == 'SEED': rotations = [runtime.render_rotations(raw)] completed += 1 if first_frame_seconds is None: first_frame_seconds = time.perf_counter()-started playback_origin = time.perf_counter() seconds = time.perf_counter()-tick compute_seconds += seconds if raw is not None: frame_seconds.append(seconds) if frames or runtime.status()['steps'] % 6 == 0: yield packet('running' if completed else 'start', frames, metas, rotations) if playback_origin is not None: delay = playback_origin+completed/fps-time.perf_counter() if delay > 0: time.sleep(min(delay, .04)) STORE.patch(session, browser, state='paused', stop=True, stream_active=False, x=0., z=0., keys_updated_at=0.) yield packet('paused') except GeneratorExit: raise except Exception as exc: import traceback traceback.print_exc() message = f'{type(exc).__name__}: {exc}' STORE.patch(session, browser, state='error', stop=True, stream_active=False, error=message) yield packet('error', error=message) finally: try: runtime.close() finally: try: state = STORE.read(session, browser) STORE.patch(session, browser, state='error' if state['error'] else 'paused', stop=True, stream_active=False, x=0., z=0., keys_updated_at=0.) finally: if acquired: LOCAL_GPU_LOCK.release() with gr.Blocks(title='FloodDiffusion 2') as demo: gr.Markdown('# FloodDiffusion 2') session = gr.Textbox(value='', visible=False) body_loaded = gr.State(False) preparing = gr.State(False) model = gr.Radio(MODELS, value='SEED', show_label=False, container=False) with gr.Row(): start = gr.Button('Start', variant='primary', interactive=False) stop = gr.Button('Stop', interactive=False) prompts, speeds = [], [] with gr.Accordion('Prompts & speeds', open=False): for index, (text, value) in enumerate(DEFAULTS, 1): with gr.Row(): prompts.append(gr.Textbox(value=text, label=str(index), lines=1, max_length=500, scale=5)) speeds.append(gr.Number(value=value, label='Speed (m/s)', minimum=0, maximum=5, step=.05, scale=1, min_width=120)) status = gr.Markdown('Loading…') viewer = create_viewer(require_body_ready=True, action_slots=True) def sync_controls(session_id, loaded, preparing_now, request: gr.Request): info = session_status(session_id, request) phase = info['state'] busy = bool(preparing_now) or phase in ('starting', 'running', 'pausing') message = {'ready':'', 'paused':'Stopped', 'starting':'Preparing…', 'running':'', 'pausing':'Stopping…', 'error':info.get('error', '')}.get(phase, '') if not loaded: message = 'Loading…' elif preparing_now: message = 'Preparing…' return (gr.Button(interactive=bool(loaded) and not busy and phase in ('ready', 'paused', 'error')), gr.Button(interactive=busy and not preparing_now and phase != 'pausing'), gr.Radio(interactive=not busy), *(gr.Textbox(interactive=not busy) for _ in prompts), *(gr.Number(interactive=not busy) for _ in speeds), message) ui_outputs = [start, stop, model, *prompts, *speeds, status] loaded = viewer.body_ready(lambda: True, outputs=body_loaded, queue=False, api_name=False) ui_inputs = [session, body_loaded, preparing] loaded.then(sync_controls, ui_inputs, ui_outputs, queue=False, api_name=False) gr.Timer(.5).tick(sync_controls, ui_inputs, ui_outputs, queue=False, trigger_mode='always_last', api_name=False) def lock_controls(): return (gr.Button(interactive=False), gr.Button(interactive=False), gr.Radio(interactive=False), *(gr.Textbox(interactive=False) for _ in prompts), *(gr.Number(interactive=False) for _ in speeds), 'Preparing…', True) locking = start.click(lock_controls, outputs=[*ui_outputs, preparing], queue=False, api_name=False) beginning = locking.success(prepare, [session, model, *prompts, *speeds], session, api_name='prepare', concurrency_limit=GPU_CONCURRENCY, concurrency_id='gpu' if GPU_CONCURRENCY else None) prepared = beginning.success(lambda: False, outputs=preparing, queue=False, api_name=False) prepared.success(sync_controls, ui_inputs, ui_outputs, queue=False, api_name=False) running = prepared.success(stream, session, viewer, api_name='stream', concurrency_limit=GPU_CONCURRENCY, stream_every=.05, concurrency_id='gpu' if GPU_CONCURRENCY else None) running.then(sync_controls, ui_inputs, ui_outputs, queue=False, api_name=False) failed = beginning.failure(lambda: False, outputs=preparing, queue=False, api_name=False) failed.then(sync_controls, ui_inputs, ui_outputs, queue=False, api_name=False) stop.click(stop_session, session, status, queue=False, api_name='stop') viewer.control(keyboard_control, inputs=[], outputs=[], queue=False, trigger_mode='multiple', api_name=False) with gr.Row(visible=False): control_sequence = gr.Number(value=0) control_x, control_z = gr.Number(value=0), gr.Number(value=0) control_shift, control_slot = gr.Checkbox(value=False), gr.Number(value=0) control_status = gr.Textbox() gr.Button('Control API').click(control, [session, control_sequence, control_x, control_z, control_shift, control_slot], control_status, api_name='control', queue=False) inspect_status = gr.JSON() gr.Button('Session status').click(session_status, session, inspect_status, queue=False, api_name='session_status') demo.queue(max_size=32) if __name__ == '__main__': demo.launch(server_name='0.0.0.0', server_port=int(os.getenv('PORT', '7860')), show_error=True, footer_links=[])