"""CPU-capable, renderer-constrained speech-to-piano demonstration.""" import json import os from pathlib import Path import subprocess import sys import tempfile import shutil import time os.environ.setdefault('OMP_NUM_THREADS', '2') os.environ.setdefault('MPLBACKEND', 'Agg') import gradio as gr import numpy as np import soundfile as sf from player import player_component, player_payload ROOT = Path(__file__).resolve().parent VOICE = '一位温柔自信的年轻女性，声音清晰，语气亲切，表达轻快而富有感染力。' MODEL = 'Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign' DEVICE = os.getenv('INFERENCE_DEVICE', 'cpu') if DEVICE not in {'cpu', 'cuda', 'auto', 'zerogpu'}: raise ValueError('INFERENCE_DEVICE must be cpu, cuda, auto or zerogpu') if DEVICE == 'zerogpu': from zero_tts import generate_speech as generate_zero_speech RUNS = ROOT / 'outputs' / 'runs' RUNS.mkdir(parents=True, exist_ok=True) def worker(command, log, timeout=1800): """Bound each job and release all model memory when the child exits.""" env = os.environ.copy() if DEVICE in {'cpu', 'zerogpu'}: env['CUDA_VISIBLE_DEVICES'] = '' env['SPACES_ZERO_GPU'] = '0' with log.open('w') as stream: process = subprocess.Popen([sys.executable, *command], cwd=ROOT, env=env, stdout=stream, stderr=subprocess.STDOUT) try: process.wait(timeout=timeout) except subprocess.TimeoutExpired: process.kill() process.wait() raise gr.Error('This job exceeded 30 minutes. Try a shorter clip or fewer fitting iterations.') from None except BaseException: process.kill() process.wait() raise if process.returncode: print(log.read_text()[-6000:], flush=True) raise gr.Error('Generation failed. Please try a shorter clip; server logs contain details.') def prepare_audio(path, out, seconds): if not path: raise gr.Error('Upload or record an audio clip first.') info = sf.info(path) if not 8000 <= info.samplerate <= 192000 or info.channels > 8: raise gr.Error('Use audio with 1–8 channels and an 8–192 kHz sample rate.') # Read only the selected prefix, bounding memory even for long uploads. audio, rate = sf.read(path, frames=int(float(seconds) * info.samplerate), always_2d=True) audio = audio.mean(axis=1) if len(audio) < rate * .2 or not np.isfinite(audio).all() or np.max(np.abs(audio)) < 1e-7: raise gr.Error('Audio must contain at least 0.2 seconds of finite, non-silent sound.') sf.write(out, audio, rate, subtype='FLOAT') return len(audio) / rate def result(directory, status, stem='dense'): return (status, str(directory/'target.wav'), player_payload(directory, stem=stem), str(directory/f'{stem}.mid')) def run(mode, audio, text, language, description, seconds, steps, fitting_style='Speech (dense)', progress=gr.Progress()): if mode not in {'Audio → MIDI', 'Text → speech → MIDI'} or not .5 <= float(seconds) <= 30 or not 200 <= int(steps) <= 1200: raise gr.Error('Choose a valid mode, duration (0.5–30 s), and iteration count (200–1200).') if fitting_style not in {'Speech (dense)', 'Instrumental (sparse)'}: raise gr.Error('Choose Speech (dense) or Instrumental (sparse) fitting.') stem = 'instrumental' if fitting_style == 'Instrumental (sparse)' else 'dense' if language not in {'Auto', 'English', 'Chinese', 'Japanese', 'Korean', 'French', 'German', 'Spanish', 'Portuguese', 'Russian'}: raise gr.Error('Choose a supported language.') for old in RUNS.glob('job-*'): if old.is_dir() and time.time() - old.stat().st_mtime > 86400: shutil.rmtree(old) directory = Path(tempfile.mkdtemp(prefix='job-', dir=RUNS)) empty = (None, '', None) started = time.monotonic() if mode == 'Text → speech → MIDI': if not text.strip() or len(text) > 300 or not description.strip() or len(description) > 600: raise gr.Error('Enter 1–300 text characters and a voice description of 1–600 characters.') yield ('Generating speech. '+('Waiting for a shared GPU.' if DEVICE == 'zerogpu' else 'CPU VoiceDesign can take several minutes on first use.'), *empty) progress(.05, desc='Loading VoiceDesign and generating speech') prompts = directory/'prompts.json' prompts.write_text(json.dumps([dict(name='speech', text=text.strip(), language=language, instruct=description.strip())])) command = ['scripts/generate_speech.py', '--device', DEVICE, '--model', MODEL, '--samples', str(prompts), '--out', str(directory)] if os.getenv('QWEN_MODEL_PATH'): command += ['--model-path', os.environ['QWEN_MODEL_PATH']] if DEVICE == 'zerogpu': wav, rate = generate_zero_speech(text.strip(), language, description.strip()) sf.write(directory/'speech.wav', wav, rate, subtype='FLOAT') else: worker(command, directory/'tts.log') audio = str(directory/'speech.wav') duration = prepare_audio(audio, directory/'input.wav', seconds) yield (f'Fitting the first {duration:.2f} s · {fitting_style}.', str(directory/'input.wav'), '', None) progress(.3, desc=f'Fitting {stem} piano notes and rendering the MIDI') module = 'speaking_midi.instrumental_demo' if stem == 'instrumental' else 'speaking_midi.dense' command = ['-m', module, str(directory/'input.wav'), '--out', str(directory), '--soundfont', str(ROOT/'assets/gm.sf3'), '--steps', str(int(steps)), '--device', 'cpu'] worker(command, directory/'fit.log') progress(.95, desc='Preparing the MIDI player') report = json.loads((directory/'report.json').read_text()) yield result(directory, f"Complete in {time.monotonic()-started:.1f} s · {report['notes']:,} piano notes · {fitting_style}.", stem=stem) def example(language): name = {'en': 'the-tyger', 'zh': 'quiet-night'}[language] return result(ROOT/'examples/dense'/name, 'Precomputed poetry voice · dense piano fitting.') def build_ui(): with gr.Blocks(title='Speaking MIDI') as demo: gr.Markdown('# Speaking MIDI\nTurn speech or music into piano notes. Listen, follow the piano roll, and download the MIDI.\n' 'This is experimental sound approximation: words may become difficult to understand.') with gr.Row(): english = gr.Button('Listen to The Tyger · English') chinese = gr.Button('Listen to Quiet Night · Chinese') with gr.Row(): with gr.Column(scale=1): mode = gr.Radio(['Audio → MIDI', 'Text → speech → MIDI'], value='Audio → MIDI', label='Start with') audio = gr.Audio(sources=['upload', 'microphone'], type='filepath', label='Input audio') with gr.Group(visible=False) as speech_controls: text = gr.Textbox(value='Hello, can you hear the piano speaking?', label='Words to speak', lines=3, max_length=300) language = gr.Dropdown(['Auto', 'English', 'Chinese', 'Japanese', 'Korean', 'French', 'German', 'Spanish', 'Portuguese', 'Russian'], value='English', label='Language') description = gr.Textbox(value=VOICE, label='Voice description', lines=3, max_length=600) fitting_style = gr.Radio(['Speech (dense)', 'Instrumental (sparse)'], value='Speech (dense)', label='Fitting style', info='Speech keeps dense short notes. Instrumental uses fewer, longer notes.') with gr.Accordion('Fitting options', open=False): seconds = gr.Slider(.5, 30, value=30, step=.5, label='Fit the first N seconds') steps = gr.Slider(200, 1200, value=600, step=200, label='Fitting iterations', info='600 is the tested setting. More iterations take longer and may not improve clarity.') submit = gr.Button('Generate piano MIDI', variant='primary') gr.Markdown('CPU inference is supported. Speech generation can take several minutes; piano fitting runs on CPU. ' 'Try the ready-made examples for instant playback. Clips are processed up to the selected duration.') with gr.Column(scale=2): status = gr.Markdown('Choose an example or create a new fit.') source = gr.Audio(label='1 · Source used for fitting', interactive=False) player = player_component(gr) midi = gr.File(label='Download MIDI', interactive=False) poetry = json.loads((ROOT/'examples/poetry/prompts.json').read_text()) gr.Examples( examples=[['Audio → MIDI', str(ROOT/'examples/poetry'/f"{row['name']}.wav"), row['text'], row['language'], 12, 'Speech (dense)'] for row in poetry], inputs=[mode, audio, text, language, seconds, fitting_style], cache_examples=False, label='Poetry voices · select to listen, then generate piano MIDI', example_labels=[f"{row['title']} — {row['author']} ({row['language']})" for row in poetry], ) selected_text = gr.Textbox(label='Selected text', interactive=False, lines=2) text.change(lambda value: value, text, selected_text, queue=False) gr.Markdown('**How it works:** render piano templates → fit dense short notes or sparse held notes → ' 'convert strengths to MIDI velocities → render the MIDI with the same piano sound. ' '[Algorithm and experiments](https://github.com/k-l-lambda/speaking-midi) · ' '[Qwen3-TTS](https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign)') outputs = [status, source, player, midi] mode.change(lambda value: (gr.update(visible=value=='Audio → MIDI'), gr.update(visible=value!='Audio → MIDI')), mode, [audio, speech_controls], queue=False) submit.click(run, [mode,audio,text,language,description,seconds,steps,fitting_style], outputs, concurrency_limit=1, concurrency_id='compute', api_name='generate') english.click(lambda: example('en'), outputs=outputs, api_name='english_example') chinese.click(lambda: example('zh'), outputs=outputs, api_name='chinese_example') return demo.queue(max_size=8, default_concurrency_limit=1) if __name__ == '__main__': build_ui().launch(server_name='0.0.0.0', server_port=int(os.getenv('PORT', '7860')), max_file_size='25mb', allowed_paths=[str(RUNS), str(ROOT/'examples')])