'use client'; import {useCallback,useRef,useState} from 'react'; import {type Clip,type Project,type Range,type Selection,addTrack,clipEnd,clipsIn,muteRange,projectEnd,soleClip,uid} from '@/lib/studio/model'; import {type Player,renderMix,sliceBuffer,spliceBuffer} from '@/lib/studio/audio'; import {SignedOut,callEngine,fetchResult,wavFile} from '@/lib/studio/engine'; import type {Analysis,BandResult,CoverResult,Download,LyricsResult,PartResult,ScoreRead,SongResult,StemsResult} from '@/lib/types'; import {type Model,modelLabel} from '@/lib/studio/models'; export type Job={id:string;label:string;status:string;start:number;end:number;error?:string}; /** * A score the user read from a selection and edited: the engine plays this * instead of reading the audio again. `clock` is when each bar starts in the * selection, which is what the result is locked onto. */ export type ScoreDraft={abc:string;clock:number[];beats?:number[];edits:{transpose?:number;bpm?:number;chords?:Record}}; function addScore(form:FormData,score?:ScoreDraft){ if(!score)return; form.append('abc',score.abc);form.append('clock',JSON.stringify({bars:score.clock,beats:score.beats||[]}));form.append('edits',JSON.stringify(score.edits)); } /** What an AI action works on: the clips it reads and the stretch of time it covers. */ export type Target={start:number;end:number;clips:Clip[];tracks:string[];range:Range|null; /** The single clip that covers the whole target, so a result can be spliced straight into it. */ sole:Clip|null}; export function targetOf(project:Project,selection:Selection):Target|null{ const range=selection.range; if(range&&range.end-range.start>0.05){ const clips=clipsIn(project,range).filter(c=>!c.muted); if(!clips.length)return null; // A range drawn past the audio would send silence the engine then has to fill. const start=Math.max(range.start,Math.min(...clips.map(c=>c.start))); const end=Math.min(range.end,Math.max(...clips.map(clipEnd))); const clamped={...range,start,end}; return {start,end,clips,tracks:range.tracks,range:clamped,sole:soleClip(project,clamped)}; } const clips=project.clips.filter(c=>selection.clips.includes(c.id)); if(!clips.length)return null; return {start:Math.min(...clips.map(c=>c.start)),end:Math.max(...clips.map(clipEnd)),clips, tracks:[...new Set(clips.map(c=>c.trackId))],range:null,sole:clips.length===1?clips[0]:null}; } type Deps={ player:Player;buffers:Map; project:()=>Project;playhead:()=>number; commit:(fn:(p:Project)=>Project)=>void; signedOut:()=>void; }; type Placed={name:string;buffer:AudioBuffer;muted?:boolean}; const MAX=240; /** Score jobs (YuE2 covers and bands, reading scores and words) take a whole song. */ const SCORE_MAX=300; export function useJobs({player,buffers,project,playhead,commit,signedOut}:Deps){ const [jobs,setJobs]=useState([]); const controllers=useRef(new Map()); const store=useCallback((buffer:AudioBuffer)=>{const id=uid();buffers.set(id,buffer);return id;},[buffers]); const load=useCallback(async(file:Download|undefined,signal:AbortSignal)=>{ if(!file)throw new Error('The engine did not return that audio.'); return player.decode(await fetchResult(file,signal)); },[player]); /** * Run one engine request as a job. Jobs are independent: the user keeps * editing while the GPU works, so each result is placed against the project * as it is when the job finishes, not as it was when it started. */ const run=useCallback((label:string,start:number,end:number,work:(signal:AbortSignal,status:(t:string)=>void)=>Promise)=>{ const id=uid(),controller=new AbortController(); controllers.current.set(id,controller); setJobs(list=>[...list,{id,label,status:'Starting…',start,end}]); const status=(text:string)=>setJobs(list=>list.map(j=>j.id===id?{...j,status:text}:j)); work(controller.signal,status).then(()=>{ setJobs(list=>list.filter(j=>j.id!==id)); },error=>{ if(controller.signal.aborted){setJobs(list=>list.filter(j=>j.id!==id));return;} if(error instanceof SignedOut)signedOut(); setJobs(list=>list.map(j=>j.id===id?{...j,error:error instanceof Error?error.message:'That could not finish.'}:j)); }).finally(()=>controllers.current.delete(id)); },[signedOut]); const cancel=useCallback((id:string)=>{controllers.current.get(id)?.abort();setJobs(list=>list.filter(j=>j.id!==id));},[]); /** Put results on new tracks below the source and, when they replace it, mute what they replace. */ const addBelow=useCallback((target:Target|null,at:number,items:Placed[],replace:boolean)=>{ const made=items.map(item=>({...item,id:store(item.buffer)})); commit(p=>{ let next=p; let after=target?[...p.tracks].reverse().find(t=>target.tracks.includes(t.id))?.id:undefined; for(const item of made){ const [withTrack,track]=addTrack(next,item.name,after);after=track.id; next={...withTrack,clips:[...withTrack.clips,{id:uid(),trackId:track.id,buffer:item.id,start:at,offset:0, duration:item.buffer.duration,name:item.name,muted:item.muted}]}; } if(replace&&target){ next=target.range?muteRange(next,target.range) :{...next,clips:next.clips.map(c=>target.clips.some(t=>t.id===c.id)?{...c,muted:true}:c)}; } return next; }); },[commit,store]); /** * Place a result that covers exactly the target: splice it into the one clip * it came from when there is one, otherwise lay it on a new track. */ const place=useCallback((target:Target,result:AudioBuffer,name:string)=>{ const length=target.end-target.start; const piece=result.duration>length+0.01?sliceBuffer(result,0,length):result; const sole=target.sole&&project().clips.find(c=>c.id===target.sole!.id); const source=sole&&buffers.get(sole.buffer); if(sole&&source){ const at=target.sole!.offset+(target.start-target.sole!.start); const id=store(spliceBuffer(source,at,piece)); commit(p=>({...p,clips:p.clips.map(c=>c.id===sole.id?{...c,buffer:id}:c)})); return; } addBelow(target,target.start,[{name,buffer:piece}],true); },[addBelow,buffers,commit,project,store]); /** Audio for a target. One clip goes raw; several are mixed as the user hears them against each other. */ const render=useCallback((target:Target,start=target.start,end=target.end)=> renderMix(project(),buffers,{start,end,clips:target.clips.map(c=>c.id),mixer:target.clips.length>1}),[buffers,project]); const check=(target:Target|null,min=3,max=MAX)=>{ if(!target)throw new Error('Select a clip, or drag across one, first.'); const length=target.end-target.start; if(lengthmax)throw new Error(`Your selection covers ${Math.floor(length/60)}:${String(Math.floor(length%60)).padStart(2,'0')}` +`${target.clips.length>1?` across ${target.clips.length} clips`:''}. Select ${max/60} minutes or less at a time.`); return target; }; const restyle=useCallback((selection:Selection,options:{style:string;change:number;lyrics:string;model:Model})=>{ const target=check(targetOf(project(),selection)); run(`Restyle · ${modelLabel(options.model)}`,target.start,target.end,async(signal,status)=>{ const form=new FormData(); form.append('audio',wavFile(await render(target))); form.append('model',options.model);form.append('style',options.style);form.append('change',String(options.change));form.append('lyrics',options.lyrics); const result=await callEngine<{audio?:Download}>('/api/remix',form,{signal,status}); status('Placing the result…'); place(target,await load(result.audio,signal),'Restyled'); }); },[load,place,project,render,run]); const rewrite=useCallback((selection:Selection,options:{style:string;variance:number;lyrics:string;model:Model})=>{ const target=check(targetOf(project(),selection),1); // Send some audio either side so the model hears what the new part must join. const from=Math.min(...target.clips.map(c=>c.start)),to=Math.max(...target.clips.map(clipEnd)); let lo=Math.max(from,target.start-8),hi=Math.min(to,target.end+8); if(hi-lo<3){lo=Math.max(0,Math.min(lo,target.end-3));hi=Math.max(hi,lo+3);} if(hi-lo>MAX)throw new Error('Select four minutes or less at a time.'); run(`Rewrite · ${modelLabel(options.model)}`,target.start,target.end,async(signal,status)=>{ const form=new FormData(); form.append('audio',wavFile(await render(target,lo,hi))); form.append('model',options.model);form.append('style',options.style);form.append('lyrics',options.lyrics); form.append('start',String(target.start-lo));form.append('end',String(target.end-lo)); form.append('variance',String(options.variance)); const result=await callEngine<{audio?:Download}>('/api/repaint',form,{signal,status}); status('Placing the result…'); const full=await load(result.audio,signal); place(target,sliceBuffer(full,target.start-lo,target.end-lo),'Rewritten'); }); },[load,place,project,render,run]); /** Adopt the tempo, key and chords the engine measured so the grid and later parts line up. */ const learn=useCallback((analysis:Analysis|undefined,offset:number)=>{ if(!analysis)return; commit(p=>({...p, bpm:analysis.tempo>0?Math.round(analysis.tempo*10)/10:p.bpm, key:analysis.key||p.key, chords:analysis.chords.length?analysis.chords.map(c=>({time:c.time+offset,name:c.name})):p.chords})); },[commit]); const band=useCallback((selection:Selection,options:{style:string;separate:string;strength:number;split:boolean;useTempo:boolean;model:Model;score?:ScoreDraft})=>{ // YuE2 and ACE-Step 1.5 bands take a whole song; ACE-Step 1 stays at four minutes. const target=check(targetOf(project(),selection),3,options.model==='ace1'?MAX:SCORE_MAX); const bpm=options.useTempo?project().bpm:0; run(`Band · ${modelLabel(options.model)}`,target.start,target.end,async(signal,status)=>{ const form=new FormData(); form.append('audio',wavFile(await render(target))); form.append('model',options.model);form.append('style',options.style);form.append('separate',options.separate); form.append('strength',String(options.strength));form.append('split',String(options.split));form.append('bpm',String(bpm)); if(options.model==='yue2')addScore(form,options.score); const result=await callEngine('/api/accompany',form,{signal,status}); status('Laying out your band…'); const items:Placed[]=[]; const lifted=!!result.analysis?.separated&&!!result.vocal; if(lifted){ items.push({name:'Lifted vocal',buffer:await load(result.vocal,signal)}); if(result.background)items.push({name:'Original backing',buffer:await load(result.background,signal),muted:true}); } if(result.drums&&result.bass&&result.other){ // Named apart from stem splits, which also produce drums and bass tracks. items.push({name:'Band drums',buffer:await load(result.drums,signal)}); items.push({name:'Band bass',buffer:await load(result.bass,signal)}); items.push({name:'Band keys & guitars',buffer:await load(result.other,signal)}); }else items.push({name:'Band',buffer:await load(result.backing,signal)}); learn(result.analysis,target.start); addBelow(target,target.start,items,lifted); }); },[addBelow,learn,load,project,render,run]); /** * Perform the selection's tune in a new style with YuE2. The result comes * back locked to the selection's bars and exactly as long, so it goes on a * new track in place and the source is muted. */ const cover=useCallback((selection:Selection,options:{style:string;lyrics:string;voice:'sing'|'play';harmony:'keep'|'free';score?:ScoreDraft})=>{ const target=check(targetOf(project(),selection),3,SCORE_MAX); run('Cover · YuE2',target.start,target.end,async(signal,status)=>{ const form=new FormData(); form.append('audio',wavFile(await render(target))); form.append('model','yue2');form.append('style',options.style);form.append('lyrics',options.lyrics); form.append('voice',options.voice);form.append('harmony',options.harmony); addScore(form,options.score); const result=await callEngine('/api/cover',form,{signal,status}); status('Placing the cover…'); const buffer=await load(result.audio,signal),length=target.end-target.start; addBelow(target,target.start,[{name:options.voice==='sing'?'Cover':'Instrumental cover', buffer:buffer.duration>length+0.01?sliceBuffer(buffer,0,length):buffer}],true); }); },[addBelow,load,project,render,run]); /** Read the selection's bars, chords, key and (optionally) words, and teach them to the project. */ const readScore=useCallback((selection:Selection,options:{words:boolean},done:(read:ScoreRead,target:Target)=>void)=>{ const target=check(targetOf(project(),selection),3,SCORE_MAX); run('Reading the score',target.start,target.end,async(signal,status)=>{ const form=new FormData(); form.append('audio',wavFile(await render(target))); form.append('words',String(options.words)); const read=await callEngine('/api/score',form,{signal,status}); const summary=read.summary; if(!summary||!read.abc)throw new Error('No score came back. Try a longer stretch.'); // The score repeats the sounding chord at every bar; the lane marks only the changes. const chords=summary.bars.flatMap(bar=>bar.chords.map((name,i)=> ({time:bar.start+i*(bar.end-bar.start)/bar.chords.length,name}))) .filter((chord,i,all)=>i===0||all[i-1].name!==chord.name); learn({tempo:summary.bpm,key:summary.keyName,chords},target.start); done(read,target); }); },[learn,project,render,run]); /** The words sung in the selection, grouped by section. */ const words=useCallback((selection:Selection,done:(lyrics:string)=>void)=>{ const target=check(targetOf(project(),selection),3,SCORE_MAX); run('Reading the words',target.start,target.end,async(signal,status)=>{ const form=new FormData(); form.append('audio',wavFile(await render(target))); const result=await callEngine('/api/lyrics',form,{signal,status}); if(!result.lyrics.trim())throw new Error('No words were found in this stretch.'); done(result.lyrics); }); },[project,render,run]); const stems=useCallback((selection:Selection)=>{ const target=check(targetOf(project(),selection)); run('Stems',target.start,target.end,async(signal,status)=>{ const form=new FormData(); form.append('audio',wavFile(await render(target))); const result=await callEngine('/api/split',form,{signal,status}); status('Laying out the stems…'); addBelow(target,target.start,[ {name:'Vocals',buffer:await load(result.vocals,signal)},{name:'Drums',buffer:await load(result.drums,signal)}, {name:'Bass',buffer:await load(result.bass,signal)},{name:'Other',buffer:await load(result.other,signal)}],true); }); },[addBelow,load,project,render,run]); const extend=useCallback((selection:Selection,options:{style:string;seconds:number;lyrics:string;model:Model})=>{ const target=check(targetOf(project(),selection)); if(target.range||!target.sole)throw new Error('Select one clip to extend.'); if(target.end-target.start>200)throw new Error('Extend works on clips up to 200 seconds.'); const clip=target.sole; run(`Extend · ${modelLabel(options.model)}`,clipEnd(clip),clipEnd(clip)+options.seconds,async(signal,status)=>{ const form=new FormData(); form.append('audio',wavFile(await render(target))); form.append('model',options.model);form.append('style',options.style);form.append('seconds',String(options.seconds));form.append('lyrics',options.lyrics); const result=await callEngine<{audio?:Download}>('/api/extend',form,{signal,status}); status('Placing the result…'); const full=await load(result.audio,signal); const now=project().clips.find(c=>c.id===clip.id),source=now&&buffers.get(now.buffer); if(!now||!source){addBelow(target,clip.start,[{name:'Extended',buffer:full}],false);return;} // Keep the original samples up to the join and crossfade into the model's continuation. const join=Math.max(0.5,clip.duration-0.3); const kept=sliceBuffer(source,now.offset,now.offset+now.duration); const joined=spliceBuffer(kept,join,sliceBuffer(full,join,full.duration),0.3); const id=store(joined); commit(p=>({...p,clips:p.clips.map(c=>c.id===clip.id?{...c,buffer:id,offset:0,duration:joined.duration}:c)})); }); },[addBelow,buffers,commit,load,project,render,run,store]); const part=useCallback((options:{instrument:string;label:string;style:string;follow:boolean;duration:number;strength:number;model:Model})=>{ const current=project(); const end=projectEnd(current); const follow=options.follow&¤t.clips.some(c=>!c.muted); if(follow&&end<3)throw new Error('The project is too short to follow. Add at least three seconds.'); // A part follows at most four minutes: from the playhead, or from the top // when the playhead sits too near the end to leave three seconds to follow. const cursor=playhead(); const at=follow?(end>MAX&&cursor{ const form=new FormData(); if(follow)form.append('audio',wavFile(await renderMix(current,buffers,{start:at,end:until}))); form.append('model',options.model);form.append('instrument',options.instrument);form.append('style',options.style); // Once a take has been analysed its tempo is known, and every part should share that grid. form.append('bpm',String(current.chords.length||!follow?current.bpm:0)); form.append('key',current.key);form.append('duration',String(options.duration)); form.append('strength',String(options.strength)); const result=await callEngine('/api/part',form,{signal,status}); status('Placing the part…'); const buffer=await load(result.audio,signal); if(!project().chords.length)learn(result.analysis,at); addBelow(null,at,[{name:options.label,buffer}],false); }); },[addBelow,buffers,learn,load,playhead,project,run]); const generate=useCallback((options:{kind:'instrumental'|'song';style:string;lyrics:string;duration:number;model:Model})=>{ if(!options.style.trim())throw new Error('Describe the style you want.'); if(options.kind==='song'&&!options.lyrics.trim())throw new Error('Add lyrics, marked with [verse] and [chorus].'); const at=playhead(),current=project(); const style=options.kind==='instrumental'&¤t.clips.length ?[options.style,`${Math.round(current.bpm)} bpm`,current.key&&`key of ${current.key}`].filter(Boolean).join(', '):options.style; run(`${options.kind==='song'?'Song':'Instrumental'} · ${modelLabel(options.model)}`,at,at+(options.kind==='song'?60:options.duration),async(signal,status)=>{ const result=options.kind==='song' ?await callEngine('/api/generate',{style,lyrics:options.lyrics,cot:'full',model:options.model},{signal,status}) :await callEngine<{audio?:Download}>('/api/instrumental',{style,lyrics:'',duration:options.duration,model:options.model},{signal,status}); status('Placing the result…'); addBelow(null,at,[{name:options.kind==='song'?'Song':'Instrumental',buffer:await load(result.audio,signal)}],false); }); },[addBelow,load,playhead,project,run]); return {jobs,cancel,restyle,rewrite,band,cover,readScore,words,stems,extend,part,generate, dismiss:(id:string)=>setJobs(list=>list.filter(j=>j.id!==id))}; }