OneAlign / web /src /pages /Create.tsx
loveaa's picture
Deploy Home Pulse: grounded setup, isolated workspaces and shared HF visual inference
90d354b verified
Raw History Blame Contribute Delete
18 kB
import {useEffect,useRef,useState} from 'react'
import {Link,useNavigate,useSearchParams} from 'react-router'
import {ArrowLeft,ArrowRight,Check,CheckCircle2,Camera,Loader2,RefreshCw,ScanLine,Upload} from 'lucide-react'
import {SCENES,TEMPLATES} from '@/lib/templates'
import {store} from '@/lib/engine'
import {annotate,captureSetupFrame,cropImage,defaultBox,fullBox,grabRegisteredFrames,groundedApi,makeFrame,type Binding,type Box,type Draft,type MediaSource,type ObjectMemory,type Setup,type Trial} from '@/lib/grounded'
import {Badge,PageHead,Steps} from '@/components/Design'
import RegionPicker from '@/components/RegionPicker'
const examples=['帮我盯着小狗,整个身体出了笼子就提醒我','我出门时看看钥匙是不是还在托盘里','门口有人连续停留30秒告诉我','晚上8点到8点半猫有没有吃饭,到点告诉我']
const defaultSources:MediaSource[]=SCENES.map(s=>({id:s.id,name:s.name,camera:s.camera,url:s.media,type:'video'}))
export default function Create(){
const [params]=useSearchParams(),navigate=useNavigate()
const [step,setStep]=useState(0),[request,setRequest]=useState(''),[name,setName]=useState(''),[source,setSource]=useState<MediaSource>(defaultSources[0]),[sources,setSources]=useState(defaultSources)
const [matchMode,setMatchMode]=useState<'instance'|'category'>('instance')
const [draft,setDraft]=useState<Draft|null>(null),[reference,setReference]=useState(''),[referenceTime,setReferenceTime]=useState(0),[targetBox,setTargetBox]=useState<Box>(defaultBox),[regionBox,setRegionBox]=useState<Box>(fullBox)
const [targetName,setTargetName]=useState(''),[regionName,setRegionName]=useState(''),[answers,setAnswers]=useState<Record<string,string>>({}),[targetConfirmed,setTargetConfirmed]=useState(false)
const [setup,setSetup]=useState<Setup|null>(null),[trial,setTrial]=useState<Trial|null>(null),[trialImages,setTrialImages]=useState<string[]>([]),[memory,setMemory]=useState<ObjectMemory|null>(null),[memories,setMemories]=useState<ObjectMemory[]>([])
const [mode,setMode]=useState<'auto'|'quiet'>('auto'),[interval,setIntervalValue]=useState(3),[remember,setRemember]=useState(true),[ruleConfirmed,setRuleConfirmed]=useState(false),[busy,setBusy]=useState(''),[error,setError]=useState(''),[ready,setReady]=useState(false),[crop,setCrop]=useState('')
const player=useRef<HTMLVideoElement>(null),upload=useRef<HTMLInputElement>(null)
useEffect(()=>{void groundedApi<MediaSource[]>('/api/media/catalog').then(rows=>setSources([...defaultSources,...rows])).catch(()=>{});void groundedApi<ObjectMemory[]>('/api/memories').then(setMemories).catch(()=>{})},[])
useEffect(()=>{const id=params.get('memory'),edit=params.get('edit');if(id)void groundedApi<ObjectMemory>('/api/memories/'+id).then(m=>{setMemory(m);setRequest(m.request);setSource(m.source)}).catch(e=>setError(e.message));if(edit){const item=store.getState().items.find(i=>i.id===edit);if(item?.contractId)void groundedApi<Setup>('/api/contracts/'+item.contractId).then(s=>{setRequest(s.request);setSource(s.binding.source)}).catch(e=>setError(e.message))}},[params])
useEffect(()=>{setReady(false)},[source.url])
useEffect(()=>{let stopped=false;if(reference)void cropImage(reference,targetBox).then(im=>{if(!stopped)setCrop(im)});return()=>{stopped=true}},[reference,targetBox])
const fail=(e:unknown)=>setError((e as Error).message||'暂未完成,请重试')
const understand=async()=>{if(!player.current||busy)return;setBusy('正在结合画面理解这份关注');setError('');try{player.current.pause();const im=captureSetupFrame(player.current),time=player.current.currentTime;setReference(im);setReferenceTime(time);const r=await groundedApi<{draft:Draft}>('/api/setup/draft',{request,image:im,source,memoryId:memory?.id});setDraft(r.draft);setName(r.draft.title);setTargetName(r.draft.targetName);setMatchMode(r.draft.targetBox?'instance':'category');setRegionName(r.draft.regionName);setTargetBox(r.draft.targetBox||defaultBox);setRegionBox(r.draft.regionBox||fullBox);setAnswers({});setTargetConfirmed(false);setRuleConfirmed(false);setSetup(null);setTrial(null);setStep(1)}catch(e){fail(e)}finally{setBusy('')}}
const binding=async():Promise<Binding>=>({source,referenceImage:reference,annotatedReference:await annotate(reference,regionBox,targetBox),targetCrop:await cropImage(reference,targetBox),targetBox,regionBox,targetName,regionName,targetDescriptionEn:draft!.targetDescriptionEn,regionDescriptionEn:draft!.regionDescriptionEn,referenceMediaTime:referenceTime,memoryId:memory?.id,matchMode})
const prepare=async()=>{if(!draft||busy)return;setBusy('正在整理你确认的观察规则');setError('');try{const r=await groundedApi<Setup>('/api/setup/prepare',{request,binding:await binding(),taskKind:draft.taskKind,answers:draft.questions.map(q=>({question:q.question,answer:answers[q.id]||''})),notification:mode,confirmed:true});setSetup(r);setTrial(null);setRuleConfirmed(false);if(r.plan.status==='ready')setStep(2);else{const added=r.plan.clarification_questions.map((question,i)=>({id:'extra-'+Date.now()+'-'+i,question,options:[]}));setDraft({...draft,questions:[...draft.questions,...added]});setError(r.plan.summary_zh||'请补充下面的条件')}}catch(e){fail(e)}finally{setBusy('')}}
const test=async()=>{if(!setup||busy)return;setBusy('正在用三帧画面试观察');setError('');setTrial(null);try{const captured=await grabRegisteredFrames(source.url,[referenceTime,referenceTime+1.5,referenceTime+3]);setTrialImages(captured.map(f=>f.image));const frames=await Promise.all(captured.map(f=>makeFrame(f.image,f.mediaTime,setup.binding)));const r=await groundedApi<Trial>('/api/setup/trial',{setupId:setup.id,frames});setTrial(r)}catch(e){fail(e)}finally{setBusy('')}}
const activate=async()=>{if(!setup||!trial?.passed||!ruleConfirmed||busy)return;setBusy('正在保存这份关注');setError('');try{const r=await groundedApi<Setup>('/api/setup/activate',{setupId:setup.id,confirmation:{target:targetConfirmed,rule:ruleConfirmed},remember,interval,taskKind:draft?.taskKind});const item=store.addConfirmed(r,name,interval,params.get('edit')||undefined);navigate('/observe/'+item.id)}catch(e){fail(e)}finally{setBusy('')}}
const uploadVideo=async(file?:File)=>{if(!file)return;setBusy('正在导入画面');setError('');try{const r=await fetch('/api/media/upload',{method:'POST',headers:{'Content-Type':file.type,'X-File-Name':encodeURIComponent(file.name)},body:file});const s=await r.json();if(!r.ok)throw new Error(s.error);setSources(old=>[s,...old]);setSource(s)}catch(e){fail(e)}finally{setBusy('')}}
const chooseMemory=async(m:ObjectMemory)=>{setBusy('正在读取确认记录');try{const full=await groundedApi<ObjectMemory>('/api/memories/'+m.id);setMemory(full);setSource(full.source);if(!request)setRequest(full.request)}catch(e){fail(e)}finally{setBusy('')}}
const unanswered=draft?.questions.some(q=>!answers[q.id]?.trim())
return <div className="page grounded-create"><Link className="back-link" to="/watches"><ArrowLeft size={15}/>全部关注</Link><PageHead eyebrow="A SHARED UNDERSTANDING" title={step===0?'先说清楚,再替你留意。':step===1?'让我们看的是同一件事。':'先看懂,再开始守候。'} description={step===0?'告诉我你的牵挂,再一起确认画面里的对象与范围。':step===1?'确认目标、位置,以及什么变化值得一条提醒。':'用实际画面试一小段,把执行方式留给你确认。'}/><Steps current={step} labels={['说出牵挂','确认目标与条件','试观察与启用']}/>
{busy&&<div className="setup-busy" role="status"><Loader2 size={18} className="spin"/><div><b>{busy}</b><span>{step===0?'Kimi 正在查看这张画面并整理需要确认的细节。':'完成后,结果会出现在这里。'}</span></div></div>}{error&&<div className="inline-error setup-error" role="alert">{error}</div>}
{step===0&&<div className="setup-intro-grid"><section><div className="intent-card"><span className="eyebrow">WHAT MATTERS TO YOU</span><label htmlFor="intent">这次,想让我帮你留意什么?</label><textarea id="intent" value={request} onChange={e=>setRequest(e.target.value)} rows={4} placeholder="比如,我出门时,看看钥匙是不是还在托盘里。" disabled={!!busy}/><div className="intent-suggestions">{examples.map(ex=><button key={ex} disabled={!!busy} onClick={()=>setRequest(ex)}>{ex}<ArrowRight size={12}/></button>)}</div><div className="intent-footer"><span>从眼前的画面,理解你说的那件事</span><button className="button primary" onClick={understand} disabled={!!busy||!ready||request.trim().length<2}>一起确认<ArrowRight size={16}/></button></div></div>
{memories.length>0&&<div className="memory-suggestions"><div className="section-head"><h2>我们已经认识的</h2><Link to="/memories" className="text-link">全部记忆<ArrowRight size={14}/></Link></div><div className="memory-chips">{memories.slice(0,5).map(m=><button className={memory?.id===m.id?'active':''} onClick={()=>chooseMemory(m)} key={m.id} disabled={!!busy}><img src={m.thumbnail} alt={m.targetName}/><span>{m.targetName}<small>{m.regionName}</small></span>{memory?.id===m.id&&<Check size={14}/>}</button>)}</div></div>}
<details className="subtle-details"><summary>从已有关注场景开始</summary><div className="intent-suggestions">{TEMPLATES.map(t=><button key={t.id} onClick={()=>setRequest(t.itemName+','+t.hint)}>{t.name}</button>)}</div></details></section>
<aside className="surface source-picker"><div className="section-head"><div><span className="eyebrow">THE VIEW</span><h2>先选一处观察视野</h2></div><button className="icon-btn" aria-label="导入观察视频" disabled={!!busy} onClick={()=>upload.current?.click()}><Upload size={18}/></button></div><label className="field-label" htmlFor="setup-source">画面来源</label><select id="setup-source" className="field" value={source.id} disabled={!!busy} onChange={e=>{const s=sources.find(x=>x.id===e.target.value);if(s)setSource(s)}}>{!sources.some(s=>s.id===source.id)&&<option value={source.id}>{source.name}</option>}{sources.map(s=><option key={s.id} value={s.id}>{s.camera} · {s.name}</option>)}</select><div className="setup-video"><video key={source.url} ref={player} src={source.url} controls muted playsInline preload="auto" onLoadedData={()=>setReady(true)} onError={()=>{setReady(false);setError('画面无法读取,请重新选择或导入')}}/></div><p className="field-note">停在能看清目标的一刻,再点击“一起确认”。</p>{memory&&<div className="memory-reuse"><img src={memory.targetCrop||memory.thumbnail} alt="上次确认的目标" style={{width:42,height:42,objectFit:'contain',borderRadius:6}}/><CheckCircle2 size={17}/><span>参考上次确认的「{memory.targetName}」</span><button className="text-link" onClick={()=>setMemory(null)}>取消复用</button></div>}<input ref={upload} className="visually-hidden" type="file" accept="video/mp4,video/webm,video/quicktime" onChange={e=>{void uploadVideo(e.target.files?.[0]);e.target.value=''}}/></aside></div>}
{step===1&&draft&&<div className="grounding-layout"><section className="surface grounding-canvas"><div className="section-head"><div><span className="eyebrow">POINT TO WHAT YOU MEAN</span><h2>是这里的这个对象吗?</h2></div><Badge tone="green">{source.camera}</Badge></div><RegionPicker image={reference} target={targetBox} region={regionBox} onChange={(t,r)=>{setTargetBox(t);setRegionBox(r);setTargetConfirmed(false);setSetup(null);setTrial(null)}}/>{draft.viewAdvice&&<p className="inline-note">{draft.viewAdvice}</p>}<div className="registration-preview"><img src={crop} alt="已圈出的目标局部图"/><div><label className="field-label" htmlFor="target-name">我想关注的是</label><input id="target-name" className="field" value={targetName} onChange={e=>{setTargetName(e.target.value);setTargetConfirmed(false)}}/><label className="field-label" htmlFor="region-name">它所在的位置 / 观察范围</label><input id="region-name" className="field" value={regionName} onChange={e=>{setRegionName(e.target.value);setTargetConfirmed(false)}}/></div></div><div className="answer-options"><button className={matchMode==='instance'?'selected':''} onClick={()=>{setMatchMode('instance');setTargetConfirmed(false)}}>关注这个具体对象</button><button className={matchMode==='category'?'selected':''} onClick={()=>{setMatchMode('category');setTargetConfirmed(false)}}>关注这一类对象</button></div><label className="confirmation-check"><input type="checkbox" checked={targetConfirmed} onChange={e=>setTargetConfirmed(e.target.checked)}/><span>目标选对了,观察范围也符合我的意思</span></label></section>
<aside className="setup-questions"><span className="eyebrow">MAKE IT PRECISE</span><h2>把关键的细节,约定好。</h2><p>{draft.summary}</p>{draft.questions.map((q,i)=><div className="clarification" key={q.id}><label htmlFor={q.id}><span>{String(i+1).padStart(2,'0')}</span>{q.question}</label>{q.options.length>0&&<div className="answer-options">{q.options.map(a=><button key={a} className={answers[q.id]===a?'selected':''} onClick={()=>setAnswers({...answers,[q.id]:a})}>{a}</button>)}</div>}<input className="field" id={q.id} value={answers[q.id]||''} placeholder="也可以用自己的话说明" onChange={e=>setAnswers({...answers,[q.id]:e.target.value})}/></div>)}<label className="field-label" htmlFor="watch-name">这份关注的名字</label><input id="watch-name" className="field" value={name} onChange={e=>setName(e.target.value)} maxLength={60}/><div className="field-label">希望怎样告诉你</div><div className="answer-options">{[['auto','有变化时提醒'],['quiet','先安静记录']].map(([v,l])=><button key={v} className={mode===v?'selected':''} onClick={()=>setMode(v as 'auto'|'quiet')}>{l}</button>)}</div><div className="form-actions"><button className="button secondary" disabled={!!busy} onClick={()=>{setStep(0);setSetup(null);setTrial(null)}}>重新选画面</button><button className="button primary" disabled={!!busy||!targetConfirmed||!!unanswered||!targetName.trim()||!regionName.trim()||!name.trim()} onClick={prepare}>生成执行约定<ArrowRight size={15}/></button></div></aside></div>}
{step===2&&setup&&<div className="setup-trial-layout"><section className="surface agreement-card"><span className="eyebrow">YOUR OBSERVATION AGREEMENT</span><h2>{name}</h2><p className="agreement-summary">{setup.plan.summary_zh}</p><div className="agreement-target"><img src={setup.binding.targetCrop} alt={targetName}/><div><b>{targetName}</b><span>{regionName} · {source.camera}</span><small>目标和范围已由你确认</small></div><CheckCircle2 size={22}/></div><div className="agreement-rules">{setup.plan.rules.map(r=><div key={r.id}><Check size={16}/><span>{r.title}{r.duration_seconds>0&&<small>持续 {r.duration_seconds} 秒</small>}{r.time_window&&<small>{r.time_window.start} — {r.time_window.end}</small>}{r.requires.length>0&&<small>联动条件:{r.requires.map(k=>({away:'离家',rain:'下雨',doorbell:'门铃'}[k]||k)).join('、')} · 由你在观察页触发</small>}</span><Badge tone="green" dot={false}>{r.notify==='summary'?'到点汇总':r.level===0||r.notify==='silent'?'安静记录':'变化提醒'}</Badge></div>)}</div>{setup.plan.assumptions.length>0&&<div className="agreement-assumptions">{setup.plan.assumptions.map(a=><p key={a}>{a}</p>)}</div>}<label className="field-label" htmlFor="setup-interval">观察间隔</label><div className="interval-options" id="setup-interval">{[1,3,5,10].map(v=><button className={interval===v?'selected':''} onClick={()=>setIntervalValue(v)} key={v}>{v} 秒</button>)}</div><label className="confirmation-check"><input type="checkbox" checked={remember} onChange={e=>setRemember(e.target.checked)}/><span>记住这个对象与位置,下次可以直接复用</span></label><label className="confirmation-check"><input type="checkbox" checked={ruleConfirmed} onChange={e=>setRuleConfirmed(e.target.checked)}/><span>以上执行方式符合我的意思</span></label><button className="text-link" disabled={!!busy} onClick={()=>{setStep(1);setSetup(null);setTrial(null)}}><ArrowLeft size={14}/>调整目标或条件</button></section>
<section className="surface trial-card"><div className="section-head"><div><span className="eyebrow">TRY IT TOGETHER</span><h2>先试着看懂眼前这几秒。</h2></div><ScanLine size={23}/></div><p>检查目标能否看清,以及约定的观察项能否得到明确答案。</p><div className="trial-frames">{(trialImages.length?trialImages:[setup.binding.referenceImage]).map((im,i)=><div key={i}><img src={im} alt={'试观察画面 '+(i+1)}/><span>{i===0?'登记时刻':'后续画面 '+i}</span></div>)}</div>{trial&&<div className="trial-checks" aria-live="polite">{trial.checks.map((c,i)=><div key={i}><span className={c.passed?'pass':'fail'}>{c.passed?<Check size={14}/>:<RefreshCw size={14}/>}</span><b>{c.label}</b><small>{c.passed?'可观察':'需要调整'}</small></div>)}<p className={trial.passed?'trial-success':'inline-error'}>{trial.passed?'目标和观察范围已就位,可以开始。':'请调整选区或选择更清晰的时刻,再试一段。'}</p><details className="subtle-details"><summary>查看试观察结果 · {trial.latencyMs.toLocaleString()} ms</summary>{trial.observations.map((o,i)=><p key={i}>{o.mediaTime.toFixed(1)}s · {o.details.map(d=>d.label).join(';')||'等待前一帧'} · {o.latencyMs} ms</p>)}</details></div>}<div className="form-actions"><button className="button secondary" onClick={test} disabled={!!busy}><Camera size={16}/>{trial?'再试一段':'开始试观察'}</button><button className="button primary" onClick={activate} disabled={!!busy||!trial?.passed||!ruleConfirmed}>启用这份关注<ArrowRight size={16}/></button></div></section></div>}
</div>
}