OneAlign / web /src /pulse /Agent.tsx
loveaa's picture
Home Pulse 1.2: app interface, recommended event policies and independent location reminders
cdd19e1 verified
Raw History Blame Contribute Delete
22.8 kB
import {useEffect,useRef,useState} from 'react'
import {useNavigate,useSearchParams} from 'react-router'
import {ArrowLeft,ArrowUp,Check,CheckCircle2,ChevronRight,ImagePlus,Loader2,ScanLine,Sparkles,Upload} from 'lucide-react'
import RegionPicker from '@/components/RegionPicker'
import {annotate,captureSetupFrame,cropImage,defaultBox,fullBox,makeFrame,type Box,type CapturePolicy,type Draft,type LocationAlerts,type MediaSource,type ObjectMemory,type Setup,type Trial} from '@/lib/grounded'
import {api,categoryMeta,levelNames,policyName,type Category,type Watch} from './api'
import {useMediaConnection,usePulse} from './context'
const prompts=[{icon:'🐶',title:'小狗围栏守护',text:'帮我留意这只小狗是否还在围栏里。暂时没看到和明确在围栏外,可以分别提醒;被人抱出来不提醒。请推荐合适的检查和提醒方式。',match:/幼犬翻越围栏/},{icon:'🔑',title:'出门钥匙确认',text:'我说准备出门时,帮我看看这串钥匙是不是还在玄关托盘里。',match:/钥匙/},{icon:'📦',title:'门口包裹守护',text:'包裹放在门口后,如果被拿走了就提醒我。',match:/包裹/},{icon:'🐾',title:'Buddy 客厅活动',text:'帮我留意这只小狗是否在客厅活动,安静记录就好。',match:/幼犬翻越围栏/},{icon:'🥣',title:'Buddy 午餐观察',text:'中午12点到1点,帮我看看小狗有没有在食盆吃饭,到点告诉我。',match:/进食|食盆/},{icon:'🚪',title:'离家门窗确认',text:'我说准备出门时,帮我确认画面里的窗户是否关好。',match:/窗户/},{icon:'👜',title:'出门公文包提醒',text:'我说准备出门时,帮我看看公文包是不是还在玄关原处。',match:/公文包|玄关/},{icon:'🌿',title:'阳台状态检查',text:'帮我留意阳台这扇窗户的开关状态,持续打开时提醒我。',match:/窗户|阳台/},{icon:'🍪',title:'零食盘位置关注',text:'帮我留意这盘零食是否被移动,位置变化时告诉我。',match:/零食|食物/},{icon:'🐢',title:'鱼缸乌龟越线',text:'帮我留意这只乌龟有没有整个身体离开划定区域。',match:/乌龟|鱼缸/},{icon:'✨',title:'Buddy 精彩瞬间',text:'帮我记录小狗在画面里玩耍的时刻,安静记录就好。',match:/小狗|宠物/}]
export default function Agent({base}:{base:string}){
const {sources,catalogReady,clientId,addSource,refresh,notify}=usePulse(),navigate=useNavigate(),[params]=useSearchParams()
const [source,setSource]=useState<MediaSource|undefined>(),[request,setRequest]=useState(''),[stage,setStage]=useState(0),[busy,setBusy]=useState(''),[error,setError]=useState('')
const [draft,setDraft]=useState<Draft|null>(null),[reference,setReference]=useState(''),[referenceTime,setReferenceTime]=useState(0),[targetBox,setTargetBox]=useState<Box>(defaultBox),[regionBox,setRegionBox]=useState<Box>(fullBox),[targetName,setTargetName]=useState(''),[regionName,setRegionName]=useState(''),[crop,setCrop]=useState('')
const [sourceOpen,setSourceOpen]=useState(false),[pendingSetup,setPendingSetup]=useState('')
const [capturePolicy,setCapturePolicy]=useState<CapturePolicy>({mode:'event',intervalSeconds:300,comparisonMaxAgeSeconds:86400}),[locationAlerts,setLocationAlerts]=useState<LocationAlerts>({notSeen:{enabled:false,confirmations:2,attentionLevel:2},outside:{enabled:true,confirmations:1,attentionLevel:3},excludeCarried:true}),[recommendationReason,setRecommendationReason]=useState('')
const [answers,setAnswers]=useState<Record<string,string>>({}),[targetConfirmed,setTargetConfirmed]=useState(false),[setup,setSetup]=useState<Setup|null>(null),[trial,setTrial]=useState<Trial|null>(null),[remember,setRemember]=useState(false),[ruleConfirmed,setRuleConfirmed]=useState(false),[category,setCategory]=useState<Category>('care'),[level,setLevel]=useState(3),[memory,setMemory]=useState<ObjectMemory|null>(null)
const player=useRef<HTMLVideoElement>(null),upload=useRef<HTMLInputElement>(null),running=useRef(false)
const connection=useMediaConnection(source,player,source?.id.startsWith('media-')?source.id:undefined)
useEffect(()=>{if(catalogReady&&!source&&sources.length)setSource(sources.find(s=>/幼犬翻越围栏/.test(s.name))||sources[0])},[source,sources,catalogReady])
useEffect(()=>{setPendingSetup(sessionStorage.getItem('pulse-pending-setup')||'')},[])
useEffect(()=>{const id=params.get('memory'),edit=params.get('edit');if(id)void api<ObjectMemory>('/memories/'+id).then(m=>{setMemory(m);setRequest(m.request);setSource(m.source)}).catch(e=>setError(e.message));if(edit)void api<Watch>('/watches/'+edit).then(w=>{setSource(w.binding.source);setRequest(w.request)}).catch(e=>setError(e.message))},[params])
useEffect(()=>{let active=true;if(reference)void cropImage(reference,targetBox).then(value=>{if(active)setCrop(value)}).catch(()=>{});return()=>{active=false}},[reference,targetBox])
async function task(label:string,fn:()=>Promise<void>){if(running.current)return;running.current=true;setBusy(label);setError('');try{await fn()}catch(e){setError((e as Error).message)}finally{running.current=false;setBusy('');void refresh()}}
const choosePrompt=(prompt:typeof prompts[number])=>{setRequest(prompt.text);const match=sources.find(s=>prompt.match.test(s.name));if(match){setSource(match);setError('')}else{setSourceOpen(true);setError('请选一段能看清这个对象的画面')}setCategory(prompt.icon==='✨'?'memory':prompt.icon==='🐶'?'care':prompt.icon==='📦'?'security':'routine');setLevel(prompt.icon==='✨'?1:3)}
const understand=()=>task('正在看画面,理解你的关注',async()=>{
if(!player.current||!source||!connection.device)throw new Error('画面连接好后,就可以一起确认了')
player.current.pause();const image=captureSetupFrame(player.current);setReference(image);setReferenceTime(player.current.currentTime)
const r=await api<{draft:Draft}>('/setups/draft',{request,image,source:{...source,id:connection.device.id},viewRevision:connection.device.viewRevision,...(memory?{memoryId:memory.id}:{})})
setDraft(r.draft);setTargetName(r.draft.targetName);setRegionName(r.draft.regionName);setTargetBox(r.draft.targetBox||defaultBox);setRegionBox(r.draft.regionBox||fullBox);if(r.draft.recommendation){const rec=r.draft.recommendation;setCapturePolicy(rec.capturePolicy);setCategory(rec.category);setLevel(rec.attentionLevel);setRecommendationReason(rec.reasonZh);if(rec.locationAlerts)setLocationAlerts(rec.locationAlerts)}setAnswers({});setTargetConfirmed(false);setSetup(null);setTrial(null);setRuleConfirmed(false);setStage(1)
})
const prepare=()=>task('正在整理你确认的守护方式',async()=>{
if(!draft||!source||!connection.device)throw new Error('请先连接这处观察画面')
const binding={source:{...source,id:connection.device.id},referenceImage:reference,annotatedReference:await annotate(reference,regionBox,targetBox),targetCrop:await cropImage(reference,targetBox),targetBox,regionBox,targetName,regionName,targetDescriptionEn:draft.targetDescriptionEn,regionDescriptionEn:draft.regionDescriptionEn,referenceMediaTime:referenceTime,matchMode:draft.taskKind!=='general'||draft.targetBox?'instance':'category',...(memory?{memoryId:memory.id}:{})}
const perLocation=draft.taskKind==='dog_boundary',attentionLevel=perLocation?Math.max(0,...[locationAlerts.notSeen,locationAlerts.outside].filter(a=>a.enabled).map(a=>a.attentionLevel)):level
const result=await api<Setup>('/setups/prepare',{request,binding,taskKind:draft.taskKind,answers:draft.questions.map(q=>({question:q.question,answer:answers[q.id]||''})),notification:perLocation?'auto':level===0?'quiet':'auto',confirmed:true,viewRevision:connection.device.viewRevision,category,attentionLevel,capturePolicy,recommendationReason,...(perLocation?{locationAlerts}:{})})
const full={...result,binding} as Setup;setSetup(full);setStage(2);setTrial(null);setRuleConfirmed(false)
sessionStorage.setItem('pulse-pending-setup',result.id)
})
const runTrial=()=>task('正在检查对象、范围和当前状态能否看清',async()=>{
if(!setup||!source||!connection.device||!player.current)throw new Error('请先完成目标确认')
const frames=[await makeFrame(captureSetupFrame(player.current),player.current.currentTime,setup.binding)]
const result=await api<Trial>('/setups/'+setup.id+'/trial',{frames,viewRevision:connection.device.viewRevision});setTrial(result)
if(result.passed)notify('试观察通过,可以确认开始守护')
})
const activate=()=>task('正在保存这份关注',async()=>{
if(!setup)throw new Error('请先完成试观察')
const edit=params.get('edit'),watch=await api<Watch>(edit?'/watches/'+edit+'/replace':'/watches',{setupId:setup.id,confirmation:{target:true,rule:true},remember})
sessionStorage.removeItem('pulse-pending-setup');await refresh();notify('这份关注已建立');navigate(base+'/watch/'+watch.id)
})
const importVideo=(file?:File)=>{if(!file)return;void task('正在接入这段画面',async()=>{const response=await fetch('/api/media/upload',{method:'POST',headers:{'Content-Type':file.type,'X-File-Name':encodeURIComponent(file.name)},body:file});const next=await response.json();if(!response.ok)throw new Error(next.error||'视频导入未完成');addSource(next);setSource(next)})}
const allAnswered=draft?.questions.every(q=>answers[q.id]?.trim())??false
const resume=()=>task('正在恢复上次的确认',async()=>{const saved=await api<Setup&{category:Category;attentionLevel:number}>('/setups/'+pendingSetup);setSource(saved.binding.source);setRequest(saved.request);setReference(saved.binding.referenceImage);setReferenceTime(saved.binding.referenceMediaTime);setTargetBox(saved.binding.targetBox);setRegionBox(saved.binding.regionBox);setTargetName(saved.binding.targetName);setRegionName(saved.binding.regionName);setSetup(saved);setTrial(saved.trial||null);setCategory(saved.category||'care');setLevel(saved.attentionLevel??3);if(saved.capturePolicy)setCapturePolicy(saved.capturePolicy);if(saved.locationAlerts)setLocationAlerts(saved.locationAlerts);setRecommendationReason(saved.recommendationReason||'');setStage(2);setTargetConfirmed(true);setRuleConfirmed(false)})
return <div className="pulse-agent">
<div className={'pulse-page-heading '+(stage===0?'pulse-agent-welcome':'')}><div><span className="pulse-eyebrow">YOUR HOME, UNDERSTOOD</span><h1>{stage===0?'有什么想让我留意的?':stage===1?'我们看的是同一个吗?':'这就是我们的约定。'}</h1><p>{stage===0?'说出你在意的事,其余交给我来安排。':stage===1?'确认对象、位置和时机,让每次观察都有依据。':'先试着观察,再由你决定开始。'}</p></div><span className="pulse-agent-symbol"><Sparkles size={25}/></span></div>
<div className="pulse-stepbar" aria-label="创建进度">{['说出关注','共同确认','开始守护'].map((label,i)=><span className={stage===i?'current':stage>i?'done':''} key={label}><i>{stage>i?<Check size={12}/>:i+1}</i>{label}</span>)}</div>
{stage>0&&<button className="pulse-text-button" disabled={!!busy} onClick={()=>{setStage(stage===2&&!draft?0:stage-1);setError('')}}><ArrowLeft size={15}/>返回调整</button>}
{stage>0&&<div className="pulse-user-message"><span>你想让我留意</span><p>{request}</p></div>}
{pendingSetup&&stage===0&&<button className="pulse-text-button" disabled={!!busy} onClick={()=>void resume()}>继续上次的确认<ChevronRight size={15}/></button>}
<div className={'pulse-agent-grid '+(stage>0?'has-confirmation':'')} inert={!!busy} aria-busy={!!busy}>
<section className={'pulse-card pulse-source-card '+(sourceOpen||stage>0?'source-open':'')}>
<div className="pulse-section-title"><button className="pulse-source-toggle" aria-expanded={sourceOpen||stage>0} onClick={()=>setSourceOpen(!sourceOpen)}><h2>观察画面</h2><small>{source?.camera} · {connection.ready?'已连接':'连接中'}</small><ChevronRight size={15}/></button><button className="pulse-icon" aria-label="导入视频" disabled={!!busy||stage>0} onClick={()=>upload.current?.click()}><Upload size={17}/></button></div>
<label className="pulse-field-label">选择观察来源<select aria-label="观察来源" value={source?.id||''} disabled={!catalogReady||!!busy||stage>0} onChange={e=>{setSource(sources.find(s=>s.id===e.target.value));setError('')}}>{source&&!sources.some(s=>s.id===source.id)&&<option value={source.id}>{source.name}</option>}{sources.map(s=><option key={s.id} value={s.id}>{s.camera} · {s.name}</option>)}</select></label>
<div className="pulse-video"><video ref={player} key={source?.url} src={source?.url} muted playsInline controls preload="auto" onLoadedData={connection.onReady} onError={connection.onError}/><span className="pulse-video-label"><span className={connection.ready?'connected':''}/>{source?.camera||'等待画面'}</span></div>
<p className="pulse-helper">停在能看清对象的一刻,再一起确认。</p>
<input ref={upload} hidden type="file" accept="video/mp4,video/webm,video/quicktime" onChange={e=>{importVideo(e.target.files?.[0]);e.target.value=''}}/>
{memory&&<div className="pulse-reference-inline"><img src={memory.thumbnail||memory.targetCrop} alt={memory.targetName}/><div><b>{memory.targetName}</b><small>参考我们上次的确认</small></div><button className="pulse-text-button" onClick={()=>setMemory(null)}>取消复用</button></div>}
</section>
{stage===0?<section className="pulse-agent-composer"><div className="pulse-chat-note"><span className="pulse-mini-orb"><Sparkles size={16}/></span><p>你希望我帮忙留意什么?<br/><small>小狗、钥匙,或者家的某个角落。</small></p></div><div className="pulse-prompt-box"><textarea aria-label="说出你的关注" placeholder="比如,帮我留意小狗是否还在围栏里…" value={request} onChange={e=>setRequest(e.target.value)} maxLength={3000}/><div><span><ImagePlus size={14}/>{connection.ready?'已连接观察画面':'正在连接画面'}</span><button className="pulse-send" aria-label="一起确认" disabled={!!busy||request.trim().length<4||!connection.ready||!clientId} onClick={()=>void understand()}><ArrowUp size={22}/></button></div></div><div className="pulse-prompt-suggestions">{prompts.map(p=><button key={p.title} disabled={!catalogReady} onClick={()=>choosePrompt(p)}><span>{p.icon}</span>{p.title}<ChevronRight size={15}/></button>)}</div></section>:stage===1?<section className="pulse-card pulse-confirm-card">
<div className="pulse-chat-note"><Sparkles size={19}/><p>{draft?.summary}</p></div>
{draft?.viewAdvice&&<p className="pulse-view-advice"><ScanLine size={15}/><span>{draft.viewAdvice}</span></p>}
<RegionPicker image={reference} target={targetBox} region={regionBox} onChange={(target,region)=>{setTargetBox(target);setRegionBox(region);setTargetConfirmed(false)}}/>
<div className="pulse-confirm-target">{crop&&<img src={crop} alt="圈选的目标"/>}<div><label className="pulse-field-label">关注对象<input aria-label="关注对象" value={targetName} onChange={e=>{setTargetName(e.target.value);setTargetConfirmed(false)}}/></label><label className="pulse-field-label">观察位置<input aria-label="观察位置" value={regionName} onChange={e=>{setRegionName(e.target.value);setTargetConfirmed(false)}}/></label></div></div>
{draft?.questions.map(q=><fieldset className="pulse-question" key={q.id}><legend>{q.question}</legend><div>{q.options.map(option=><button className={answers[q.id]===option?'selected':''} key={option} onClick={()=>setAnswers(old=>({...old,[q.id]:option}))}>{option}</button>)}</div><input aria-label={q.question} placeholder="也可以补充你的想法" value={answers[q.id]||''} onChange={e=>setAnswers(old=>({...old,[q.id]:e.target.value}))}/></fieldset>)}
<section className="pulse-policy-proposal"><div className="pulse-section-title"><h2><Sparkles size={16}/>我建议这样守护</h2></div><p>{recommendationReason}</p><label className="pulse-field-label">什么时候看<select aria-label="取图方式" value={capturePolicy.mode} onChange={e=>setCapturePolicy(old=>({...old,mode:e.target.value as CapturePolicy['mode']}))}><option value="event" disabled={!connection.device?.capabilities.events}>设备有事件时检查</option><option value="manual">我想看时检查</option><option value="scheduled" disabled={!connection.device?.capabilities.scheduledCapture}>定时检查{connection.device?.capabilities.scheduledCapture?'':'(设备支持时可用)'}</option></select></label>{capturePolicy.mode==='scheduled'&&<label className="pulse-field-label">检查间隔<select aria-label="检查间隔" value={capturePolicy.intervalSeconds} onChange={e=>setCapturePolicy(old=>({...old,intervalSeconds:Number(e.target.value)}))}>{[300,600,900,1800,3600].map(seconds=><option value={seconds} key={seconds}>每 {seconds/60} 分钟</option>)}</select></label>}
{draft?.taskKind==='dog_boundary'&&<><h3>什么变化告诉你</h3>{([['notSeen','围栏内暂时没看到','位置待确认,提醒你检查一下。'],['outside','明确看到已经在围栏外','已经看清当前位置,按约定提醒。']] as const).map(([key,title,description])=><div className="pulse-alert-choice" key={key}><label className="pulse-check"><input type="checkbox" aria-label={title} checked={locationAlerts[key].enabled} onChange={e=>setLocationAlerts(old=>({...old,[key]:{...old[key],enabled:e.target.checked}}))}/><span><b>{title}</b><small>{description}</small></span></label>{locationAlerts[key].enabled&&<div className="pulse-two-fields"><label>确认方式<select aria-label={title+'确认方式'} value={locationAlerts[key].confirmations} onChange={e=>setLocationAlerts(old=>({...old,[key]:{...old[key],confirmations:Number(e.target.value)}}))}><option value={1}>本次看清后提醒</option><option value={2}>连续两次有效观察后</option><option value={3}>连续三次有效观察后</option></select></label><label>提醒方式<select aria-label={title+'提醒方式'} value={locationAlerts[key].attentionLevel} onChange={e=>setLocationAlerts(old=>({...old,[key]:{...old[key],attentionLevel:Number(e.target.value)}}))}>{levelNames.map((name,i)=><option value={i} key={name}>{name}</option>)}</select></label></div>}</div>)}<label className="pulse-check"><input type="checkbox" checked={locationAlerts.excludeCarried} onChange={e=>setLocationAlerts(old=>({...old,excludeCarried:e.target.checked}))}/>画面中正被人抱着时不提醒</label></>}
</section>
<label className="pulse-check"><input type="checkbox" checked={targetConfirmed} onChange={e=>setTargetConfirmed(e.target.checked)}/><span>对象、范围和条件都确认好了</span></label>
<details className="pulse-details"><summary>提醒表达 <span>{categoryMeta[category].label}{draft?.taskKind==='dog_boundary'?' · 按两类变化分别提醒':' · '+levelNames[level]}</span></summary><div className="pulse-two-fields"><label>场景<select aria-label="场景类型" value={category} onChange={e=>setCategory(e.target.value as Category)}>{Object.entries(categoryMeta).map(([key,c])=><option value={key} key={key}>{c.label}</option>)}</select></label>{draft?.taskKind!=='dog_boundary'&&<label>打扰程度<select aria-label="注意力等级" value={level} onChange={e=>setLevel(Number(e.target.value))}>{levelNames.map((name,i)=><option value={i} key={name}>{name}</option>)}</select></label>}</div></details>
<button className="pulse-button full" disabled={!!busy||!targetConfirmed||!allAnswered||!targetName.trim()||!regionName.trim()||!connection.ready} onClick={()=>void prepare()}>生成守护方式<ChevronRight size={17}/></button>
</section>:<section className="pulse-card pulse-plan-card">
<div className="pulse-section-title"><span className="pulse-pill mint"><CheckCircle2 size={13}/>对象已确认</span><span className="pulse-helper">{categoryMeta[category].label} · {setup?.locationAlerts?'按变化分别提醒':levelNames[level]}</span></div><h2>{setup?.plan.title}</h2><p>{setup?.plan.summary_zh}</p>
<div className="pulse-reference-inline"><img src={setup?.binding.targetCrop||crop} alt={targetName}/><div><b>{targetName||setup?.binding.targetName}</b><small>{regionName||setup?.binding.regionName}</small></div></div>
<div className="pulse-rule-list">{setup?.plan.observers.map(o=><div key={o.id}><ScanLine size={15}/><span>{o.title}</span></div>)}</div>
{setup?.plan.status!=='ready'&&<div className="pulse-error">{setup?.plan.clarification_questions.join(';')||'这份需求还需补充,请返回调整条件。'}</div>}
<div className="pulse-policy-summary"><b>{policyName(setup?.capturePolicy)}</b><details><summary>查看最初的建议</summary><p>{setup?.recommendationReason||recommendationReason}</p></details>{setup?.locationAlerts&&<small>暂未看到:{setup.locationAlerts.notSeen.enabled?`${setup.locationAlerts.notSeen.confirmations} 次有效观察 · ${levelNames[setup.locationAlerts.notSeen.attentionLevel]}`:'不提醒'}<br/>明确在外:{setup.locationAlerts.outside.enabled?`${setup.locationAlerts.outside.confirmations} 次有效观察 · ${levelNames[setup.locationAlerts.outside.attentionLevel]}`:'不提醒'}</small>}</div>
<button className="pulse-button soft full" disabled={!!busy||!connection.ready||setup?.plan.status!=='ready'} onClick={()=>void runTrial()}><ScanLine size={16}/>{trial?'重新试观察':'先试着观察'}</button>
{trial&&<div className="pulse-trial" aria-live="polite"><b>{trial.passed?'能看清,也能判断。':'还需要调整一下。'}</b>{trial.checks.map(check=><div key={check.label}><span className={check.passed?'passed':'not-passed'}>{check.passed?<Check size={14}/>:<ScanLine size={14}/>}</span>{check.label}<small>{check.passed?'通过':'待调整'}</small></div>)}</div>}
<label className="pulse-check"><input type="checkbox" checked={remember} onChange={e=>setRemember(e.target.checked)}/><span>记住这个对象,下次可以直接选用</span></label><label className="pulse-check"><input type="checkbox" checked={ruleConfirmed} onChange={e=>setRuleConfirmed(e.target.checked)}/><span>按这份约定开始守护</span></label>
<button className="pulse-button full" disabled={!!busy||!trial?.passed||!ruleConfirmed||!connection.ready} onClick={()=>void activate()}>开始守护<Check size={17}/></button>
</section>}
</div>
{(error||connection.error)&&<div className="pulse-error" role="alert">{error||connection.error}</div>}
{busy&&<div className="pulse-busy" role="status"><Loader2 size={18} className="spin"/><span>{busy}<small>已确认的内容会为你保留</small></span></div>}
</div>
}