File size: 8,003 Bytes
90d354b
 
 
133b124
 
 
 
90d354b
 
 
 
 
 
133b124
90d354b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
import type {Template,Level} from './templates'
export interface Box {x:number;y:number;w:number;h:number}
export interface MediaSource {id:string;name:string;camera:string;url:string;type:'video'|'image';duration?:number}
export interface CapturePolicy {mode:'event'|'scheduled'|'manual';intervalSeconds:number;comparisonMaxAgeSeconds:number}
export interface LocationAlerts {notSeen:{enabled:boolean;confirmations:number;attentionLevel:number};outside:{enabled:boolean;confirmations:number;attentionLevel:number};excludeCarried:boolean}
export interface Recommendation {capturePolicy:CapturePolicy;category:'security'|'care'|'memory'|'routine';attentionLevel:number;reasonZh:string;locationAlerts?:LocationAlerts}
export interface Draft {title:string;taskKind:'dog_boundary'|'keys_check'|'general';targetName:string;targetDescriptionEn:string;regionName:string;regionDescriptionEn:string;targetBox:Box|null;regionBox:Box|null;summary:string;questions:{id:string;question:string;options:string[]}[];viewAdvice:string;recommendation?:Recommendation}
export interface Observer {id:string;title:string;question_en:string;input:'single'|'pair'|'reference';options:Record<string,string>;labels_zh:Record<string,string>}
export interface Rule {id:string;title:string;mode:string;condition:{operator:string;terms:{observer:string;values:string[]}[]};then:unknown;duration_seconds:number;within_seconds:number;time_window:{start:string;end:string}|null;exclude_window:{start:string;end:string}|null;requires:string[];notify:string;level:Level;confirmation_frames:number;cooldown_seconds:number}
export interface Plan {title:string;summary_zh:string;status:'ready'|'needs_clarification'|'unsupported';clarification_questions:string[];assumptions:string[];observers:Observer[];rules:Rule[]}
export interface Binding {source:MediaSource;referenceImage:string;annotatedReference:string;targetCrop:string;targetBox:Box;regionBox:Box;targetName:string;regionName:string;targetDescriptionEn:string;regionDescriptionEn:string;referenceMediaTime:number;memoryId?:string;matchMode?:'instance'|'category'}
export interface Detail {observer:string;title:string;label:string;answer:{choice:string|null;abstained:boolean};recordId:string;latencyMs:number}
export interface Trial {passed:boolean;at:string;checks:{label:string;passed:boolean;usable?:number;needed?:number}[];observations:{mediaTime:number;details:Detail[];latencyMs:number}[];latencyMs:number}
export interface Setup {id:string;request:string;plan:Plan;binding:Binding;trial?:Trial;confirmedAt?:string;memoryId?:string;notification?:'auto'|'quiet';answers?:{question:string;answer:string}[];capturePolicy?:CapturePolicy;locationAlerts?:LocationAlerts;recommendationReason?:string}
export interface ObjectMemory extends Binding {id:string;confirmedAt:string;thumbnail:string;active:boolean;taskSummary:string;request:string;taskKind:Draft['taskKind']}
export interface Frame {image:string;annotated:string;pairImage:string;mediaTime:number;capturedAt:string;frameId:string;sourceId:string}
export interface ContractResult {answers:Record<string,{choice:string|null;abstained:boolean}>;details:Detail[];latencyMs:number;engineMs:number;notices:{eventId:string;title:string;level:Level;at:number}[];text:string;unknown?:boolean;pending?:boolean;stateChanged?:boolean;outcome:string;duplicate?:boolean;at?:string}
export async function groundedApi<T>(url:string,body?:unknown):Promise<T>{
 const r=await fetch(url,{method:body===undefined?'GET':'POST',headers:body===undefined?undefined:{'Content-Type':'application/json'},body:body===undefined?undefined:JSON.stringify(body),signal:AbortSignal.timeout(240000)})
 const data=await r.json();if(!r.ok)throw new Error(data.error||'暂未完成,请重试');return data
}
export const defaultBox:Box={x:.3,y:.3,w:.3,h:.3}
export const fullBox:Box={x:0,y:0,w:1,h:1}
function loadImage(src:string):Promise<HTMLImageElement>{return new Promise((resolve,reject)=>{const i=new Image();i.onload=()=>resolve(i);i.onerror=()=>reject(new Error('图片加载失败'));i.src=src})}
function canvas(w:number,h:number){const c=document.createElement('canvas');c.width=w;c.height=h;return c}
export function captureSetupFrame(video:HTMLVideoElement){if(video.readyState<2||!video.videoWidth)throw new Error('请等待画面加载');const scale=Math.min(1,1280/Math.max(video.videoWidth,video.videoHeight));const c=canvas(Math.round(video.videoWidth*scale),Math.round(video.videoHeight*scale));c.getContext('2d')!.drawImage(video,0,0,c.width,c.height);return c.toDataURL('image/jpeg',.85)}
export async function cropImage(src:string,box:Box){const im=await loadImage(src),w=box.w*im.naturalWidth,h=box.h*im.naturalHeight;const scale=Math.min(1,500/Math.max(w,h));const c=canvas(Math.max(1,Math.round(w*scale)),Math.max(1,Math.round(h*scale)));c.getContext('2d')!.drawImage(im,box.x*im.naturalWidth,box.y*im.naturalHeight,w,h,0,0,c.width,c.height);return c.toDataURL('image/jpeg',.9)}
export async function annotate(src:string,region:Box,target?:Box){
 const im=await loadImage(src),c=canvas(im.naturalWidth,im.naturalHeight),ctx=c.getContext('2d')!;ctx.drawImage(im,0,0)
 for(const [b,color,label] of [[region,'#34d399','REGION'],...(target?[[target,'#f87171','TARGET']]:[])] as [Box,string,string][]){ctx.strokeStyle=color;ctx.lineWidth=Math.max(3,c.width/320);ctx.strokeRect(b.x*c.width,b.y*c.height,b.w*c.width,b.h*c.height);ctx.font=`bold ${Math.max(12,c.width/70)}px sans-serif`;ctx.fillStyle=color;ctx.fillText(label,Math.max(4,b.x*c.width+4),Math.max(18,b.y*c.height-6))}
 return c.toDataURL('image/jpeg',.87)
}
export async function makeFrame(raw:string,mediaTime:number,b:Binding,capturedAt=new Date().toISOString()):Promise<Frame>{
 const annotated=await annotate(raw,b.regionBox),im=await loadImage(annotated),ref=await loadImage(b.targetCrop)
 const c=canvas(im.naturalWidth,im.naturalHeight+128),ctx=c.getContext('2d')!;ctx.fillStyle='#f5f5ef';ctx.fillRect(0,0,c.width,c.height);ctx.drawImage(im,0,128);ctx.drawImage(ref,8,6,116,116);ctx.fillStyle='#163426';ctx.font='18px sans-serif';ctx.fillText('REGISTERED TARGET (reference only)',138,45);ctx.fillText('Current scene below',138,78)
 return {image:raw,annotated,pairImage:c.toDataURL('image/jpeg',.85),mediaTime,capturedAt,frameId:b.source.id+':'+mediaTime.toFixed(3),sourceId:b.source.id}
}
export function grabRegisteredFrames(src:string,times:number[]):Promise<{image:string;mediaTime:number}[]>{
 return new Promise((resolve,reject)=>{const v=document.createElement('video');v.muted=true;v.preload='auto';const results:{image:string;mediaTime:number}[]=[];let index=0;const timeout=setTimeout(()=>finish(new Error('素材加载超时')),30000)
 const finish=(error?:Error)=>{clearTimeout(timeout);v.onseeked=null;v.onloadeddata=null;v.onerror=null;v.removeAttribute('src');v.load();if(error)reject(error);else resolve(results)}
 const seek=()=>{const t=Math.max(.001,Math.min(times[index],v.duration-.01));if(Math.abs(v.currentTime-t)<.0005){take();return}v.currentTime=t}
 const take=()=>{try{results.push({image:captureSetupFrame(v),mediaTime:v.currentTime});index++;if(index===times.length)finish();else seek()}catch(e){finish(e as Error)}}
 v.onerror=()=>finish(new Error('素材无法读取'));v.onseeked=take;v.onloadeddata=()=>{v.onloadeddata=null;seek()};v.src=src;v.load()
 })
}
export function contractTemplate(s:Setup):Template{return {id:'contract',name:s.plan.title,itemName:s.plan.title,desc:s.plan.summary_zh,camera:s.binding.source.camera,groundClip:s.binding.source.url,groundQuestion:s.binding.targetDescriptionEn,groundYes:'clear',question:s.plan.observers.map(o=>o.title).join(' · '),choices:{quiet:'已观察',attention:'需要关注',unknown:'待补证'},labels:{quiet:'已记录当前状态',attention:'发现需要关注的变化',unknown:'等待清晰画面'},levels:{quiet:0,attention:2,unknown:0},normal:['quiet'],hint:s.plan.summary_zh,keywords:[],icon:/狗|猫|宠物/.test(s.binding.targetName)?'cat':'package',confirmation:[]}}