| import { UI_THEME as T } from "../../shared/ui-theme.mjs"; |
| import { blockoutColor } from "../../shared/categories.mjs"; |
| import { TAKE_LIMITS } from "../../shared/take-limits.mjs"; |
| import * as THREE from "three"; |
| import { createActorLayer } from "./actor-view.mjs"; |
| import { doorTransform } from "../../shared/doors.mjs"; |
| import { createHandLayer } from "../hand-view.mjs"; |
| import { createCinemaLayer } from "./cinema-layer.mjs"; |
| import { isRig, objectQuaternion } from "../../shared/cinema.mjs"; |
| import { objectGeometry } from "../../shared/object-geometry.mjs"; |
| import { objectGeometry as objectGeometry2 } from "../../shared/object-geometry.mjs"; |
| import { createFloodLayer } from "./flood-layer.mjs"; |
| import { HANDS_METADATA, validateHandsSample, validateHandsMetadata } from "../../shared/hand-frame.mjs"; |
| function createVirtualScene(definition, matrix = new THREE.Matrix4()) { |
| const scene = new THREE.Scene(); |
| scene.background = new THREE.Color(T.background); |
| const world = new THREE.Group(); |
| world.matrixAutoUpdate = false; |
| world.matrix.copy(matrix); |
| scene.add(world); |
| scene.userData.definitionObjects = definition.objects; |
| scene.userData.contentWorld = world; |
| scene.userData.objectMeshes = new Map(); |
| scene.userData.baseTransforms = definition.objects.filter((o) => !isRig(o)).map((o) => ({ id: o.id, position: [...o.position], rotation: o.rotation, quaternion: objectQuaternion(o) })); |
| scene.add(new THREE.HemisphereLight("#ffffff", "#dce2ef", 2)); |
| const sun = new THREE.DirectionalLight("#ffffff", 0.9); |
| sun.position.set(-10, 20, 12); |
| scene.add(sun); |
| const rim = new THREE.DirectionalLight("#ffffff", 0.4); |
| rim.position.set(15, 8, -15); |
| scene.add(rim); |
| for (const o of definition.objects) { |
| if (isRig(o)) continue; |
| const mesh = new THREE.Mesh(objectGeometry(o), new THREE.MeshStandardMaterial({ color: blockoutColor(o, definition), roughness: o.roughness, metalness: o.metalness, opacity: 1, transparent: false })); |
| mesh.name = o.id; |
| mesh.position.fromArray(o.position); |
| mesh.scale.fromArray(o.size); |
| mesh.quaternion.fromArray(objectQuaternion(o)); |
| world.add(mesh); |
| scene.userData.objectMeshes.set(o.id, mesh); |
| } |
| if (definition.objects.some((o) => isRig(o) || o.track)) { |
| scene.userData.cinema = createCinemaLayer(world); |
| scene.userData.cinema.sync(definition); |
| scene.userData.cinema.frame(null); |
| } |
| if (definition.floods?.length) { |
| scene.userData.flood = createFloodLayer(world, { getObjectPose: (id) => { |
| const mesh = scene.userData.objectMeshes.get(id); |
| return mesh ? { position: mesh.position.toArray(), quaternion: mesh.quaternion.toArray() } : null; |
| } }); |
| scene.userData.flood.sync(definition); |
| scene.userData.flood.frame({ states: [] }, { showRegions: false }); |
| } |
| if (definition.objects.some((o) => o.kind === "light")) for (const light of scene.children.filter((o) => o.isLight)) light.intensity *= 0.18; |
| applyRecordedTransforms(scene, (definition.doorEffects || []).map((d) => doorTransform(definition.objects.find((o) => o.id === d.objectId), d))); |
| return scene; |
| } |
| function applyRecordedTransforms(scene, updates) { |
| const overrides = new Map(updates.map((o) => [o.id, o])); |
| for (const base of scene.userData.baseTransforms) { |
| const o = overrides.get(base.id) || base, mesh = scene.userData.objectMeshes.get(base.id); |
| if (!mesh) continue; |
| mesh.position.fromArray(o.position); |
| if (o.quaternion) mesh.quaternion.fromArray(o.quaternion); |
| else mesh.rotation.set(0, o.rotation || 0, 0); |
| } |
| } |
| function createVirtualRecorder({ onChange = () => { |
| }, upload, sink = null, now = () => performance.now(), assets = /* @__PURE__ */ new Map(), includeActorPreviews = () => false, contentFactory = null, Renderer = THREE.WebGLRenderer, Recorder = globalThis.MediaRecorder, makeCanvas = () => document.createElement("canvas") }) { |
| let status = "idle", message = "B: start / stop camera recording", renderer, scene, camera, stream, recorder, chunks = [], bytes = 0, startTime = 0, endTime = 0, lastFrame = -Infinity, frames = 0, result = null, stopPromise = null, resolveStop, rejectStop, stopTimer, generation = 0, pendingUpload = null; |
| let limitTimer, pumpTimer, stopReason = "user", storageError = null; |
| let content = null, actors = null, hands = null, timeline = null, definitionKey = null, recordPreviews = false, handMetadata = HANDS_METADATA; |
| const emit = () => onChange(snapshot()); |
| const snapshot = () => ({ status, message: status === "saving" && sink ? sink.snapshot().stage === "finalizing" ? "Processing and verifying video…" : "Uploading recorded chunks…" : status === "recording" && sink?.snapshot().lastError ? "Recording · Chunks stored in browser; upload will retry" : message, takeId: sink?.snapshot().takeId, storage: sink?.snapshot(), maxSeconds: TAKE_LIMITS.seconds, stopReason, seconds: status === "recording" ? Math.floor((now() - startTime) / 1e3) : 0, frames, result, canRetry: !!pendingUpload || !!sink?.snapshot().takeId && status === "error", hands: hands?.snapshot() || null }); |
| async function uploadPending() { |
| try { |
| result = await upload(pendingUpload.blob, pendingUpload.metadata, pendingUpload.timeline); |
| pendingUpload = null; |
| status = "idle"; |
| message = "Reference video saved · Virtual scene only"; |
| emit(); |
| return result; |
| } catch (error) { |
| status = "error"; |
| message = `Video retained in this page. Retry saving: ${error.message}`; |
| emit(); |
| throw error; |
| } |
| } |
| async function retry() { |
| if (!pendingUpload && !sink || status === "saving") return result; |
| status = "saving"; |
| message = "Retrying video save…"; |
| emit(); |
| if (sink) { |
| try { |
| result = await sink.retry(); |
| status = "idle"; |
| message = "Reference video saved"; |
| emit(); |
| return result; |
| } catch (error) { |
| status = "error"; |
| message = "Recording chunks retained. Retry: " + error.message; |
| emit(); |
| throw error; |
| } |
| } |
| return uploadPending(); |
| } |
| async function recover() { |
| if (sink && await sink.recover()) { |
| status = "error"; |
| message = "Unfinished recording found. Retry saving the recorded portion."; |
| emit(); |
| return true; |
| } |
| return false; |
| } |
| function cleanup() { |
| clearTimeout(limitTimer); |
| clearInterval(pumpTimer); |
| stream?.getTracks().forEach((t) => t.stop()); |
| stream = null; |
| content?.dispose(); |
| content = null; |
| actors?.dispose(); |
| actors = null; |
| hands?.dispose(); |
| hands = null; |
| scene?.traverse((o) => { |
| o.geometry?.dispose(); |
| o.material?.dispose(); |
| }); |
| scene = null; |
| renderer?.dispose(); |
| renderer = null; |
| camera = null; |
| recorder = null; |
| clearTimeout(stopTimer); |
| } |
| function fail(error) { |
| generation++; |
| const active = recorder; |
| cleanup(); |
| if (active?.state === "recording") try { |
| active.stop(); |
| } catch { |
| } |
| status = "error"; |
| message = error.message || "Recording failed"; |
| emit(); |
| rejectStop?.(error); |
| resolveStop = rejectStop = null; |
| stopPromise = null; |
| } |
| function start(definition, matrix, initialView, initialActors = [], initialTransforms = [], initialHands, metadata = HANDS_METADATA, presentation = {}) { |
| if (["starting", "recording", "saving"].includes(status)) throw new Error("Recording or saving in progress"); |
| if (pendingUpload) throw new Error("Previous video is unsaved. Retry saving it first."); |
| validateHandsMetadata(metadata); |
| handMetadata = structuredClone(metadata); |
| if (initialHands) validateHandsSample(initialHands, handMetadata); |
| if (!Recorder) throw new Error("Video recording is unavailable in this browser"); |
| const mime = ["video/webm;codecs=vp8", "video/webm;codecs=vp9", "video/mp4"].find((type) => Recorder.isTypeSupported(type)); |
| if (!mime) throw new Error("No compatible video encoder"); |
| const token = ++generation; |
| try { |
| const canvas = makeCanvas(); |
| renderer = new Renderer({ canvas, antialias: true, alpha: false, preserveDrawingBuffer: true }); |
| renderer.setPixelRatio(1); |
| renderer.setSize(1280, 720, false); |
| renderer.outputColorSpace = THREE.SRGBColorSpace; |
| renderer.toneMapping = THREE.ACESFilmicToneMapping; |
| renderer.toneMappingExposure = 1.25; |
| scene = createVirtualScene(definition, matrix); |
| scene.userData.contentWorld.visible = presentation.visible !== false; |
| camera = new THREE.PerspectiveCamera(65, 16 / 9, 0.02, 200); |
| actors = createActorLayer(scene.userData.contentWorld, assets); |
| definitionKey = JSON.stringify(definition); |
| hands = initialHands ? createHandLayer(scene.userData.contentWorld, handMetadata) : null; |
| hands?.apply(initialHands); |
| recordPreviews = includeActorPreviews(); |
| actors.apply(initialActors, { clean: !recordPreviews }); |
| content = contentFactory?.(scene.userData.contentWorld, actors.views); |
| content?.apply(presentation.showcase); |
| if (initialTransforms.length) applyRecordedTransforms(scene, initialTransforms); |
| timeline = { schema: "vrbuild-take/1", actorPreviews: recordPreviews ? "include" : "exclude", initialScene: structuredClone(definition), sceneMatrix: matrix.toArray(), assets: [...assets.values()].map(({ template, motion, ...meta }) => meta), events: [], samples: [], frameMapping: "render samples with monotonic times; encoded frames may be resampled" }; |
| if (initialHands) { |
| timeline.handsMetadata = structuredClone(handMetadata); |
| timeline.initialHands = structuredClone(initialHands); |
| if (sink) { |
| timeline.initialHands.sampleId = 0; |
| timeline.initialHands.frameTimeMs = 0; |
| } |
| } |
| if (initialView) { |
| if (initialView.isPerspectiveCamera) { |
| camera.fov = initialView.fov; |
| camera.aspect = initialView.aspect; |
| camera.updateProjectionMatrix(); |
| } |
| initialView.getWorldPosition(camera.position); |
| initialView.getWorldQuaternion(camera.quaternion); |
| camera.updateMatrixWorld(true); |
| renderer.render(scene, camera); |
| } |
| if (!canvas.captureStream) throw new Error("Canvas recording is unavailable in this browser"); |
| stream = canvas.captureStream(30); |
| recorder = new Recorder(stream, { mimeType: mime, videoBitsPerSecond: 3e6 }); |
| chunks = []; |
| bytes = 0; |
| frames = 0; |
| lastFrame = -Infinity; |
| result = null; |
| stopPromise = null; |
| storageError = null; |
| stopReason = "user"; |
| const segment = () => ({ time: Math.min(TAKE_LIMITS.seconds, Math.max(0, ((endTime || now()) - startTime) / 1e3)), samples: timeline.samples.splice(0), events: timeline.events.splice(0) }); |
| recorder.ondataavailable = (event2) => { |
| if (token !== generation || !event2.data.size) return; |
| bytes += event2.data.size; |
| if (sink) { |
| try { |
| sink.append(event2.data, segment()).catch((error) => { |
| storageError = error; |
| stop("storage-error").catch(() => { |
| }); |
| }); |
| } catch (error) { |
| storageError = error; |
| stop("storage-error").catch(() => { |
| }); |
| } |
| } else chunks.push(event2.data); |
| if (bytes > TAKE_LIMITS.videoBytes && status === "recording") stop("storage-error").catch(() => { |
| }); |
| }; |
| recorder.onerror = (event2) => { |
| if (token === generation) fail(event2.error || new Error("Video encoding failed")); |
| }; |
| recorder.onstop = async () => { |
| if (token !== generation) return; |
| clearTimeout(stopTimer); |
| const duration = Math.min(TAKE_LIMITS.seconds, ((endTime || now()) - startTime) / 1e3), blob = sink ? null : new Blob(chunks, { type: mime.split(";")[0] }); |
| chunks = []; |
| cleanup(); |
| try { |
| if (!bytes || !frames) { |
| await sink?.discardEmpty(); |
| throw new Error("No valid frames recorded. Start a new recording."); |
| } |
| if (storageError) throw storageError; |
| if (sink) { |
| await sink.append(new Blob([]), segment()); |
| message = "Uploading and processing recording…"; |
| emit(); |
| result = await sink.finish({ duration, frames, reason: stopReason }); |
| status = "idle"; |
| message = "Reference video saved · Virtual scene only"; |
| emit(); |
| } else { |
| pendingUpload = { blob, metadata: { duration, width: 1280, height: 720, frames }, timeline }; |
| result = await uploadPending(); |
| } |
| resolveStop?.(result); |
| } catch (error) { |
| status = "error"; |
| message = "Recording is not fully saved. Retry the retained chunks: " + error.message; |
| emit(); |
| rejectStop?.(error); |
| } finally { |
| resolveStop = rejectStop = null; |
| stopPromise = null; |
| } |
| }; |
| const activate = () => { |
| recorder.start(1e3); |
| startTime = now(); |
| endTime = 0; |
| status = "recording"; |
| message = "Recording · Maximum 3 minutes · B to stop"; |
| limitTimer = setTimeout(() => stop("limit").catch(() => { |
| }), TAKE_LIMITS.seconds * 1e3); |
| pumpTimer = sink ? setInterval(() => sink.flush().catch(() => { |
| }), 3e3) : null; |
| emit(); |
| }; |
| if (sink) { |
| status = "starting"; |
| message = "Preparing local recording storage…"; |
| emit(); |
| return sink.begin(structuredClone(timeline), mime.split(";")[0]).then(activate).catch((error) => { |
| cleanup(); |
| status = "error"; |
| message = error.message; |
| emit(); |
| throw error; |
| }); |
| } |
| activate(); |
| } catch (error) { |
| cleanup(); |
| status = "error"; |
| message = error.message; |
| emit(); |
| throw error; |
| } |
| } |
| function stop(reason = "user") { |
| if (stopPromise) return stopPromise; |
| if (status !== "recording") return Promise.resolve(result); |
| stopReason = reason; |
| clearTimeout(limitTimer); |
| clearInterval(pumpTimer); |
| endTime = now(); |
| status = "saving"; |
| message = "Saving reference video…"; |
| emit(); |
| stopPromise = new Promise((resolve, reject) => { |
| resolveStop = resolve; |
| rejectStop = reject; |
| }); |
| const pending = stopPromise; |
| stopTimer = setTimeout(() => fail(new Error("Video encoder did not finish in time")), 15e3); |
| try { |
| recorder.stop(); |
| } catch (error) { |
| fail(error); |
| } |
| return pending; |
| } |
| function syncDefinition(definition, matrix) { |
| if (status !== "recording") return; |
| if (matrix) { |
| scene.userData.contentWorld.matrix.copy(matrix); |
| scene.userData.contentWorld.matrixWorldNeedsUpdate = true; |
| } |
| const key = JSON.stringify(definition); |
| if (key === definitionKey) return; |
| definitionKey = key; |
| const world = scene.userData.contentWorld; |
| for (const child of [...world.children]) if (child.isMesh) { |
| world.remove(child); |
| child.geometry.dispose(); |
| child.material.dispose(); |
| } |
| scene.userData.cinema?.dispose(); |
| scene.userData.flood?.dispose(); |
| const fresh = createVirtualScene(definition, matrix); |
| fresh.userData.cinema?.dispose(); |
| fresh.userData.flood?.dispose(); |
| for (const child of [...fresh.userData.contentWorld.children]) world.add(child); |
| for (const key2 of ["objectMeshes", "baseTransforms", "definitionObjects"]) scene.userData[key2] = fresh.userData[key2]; |
| scene.userData.cinema = null; |
| scene.userData.flood = null; |
| if (definition.objects.some((o) => isRig(o) || o.track)) { |
| scene.userData.cinema = createCinemaLayer(world); |
| scene.userData.cinema.sync(definition); |
| } |
| if (definition.floods?.length) { |
| scene.userData.flood = createFloodLayer(world, { getObjectPose: (id) => { |
| const mesh = scene.userData.objectMeshes.get(id); |
| return mesh ? { position: mesh.position.toArray(), quaternion: mesh.quaternion.toArray() } : null; |
| } }); |
| scene.userData.flood.sync(definition); |
| } |
| timeline.events.push({ time: Math.min(TAKE_LIMITS.seconds, (now() - startTime) / 1e3), type: "scene-update", scene: structuredClone(definition) }); |
| } |
| function event(type, detail) { |
| if (status === "recording") timeline.events.push({ time: Math.min(TAKE_LIMITS.seconds, (now() - startTime) / 1e3), type, detail: structuredClone(detail) }); |
| } |
| function frame(time, view, actorFrames = [], objectTransforms = [], handSample, presentation = {}) { |
| if (status !== "recording") return; |
| if (now() - startTime >= TAKE_LIMITS.seconds * 1e3) { |
| stop("limit").catch(() => { |
| }); |
| return; |
| } |
| if (time - lastFrame < 1e3 / 30) return; |
| const world = scene.userData.contentWorld; |
| if (presentation.matrix) { |
| world.matrix.copy(presentation.matrix); |
| world.matrixWorldNeedsUpdate = true; |
| } |
| world.visible = presentation.visible !== false; |
| if (handSample && !hands) hands = createHandLayer(scene.userData.contentWorld, handMetadata); |
| hands?.apply(handSample); |
| if (handSample && !timeline.handsMetadata) timeline.handsMetadata = structuredClone(handMetadata); |
| lastFrame = Number.isFinite(lastFrame) ? time - (time - lastFrame) % (1e3 / 30) : time; |
| view.getWorldPosition(camera.position); |
| view.getWorldQuaternion(camera.quaternion); |
| camera.updateMatrixWorld(true); |
| actors?.apply(actorFrames, { clean: !recordPreviews }); |
| applyRecordedTransforms(scene, objectTransforms); |
| content?.apply(presentation.showcase); |
| if (view.isPerspectiveCamera) { |
| camera.fov = view.fov; |
| camera.aspect = view.aspect; |
| camera.updateProjectionMatrix(); |
| } |
| scene.userData.cinema?.frame(presentation.productionTime || 0, [...scene.userData.definitionObjects || [], ...actorFrames], [], presentation.previewObjectIds ?? null, presentation.triggerClocks || {}); |
| scene.userData.flood?.frame(presentation.floodState || { states: [] }, { showRegions: false }); |
| renderer.render(scene, camera); |
| frames++; |
| timeline.samples.push({ time: (now() - startTime) / 1e3, ...presentation.matrix ? { sceneMatrix: presentation.matrix.toArray(), phase: presentation.phase, worldVisible: world.visible } : {}, position: camera.position.toArray(), quaternion: camera.quaternion.toArray(), projection: camera.projectionMatrix.toArray(), ...presentation.productionTime !== void 0 ? { productionTime: presentation.productionTime, ...presentation.previewObjectIds ? { previewObjectIds: [...presentation.previewObjectIds], triggerClocks: structuredClone(presentation.triggerClocks || {}) } : {}, cameraId: presentation.cameraId || null, floodState: structuredClone(presentation.floodState || { states: [] }) } : {}, ...presentation.showcase ? { showcase: structuredClone(presentation.showcase), props: content?.snapshot?.() || [] } : {}, actors: actorFrames.map((f) => ({ ...f, visible: f.visible && (!f.preview || recordPreviews) })), objectTransforms: structuredClone(objectTransforms), ...handSample ? { hands: { ...structuredClone(handSample), ...sink ? { sampleId: frames, frameTimeMs: now() - startTime } : {} } } : {} }); |
| } |
| return { start, stop, retry, recover, frame, snapshot, syncDefinition, event }; |
| } |
| export { |
| applyRecordedTransforms, |
| createVirtualRecorder, |
| createVirtualScene, |
| objectGeometry2 as objectGeometry |
| }; |
|
|
