import fs from 'node:fs' import http from 'node:http' import os from 'node:os' import path from 'node:path' import { execFile } from 'node:child_process' import { createHash } from 'node:crypto' import { fileURLToPath } from 'node:url' import { promisify } from 'node:util' import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3' import { fromIni } from '@aws-sdk/credential-providers' import { getSignedUrl } from '@aws-sdk/s3-request-presigner' import { createTraceApi } from './trace-api.mjs' import { benchmarkMetadataPaths, shouldPromoteMetadata, traceMetadataPaths, } from './cloud-metadata.mjs' const root = path.dirname(fileURLToPath(import.meta.url)) const dist = path.join(root, 'dist') const execFileAsync = promisify(execFile) const cloudDownloads = new Map() const cloudBucket = process.env.AUTOCAD_BENCH_S3_BUCKET || '' const cloudPrefix = (process.env.AUTOCAD_BENCH_S3_RUNS_PREFIX || 'controller/runs') .replace(/^\/+|\/+$/g, '') const awsProfile = process.env.AWS_PROFILE || '' const awsRegion = process.env.AWS_REGION || '' const cloudArtifactsEnabled = Boolean(cloudBucket) && process.env.AUTOCAD_BENCH_CLOUD_ARTIFACTS !== 'off' const signedArtifactLifetimeSeconds = 300 const s3 = new S3Client({ ...(awsRegion ? { region: awsRegion } : {}), ...(awsProfile ? { credentials: fromIni({ profile: awsProfile }) } : {}), }) const cloudMetadataDownloads = new Map() const cloudMetadataFreshAt = new Map() const cloudArtifactAttempts = Math.max( 1, Math.min(5, Number(process.env.AUTOCAD_BENCH_CLOUD_ARTIFACT_ATTEMPTS || 3)), ) const cloudArtifactTimeoutMs = Math.max( 5_000, Math.min(60_000, Number(process.env.AUTOCAD_BENCH_CLOUD_ARTIFACT_TIMEOUT_MS || 15_000)), ) const cloudMetadataCache = path.join( os.tmpdir(), `autocad-bench-dashboard-${createHash('sha256') .update(`${cloudBucket}/${cloudPrefix}`) .digest('hex') .slice(0, 16)}`, ) const cloudMetadataConcurrency = Math.max( 1, Math.min(64, Number(process.env.AUTOCAD_BENCH_CLOUD_METADATA_CONCURRENCY || 24)), ) function promoteCloudMetadata(shadowDirectory, destinationDirectory) { if (!fs.existsSync(shadowDirectory)) return const visit = (directory) => { for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { const source = path.join(directory, entry.name) if (entry.isDirectory()) { visit(source) continue } if (!entry.isFile()) continue const relativePath = path.relative(shadowDirectory, source) const destination = path.resolve(destinationDirectory, relativePath) if (!destination.startsWith(path.resolve(destinationDirectory) + path.sep)) continue if (!shouldPromoteMetadata(source, destination)) continue fs.mkdirSync(path.dirname(destination), { recursive: true }) const temporary = `${destination}.cloud-update-${process.pid}-${Date.now()}` try { fs.copyFileSync(source, temporary) fs.renameSync(temporary, destination) } finally { try { if (fs.existsSync(temporary)) fs.unlinkSync(temporary) } catch { // A failed cleanup cannot invalidate a successful atomic promotion. } } } } visit(shadowDirectory) } function wait(milliseconds) { return new Promise((resolve) => setTimeout(resolve, milliseconds)) } async function cloudMetadataLoader({ id, destinationDirectory, relativePaths, maxAgeMs = 0, }) { if (!cloudArtifactsEnabled) return const safeRelativePaths = [...new Set(relativePaths)].filter((relativePath) => { if (!relativePath || relativePath.includes('\\')) return false return !relativePath.split('/').some((segment) => !segment || segment === '.' || segment === '..') }) if (!safeRelativePaths.length) return const downloadKey = `${id}\0${safeRelativePaths.join('\0')}` const refreshedAt = cloudMetadataFreshAt.get(downloadKey) || 0 if (maxAgeMs > 0 && Date.now() - refreshedAt < maxAgeMs) return if (cloudMetadataDownloads.has(downloadKey)) { return cloudMetadataDownloads.get(downloadKey) } const download = (async () => { const pathNamespace = createHash('sha256') .update(safeRelativePaths.join('\0')) .digest('hex') .slice(0, 12) const shadowDirectory = path.resolve(cloudMetadataCache, pathNamespace, id) if (!shadowDirectory.startsWith(path.resolve(cloudMetadataCache) + path.sep)) { throw new Error('Unsafe cloud metadata cache path') } if ( maxAgeMs > 0 && fs.existsSync(shadowDirectory) && Date.now() - fs.statSync(shadowDirectory).mtimeMs < maxAgeMs ) { cloudMetadataFreshAt.set(downloadKey, Date.now()) return } fs.mkdirSync(shadowDirectory, { recursive: true }) let nextIndex = 0 const errors = [] const workers = Array.from({ length: Math.min(cloudMetadataConcurrency, safeRelativePaths.length), }, async () => { while (nextIndex < safeRelativePaths.length) { const relativePath = safeRelativePaths[nextIndex] nextIndex += 1 const destination = path.resolve(shadowDirectory, relativePath) if (!destination.startsWith(path.resolve(shadowDirectory) + path.sep)) continue const key = [cloudPrefix, id, relativePath] .join('/') .replaceAll('\\', '/') .replace(/\/{2,}/g, '/') try { const object = await s3.send(new GetObjectCommand({ Bucket: cloudBucket, Key: key, })) if (!object.Body) continue const bytes = await object.Body.transformToByteArray() fs.mkdirSync(path.dirname(destination), { recursive: true }) const temporary = `${destination}.cloud-download-${process.pid}-${Date.now()}` try { fs.writeFileSync(temporary, bytes) fs.renameSync(temporary, destination) } finally { try { if (fs.existsSync(temporary)) fs.unlinkSync(temporary) } catch { // A failed temporary-file cleanup cannot invalidate the cache. } } } catch (error) { const statusCode = error?.$metadata?.httpStatusCode if (statusCode === 404 || ['NoSuchKey', 'NotFound'].includes(error?.name)) continue errors.push(error) } } }) await Promise.all(workers) promoteCloudMetadata(shadowDirectory, destinationDirectory) if (errors.length) throw errors[0] const refreshed = new Date() fs.utimesSync(shadowDirectory, refreshed, refreshed) cloudMetadataFreshAt.set(downloadKey, refreshed.valueOf()) })() cloudMetadataDownloads.set(downloadKey, download) try { await download } finally { cloudMetadataDownloads.delete(downloadKey) } } async function cloudBenchmarkLoader({ benchmarkId, benchmarkDirectory }) { let state = null try { state = JSON.parse(fs.readFileSync( path.join(benchmarkDirectory, 'batch-state.json'), 'utf8', )) } catch { // A new cloud-only benchmark still needs a blocking first bootstrap. } if (!state) { // Fetch the one authoritative suite index directly. Returning as soon as // this file is present makes a cold benchmark visible immediately; the // rollout metadata is hydrated concurrently below. await cloudMetadataLoader({ id: benchmarkId, destinationDirectory: benchmarkDirectory, relativePaths: ['batch-state.json'], }) try { state = JSON.parse(fs.readFileSync( path.join(benchmarkDirectory, 'batch-state.json'), 'utf8', )) } catch { return } } const terminal = !['starting', 'running'].includes(String(state?.state || '')) const refresh = cloudMetadataLoader({ id: benchmarkId, destinationDirectory: benchmarkDirectory, relativePaths: benchmarkMetadataPaths(state), maxAgeMs: state ? (terminal ? 5 * 60_000 : 10_000) : 0, }) // Benchmark detail is always local-first. The direct metadata GETs continue // in the background and the next 15-second UI refresh promotes the snapshot. void refresh.catch(() => {}) } async function cloudTraceLoader({ runId, runDirectory }) { let state = null try { state = JSON.parse(fs.readFileSync(path.join(runDirectory, 'run-state.json'), 'utf8')) } catch { // A not-yet-mirrored rollout should always attempt its first refresh. } const terminal = [ 'completed', 'recovered', 'budget_exhausted', 'model_failed', 'infrastructure_failed', 'artifact_missing', 'interrupted', ].includes(String(state?.state || state?.execution_status || '')) const refresh = cloudMetadataLoader({ id: runId, destinationDirectory: runDirectory, relativePaths: traceMetadataPaths(), maxAgeMs: state ? (terminal ? 5 * 60_000 : 5_000) : 0, }) if (state) { void refresh.catch(() => {}) return } await refresh } async function cloudArtifactLoader({ runId, relativePath, artifact, expectedSha256 = null, force = false, }) { if (!cloudArtifactsEnabled) return if (!force && fs.existsSync(artifact) && fs.statSync(artifact).isFile()) return if (cloudDownloads.has(artifact)) return cloudDownloads.get(artifact) const download = (async () => { fs.mkdirSync(path.dirname(artifact), { recursive: true }) let lastError = null const key = [cloudPrefix, runId, relativePath] .join('/') .replaceAll('\\', '/') .replace(/\/{2,}/g, '/') for (let attempt = 1; attempt <= cloudArtifactAttempts; attempt += 1) { const temporary = `${artifact}.download-${process.pid}-${Date.now()}-${attempt}` try { const awsArguments = [ 's3', 'cp', `s3://${cloudBucket}/${key}`, temporary, '--only-show-errors', ] if (awsProfile) awsArguments.push('--profile', awsProfile) if (awsRegion) awsArguments.push('--region', awsRegion) await execFileAsync('aws', awsArguments, { timeout: cloudArtifactTimeoutMs, maxBuffer: 1024 * 1024, }) if (expectedSha256) { const actualSha256 = createHash('sha256') .update(fs.readFileSync(temporary)) .digest('hex') if (actualSha256 !== expectedSha256) { throw new Error( `Cloud artifact checksum mismatch for ${relativePath}: expected ${expectedSha256}, got ${actualSha256}`, ) } } fs.renameSync(temporary, artifact) return } catch (error) { lastError = error } finally { try { if (fs.existsSync(temporary)) fs.unlinkSync(temporary) } catch { // A failed cleanup cannot change the artifact response. } } if (attempt < cloudArtifactAttempts) { await wait(150 * attempt * attempt) } } throw lastError || new Error(`Cloud artifact download failed for ${relativePath}`) })() cloudDownloads.set(artifact, download) try { await download } finally { cloudDownloads.delete(artifact) } } async function cloudArtifactUrl({ runId, relativePath }) { if (!cloudArtifactsEnabled) return null const key = [cloudPrefix, runId, relativePath] .join('/') .replaceAll('\\', '/') .replace(/\/{2,}/g, '/') const extension = path.extname(relativePath).toLowerCase() const responseContentType = { '.png': 'image/png', '.json': 'application/json', '.jsonl': 'application/x-ndjson', '.dwg': 'application/acad', }[extension] || 'application/octet-stream' return getSignedUrl( s3, new GetObjectCommand({ Bucket: cloudBucket, Key: key, ResponseCacheControl: 'private, max-age=31536000, immutable', ResponseContentType: responseContentType, }), { expiresIn: signedArtifactLifetimeSeconds }, ) } const api = createTraceApi({ cloudArtifactLoader, cloudArtifactUrl, cloudBenchmarkLoader, cloudTraceLoader, }) const host = process.env.HOST || '127.0.0.1' const port = Number(process.env.PORT || 4173) const mime = { '.css': 'text/css; charset=utf-8', '.html': 'text/html; charset=utf-8', '.js': 'text/javascript; charset=utf-8', '.json': 'application/json; charset=utf-8', '.png': 'image/png', '.svg': 'image/svg+xml', '.woff2': 'font/woff2', } http .createServer(async (request, response) => { if (await api(request, response)) return const pathname = decodeURIComponent(new URL(request.url || '/', `http://${host}`).pathname) const requested = path.resolve(dist, pathname.replace(/^\/+/, '')) const safe = requested.startsWith(dist + path.sep) ? requested : null const filePath = safe && fs.existsSync(safe) && fs.statSync(safe).isFile() ? safe : path.join(dist, 'index.html') response.statusCode = 200 response.setHeader('Content-Type', mime[path.extname(filePath)] || 'application/octet-stream') fs.createReadStream(filePath).pipe(response) }) .listen(port, host, () => { process.stdout.write(`AutoCAD Trace Console: http://${host}:${port}\n`) })