| import fs from 'node:fs' |
| import http from 'node:http' |
| import os from 'node:os' |
| import path from 'node:path' |
| import { execFile } from 'node:child_process' |
| import { createHash } from 'node:crypto' |
| import { fileURLToPath } from 'node:url' |
| import { promisify } from 'node:util' |
| import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3' |
| import { fromIni } from '@aws-sdk/credential-providers' |
| import { getSignedUrl } from '@aws-sdk/s3-request-presigner' |
| import { createTraceApi } from './trace-api.mjs' |
| import { |
| benchmarkMetadataPaths, |
| shouldPromoteMetadata, |
| traceMetadataPaths, |
| } from './cloud-metadata.mjs' |
|
|
| const root = path.dirname(fileURLToPath(import.meta.url)) |
| const dist = path.join(root, 'dist') |
| const execFileAsync = promisify(execFile) |
| const cloudDownloads = new Map() |
| const cloudBucket = process.env.AUTOCAD_BENCH_S3_BUCKET || '' |
| const cloudPrefix = (process.env.AUTOCAD_BENCH_S3_RUNS_PREFIX || 'controller/runs') |
| .replace(/^\/+|\/+$/g, '') |
| const awsProfile = process.env.AWS_PROFILE || '' |
| const awsRegion = process.env.AWS_REGION || '' |
| const cloudArtifactsEnabled = Boolean(cloudBucket) |
| && process.env.AUTOCAD_BENCH_CLOUD_ARTIFACTS !== 'off' |
| const signedArtifactLifetimeSeconds = 300 |
| const s3 = new S3Client({ |
| ...(awsRegion ? { region: awsRegion } : {}), |
| ...(awsProfile ? { credentials: fromIni({ profile: awsProfile }) } : {}), |
| }) |
| const cloudMetadataDownloads = new Map() |
| const cloudMetadataFreshAt = new Map() |
| const cloudArtifactAttempts = Math.max( |
| 1, |
| Math.min(5, Number(process.env.AUTOCAD_BENCH_CLOUD_ARTIFACT_ATTEMPTS || 3)), |
| ) |
| const cloudArtifactTimeoutMs = Math.max( |
| 5_000, |
| Math.min(60_000, Number(process.env.AUTOCAD_BENCH_CLOUD_ARTIFACT_TIMEOUT_MS || 15_000)), |
| ) |
| const cloudMetadataCache = path.join( |
| os.tmpdir(), |
| `autocad-bench-dashboard-${createHash('sha256') |
| .update(`${cloudBucket}/${cloudPrefix}`) |
| .digest('hex') |
| .slice(0, 16)}`, |
| ) |
|
|
| const cloudMetadataConcurrency = Math.max( |
| 1, |
| Math.min(64, Number(process.env.AUTOCAD_BENCH_CLOUD_METADATA_CONCURRENCY || 24)), |
| ) |
|
|
| function promoteCloudMetadata(shadowDirectory, destinationDirectory) { |
| if (!fs.existsSync(shadowDirectory)) return |
| const visit = (directory) => { |
| for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { |
| const source = path.join(directory, entry.name) |
| if (entry.isDirectory()) { |
| visit(source) |
| continue |
| } |
| if (!entry.isFile()) continue |
| const relativePath = path.relative(shadowDirectory, source) |
| const destination = path.resolve(destinationDirectory, relativePath) |
| if (!destination.startsWith(path.resolve(destinationDirectory) + path.sep)) continue |
| if (!shouldPromoteMetadata(source, destination)) continue |
| fs.mkdirSync(path.dirname(destination), { recursive: true }) |
| const temporary = `${destination}.cloud-update-${process.pid}-${Date.now()}` |
| try { |
| fs.copyFileSync(source, temporary) |
| fs.renameSync(temporary, destination) |
| } finally { |
| try { |
| if (fs.existsSync(temporary)) fs.unlinkSync(temporary) |
| } catch { |
| |
| } |
| } |
| } |
| } |
| visit(shadowDirectory) |
| } |
|
|
| function wait(milliseconds) { |
| return new Promise((resolve) => setTimeout(resolve, milliseconds)) |
| } |
|
|
| async function cloudMetadataLoader({ |
| id, |
| destinationDirectory, |
| relativePaths, |
| maxAgeMs = 0, |
| }) { |
| if (!cloudArtifactsEnabled) return |
| const safeRelativePaths = [...new Set(relativePaths)].filter((relativePath) => { |
| if (!relativePath || relativePath.includes('\\')) return false |
| return !relativePath.split('/').some((segment) => !segment || segment === '.' || segment === '..') |
| }) |
| if (!safeRelativePaths.length) return |
| const downloadKey = `${id}\0${safeRelativePaths.join('\0')}` |
| const refreshedAt = cloudMetadataFreshAt.get(downloadKey) || 0 |
| if (maxAgeMs > 0 && Date.now() - refreshedAt < maxAgeMs) return |
| if (cloudMetadataDownloads.has(downloadKey)) { |
| return cloudMetadataDownloads.get(downloadKey) |
| } |
| const download = (async () => { |
| const pathNamespace = createHash('sha256') |
| .update(safeRelativePaths.join('\0')) |
| .digest('hex') |
| .slice(0, 12) |
| const shadowDirectory = path.resolve(cloudMetadataCache, pathNamespace, id) |
| if (!shadowDirectory.startsWith(path.resolve(cloudMetadataCache) + path.sep)) { |
| throw new Error('Unsafe cloud metadata cache path') |
| } |
| if ( |
| maxAgeMs > 0 |
| && fs.existsSync(shadowDirectory) |
| && Date.now() - fs.statSync(shadowDirectory).mtimeMs < maxAgeMs |
| ) { |
| cloudMetadataFreshAt.set(downloadKey, Date.now()) |
| return |
| } |
| fs.mkdirSync(shadowDirectory, { recursive: true }) |
| let nextIndex = 0 |
| const errors = [] |
| const workers = Array.from({ |
| length: Math.min(cloudMetadataConcurrency, safeRelativePaths.length), |
| }, async () => { |
| while (nextIndex < safeRelativePaths.length) { |
| const relativePath = safeRelativePaths[nextIndex] |
| nextIndex += 1 |
| const destination = path.resolve(shadowDirectory, relativePath) |
| if (!destination.startsWith(path.resolve(shadowDirectory) + path.sep)) continue |
| const key = [cloudPrefix, id, relativePath] |
| .join('/') |
| .replaceAll('\\', '/') |
| .replace(/\/{2,}/g, '/') |
| try { |
| const object = await s3.send(new GetObjectCommand({ |
| Bucket: cloudBucket, |
| Key: key, |
| })) |
| if (!object.Body) continue |
| const bytes = await object.Body.transformToByteArray() |
| fs.mkdirSync(path.dirname(destination), { recursive: true }) |
| const temporary = `${destination}.cloud-download-${process.pid}-${Date.now()}` |
| try { |
| fs.writeFileSync(temporary, bytes) |
| fs.renameSync(temporary, destination) |
| } finally { |
| try { |
| if (fs.existsSync(temporary)) fs.unlinkSync(temporary) |
| } catch { |
| |
| } |
| } |
| } catch (error) { |
| const statusCode = error?.$metadata?.httpStatusCode |
| if (statusCode === 404 || ['NoSuchKey', 'NotFound'].includes(error?.name)) continue |
| errors.push(error) |
| } |
| } |
| }) |
| await Promise.all(workers) |
| promoteCloudMetadata(shadowDirectory, destinationDirectory) |
| if (errors.length) throw errors[0] |
| const refreshed = new Date() |
| fs.utimesSync(shadowDirectory, refreshed, refreshed) |
| cloudMetadataFreshAt.set(downloadKey, refreshed.valueOf()) |
| })() |
| cloudMetadataDownloads.set(downloadKey, download) |
| try { |
| await download |
| } finally { |
| cloudMetadataDownloads.delete(downloadKey) |
| } |
| } |
|
|
| async function cloudBenchmarkLoader({ benchmarkId, benchmarkDirectory }) { |
| let state = null |
| try { |
| state = JSON.parse(fs.readFileSync( |
| path.join(benchmarkDirectory, 'batch-state.json'), |
| 'utf8', |
| )) |
| } catch { |
| |
| } |
| if (!state) { |
| |
| |
| |
| await cloudMetadataLoader({ |
| id: benchmarkId, |
| destinationDirectory: benchmarkDirectory, |
| relativePaths: ['batch-state.json'], |
| }) |
| try { |
| state = JSON.parse(fs.readFileSync( |
| path.join(benchmarkDirectory, 'batch-state.json'), |
| 'utf8', |
| )) |
| } catch { |
| return |
| } |
| } |
| const terminal = !['starting', 'running'].includes(String(state?.state || '')) |
| const refresh = cloudMetadataLoader({ |
| id: benchmarkId, |
| destinationDirectory: benchmarkDirectory, |
| relativePaths: benchmarkMetadataPaths(state), |
| maxAgeMs: state ? (terminal ? 5 * 60_000 : 10_000) : 0, |
| }) |
| |
| |
| void refresh.catch(() => {}) |
| } |
|
|
| async function cloudTraceLoader({ runId, runDirectory }) { |
| let state = null |
| try { |
| state = JSON.parse(fs.readFileSync(path.join(runDirectory, 'run-state.json'), 'utf8')) |
| } catch { |
| |
| } |
| const terminal = [ |
| 'completed', 'recovered', 'budget_exhausted', 'model_failed', |
| 'infrastructure_failed', 'artifact_missing', 'interrupted', |
| ].includes(String(state?.state || state?.execution_status || '')) |
| const refresh = cloudMetadataLoader({ |
| id: runId, |
| destinationDirectory: runDirectory, |
| relativePaths: traceMetadataPaths(), |
| maxAgeMs: state ? (terminal ? 5 * 60_000 : 5_000) : 0, |
| }) |
| if (state) { |
| void refresh.catch(() => {}) |
| return |
| } |
| await refresh |
| } |
|
|
| async function cloudArtifactLoader({ |
| runId, |
| relativePath, |
| artifact, |
| expectedSha256 = null, |
| force = false, |
| }) { |
| if (!cloudArtifactsEnabled) return |
| if (!force && fs.existsSync(artifact) && fs.statSync(artifact).isFile()) return |
| if (cloudDownloads.has(artifact)) return cloudDownloads.get(artifact) |
| const download = (async () => { |
| fs.mkdirSync(path.dirname(artifact), { recursive: true }) |
| let lastError = null |
| const key = [cloudPrefix, runId, relativePath] |
| .join('/') |
| .replaceAll('\\', '/') |
| .replace(/\/{2,}/g, '/') |
| for (let attempt = 1; attempt <= cloudArtifactAttempts; attempt += 1) { |
| const temporary = `${artifact}.download-${process.pid}-${Date.now()}-${attempt}` |
| try { |
| const awsArguments = [ |
| 's3', 'cp', `s3://${cloudBucket}/${key}`, temporary, |
| '--only-show-errors', |
| ] |
| if (awsProfile) awsArguments.push('--profile', awsProfile) |
| if (awsRegion) awsArguments.push('--region', awsRegion) |
| await execFileAsync('aws', awsArguments, { |
| timeout: cloudArtifactTimeoutMs, |
| maxBuffer: 1024 * 1024, |
| }) |
| if (expectedSha256) { |
| const actualSha256 = createHash('sha256') |
| .update(fs.readFileSync(temporary)) |
| .digest('hex') |
| if (actualSha256 !== expectedSha256) { |
| throw new Error( |
| `Cloud artifact checksum mismatch for ${relativePath}: expected ${expectedSha256}, got ${actualSha256}`, |
| ) |
| } |
| } |
| fs.renameSync(temporary, artifact) |
| return |
| } catch (error) { |
| lastError = error |
| } finally { |
| try { |
| if (fs.existsSync(temporary)) fs.unlinkSync(temporary) |
| } catch { |
| |
| } |
| } |
| if (attempt < cloudArtifactAttempts) { |
| await wait(150 * attempt * attempt) |
| } |
| } |
| throw lastError || new Error(`Cloud artifact download failed for ${relativePath}`) |
| })() |
| cloudDownloads.set(artifact, download) |
| try { |
| await download |
| } finally { |
| cloudDownloads.delete(artifact) |
| } |
| } |
|
|
| async function cloudArtifactUrl({ runId, relativePath }) { |
| if (!cloudArtifactsEnabled) return null |
| const key = [cloudPrefix, runId, relativePath] |
| .join('/') |
| .replaceAll('\\', '/') |
| .replace(/\/{2,}/g, '/') |
| const extension = path.extname(relativePath).toLowerCase() |
| const responseContentType = { |
| '.png': 'image/png', |
| '.json': 'application/json', |
| '.jsonl': 'application/x-ndjson', |
| '.dwg': 'application/acad', |
| }[extension] || 'application/octet-stream' |
| return getSignedUrl( |
| s3, |
| new GetObjectCommand({ |
| Bucket: cloudBucket, |
| Key: key, |
| ResponseCacheControl: 'private, max-age=31536000, immutable', |
| ResponseContentType: responseContentType, |
| }), |
| { expiresIn: signedArtifactLifetimeSeconds }, |
| ) |
| } |
|
|
| const api = createTraceApi({ |
| cloudArtifactLoader, |
| cloudArtifactUrl, |
| cloudBenchmarkLoader, |
| cloudTraceLoader, |
| }) |
| const host = process.env.HOST || '127.0.0.1' |
| const port = Number(process.env.PORT || 4173) |
|
|
| const mime = { |
| '.css': 'text/css; charset=utf-8', |
| '.html': 'text/html; charset=utf-8', |
| '.js': 'text/javascript; charset=utf-8', |
| '.json': 'application/json; charset=utf-8', |
| '.png': 'image/png', |
| '.svg': 'image/svg+xml', |
| '.woff2': 'font/woff2', |
| } |
|
|
| http |
| .createServer(async (request, response) => { |
| if (await api(request, response)) return |
| const pathname = decodeURIComponent(new URL(request.url || '/', `http://${host}`).pathname) |
| const requested = path.resolve(dist, pathname.replace(/^\/+/, '')) |
| const safe = requested.startsWith(dist + path.sep) ? requested : null |
| const filePath = safe && fs.existsSync(safe) && fs.statSync(safe).isFile() |
| ? safe |
| : path.join(dist, 'index.html') |
| response.statusCode = 200 |
| response.setHeader('Content-Type', mime[path.extname(filePath)] || 'application/octet-stream') |
| fs.createReadStream(filePath).pipe(response) |
| }) |
| .listen(port, host, () => { |
| process.stdout.write(`AutoCAD Trace Console: http://${host}:${port}\n`) |
| }) |
|
|
