import {createRequire} from 'node:module'; import {existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync} from 'node:fs'; import {join} from 'node:path'; import {deleteFiles, downloadFile, uploadFiles} from '@huggingface/hub'; const Database = createRequire('/app/package.json')('better-sqlite3'); const { HF_TOKEN: accessToken, HF_BUCKET_ID = 'hank-jay/9router-storage', OMNI_BUCKET_PREFIX = 'omniroute', OMNI_SYNC_INTERVAL = '120', DATA_DIR = '/app/data', } = process.env; const repo = {type: 'bucket', name: HF_BUCKET_ID}; const dbFile = join(DATA_DIR, 'storage.sqlite'); const remote = (name) => `${OMNI_BUCKET_PREFIX}/${name}`; const log = (...args) => console.log(`[sync ${new Date().toISOString().slice(11, 19)}]`, ...args); const VOLATILE = [ 'call_logs', 'proxy_logs', 'request_detail_logs', 'relay_logs', 'middleware_logs', 'mcp_tool_audit', 'usage_history', 'hourly_usage_summary', 'request_cost_ledger', 'token_ledger', 'routing_decisions', 'compression_analytics', 'compression_cache_stats', 'semantic_cache', 'reasoning_cache', 'radar_feed_cache', 'radar_intel_cache', 'radar_offers_cache', 'radar_referrals_cache', 'inspector_sessions', 'inspector_session_requests', 'job_runs', 'webhook_deliveries', 'session_model_history', 'config_audit_log', 'xp_audit_log', 'domain_cost_history', 'discovery_results', 'plugin_metrics', 'plugin_analytics', 'quota_snapshots', 'provider_quota_reset_events', 'api_key_token_limit_reset_logs', 'free_proxy_sync_errors', 'a2a_task_events', 'server_tool_executions', 'skill_executions', ]; const fetchSnapshot = (name) => downloadFile({repo, path: remote(name), accessToken}).catch((error) => { log(`download ${name} failed:`, error.message); return null; }); const restore = async () => { if (existsSync(dbFile)) return log('local DB exists, skip restore'); const override = await fetchSnapshot('restore.sqlite'); const blob = override ?? (await fetchSnapshot('storage.sqlite')); if (!blob) return log('no snapshot in bucket, starting clean'); mkdirSync(DATA_DIR, {recursive: true}); writeFileSync(dbFile, Buffer.from(await blob.arrayBuffer())); log(`restored ${statSync(dbFile).size} bytes from ${remote(override ? 'restore.sqlite' : 'storage.sqlite')}`); if (override) await deleteFiles({repo, accessToken, paths: [remote('restore.sqlite')]}); }; const compact = (file) => { const db = new Database(file); const tables = new Set(db.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").pluck().all()); db.pragma('foreign_keys = OFF'); for (const table of VOLATILE) if (tables.has(table)) db.exec(`DELETE FROM "${table}"`); db.exec('VACUUM'); const check = db.pragma('integrity_check', {simple: true}); db.close(); if (check !== 'ok') throw new Error(`integrity_check: ${check}`); }; const backup = async () => { if (!existsSync(dbFile)) return log('no local DB yet, skip backup'); const snapshot = join(DATA_DIR, 'storage.sqlite.snapshot'); rmSync(snapshot, {force: true}); const live = new Database(dbFile, {readonly: true, fileMustExist: true}); await live.backup(snapshot); live.close(); compact(snapshot); const content = readFileSync(snapshot); const health = {at: new Date().toISOString(), bytes: content.length, liveBytes: statSync(dbFile).size}; await uploadFiles({ repo, accessToken, files: [ {path: remote('storage.sqlite'), content: new Blob([content])}, {path: remote('health.json'), content: new Blob([JSON.stringify(health)])}, ], }); rmSync(snapshot, {force: true}); log(`backup ok: ${health.bytes} bytes (live ${health.liveBytes})`); }; const loop = async () => { const seconds = Number(OMNI_SYNC_INTERVAL); log(`backup every ${seconds}s to ${HF_BUCKET_ID}/${OMNI_BUCKET_PREFIX}`); for (;;) { await new Promise((done) => setTimeout(done, seconds * 1000)); await backup().catch((error) => log('backup failed:', error.message)); } }; if (!accessToken) { log('HF_TOKEN missing, persistence disabled'); process.exit(0); } await {restore, backup, loop}[process.argv[2]]();