| import {createRequire} from 'node:module'; |
| import {existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync} from 'node:fs'; |
| import {join} from 'node:path'; |
| import {deleteFiles, downloadFile, uploadFiles} from '@huggingface/hub'; |
|
|
| const Database = createRequire('/app/package.json')('better-sqlite3'); |
|
|
| const { |
| HF_TOKEN: accessToken, |
| HF_BUCKET_ID = 'hank-jay/9router-storage', |
| OMNI_BUCKET_PREFIX = 'omniroute', |
| OMNI_SYNC_INTERVAL = '120', |
| DATA_DIR = '/app/data', |
| } = process.env; |
|
|
| const repo = {type: 'bucket', name: HF_BUCKET_ID}; |
| const dbFile = join(DATA_DIR, 'storage.sqlite'); |
| const remote = (name) => `${OMNI_BUCKET_PREFIX}/${name}`; |
| const log = (...args) => console.log(`[sync ${new Date().toISOString().slice(11, 19)}]`, ...args); |
|
|
| const VOLATILE = [ |
| 'call_logs', 'proxy_logs', 'request_detail_logs', 'relay_logs', 'middleware_logs', 'mcp_tool_audit', |
| 'usage_history', 'hourly_usage_summary', 'request_cost_ledger', 'token_ledger', 'routing_decisions', |
| 'compression_analytics', 'compression_cache_stats', 'semantic_cache', 'reasoning_cache', |
| 'radar_feed_cache', 'radar_intel_cache', 'radar_offers_cache', 'radar_referrals_cache', |
| 'inspector_sessions', 'inspector_session_requests', 'job_runs', 'webhook_deliveries', |
| 'session_model_history', 'config_audit_log', 'xp_audit_log', 'domain_cost_history', |
| 'discovery_results', 'plugin_metrics', 'plugin_analytics', 'quota_snapshots', |
| 'provider_quota_reset_events', 'api_key_token_limit_reset_logs', 'free_proxy_sync_errors', |
| 'a2a_task_events', 'server_tool_executions', 'skill_executions', |
| ]; |
|
|
| const fetchSnapshot = (name) => |
| downloadFile({repo, path: remote(name), accessToken}).catch((error) => { |
| log(`download ${name} failed:`, error.message); |
| return null; |
| }); |
|
|
| const restore = async () => { |
| if (existsSync(dbFile)) return log('local DB exists, skip restore'); |
|
|
| const override = await fetchSnapshot('restore.sqlite'); |
| const blob = override ?? (await fetchSnapshot('storage.sqlite')); |
| if (!blob) return log('no snapshot in bucket, starting clean'); |
|
|
| mkdirSync(DATA_DIR, {recursive: true}); |
| writeFileSync(dbFile, Buffer.from(await blob.arrayBuffer())); |
| log(`restored ${statSync(dbFile).size} bytes from ${remote(override ? 'restore.sqlite' : 'storage.sqlite')}`); |
|
|
| if (override) await deleteFiles({repo, accessToken, paths: [remote('restore.sqlite')]}); |
| }; |
|
|
| const compact = (file) => { |
| const db = new Database(file); |
| const tables = new Set(db.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").pluck().all()); |
| db.pragma('foreign_keys = OFF'); |
| for (const table of VOLATILE) if (tables.has(table)) db.exec(`DELETE FROM "${table}"`); |
| db.exec('VACUUM'); |
| const check = db.pragma('integrity_check', {simple: true}); |
| db.close(); |
| if (check !== 'ok') throw new Error(`integrity_check: ${check}`); |
| }; |
|
|
| const backup = async () => { |
| if (!existsSync(dbFile)) return log('no local DB yet, skip backup'); |
|
|
| const snapshot = join(DATA_DIR, 'storage.sqlite.snapshot'); |
| rmSync(snapshot, {force: true}); |
| const live = new Database(dbFile, {readonly: true, fileMustExist: true}); |
| await live.backup(snapshot); |
| live.close(); |
| compact(snapshot); |
|
|
| const content = readFileSync(snapshot); |
| const health = {at: new Date().toISOString(), bytes: content.length, liveBytes: statSync(dbFile).size}; |
| await uploadFiles({ |
| repo, |
| accessToken, |
| files: [ |
| {path: remote('storage.sqlite'), content: new Blob([content])}, |
| {path: remote('health.json'), content: new Blob([JSON.stringify(health)])}, |
| ], |
| }); |
| rmSync(snapshot, {force: true}); |
| log(`backup ok: ${health.bytes} bytes (live ${health.liveBytes})`); |
| }; |
|
|
| const loop = async () => { |
| const seconds = Number(OMNI_SYNC_INTERVAL); |
| log(`backup every ${seconds}s to ${HF_BUCKET_ID}/${OMNI_BUCKET_PREFIX}`); |
| for (;;) { |
| await new Promise((done) => setTimeout(done, seconds * 1000)); |
| await backup().catch((error) => log('backup failed:', error.message)); |
| } |
| }; |
|
|
| if (!accessToken) { |
| log('HF_TOKEN missing, persistence disabled'); |
| process.exit(0); |
| } |
|
|
| await {restore, backup, loop}[process.argv[2]](); |
|
|
