Spaces:

Mike0021
/

open-art-search-api

Paused

open-art-search-api / server /server.mjs

Deploy API for index met-20261007b (259,561 artworks) (part 3)

6ab11ea verified 16 days ago

8.74 kB

// Open Art Search — server-side search API (read-only JSON over HTTP) for agents and scripts.
//   node server/server.mjs        PORT (7860), DATA_DIR (public/data), INDEX_BUILD (the only build present),
//                                 APP_URL (the static app, for self-hosted thumbnails and links), PUBLIC_URL (optional)
// Everything is loaded before the port opens, so a running server always has the full index (and normally the model).
// Query text is processed in memory to rank results and is never logged or stored.
import http from 'node:http';
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { loadCollection } from './collection.mjs';
import { loadEmbedder } from './embed.mjs';
import { ROUTES, openapi } from './routes.mjs';

const here = path.dirname(fileURLToPath(import.meta.url));
const PORT = +process.env.PORT || 7860;
const DATA_DIR = path.resolve(here, '..', process.env.DATA_DIR || 'public/data');
const APP_URL = process.env.APP_URL || 'https://mike0021-open-art-search.static.hf.space/';
const RATE_PER_MIN = +process.env.RATE_PER_MIN || 120; // per client IP
const MAX_QUEUE = 16; // semantic searches waiting for the (serialised) query model before we answer 503

const builds = fs.readdirSync(DATA_DIR).filter((d) => !d.startsWith('.')).sort();
const BUILD = process.env.INDEX_BUILD || (builds.length === 1 ? builds[0] : null);
if (!BUILD || !builds.includes(BUILD)) throw new Error(`set INDEX_BUILD: ${DATA_DIR} has ${builds.join(', ') || 'no builds'}`);

const t0 = performance.now();
const embedder = loadEmbedder();
const { M, tools, indexMs, model } = await loadCollection({ dir: path.join(DATA_DIR, BUILD), appUrl: APP_URL, embedder });
await embedder.catch((e) => console.error('query model failed to load; serving title/artist keyword matching only:', e));
const exampleSearch = await tools.search_artworks.execute({ query: 'a cat sleeping', limit: 2 }); // shown in the docs
console.log(JSON.stringify({ build: BUILD, artworks: M.count, indexMs: Math.round(indexMs), model: model.state, modelLoadMs: Math.round(model.s?.loadMs ?? 0), startupMs: Math.round(performance.now() - t0) }));

const routes = ROUTES.map((r) => ({ ...r, re: new RegExp('^' + r.path.replace('{id}', '([^/]+)') + '/?$') }));
const docsTemplate = fs.readFileSync(path.join(here, 'docs.html'), 'utf8');

function publicUrl(req) {
  if (process.env.PUBLIC_URL) return process.env.PUBLIC_URL.replace(/\/$/, '');
  if (process.env.SPACE_HOST) return `https://${process.env.SPACE_HOST}`; // set by Hugging Face Spaces
  return `${req.headers['x-forwarded-proto'] || 'http'}://${req.headers['x-forwarded-host'] || req.headers.host}`;
}

class HttpError extends Error {
  constructor(status, code, message, headers = {}) { super(message); Object.assign(this, { status, code, headers }); }
}

const withPrefix = (id) => (/^\d{1,9}$/.test(id) ? `met:${id}` : id); // bare Met object IDs are accepted too

// query string -> tool arguments, typed per the tool's input schema (validation itself stays in agent-tools.js)
function argsFrom(route, params, idParam) {
  const props = tools[route.tool].inputSchema.properties;
  const args = {};
  if (idParam != null) {
    let id;
    try { id = decodeURIComponent(idParam); } catch { throw new HttpError(400, 'invalid_argument', 'malformed artwork id'); }
    args.id = withPrefix(id);
  }
  for (const key of new Set(params.keys())) {
    const name = key === 'q' && route.query.includes('query') ? 'query' : key;
    if (!route.query.includes(name)) {
      throw new HttpError(400, 'invalid_argument', `unknown parameter "${key}"` + (route.query.length ? ` (accepted: ${route.query.join(', ')})` : ' (this endpoint takes no query parameters)'));
    }
    const all = params.getAll(key);
    if (all.length > 1 || name in args) throw new HttpError(400, 'invalid_argument', `parameter "${name}" given more than once`);
    const raw = all[0];
    if (raw === '' && name !== 'query') continue; // empty = not set
    const t = props[name].type;
    args[name] = t === 'integer' && /^[+-]?\d+$/.test(raw.trim()) ? Number(raw)
      : t === 'array' ? raw.split(',').map((s) => withPrefix(s.trim())).filter(Boolean) // ids
      : raw;
  }
  return args;
}

const buckets = new Map(); // ip -> { tokens, t }
function rateLimit(ip) {
  const now = Date.now();
  const b = buckets.get(ip) || { tokens: RATE_PER_MIN, t: now };
  b.tokens = Math.min(RATE_PER_MIN, b.tokens + ((now - b.t) / 60000) * RATE_PER_MIN);
  b.t = now;
  buckets.set(ip, b);
  if (b.tokens < 1) throw new HttpError(429, 'rate_limited', `more than ${RATE_PER_MIN} requests per minute`, { 'Retry-After': String(Math.ceil(((1 - b.tokens) * 60) / RATE_PER_MIN)) });
  b.tokens -= 1;
}
setInterval(() => { const old = Date.now() - 120000; for (const [ip, b] of buckets) if (b.t < old) buckets.delete(ip); }, 60000).unref();

let queued = 0;
async function callTool(route, args) {
  const semantic = route.tool === 'search_artworks' && (args.query || '').trim() && model.state === 'ready';
  if (semantic && queued >= MAX_QUEUE) throw new HttpError(503, 'busy', 'too many searches in progress, retry shortly', { 'Retry-After': '2' });
  if (semantic) queued++;
  try { return await tools[route.tool].execute(args); }
  catch (e) {
    if (e.code === 'invalid_argument') throw new HttpError(400, e.code, e.message);
    if (e.code === 'not_found') throw new HttpError(404, e.code, e.message);
    throw e;
  } finally { if (semantic) queued--; }
}

const esc = (s) => String(s).replace(/[&<>"']/g, (c) => `&#${c.charCodeAt(0)};`);
function docsPage(base) {
  const vars = {
    BASE: base, APP: APP_URL, BUILD: M.build, COUNT: M.count.toLocaleString('en-US'), RATE: RATE_PER_MIN,
    EXAMPLE: JSON.stringify(exampleSearch, null, 2),
  };
  return docsTemplate.replace(/\{\{(\w+)\}\}/g, (_, k) => esc(vars[k] ?? ''));
}

const COMMON = { 'Access-Control-Allow-Origin': '*', 'X-Content-Type-Options': 'nosniff', 'Referrer-Policy': 'no-referrer' };
function send(res, status, body, type, headers = {}) {
  res.writeHead(status, { ...COMMON, 'Content-Type': type, ...headers });
  res.end(res.req.method === 'HEAD' ? undefined : body);
}
const sendJSON = (res, status, obj, headers) => send(res, status, JSON.stringify(obj, null, 2) + '\n', 'application/json; charset=utf-8', headers);

async function handle(req, res) {
  const url = new URL(req.url, 'http://x');
  const base = publicUrl(req);
  if (req.method === 'OPTIONS') return send(res, 204, '', 'text/plain', { 'Access-Control-Allow-Methods': 'GET, HEAD, OPTIONS', 'Access-Control-Allow-Headers': '*', 'Access-Control-Max-Age': '86400' });
  if (req.method !== 'GET' && req.method !== 'HEAD') throw new HttpError(405, 'method_not_allowed', 'the API is read-only: use GET', { Allow: 'GET, HEAD, OPTIONS' });
  if (url.pathname === '/' || url.pathname === '/docs') {
    return send(res, 200, docsPage(base), 'text/html; charset=utf-8', {
      'Content-Security-Policy': "default-src 'none'; style-src 'unsafe-inline'; img-src https: data:; base-uri 'none'; form-action 'none'",
      'Cache-Control': 'public, max-age=300',
    });
  }
  if (url.pathname === '/openapi.json') return sendJSON(res, 200, openapi({ tools, M, baseUrl: base, appUrl: APP_URL }), { 'Cache-Control': 'public, max-age=300' });
  if (url.pathname === '/favicon.ico') return send(res, 204, '', 'image/x-icon');
  const ip = String(req.headers['x-forwarded-for'] || req.socket.remoteAddress || '').split(',')[0].trim();
  for (const r of routes) {
    const m = url.pathname.match(r.re);
    if (!m) continue;
    rateLimit(ip);
    return sendJSON(res, 200, await callTool(r, argsFrom(r, url.searchParams, m[1])), { 'Cache-Control': 'no-store' });
  }
  throw new HttpError(404, 'not_found', `no such endpoint ${url.pathname}; see ${base}/openapi.json`);
}

const server = http.createServer(async (req, res) => {
  const t = performance.now();
  try { await handle(req, res); }
  catch (e) {
    const status = e instanceof HttpError ? e.status : 500;
    if (status === 500) console.error(e);
    sendJSON(res, status, { error: { status, code: e.code || 'internal', message: status === 500 ? 'internal error' : e.message }, docs: `${publicUrl(req)}/` }, e.headers);
  } finally {
    // method, path (no query string: queries are not logged), status, duration
    console.log(`${req.method} ${new URL(req.url, 'http://x').pathname.slice(0, 120)} ${res.statusCode} ${Math.round(performance.now() - t)}ms`);
  }
});
server.requestTimeout = 30000;
server.headersTimeout = 15000;
server.listen(PORT, () => console.log(`listening on :${PORT}`));
for (const sig of ['SIGTERM', 'SIGINT']) process.on(sig, () => server.close(() => process.exit(0)));

	// Open Art Search — server-side search API (read-only JSON over HTTP) for agents and scripts.
	// node server/server.mjs PORT (7860), DATA_DIR (public/data), INDEX_BUILD (the only build present),
	// APP_URL (the static app, for self-hosted thumbnails and links), PUBLIC_URL (optional)
	// Everything is loaded before the port opens, so a running server always has the full index (and normally the model).
	// Query text is processed in memory to rank results and is never logged or stored.
	import http from 'node:http';
	import fs from 'node:fs';
	import path from 'node:path';
	import { fileURLToPath } from 'node:url';
	import { loadCollection } from './collection.mjs';
	import { loadEmbedder } from './embed.mjs';
	import { ROUTES, openapi } from './routes.mjs';

	const here = path.dirname(fileURLToPath(import.meta.url));
	const PORT = +process.env.PORT \|\| 7860;
	const DATA_DIR = path.resolve(here, '..', process.env.DATA_DIR \|\| 'public/data');
	const APP_URL = process.env.APP_URL \|\| 'https://mike0021-open-art-search.static.hf.space/';
	const RATE_PER_MIN = +process.env.RATE_PER_MIN \|\| 120; // per client IP
	const MAX_QUEUE = 16; // semantic searches waiting for the (serialised) query model before we answer 503

	const builds = fs.readdirSync(DATA_DIR).filter((d) => !d.startsWith('.')).sort();
	const BUILD = process.env.INDEX_BUILD \|\| (builds.length === 1 ? builds[0] : null);
	if (!BUILD \|\| !builds.includes(BUILD)) throw new Error(`set INDEX_BUILD: ${DATA_DIR} has ${builds.join(', ') \|\| 'no builds'}`);

	const t0 = performance.now();
	const embedder = loadEmbedder();
	const { M, tools, indexMs, model } = await loadCollection({ dir: path.join(DATA_DIR, BUILD), appUrl: APP_URL, embedder });
	await embedder.catch((e) => console.error('query model failed to load; serving title/artist keyword matching only:', e));
	const exampleSearch = await tools.search_artworks.execute({ query: 'a cat sleeping', limit: 2 }); // shown in the docs
	console.log(JSON.stringify({ build: BUILD, artworks: M.count, indexMs: Math.round(indexMs), model: model.state, modelLoadMs: Math.round(model.s?.loadMs ?? 0), startupMs: Math.round(performance.now() - t0) }));

	const routes = ROUTES.map((r) => ({ ...r, re: new RegExp('^' + r.path.replace('{id}', '([^/]+)') + '/?$') }));
	const docsTemplate = fs.readFileSync(path.join(here, 'docs.html'), 'utf8');

	function publicUrl(req) {
	if (process.env.PUBLIC_URL) return process.env.PUBLIC_URL.replace(/\/$/, '');
	if (process.env.SPACE_HOST) return `https://${process.env.SPACE_HOST}`; // set by Hugging Face Spaces
	return `${req.headers['x-forwarded-proto'] \|\| 'http'}://${req.headers['x-forwarded-host'] \|\| req.headers.host}`;
	}

	class HttpError extends Error {
	constructor(status, code, message, headers = {}) { super(message); Object.assign(this, { status, code, headers }); }
	}

	const withPrefix = (id) => (/^\d{1,9}$/.test(id) ? `met:${id}` : id); // bare Met object IDs are accepted too

	// query string -> tool arguments, typed per the tool's input schema (validation itself stays in agent-tools.js)
	function argsFrom(route, params, idParam) {
	const props = tools[route.tool].inputSchema.properties;
	const args = {};
	if (idParam != null) {
	let id;
	try { id = decodeURIComponent(idParam); } catch { throw new HttpError(400, 'invalid_argument', 'malformed artwork id'); }
	args.id = withPrefix(id);
	}
	for (const key of new Set(params.keys())) {
	const name = key === 'q' && route.query.includes('query') ? 'query' : key;
	if (!route.query.includes(name)) {
	throw new HttpError(400, 'invalid_argument', `unknown parameter "${key}"` + (route.query.length ? ` (accepted: ${route.query.join(', ')})` : ' (this endpoint takes no query parameters)'));
	}
	const all = params.getAll(key);
	if (all.length > 1 \|\| name in args) throw new HttpError(400, 'invalid_argument', `parameter "${name}" given more than once`);
	const raw = all[0];
	if (raw === '' && name !== 'query') continue; // empty = not set
	const t = props[name].type;
	args[name] = t === 'integer' && /^[+-]?\d+$/.test(raw.trim()) ? Number(raw)
	: t === 'array' ? raw.split(',').map((s) => withPrefix(s.trim())).filter(Boolean) // ids
	: raw;
	}
	return args;
	}

	const buckets = new Map(); // ip -> { tokens, t }
	function rateLimit(ip) {
	const now = Date.now();
	const b = buckets.get(ip) \|\| { tokens: RATE_PER_MIN, t: now };
	b.tokens = Math.min(RATE_PER_MIN, b.tokens + ((now - b.t) / 60000) * RATE_PER_MIN);
	b.t = now;
	buckets.set(ip, b);
	if (b.tokens < 1) throw new HttpError(429, 'rate_limited', `more than ${RATE_PER_MIN} requests per minute`, { 'Retry-After': String(Math.ceil(((1 - b.tokens) * 60) / RATE_PER_MIN)) });
	b.tokens -= 1;
	}
	setInterval(() => { const old = Date.now() - 120000; for (const [ip, b] of buckets) if (b.t < old) buckets.delete(ip); }, 60000).unref();

	let queued = 0;
	async function callTool(route, args) {
	const semantic = route.tool === 'search_artworks' && (args.query \|\| '').trim() && model.state === 'ready';
	if (semantic && queued >= MAX_QUEUE) throw new HttpError(503, 'busy', 'too many searches in progress, retry shortly', { 'Retry-After': '2' });
	if (semantic) queued++;
	try { return await tools[route.tool].execute(args); }
	catch (e) {
	if (e.code === 'invalid_argument') throw new HttpError(400, e.code, e.message);
	if (e.code === 'not_found') throw new HttpError(404, e.code, e.message);
	throw e;
	} finally { if (semantic) queued--; }
	}

	const esc = (s) => String(s).replace(/[&<>"']/g, (c) => `&#${c.charCodeAt(0)};`);
	function docsPage(base) {
	const vars = {
	BASE: base, APP: APP_URL, BUILD: M.build, COUNT: M.count.toLocaleString('en-US'), RATE: RATE_PER_MIN,
	EXAMPLE: JSON.stringify(exampleSearch, null, 2),
	};
	return docsTemplate.replace(/\{\{(\w+)\}\}/g, (_, k) => esc(vars[k] ?? ''));
	}

	const COMMON = { 'Access-Control-Allow-Origin': '*', 'X-Content-Type-Options': 'nosniff', 'Referrer-Policy': 'no-referrer' };
	function send(res, status, body, type, headers = {}) {
	res.writeHead(status, { ...COMMON, 'Content-Type': type, ...headers });
	res.end(res.req.method === 'HEAD' ? undefined : body);
	}
	const sendJSON = (res, status, obj, headers) => send(res, status, JSON.stringify(obj, null, 2) + '\n', 'application/json; charset=utf-8', headers);

	async function handle(req, res) {
	const url = new URL(req.url, 'http://x');
	const base = publicUrl(req);
	if (req.method === 'OPTIONS') return send(res, 204, '', 'text/plain', { 'Access-Control-Allow-Methods': 'GET, HEAD, OPTIONS', 'Access-Control-Allow-Headers': '*', 'Access-Control-Max-Age': '86400' });
	if (req.method !== 'GET' && req.method !== 'HEAD') throw new HttpError(405, 'method_not_allowed', 'the API is read-only: use GET', { Allow: 'GET, HEAD, OPTIONS' });
	if (url.pathname === '/' \|\| url.pathname === '/docs') {
	return send(res, 200, docsPage(base), 'text/html; charset=utf-8', {
	'Content-Security-Policy': "default-src 'none'; style-src 'unsafe-inline'; img-src https: data:; base-uri 'none'; form-action 'none'",
	'Cache-Control': 'public, max-age=300',
	});
	}
	if (url.pathname === '/openapi.json') return sendJSON(res, 200, openapi({ tools, M, baseUrl: base, appUrl: APP_URL }), { 'Cache-Control': 'public, max-age=300' });
	if (url.pathname === '/favicon.ico') return send(res, 204, '', 'image/x-icon');
	const ip = String(req.headers['x-forwarded-for'] \|\| req.socket.remoteAddress \|\| '').split(',')[0].trim();
	for (const r of routes) {
	const m = url.pathname.match(r.re);
	if (!m) continue;
	rateLimit(ip);
	return sendJSON(res, 200, await callTool(r, argsFrom(r, url.searchParams, m[1])), { 'Cache-Control': 'no-store' });
	}
	throw new HttpError(404, 'not_found', `no such endpoint ${url.pathname}; see ${base}/openapi.json`);
	}

	const server = http.createServer(async (req, res) => {
	const t = performance.now();
	try { await handle(req, res); }
	catch (e) {
	const status = e instanceof HttpError ? e.status : 500;
	if (status === 500) console.error(e);
	sendJSON(res, status, { error: { status, code: e.code \|\| 'internal', message: status === 500 ? 'internal error' : e.message }, docs: `${publicUrl(req)}/` }, e.headers);
	} finally {
	// method, path (no query string: queries are not logged), status, duration
	console.log(`${req.method} ${new URL(req.url, 'http://x').pathname.slice(0, 120)} ${res.statusCode} ${Math.round(performance.now() - t)}ms`);
	}
	});
	server.requestTimeout = 30000;
	server.headersTimeout = 15000;
	server.listen(PORT, () => console.log(`listening on :${PORT}`));
	for (const sig of ['SIGTERM', 'SIGINT']) process.on(sig, () => server.close(() => process.exit(0)));