Spaces:
Running
Running
File size: 3,547 Bytes
8b916da | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | /* Next Match compute core: port of match_core.py (tokenizer, text/image encoders, fusion, ranking). */
(function (root, factory) {
if (typeof module === "object" && module.exports) module.exports = factory();
else root.Core = factory();
})(typeof self !== "undefined" ? self : this, function () {
"use strict";
const ASSETS = ["model.json", "catalog.json"];
function lowerTr(s) { return s.replace(/I/g, "ı").replace(/İ/g, "i").toLowerCase(); }
function tokenize(text, vocab) {
const clean = Array.from(lowerTr(text).replace(/-/g, "")).map(ch => (/[\p{L}\p{N}]/u.test(ch) || ch === " " ? ch : " ")).join("");
const ids = [];
for (const w of clean.split(/\s+/).filter(Boolean)) {
if (w in vocab) { ids.push(vocab[w]); continue; }
const chars = Array.from(w);
for (let k = chars.length - 1; k > 3; k--) { const p = chars.slice(0, k).join(""); if (p in vocab) { ids.push(vocab[p]); break; } }
}
return ids;
}
function textVector(ids, emb) {
const [V, D] = emb.shape, out = new Float64Array(D);
if (!ids.length) return out;
for (const i of ids) for (let d = 0; d < D; d++) out[d] += emb.data[i * D + d];
for (let d = 0; d < D; d++) out[d] /= ids.length;
return out;
}
function l2n(x) { let s = 0; for (const v of x) s += v * v; const n = Math.max(Math.sqrt(s), 1e-12); return Float64Array.from(x, v => v / n); }
function init(assets) {
const m = NLRT.decode(assets["model.json"]), cat = assets["catalog.json"];
const keys = m.tensors.gallery_keys;
return { m, vocab: m.meta.vocab, cat, keys, D: keys.shape[1] };
}
function imageFeatures(ctx, pixels, net) { // pixels: Float64Array 3*48*48 in [0,1] (CHW)
return NLRT.forward(ctx.m.nets[net], NLRT.T([3, 48, 48], pixels)).data;
}
function encodeQuery(ctx, pixels, text, mode) {
const n = ctx.m.nets, t = ctx.m.tensors;
if (mode === "combined") {
const fi = imageFeatures(ctx, pixels, "img");
const ft = NLRT.forward(n.txt, NLRT.T([t.emb.shape[1]], textVector(tokenize(text, ctx.vocab), t.emb))).data;
const cat = new Float64Array(fi.length + ft.length); cat.set(fi); cat.set(ft, fi.length);
const delta = NLRT.forward(n.fuse, NLRT.T([cat.length], cat)).data;
return l2n(fi.map((v, i) => v + delta[i]));
}
if (mode === "image") {
const fi = imageFeatures(ctx, pixels, "img_only");
return l2n(NLRT.forward(n.img_only_head, NLRT.T([fi.length], fi)).data);
}
return l2n(NLRT.forward(n.txt_only, NLRT.T([t.emb_only.shape[1]], textVector(tokenize(text, ctx.vocab), t.emb_only))).data);
}
function rank(ctx, q, mode) {
const K = mode === "image" ? ctx.m.tensors.gallery_keys_img_only : mode === "text" ? ctx.m.tensors.gallery_keys_txt_only : ctx.keys;
const N = K.shape[0], D = K.shape[1], s = new Float64Array(N);
for (let i = 0; i < N; i++) { let a = 0; for (let d = 0; d < D; d++) a += K.data[i * D + d] * q[d]; s[i] = a; }
return Array.from({ length: N }, (_, i) => i).sort((a, b) => (s[b] - s[a]) || (a - b)).map(i => [i, s[i]]);
}
function runCase(ctx, input) {
const px = Float64Array.from(input.pixels, v => v / 255);
const out = {};
for (const mode of ["combined", "image", "text"]) {
const q = encodeQuery(ctx, px, input.text, mode);
out[mode] = rank(ctx, q, mode).slice(0, 5).map(r => r[0]);
out[mode + "_q0"] = q[0];
}
out.tokens = tokenize(input.text, ctx.vocab);
return out;
}
return { ASSETS, init, tokenize, encodeQuery, rank, runCase };
});
|