File size: 2,654 Bytes
50f3d57
f561688
 
 
803273b
 
 
 
 
 
89f44e0
5afc3ae
 
abdf565
5afc3ae
abdf565
4faee51
5afc3ae
 
4faee51
5afc3ae
4faee51
 
5afc3ae
 
4faee51
5afc3ae
4faee51
4d5457d
5afc3ae
 
4d5457d
5afc3ae
4d5457d
f561688
 
4b66cfe
50f3d57
f561688
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
// The model part of the WhatDoWeSee Space, free of DOM code so it can be tested outside a browser.

/** Models the page can run. Each loads its files from its own Hub repo at a fixed revision. */
export const MODELS = [
  {
    id: "efficientnetv2_b0-v0.4.0",
    name: "EfficientNetV2-B0 · v0.4.0",
    repo: "StephanMeijer/WhatDoWeSee-EfficientNetV2-B0",
    revision: "v0.4.0",
  },
  {
    id: "mobilenetv4-v0.3.0",
    name: "MobileNetV4 · v0.3.0",
    repo: "StephanMeijer/WhatDoWeSee-MobileNetV4",
    revision: "v0.3.0",
  },
  {
    id: "efficientnet_b0-v0.3.0",
    name: "EfficientNet-B0 · v0.3.0",
    repo: "StephanMeijer/WhatDoWeSee-EfficientNet-B0",
    revision: "v0.3.0",
  },
  {
    id: "mobilenetv4-v0.2",
    name: "MobileNetV4 · v0.2",
    repo: "StephanMeijer/WhatDoWeSee-MobileNetV4",
    revision: "v0.2",
  },
  {
    id: "efficientnet_b0-v0.2",
    name: "EfficientNet-B0 · v0.2",
    repo: "StephanMeijer/WhatDoWeSee-EfficientNet-B0",
    revision: "v0.2",
  },
  {
    id: "efficientnet_b0-v0.1",
    name: "EfficientNet-B0 · v0.1",
    repo: "StephanMeijer/WhatDoWeSee-EfficientNet-B0",
    revision: "v0.1",
  },
];

export function fileUrl(model, file) {
  return `https://huggingface.co/${model.repo}/resolve/${model.revision}/${file}`;
}

/**
 * RGBA pixels already resized to size × size (no crop) to the model input: RGB scaled to [0, 1],
 * normalised with the config's mean and std, NCHW float32.
 */
export function toTensorData(rgba, size, mean, std) {
  const plane = size * size;
  const out = new Float32Array(3 * plane);
  for (let i = 0; i < plane; i++) {
    for (let c = 0; c < 3; c++) {
      out[c * plane + i] = (rgba[i * 4 + c] / 255 - mean[c]) / std[c];
    }
  }
  return out;
}

/** Every label with its probability and threshold, the present ones first, highest first. */
export function tags(logits, config) {
  return config.labels
    .map((id, i) => {
      const p = 1 / (1 + Math.exp(-logits[i]));
      const threshold = config.thresholds[id] ?? 0.5;
      return { id, name: config.names[i], definition: config.definitions[id], p, threshold, present: p >= threshold };
    })
    .sort((a, b) => Number(b.present) - Number(a.present) || b.p - a.p);
}

/** Run the model on prepared pixels; `ort` is onnxruntime-web (or -node in tests). */
export async function run(ort, session, config, rgba) {
  const size = config.input.shape[3];
  const data = toTensorData(rgba, size, config.input.mean, config.input.std);
  const output = await session.run({ [config.input.name]: new ort.Tensor("float32", data, [1, 3, size, size]) });
  return tags(output[config.output.name].data, config);
}