ai999x-encoder (ai999x-encoder-3)

README.md

@@ -8,15 +8,26 @@ tags:
 - sentence-similarity
 - feature-extraction
 - semantic-search
+- text-classification
 - onnx
 - safetensors
 - capability-search
+- question-routing
 - science
-- physics
 - mathematics
+- engineering
+- data-science-machine-learning
+- economics-finance
+- engineering
+- geospatial
+- life-sciences
+- mathematics-statistics
+- physical-sciences
 - tiny
 - cpu
 - browser
+- edge-ai
+- offline
 - ai999x
 model-index:
 - name: ai999x-encoder
@@ -25,87 +36,135 @@ model-index:
       type: sentence-similarity
       name: capability search
     dataset:
-      name: "AI999X held-out verified questions: physics"
+      name: "AI999X held-out verified questions, all models"
       type: ai999x-evaluation
     metrics:
     - type: recall_at_1
-      value: 0.953
+      value: 0.918
       name: right capability first
     - type: recall_at_5
-      value: 0.991
+      value: 0.988
       name: right capability in the top five
   - task:
       type: sentence-similarity
       name: capability search
     dataset:
-      name: "AI999X held-out verified questions: mechanics"
+      name: "AI999X stress set: 12 wording styles, never trained on"
       type: ai999x-evaluation
     metrics:
     - type: recall_at_1
-      value: 0.8
+      value: 0.892
       name: right capability first
     - type: recall_at_5
-      value: 1.0
-      name: right capability in the top five
-  - task:
-      type: sentence-similarity
-      name: capability search
-    dataset:
-      name: "AI999X held-out verified questions: algebra"
-      type: ai999x-evaluation
-    metrics:
-    - type: recall_at_1
-      value: 0.732
-      name: right capability first
-    - type: recall_at_5
-      value: 0.917
-      name: right capability in the top five
-  - task:
-      type: sentence-similarity
-      name: capability search
-    dataset:
-      name: "AI999X held-out verified questions: overall"
-      type: ai999x-evaluation
-    metrics:
-    - type: recall_at_1
-      value: 0.89
-      name: right capability first
-    - type: recall_at_5
-      value: 0.977
+      value: 0.988
       name: right capability in the top five
 ---
 
-# ai999x-encoder — a 4.9 MB capability encoder for science and mathematics questions
+# ai999x-encoder — a 4.9 MB encoder that finds the right capability for a science, mathematics or engineering question
 
-**[AI999X on Hugging Face](https://huggingface.co/AI999X)** · **[Capability orb (runs this model in your browser)](https://www.ai999x.com/orb/)** · **[Website](https://www.ai999x.com)**
+**[AI999X on Hugging Face](https://huggingface.co/AI999X)** · **[Capability orb](https://www.ai999x.com/orb/)** · **[Website](https://www.ai999x.com/models/ai999x-encoder/)**
 
-License: Apache-2.0 · Author: Ibnul Jaif Farabi (creator and engineer) · Organization: AI999X · 4,715,520 parameters
+License: Apache-2.0 · Author: Ibnul Jaif Farabi (creator and engineer) · Organization: AI999X · 4,715,520 parameters · version 3
 
-ai999x-encoder turns a question written in plain English into a vector and finds which certified AI999X capability it is about — a formula, a procedure, a law — across every AI999X model at once. It is small enough to run offline on any CPU and in the browser, and it powers `search()` in every AI999X Python package and the [capability orb](https://www.ai999x.com/orb/).
+ai999x-encoder turns a question written in plain English into a 256-number vector and finds which AI999X capability it is about — a formula, a procedure, an algorithm — among **2,696 capabilities of 31 models** at once. It is small enough to run offline on any CPU, on a phone and in the browser, and it powers `search()` in every AI999X package and the search on the capability orb.
 
 ## Results
 
-On 776 held-out questions it never saw in training (verified questions of the AI999X physics, mechanics and algebra models; the task: put the right one of 171 capabilities first):
+**Held-out questions.** 12,998 verified questions it never saw in training, across all 31 models; the task: put the right one of 2,696 capabilities first.
 
 | Model | Right capability first | In the top five |
 |---|---|---|
-| **ai999x-encoder (4.7M parameters)** | 0.890 | 0.977 |
-| A commercial embedding API (its teacher) | 0.838 | 0.963 |
-| bge-small-en-v1.5 (33M parameters) | 0.705 | 0.876 |
+| **ai999x-encoder v3 (4.7M parameters, 4.9 MB)** | **91.8%** | **98.8%** |
+| A commercial embedding API (its teacher) | 85.0% | 97.6% |
+| bge-small-en-v1.5 (33M parameters) | 80.1% | 95.0% |
 
-| Domain | first | top five |
+**Stress set.** 10,800 questions from 27 models, written in 12 different styles (beginner, expert, story, terse, two-part, negation, …) and never used for training or tuning.
+
+| Model | Right capability first | In the top five |
 |---|---|---|
-| physics | 0.953 | 0.991 |
-| mechanics | 0.800 | 1.000 |
-| algebra | 0.732 | 0.917 |
+| **ai999x-encoder v3** | **89.2%** | **98.8%** |
+| ai999x-encoder v2 (previous version) | 50.9% | 73.8% |
+
+<details><summary>By wording style (v3)</summary>
+
+| Style | First | Top five |
+|---|---|---|
+| asked-first | 93.8% | 100.0% |
+| asked-last | 90.9% | 99.3% |
+| beginner | 78.8% | 94.6% |
+| comparison | 88.6% | 99.1% |
+| expert | 88.5% | 98.9% |
+| imperative | 92.4% | 99.8% |
+| list-like | 91.5% | 99.4% |
+| negation | 91.0% | 100.0% |
+| story | 87.9% | 98.6% |
+| terse | 87.4% | 98.2% |
+| two-part | 91.6% | 98.9% |
+| verbose | 87.3% | 98.7% |
+
+</details>
+
+**By field** (held-out questions)
+
+| Field | Questions | First | Top five |
+|---|---:|---:|---:|
+| Mathematics & Statistics | 4,089 | 88.4% | 97.3% |
+| Physical Sciences | 2,721 | 96.1% | 100.0% |
+| Engineering | 4,006 | 94.9% | 99.8% |
+| Life Sciences | 803 | 94.1% | 99.9% |
+| Economics & Finance | 416 | 86.3% | 98.3% |
+| Geospatial | 448 | 78.8% | 95.1% |
+| Data Science & Machine Learning | 515 | 85.0% | 97.5% |
+
+<details><summary>By model (held-out questions)</summary>
+
+| Model | Field | First | Top five |
+|---|---|---:|---:|
+| [bayesian-statistics-ai999x](https://huggingface.co/AI999X/bayesian-statistics-ai999x) | Mathematics & Statistics | 94.9% | 99.2% |
+| [calculus-ai999x](https://huggingface.co/AI999X/calculus-ai999x) | Mathematics & Statistics | 95.5% | 100.0% |
+| [differential-equations-ai999x](https://huggingface.co/AI999X/differential-equations-ai999x) | Mathematics & Statistics | 93.1% | 99.5% |
+| [discrete-math-ai999x](https://huggingface.co/AI999X/discrete-math-ai999x) | Mathematics & Statistics | 88.3% | 98.2% |
+| [geometry-ai999x](https://huggingface.co/AI999X/geometry-ai999x) | Mathematics & Statistics | 88.9% | 98.3% |
+| [linear-algebra-ai999x](https://huggingface.co/AI999X/linear-algebra-ai999x) | Mathematics & Statistics | 95.0% | 99.5% |
+| [math-foundations-ai999x](https://huggingface.co/AI999X/math-foundations-ai999x) | Mathematics & Statistics | 88.1% | 98.5% |
+| [optimization-ai999x](https://huggingface.co/AI999X/optimization-ai999x) | Mathematics & Statistics | 93.2% | 99.4% |
+| [statistics-ai999x](https://huggingface.co/AI999X/statistics-ai999x) | Mathematics & Statistics | 83.3% | 97.0% |
+| [time-series-ai999x](https://huggingface.co/AI999X/time-series-ai999x) | Mathematics & Statistics | 64.2% | 82.3% |
+| [astronomy-ai999x](https://huggingface.co/AI999X/astronomy-ai999x) | Physical Sciences | 97.9% | 100.0% |
+| [chemistry-ai999x](https://huggingface.co/AI999X/chemistry-ai999x) | Physical Sciences | 92.0% | 100.0% |
+| [classical-physics-ai999x](https://huggingface.co/AI999X/classical-physics-ai999x) | Physical Sciences | 98.2% | 100.0% |
+| [earth-science-ai999x](https://huggingface.co/AI999X/earth-science-ai999x) | Physical Sciences | 96.2% | 100.0% |
+| [materials-science-ai999x](https://huggingface.co/AI999X/materials-science-ai999x) | Physical Sciences | 94.6% | 100.0% |
+| [quantum-physics-ai999x](https://huggingface.co/AI999X/quantum-physics-ai999x) | Physical Sciences | 98.3% | 99.7% |
+| [aerospace-engineering-ai999x](https://huggingface.co/AI999X/aerospace-engineering-ai999x) | Engineering | 94.5% | 100.0% |
+| [chemical-engineering-ai999x](https://huggingface.co/AI999X/chemical-engineering-ai999x) | Engineering | 95.9% | 99.8% |
+| [civil-engineering-ai999x](https://huggingface.co/AI999X/civil-engineering-ai999x) | Engineering | 94.2% | 99.8% |
+| [electrical-engineering-ai999x](https://huggingface.co/AI999X/electrical-engineering-ai999x) | Engineering | 94.4% | 100.0% |
+| [energy-systems-ai999x](https://huggingface.co/AI999X/energy-systems-ai999x) | Engineering | 88.1% | 100.0% |
+| [mechanical-engineering-ai999x](https://huggingface.co/AI999X/mechanical-engineering-ai999x) | Engineering | 96.9% | 99.8% |
+| [nuclear-engineering-ai999x](https://huggingface.co/AI999X/nuclear-engineering-ai999x) | Engineering | 97.1% | 100.0% |
+| [signal-processing-ai999x](https://huggingface.co/AI999X/signal-processing-ai999x) | Engineering | 93.0% | 98.9% |
+| [thermofluids-ai999x](https://huggingface.co/AI999X/thermofluids-ai999x) | Engineering | 97.4% | 100.0% |
+| [transportation-ai999x](https://huggingface.co/AI999X/transportation-ai999x) | Engineering | 96.0% | 100.0% |
+| [biomedical-science-ai999x](https://huggingface.co/AI999X/biomedical-science-ai999x) | Life Sciences | 93.4% | 99.8% |
+| [computational-biology-ai999x](https://huggingface.co/AI999X/computational-biology-ai999x) | Life Sciences | 94.9% | 100.0% |
+| [economics-finance-ai999x](https://huggingface.co/AI999X/economics-finance-ai999x) | Economics & Finance | 86.3% | 98.3% |
+| [geospatial-analytics-ai999x](https://huggingface.co/AI999X/geospatial-analytics-ai999x) | Geospatial | 78.8% | 95.1% |
+| [data-science-ml-ai999x](https://huggingface.co/AI999X/data-science-ml-ai999x) | Data Science & Machine Learning | 85.0% | 97.5% |
+
+</details>
+
+## How it was made
+
+Its own 8,000-token vocabulary was learned from AI999X's science and mathematics texts. A 4-layer, 256-wide encoder first learned general meaning from a larger embedding model's vectors of those texts, then learned the task itself: a question close to its own capability's description and far from the others. The held-out and stress questions were removed from every training stage, the vocabulary included.
 
 ## Use it
 
 The easiest way is through any AI999X package:
 
 ```python
-# pip install "physics-ai999x[search]"
-import physics_ai999x as physics
+# pip install "classical-physics-ai999x[search]"
+import classical_physics_ai999x as physics
 physics.search("How long does a pendulum take to swing back and forth?", k=3)
 ```
 
@@ -122,6 +181,24 @@ v = sess.run(None, {"input_ids": np.array([e.ids], dtype=np.int64),
 # v is a unit vector (256 numbers); capability_index.json holds the vectors of every capability
 ```
 
+In the browser with onnxruntime-web (this is how the capability orb searches):
+
+```js
+import * as ort from 'onnxruntime-web';
+const session = await ort.InferenceSession.create('onnx/encoder.int8.onnx');
+// ids and mask from tokenizer.json (WordPiece, lower-cased, max 96 tokens)
+const out = await session.run({ input_ids: new ort.Tensor('int64', ids, [1, ids.length]),
+                                attention_mask: new ort.Tensor('int64', mask, [1, mask.length]) });
+// out[Object.keys(out)[0]].data: the 256-number vector; rank capability_index.json by dot product
+```
+
+## Versions
+
+| Version | Covers | Held-out: first / top five |
+|---|---|---|
+| **3** (current) | 2,696 capabilities, 31 models | 91.8% / 98.8% |
+| 2 | 171 capabilities of the first 3 models | 89.0% / 97.7% (on those models only) |
+
 ## Files
 
 | File | What it is |

evaluation/report.json

@@ -5,143 +5,1151 @@
  "hidden": 256,
  "layers": 4,
  "device": "cuda",
- "train_minutes": 4.0,
+ "train_minutes": 24.9,
  "student": {
-  "physics": {
+  "bayesian_statistics": {
    "within_domain": {
-    "questions": 529,
-    "recall@1": 0.953,
+    "questions": 375,
+    "recall@1": 0.965,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 375,
+    "recall@1": 0.949,
+    "recall@5": 0.992
+   }
+  },
+  "calculus": {
+   "within_domain": {
+    "questions": 355,
+    "recall@1": 0.966,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 355,
+    "recall@1": 0.955,
+    "recall@5": 1.0
+   }
+  },
+  "differential_equations": {
+   "within_domain": {
+    "questions": 376,
+    "recall@1": 0.949,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 376,
+    "recall@1": 0.931,
+    "recall@5": 0.995
+   }
+  },
+  "discrete_math": {
+   "within_domain": {
+    "questions": 488,
+    "recall@1": 0.895,
+    "recall@5": 0.986
+   },
+   "all_domains": {
+    "questions": 488,
+    "recall@1": 0.883,
+    "recall@5": 0.982
+   }
+  },
+  "geometry": {
+   "within_domain": {
+    "questions": 361,
+    "recall@1": 0.903,
     "recall@5": 0.994
    },
    "all_domains": {
-    "questions": 529,
-    "recall@1": 0.953,
+    "questions": 361,
+    "recall@1": 0.889,
+    "recall@5": 0.983
+   }
+  },
+  "linear_algebra": {
+   "within_domain": {
+    "questions": 441,
+    "recall@1": 0.966,
+    "recall@5": 0.998
+   },
+   "all_domains": {
+    "questions": 441,
+    "recall@1": 0.95,
+    "recall@5": 0.995
+   }
+  },
+  "math_foundations": {
+   "within_domain": {
+    "questions": 470,
+    "recall@1": 0.906,
     "recall@5": 0.991
+   },
+   "all_domains": {
+    "questions": 470,
+    "recall@1": 0.881,
+    "recall@5": 0.985
+   }
+  },
+  "optimization": {
+   "within_domain": {
+    "questions": 336,
+    "recall@1": 0.94,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 336,
+    "recall@1": 0.932,
+    "recall@5": 0.994
    }
   },
-  "mechanics": {
+  "statistics": {
    "within_domain": {
-    "questions": 90,
+    "questions": 532,
+    "recall@1": 0.848,
+    "recall@5": 0.97
+   },
+   "all_domains": {
+    "questions": 532,
     "recall@1": 0.833,
+    "recall@5": 0.97
+   }
+  },
+  "time_series": {
+   "within_domain": {
+    "questions": 355,
+    "recall@1": 0.651,
+    "recall@5": 0.839
+   },
+   "all_domains": {
+    "questions": 355,
+    "recall@1": 0.642,
+    "recall@5": 0.823
+   }
+  },
+  "astronomy": {
+   "within_domain": {
+    "questions": 335,
+    "recall@1": 0.982,
     "recall@5": 1.0
    },
    "all_domains": {
-    "questions": 90,
-    "recall@1": 0.8,
+    "questions": 335,
+    "recall@1": 0.979,
     "recall@5": 1.0
    }
   },
-  "algebra": {
+  "chemistry": {
    "within_domain": {
-    "questions": 157,
-    "recall@1": 0.783,
-    "recall@5": 0.949
+    "questions": 511,
+    "recall@1": 0.924,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 511,
+    "recall@1": 0.92,
+    "recall@5": 1.0
+   }
+  },
+  "classical_physics": {
+   "within_domain": {
+    "questions": 670,
+    "recall@1": 0.991,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 670,
+    "recall@1": 0.982,
+    "recall@5": 1.0
+   }
+  },
+  "earth_science": {
+   "within_domain": {
+    "questions": 420,
+    "recall@1": 0.976,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 420,
+    "recall@1": 0.962,
+    "recall@5": 1.0
+   }
+  },
+  "materials_science": {
+   "within_domain": {
+    "questions": 424,
+    "recall@1": 0.962,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 424,
+    "recall@1": 0.946,
+    "recall@5": 1.0
+   }
+  },
+  "quantum_physics": {
+   "within_domain": {
+    "questions": 361,
+    "recall@1": 0.983,
+    "recall@5": 0.997
+   },
+   "all_domains": {
+    "questions": 361,
+    "recall@1": 0.983,
+    "recall@5": 0.997
+   }
+  },
+  "aerospace_engineering": {
+   "within_domain": {
+    "questions": 397,
+    "recall@1": 0.947,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 397,
+    "recall@1": 0.945,
+    "recall@5": 1.0
+   }
+  },
+  "chemical_engineering": {
+   "within_domain": {
+    "questions": 435,
+    "recall@1": 0.959,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 435,
+    "recall@1": 0.959,
+    "recall@5": 0.998
+   }
+  },
+  "civil_engineering": {
+   "within_domain": {
+    "questions": 415,
+    "recall@1": 0.945,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 415,
+    "recall@1": 0.942,
+    "recall@5": 0.998
+   }
+  },
+  "electrical_engineering": {
+   "within_domain": {
+    "questions": 301,
+    "recall@1": 0.973,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 301,
+    "recall@1": 0.944,
+    "recall@5": 1.0
+   }
+  },
+  "energy_systems": {
+   "within_domain": {
+    "questions": 394,
+    "recall@1": 0.886,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 394,
+    "recall@1": 0.881,
+    "recall@5": 1.0
+   }
+  },
+  "mechanical_engineering": {
+   "within_domain": {
+    "questions": 426,
+    "recall@1": 0.977,
+    "recall@5": 0.998
+   },
+   "all_domains": {
+    "questions": 426,
+    "recall@1": 0.969,
+    "recall@5": 0.998
+   }
+  },
+  "nuclear_engineering": {
+   "within_domain": {
+    "questions": 414,
+    "recall@1": 0.971,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 414,
+    "recall@1": 0.971,
+    "recall@5": 1.0
+   }
+  },
+  "signal_processing": {
+   "within_domain": {
+    "questions": 357,
+    "recall@1": 0.933,
+    "recall@5": 0.997
+   },
+   "all_domains": {
+    "questions": 357,
+    "recall@1": 0.93,
+    "recall@5": 0.989
+   }
+  },
+  "thermofluids": {
+   "within_domain": {
+    "questions": 495,
+    "recall@1": 0.984,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 495,
+    "recall@1": 0.974,
+    "recall@5": 1.0
+   }
+  },
+  "transportation": {
+   "within_domain": {
+    "questions": 372,
+    "recall@1": 0.965,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 372,
+    "recall@1": 0.96,
+    "recall@5": 1.0
+   }
+  },
+  "biomedical_science": {
+   "within_domain": {
+    "questions": 410,
+    "recall@1": 0.944,
+    "recall@5": 0.998
    },
    "all_domains": {
-    "questions": 157,
-    "recall@1": 0.732,
-    "recall@5": 0.917
+    "questions": 410,
+    "recall@1": 0.934,
+    "recall@5": 0.998
+   }
+  },
+  "computational_biology": {
+   "within_domain": {
+    "questions": 393,
+    "recall@1": 0.952,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 393,
+    "recall@1": 0.949,
+    "recall@5": 1.0
+   }
+  },
+  "economics_finance": {
+   "within_domain": {
+    "questions": 416,
+    "recall@1": 0.877,
+    "recall@5": 0.99
+   },
+   "all_domains": {
+    "questions": 416,
+    "recall@1": 0.863,
+    "recall@5": 0.983
+   }
+  },
+  "geospatial_analytics": {
+   "within_domain": {
+    "questions": 448,
+    "recall@1": 0.795,
+    "recall@5": 0.955
+   },
+   "all_domains": {
+    "questions": 448,
+    "recall@1": 0.788,
+    "recall@5": 0.951
+   }
+  },
+  "data_science_ml": {
+   "within_domain": {
+    "questions": 515,
+    "recall@1": 0.858,
+    "recall@5": 0.984
+   },
+   "all_domains": {
+    "questions": 515,
+    "recall@1": 0.85,
+    "recall@5": 0.975
    }
   },
   "overall": {
-   "questions": 776,
-   "recall@1": 0.89,
-   "recall@5": 0.977
+   "questions": 12998,
+   "recall@1": 0.918,
+   "recall@5": 0.988
   }
  },
  "teacher_itself": {
-  "physics": {
+  "bayesian_statistics": {
    "within_domain": {
-    "questions": 529,
-    "recall@1": 0.864,
-    "recall@5": 0.987
+    "questions": 375,
+    "recall@1": 0.848,
+    "recall@5": 0.981
    },
    "all_domains": {
-    "questions": 529,
-    "recall@1": 0.858,
+    "questions": 375,
+    "recall@1": 0.832,
+    "recall@5": 0.976
+   }
+  },
+  "calculus": {
+   "within_domain": {
+    "questions": 355,
+    "recall@1": 0.882,
     "recall@5": 0.983
+   },
+   "all_domains": {
+    "questions": 355,
+    "recall@1": 0.772,
+    "recall@5": 0.93
    }
   },
-  "mechanics": {
+  "differential_equations": {
    "within_domain": {
-    "questions": 90,
-    "recall@1": 0.933,
+    "questions": 376,
+    "recall@1": 0.827,
+    "recall@5": 0.971
+   },
+   "all_domains": {
+    "questions": 376,
+    "recall@1": 0.737,
+    "recall@5": 0.918
+   }
+  },
+  "discrete_math": {
+   "within_domain": {
+    "questions": 488,
+    "recall@1": 0.768,
+    "recall@5": 0.955
+   },
+   "all_domains": {
+    "questions": 488,
+    "recall@1": 0.734,
+    "recall@5": 0.936
+   }
+  },
+  "geometry": {
+   "within_domain": {
+    "questions": 361,
+    "recall@1": 0.917,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 361,
+    "recall@1": 0.842,
+    "recall@5": 0.958
+   }
+  },
+  "linear_algebra": {
+   "within_domain": {
+    "questions": 441,
+    "recall@1": 0.932,
+    "recall@5": 0.998
+   },
+   "all_domains": {
+    "questions": 441,
+    "recall@1": 0.837,
+    "recall@5": 0.971
+   }
+  },
+  "math_foundations": {
+   "within_domain": {
+    "questions": 470,
+    "recall@1": 0.804,
+    "recall@5": 0.97
+   },
+   "all_domains": {
+    "questions": 470,
+    "recall@1": 0.638,
+    "recall@5": 0.845
+   }
+  },
+  "optimization": {
+   "within_domain": {
+    "questions": 336,
+    "recall@1": 0.908,
+    "recall@5": 0.994
+   },
+   "all_domains": {
+    "questions": 336,
+    "recall@1": 0.875,
+    "recall@5": 0.985
+   }
+  },
+  "statistics": {
+   "within_domain": {
+    "questions": 532,
+    "recall@1": 0.835,
+    "recall@5": 0.976
+   },
+   "all_domains": {
+    "questions": 532,
+    "recall@1": 0.763,
+    "recall@5": 0.944
+   }
+  },
+  "time_series": {
+   "within_domain": {
+    "questions": 355,
+    "recall@1": 0.904,
+    "recall@5": 0.986
+   },
+   "all_domains": {
+    "questions": 355,
+    "recall@1": 0.887,
+    "recall@5": 0.98
+   }
+  },
+  "astronomy": {
+   "within_domain": {
+    "questions": 335,
+    "recall@1": 0.857,
+    "recall@5": 0.991
+   },
+   "all_domains": {
+    "questions": 335,
+    "recall@1": 0.845,
+    "recall@5": 0.991
+   }
+  },
+  "chemistry": {
+   "within_domain": {
+    "questions": 511,
+    "recall@1": 0.759,
+    "recall@5": 0.992
+   },
+   "all_domains": {
+    "questions": 511,
+    "recall@1": 0.691,
+    "recall@5": 0.971
+   }
+  },
+  "classical_physics": {
+   "within_domain": {
+    "questions": 670,
+    "recall@1": 0.964,
+    "recall@5": 0.999
+   },
+   "all_domains": {
+    "questions": 670,
+    "recall@1": 0.931,
+    "recall@5": 0.996
+   }
+  },
+  "earth_science": {
+   "within_domain": {
+    "questions": 420,
+    "recall@1": 0.895,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 420,
+    "recall@1": 0.85,
+    "recall@5": 1.0
+   }
+  },
+  "materials_science": {
+   "within_domain": {
+    "questions": 424,
+    "recall@1": 0.953,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 424,
+    "recall@1": 0.936,
+    "recall@5": 1.0
+   }
+  },
+  "quantum_physics": {
+   "within_domain": {
+    "questions": 361,
+    "recall@1": 0.936,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 361,
+    "recall@1": 0.889,
+    "recall@5": 0.994
+   }
+  },
+  "aerospace_engineering": {
+   "within_domain": {
+    "questions": 397,
+    "recall@1": 0.869,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 397,
+    "recall@1": 0.851,
+    "recall@5": 0.997
+   }
+  },
+  "chemical_engineering": {
+   "within_domain": {
+    "questions": 435,
+    "recall@1": 0.878,
+    "recall@5": 0.989
+   },
+   "all_domains": {
+    "questions": 435,
+    "recall@1": 0.869,
+    "recall@5": 0.984
+   }
+  },
+  "civil_engineering": {
+   "within_domain": {
+    "questions": 415,
+    "recall@1": 0.93,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 415,
+    "recall@1": 0.925,
+    "recall@5": 0.995
+   }
+  },
+  "electrical_engineering": {
+   "within_domain": {
+    "questions": 301,
+    "recall@1": 0.9,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 301,
+    "recall@1": 0.85,
+    "recall@5": 0.997
+   }
+  },
+  "energy_systems": {
+   "within_domain": {
+    "questions": 394,
+    "recall@1": 0.906,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 394,
+    "recall@1": 0.896,
+    "recall@5": 1.0
+   }
+  },
+  "mechanical_engineering": {
+   "within_domain": {
+    "questions": 426,
+    "recall@1": 0.915,
+    "recall@5": 0.998
+   },
+   "all_domains": {
+    "questions": 426,
+    "recall@1": 0.904,
+    "recall@5": 0.998
+   }
+  },
+  "nuclear_engineering": {
+   "within_domain": {
+    "questions": 414,
+    "recall@1": 0.932,
+    "recall@5": 0.998
+   },
+   "all_domains": {
+    "questions": 414,
+    "recall@1": 0.932,
+    "recall@5": 0.998
+   }
+  },
+  "signal_processing": {
+   "within_domain": {
+    "questions": 357,
+    "recall@1": 0.91,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 357,
+    "recall@1": 0.905,
+    "recall@5": 1.0
+   }
+  },
+  "thermofluids": {
+   "within_domain": {
+    "questions": 495,
+    "recall@1": 0.962,
     "recall@5": 1.0
    },
    "all_domains": {
-    "questions": 90,
+    "questions": 495,
     "recall@1": 0.933,
     "recall@5": 1.0
    }
   },
-  "algebra": {
+  "transportation": {
+   "within_domain": {
+    "questions": 372,
+    "recall@1": 0.903,
+    "recall@5": 0.997
+   },
+   "all_domains": {
+    "questions": 372,
+    "recall@1": 0.901,
+    "recall@5": 0.995
+   }
+  },
+  "biomedical_science": {
+   "within_domain": {
+    "questions": 410,
+    "recall@1": 0.92,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 410,
+    "recall@1": 0.907,
+    "recall@5": 0.998
+   }
+  },
+  "computational_biology": {
+   "within_domain": {
+    "questions": 393,
+    "recall@1": 0.944,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 393,
+    "recall@1": 0.944,
+    "recall@5": 1.0
+   }
+  },
+  "economics_finance": {
+   "within_domain": {
+    "questions": 416,
+    "recall@1": 0.832,
+    "recall@5": 0.993
+   },
+   "all_domains": {
+    "questions": 416,
+    "recall@1": 0.793,
+    "recall@5": 0.986
+   }
+  },
+  "geospatial_analytics": {
    "within_domain": {
-    "questions": 157,
-    "recall@1": 0.758,
-    "recall@5": 0.917
+    "questions": 448,
+    "recall@1": 0.871,
+    "recall@5": 0.982
    },
    "all_domains": {
-    "questions": 157,
-    "recall@1": 0.713,
-    "recall@5": 0.873
+    "questions": 448,
+    "recall@1": 0.866,
+    "recall@5": 0.982
+   }
+  },
+  "data_science_ml": {
+   "within_domain": {
+    "questions": 515,
+    "recall@1": 0.887,
+    "recall@5": 0.983
+   },
+   "all_domains": {
+    "questions": 515,
+    "recall@1": 0.866,
+    "recall@5": 0.967
    }
   },
   "overall": {
-   "questions": 776,
-   "recall@1": 0.838,
-   "recall@5": 0.963
+   "questions": 12998,
+   "recall@1": 0.85,
+   "recall@5": 0.976
   }
  },
  "bge_small": {
-  "physics": {
+  "bayesian_statistics": {
+   "within_domain": {
+    "questions": 375,
+    "recall@1": 0.773,
+    "recall@5": 0.965
+   },
+   "all_domains": {
+    "questions": 375,
+    "recall@1": 0.733,
+    "recall@5": 0.909
+   }
+  },
+  "calculus": {
+   "within_domain": {
+    "questions": 355,
+    "recall@1": 0.797,
+    "recall@5": 0.977
+   },
+   "all_domains": {
+    "questions": 355,
+    "recall@1": 0.676,
+    "recall@5": 0.868
+   }
+  },
+  "differential_equations": {
+   "within_domain": {
+    "questions": 376,
+    "recall@1": 0.793,
+    "recall@5": 0.936
+   },
+   "all_domains": {
+    "questions": 376,
+    "recall@1": 0.622,
+    "recall@5": 0.827
+   }
+  },
+  "discrete_math": {
+   "within_domain": {
+    "questions": 488,
+    "recall@1": 0.727,
+    "recall@5": 0.924
+   },
+   "all_domains": {
+    "questions": 488,
+    "recall@1": 0.67,
+    "recall@5": 0.877
+   }
+  },
+  "geometry": {
+   "within_domain": {
+    "questions": 361,
+    "recall@1": 0.864,
+    "recall@5": 0.964
+   },
+   "all_domains": {
+    "questions": 361,
+    "recall@1": 0.784,
+    "recall@5": 0.909
+   }
+  },
+  "linear_algebra": {
+   "within_domain": {
+    "questions": 441,
+    "recall@1": 0.825,
+    "recall@5": 0.98
+   },
+   "all_domains": {
+    "questions": 441,
+    "recall@1": 0.773,
+    "recall@5": 0.941
+   }
+  },
+  "math_foundations": {
+   "within_domain": {
+    "questions": 470,
+    "recall@1": 0.734,
+    "recall@5": 0.921
+   },
+   "all_domains": {
+    "questions": 470,
+    "recall@1": 0.568,
+    "recall@5": 0.781
+   }
+  },
+  "optimization": {
+   "within_domain": {
+    "questions": 336,
+    "recall@1": 0.824,
+    "recall@5": 0.982
+   },
+   "all_domains": {
+    "questions": 336,
+    "recall@1": 0.795,
+    "recall@5": 0.952
+   }
+  },
+  "statistics": {
+   "within_domain": {
+    "questions": 532,
+    "recall@1": 0.735,
+    "recall@5": 0.934
+   },
+   "all_domains": {
+    "questions": 532,
+    "recall@1": 0.658,
+    "recall@5": 0.865
+   }
+  },
+  "time_series": {
+   "within_domain": {
+    "questions": 355,
+    "recall@1": 0.831,
+    "recall@5": 0.98
+   },
+   "all_domains": {
+    "questions": 355,
+    "recall@1": 0.808,
+    "recall@5": 0.961
+   }
+  },
+  "astronomy": {
+   "within_domain": {
+    "questions": 335,
+    "recall@1": 0.845,
+    "recall@5": 0.994
+   },
+   "all_domains": {
+    "questions": 335,
+    "recall@1": 0.821,
+    "recall@5": 0.991
+   }
+  },
+  "chemistry": {
    "within_domain": {
-    "questions": 529,
-    "recall@1": 0.769,
+    "questions": 511,
+    "recall@1": 0.708,
+    "recall@5": 0.978
+   },
+   "all_domains": {
+    "questions": 511,
+    "recall@1": 0.652,
     "recall@5": 0.943
+   }
+  },
+  "classical_physics": {
+   "within_domain": {
+    "questions": 670,
+    "recall@1": 0.933,
+    "recall@5": 0.996
    },
    "all_domains": {
-    "questions": 529,
-    "recall@1": 0.756,
-    "recall@5": 0.93
+    "questions": 670,
+    "recall@1": 0.899,
+    "recall@5": 0.988
    }
   },
-  "mechanics": {
+  "earth_science": {
    "within_domain": {
-    "questions": 90,
-    "recall@1": 0.833,
+    "questions": 420,
+    "recall@1": 0.91,
     "recall@5": 1.0
    },
    "all_domains": {
-    "questions": 90,
-    "recall@1": 0.833,
-    "recall@5": 0.978
+    "questions": 420,
+    "recall@1": 0.871,
+    "recall@5": 0.998
+   }
+  },
+  "materials_science": {
+   "within_domain": {
+    "questions": 424,
+    "recall@1": 0.908,
+    "recall@5": 0.995
+   },
+   "all_domains": {
+    "questions": 424,
+    "recall@1": 0.88,
+    "recall@5": 0.993
+   }
+  },
+  "quantum_physics": {
+   "within_domain": {
+    "questions": 361,
+    "recall@1": 0.917,
+    "recall@5": 0.989
+   },
+   "all_domains": {
+    "questions": 361,
+    "recall@1": 0.825,
+    "recall@5": 0.967
+   }
+  },
+  "aerospace_engineering": {
+   "within_domain": {
+    "questions": 397,
+    "recall@1": 0.882,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 397,
+    "recall@1": 0.877,
+    "recall@5": 0.997
+   }
+  },
+  "chemical_engineering": {
+   "within_domain": {
+    "questions": 435,
+    "recall@1": 0.867,
+    "recall@5": 0.993
+   },
+   "all_domains": {
+    "questions": 435,
+    "recall@1": 0.86,
+    "recall@5": 0.989
+   }
+  },
+  "civil_engineering": {
+   "within_domain": {
+    "questions": 415,
+    "recall@1": 0.918,
+    "recall@5": 0.998
+   },
+   "all_domains": {
+    "questions": 415,
+    "recall@1": 0.913,
+    "recall@5": 0.998
+   }
+  },
+  "electrical_engineering": {
+   "within_domain": {
+    "questions": 301,
+    "recall@1": 0.88,
+    "recall@5": 0.993
+   },
+   "all_domains": {
+    "questions": 301,
+    "recall@1": 0.814,
+    "recall@5": 0.957
+   }
+  },
+  "energy_systems": {
+   "within_domain": {
+    "questions": 394,
+    "recall@1": 0.901,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 394,
+    "recall@1": 0.886,
+    "recall@5": 1.0
+   }
+  },
+  "mechanical_engineering": {
+   "within_domain": {
+    "questions": 426,
+    "recall@1": 0.89,
+    "recall@5": 0.993
+   },
+   "all_domains": {
+    "questions": 426,
+    "recall@1": 0.862,
+    "recall@5": 0.977
+   }
+  },
+  "nuclear_engineering": {
+   "within_domain": {
+    "questions": 414,
+    "recall@1": 0.889,
+    "recall@5": 1.0
+   },
+   "all_domains": {
+    "questions": 414,
+    "recall@1": 0.877,
+    "recall@5": 0.998
+   }
+  },
+  "signal_processing": {
+   "within_domain": {
+    "questions": 357,
+    "recall@1": 0.846,
+    "recall@5": 0.994
+   },
+   "all_domains": {
+    "questions": 357,
+    "recall@1": 0.826,
+    "recall@5": 0.986
+   }
+  },
+  "thermofluids": {
+   "within_domain": {
+    "questions": 495,
+    "recall@1": 0.937,
+    "recall@5": 0.994
+   },
+   "all_domains": {
+    "questions": 495,
+    "recall@1": 0.919,
+    "recall@5": 0.994
+   }
+  },
+  "transportation": {
+   "within_domain": {
+    "questions": 372,
+    "recall@1": 0.898,
+    "recall@5": 0.989
+   },
+   "all_domains": {
+    "questions": 372,
+    "recall@1": 0.884,
+    "recall@5": 0.976
+   }
+  },
+  "biomedical_science": {
+   "within_domain": {
+    "questions": 410,
+    "recall@1": 0.9,
+    "recall@5": 0.998
+   },
+   "all_domains": {
+    "questions": 410,
+    "recall@1": 0.866,
+    "recall@5": 0.995
+   }
+  },
+  "computational_biology": {
+   "within_domain": {
+    "questions": 393,
+    "recall@1": 0.88,
+    "recall@5": 0.987
+   },
+   "all_domains": {
+    "questions": 393,
+    "recall@1": 0.875,
+    "recall@5": 0.985
+   }
+  },
+  "economics_finance": {
+   "within_domain": {
+    "questions": 416,
+    "recall@1": 0.805,
+    "recall@5": 0.976
+   },
+   "all_domains": {
+    "questions": 416,
+    "recall@1": 0.788,
+    "recall@5": 0.974
+   }
+  },
+  "geospatial_analytics": {
+   "within_domain": {
+    "questions": 448,
+    "recall@1": 0.804,
+    "recall@5": 0.96
+   },
+   "all_domains": {
+    "questions": 448,
+    "recall@1": 0.783,
+    "recall@5": 0.949
    }
   },
-  "algebra": {
+  "data_science_ml": {
    "within_domain": {
-    "questions": 157,
-    "recall@1": 0.548,
-    "recall@5": 0.764
+    "questions": 515,
+    "recall@1": 0.829,
+    "recall@5": 0.971
    },
    "all_domains": {
-    "questions": 157,
-    "recall@1": 0.459,
-    "recall@5": 0.637
+    "questions": 515,
+    "recall@1": 0.786,
+    "recall@5": 0.94
    }
   },
   "overall": {
-   "questions": 776,
-   "recall@1": 0.705,
-   "recall@5": 0.876
+   "questions": 12998,
+   "recall@1": 0.801,
+   "recall@5": 0.95
   }
  },
  "int8_onnx": {
   "mb": 4.89,
   "fp32_mb": 18.94,
   "overall": {
-   "questions": 776,
-   "recall@1": 0.892,
-   "recall@5": 0.977
+   "questions": 12998,
+   "recall@1": 0.918,
+   "recall@5": 0.987
   }
  }
 }

evaluation/stress_eval.json

@@ -0,0 +1,419 @@
+{
+ "questions": 10800,
+ "models": 27,
+ "cards": 2696,
+ "encoders": {
+  "ai999x-encoder-3": {
+   "overall": {
+    "questions": 10800,
+    "recall@1": 0.892,
+    "recall@5": 0.988
+   },
+   "by_model": {
+    "aerospace_engineering": {
+     "questions": 400,
+     "recall@1": 0.89,
+     "recall@5": 0.993
+    },
+    "astronomy": {
+     "questions": 400,
+     "recall@1": 0.922,
+     "recall@5": 0.995
+    },
+    "bayesian_statistics": {
+     "questions": 400,
+     "recall@1": 0.92,
+     "recall@5": 0.993
+    },
+    "calculus": {
+     "questions": 400,
+     "recall@1": 0.935,
+     "recall@5": 0.993
+    },
+    "chemical_engineering": {
+     "questions": 400,
+     "recall@1": 0.825,
+     "recall@5": 0.988
+    },
+    "chemistry": {
+     "questions": 400,
+     "recall@1": 0.858,
+     "recall@5": 0.988
+    },
+    "civil_engineering": {
+     "questions": 400,
+     "recall@1": 0.897,
+     "recall@5": 0.99
+    },
+    "classical_physics": {
+     "questions": 400,
+     "recall@1": 0.963,
+     "recall@5": 0.998
+    },
+    "computational_biology": {
+     "questions": 400,
+     "recall@1": 0.89,
+     "recall@5": 0.993
+    },
+    "differential_equations": {
+     "questions": 400,
+     "recall@1": 0.88,
+     "recall@5": 0.983
+    },
+    "discrete_math": {
+     "questions": 400,
+     "recall@1": 0.85,
+     "recall@5": 0.968
+    },
+    "earth_science": {
+     "questions": 400,
+     "recall@1": 0.945,
+     "recall@5": 1.0
+    },
+    "electrical_engineering": {
+     "questions": 400,
+     "recall@1": 0.882,
+     "recall@5": 0.993
+    },
+    "energy_systems": {
+     "questions": 400,
+     "recall@1": 0.858,
+     "recall@5": 0.99
+    },
+    "geometry": {
+     "questions": 400,
+     "recall@1": 0.93,
+     "recall@5": 0.985
+    },
+    "linear_algebra": {
+     "questions": 400,
+     "recall@1": 0.91,
+     "recall@5": 0.978
+    },
+    "materials_science": {
+     "questions": 400,
+     "recall@1": 0.94,
+     "recall@5": 0.995
+    },
+    "math_foundations": {
+     "questions": 400,
+     "recall@1": 0.848,
+     "recall@5": 0.99
+    },
+    "mechanical_engineering": {
+     "questions": 400,
+     "recall@1": 0.96,
+     "recall@5": 0.998
+    },
+    "nuclear_engineering": {
+     "questions": 400,
+     "recall@1": 0.932,
+     "recall@5": 0.995
+    },
+    "optimization": {
+     "questions": 400,
+     "recall@1": 0.875,
+     "recall@5": 0.993
+    },
+    "quantum_physics": {
+     "questions": 400,
+     "recall@1": 0.922,
+     "recall@5": 0.993
+    },
+    "signal_processing": {
+     "questions": 400,
+     "recall@1": 0.9,
+     "recall@5": 0.99
+    },
+    "statistics": {
+     "questions": 400,
+     "recall@1": 0.775,
+     "recall@5": 0.96
+    },
+    "thermofluids": {
+     "questions": 400,
+     "recall@1": 0.925,
+     "recall@5": 0.998
+    },
+    "time_series": {
+     "questions": 400,
+     "recall@1": 0.715,
+     "recall@5": 0.943
+    },
+    "transportation": {
+     "questions": 400,
+     "recall@1": 0.938,
+     "recall@5": 1.0
+    }
+   },
+   "by_style": {
+    "asked-first": {
+     "questions": 915,
+     "recall@1": 0.938,
+     "recall@5": 1.0
+    },
+    "asked-last": {
+     "questions": 903,
+     "recall@1": 0.909,
+     "recall@5": 0.993
+    },
+    "beginner": {
+     "questions": 893,
+     "recall@1": 0.788,
+     "recall@5": 0.946
+    },
+    "comparison": {
+     "questions": 754,
+     "recall@1": 0.886,
+     "recall@5": 0.991
+    },
+    "expert": {
+     "questions": 920,
+     "recall@1": 0.885,
+     "recall@5": 0.989
+    },
+    "imperative": {
+     "questions": 924,
+     "recall@1": 0.924,
+     "recall@5": 0.998
+    },
+    "list-like": {
+     "questions": 955,
+     "recall@1": 0.915,
+     "recall@5": 0.994
+    },
+    "negation": {
+     "questions": 880,
+     "recall@1": 0.91,
+     "recall@5": 1.0
+    },
+    "story": {
+     "questions": 910,
+     "recall@1": 0.879,
+     "recall@5": 0.986
+    },
+    "terse": {
+     "questions": 908,
+     "recall@1": 0.874,
+     "recall@5": 0.982
+    },
+    "two-part": {
+     "questions": 946,
+     "recall@1": 0.916,
+     "recall@5": 0.989
+    },
+    "verbose": {
+     "questions": 892,
+     "recall@1": 0.873,
+     "recall@5": 0.987
+    }
+   }
+  },
+  "ai999x-encoder-2": {
+   "overall": {
+    "questions": 10800,
+    "recall@1": 0.509,
+    "recall@5": 0.738
+   },
+   "by_model": {
+    "aerospace_engineering": {
+     "questions": 400,
+     "recall@1": 0.52,
+     "recall@5": 0.785
+    },
+    "astronomy": {
+     "questions": 400,
+     "recall@1": 0.575,
+     "recall@5": 0.825
+    },
+    "bayesian_statistics": {
+     "questions": 400,
+     "recall@1": 0.458,
+     "recall@5": 0.66
+    },
+    "calculus": {
+     "questions": 400,
+     "recall@1": 0.472,
+     "recall@5": 0.772
+    },
+    "chemical_engineering": {
+     "questions": 400,
+     "recall@1": 0.385,
+     "recall@5": 0.682
+    },
+    "chemistry": {
+     "questions": 400,
+     "recall@1": 0.55,
+     "recall@5": 0.848
+    },
+    "civil_engineering": {
+     "questions": 400,
+     "recall@1": 0.517,
+     "recall@5": 0.723
+    },
+    "classical_physics": {
+     "questions": 400,
+     "recall@1": 0.885,
+     "recall@5": 0.993
+    },
+    "computational_biology": {
+     "questions": 400,
+     "recall@1": 0.372,
+     "recall@5": 0.527
+    },
+    "differential_equations": {
+     "questions": 400,
+     "recall@1": 0.445,
+     "recall@5": 0.733
+    },
+    "discrete_math": {
+     "questions": 400,
+     "recall@1": 0.438,
+     "recall@5": 0.623
+    },
+    "earth_science": {
+     "questions": 400,
+     "recall@1": 0.682,
+     "recall@5": 0.915
+    },
+    "electrical_engineering": {
+     "questions": 400,
+     "recall@1": 0.568,
+     "recall@5": 0.818
+    },
+    "energy_systems": {
+     "questions": 400,
+     "recall@1": 0.52,
+     "recall@5": 0.797
+    },
+    "geometry": {
+     "questions": 400,
+     "recall@1": 0.512,
+     "recall@5": 0.76
+    },
+    "linear_algebra": {
+     "questions": 400,
+     "recall@1": 0.445,
+     "recall@5": 0.693
+    },
+    "materials_science": {
+     "questions": 400,
+     "recall@1": 0.575,
+     "recall@5": 0.838
+    },
+    "math_foundations": {
+     "questions": 400,
+     "recall@1": 0.555,
+     "recall@5": 0.818
+    },
+    "mechanical_engineering": {
+     "questions": 400,
+     "recall@1": 0.497,
+     "recall@5": 0.735
+    },
+    "nuclear_engineering": {
+     "questions": 400,
+     "recall@1": 0.495,
+     "recall@5": 0.74
+    },
+    "optimization": {
+     "questions": 400,
+     "recall@1": 0.403,
+     "recall@5": 0.625
+    },
+    "quantum_physics": {
+     "questions": 400,
+     "recall@1": 0.623,
+     "recall@5": 0.805
+    },
+    "signal_processing": {
+     "questions": 400,
+     "recall@1": 0.573,
+     "recall@5": 0.792
+    },
+    "statistics": {
+     "questions": 400,
+     "recall@1": 0.285,
+     "recall@5": 0.517
+    },
+    "thermofluids": {
+     "questions": 400,
+     "recall@1": 0.818,
+     "recall@5": 0.948
+    },
+    "time_series": {
+     "questions": 400,
+     "recall@1": 0.235,
+     "recall@5": 0.367
+    },
+    "transportation": {
+     "questions": 400,
+     "recall@1": 0.335,
+     "recall@5": 0.6
+    }
+   },
+   "by_style": {
+    "asked-first": {
+     "questions": 915,
+     "recall@1": 0.555,
+     "recall@5": 0.793
+    },
+    "asked-last": {
+     "questions": 903,
+     "recall@1": 0.596,
+     "recall@5": 0.796
+    },
+    "beginner": {
+     "questions": 893,
+     "recall@1": 0.376,
+     "recall@5": 0.57
+    },
+    "comparison": {
+     "questions": 754,
+     "recall@1": 0.459,
+     "recall@5": 0.686
+    },
+    "expert": {
+     "questions": 920,
+     "recall@1": 0.488,
+     "recall@5": 0.717
+    },
+    "imperative": {
+     "questions": 924,
+     "recall@1": 0.564,
+     "recall@5": 0.793
+    },
+    "list-like": {
+     "questions": 955,
+     "recall@1": 0.57,
+     "recall@5": 0.801
+    },
+    "negation": {
+     "questions": 880,
+     "recall@1": 0.52,
+     "recall@5": 0.762
+    },
+    "story": {
+     "questions": 910,
+     "recall@1": 0.484,
+     "recall@5": 0.725
+    },
+    "terse": {
+     "questions": 908,
+     "recall@1": 0.479,
+     "recall@5": 0.709
+    },
+    "two-part": {
+     "questions": 946,
+     "recall@1": 0.548,
+     "recall@5": 0.802
+    },
+    "verbose": {
+     "questions": 892,
+     "recall@1": 0.451,
+     "recall@5": 0.686
+    }
+   }
+  }
+ }
+}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:420de5981574c0272879a7c303e20c7ff44be65690371ee3d15871459bd2bf83
+oid sha256:926b2c37764c52357756fcb8338effbf4b653b512bd355b6b3288760ee5b1ce7
 size 18869624

onnx/encoder.int8.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8dfdc64b74a689092588fa02d7b7c90a09c478de71f1a4003e18dce5cd0980bd
-size 4888788
+oid sha256:521452d928a72da4d241b1835ab0f46b0e81ff58ffff0367b9a7029bd129fbd9
+size 4888789

onnx/encoder.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0c4da303bb7f807d2e850c76f6ba48884e29ba13b8170482daabd87011bc805c
+oid sha256:5beabbc9cf2bd61221b88934ffec47a593af5270b83856015c6fe6b6d02ed963
 size 18936053