tiny-search-reader-60M v3.1a: model, GGUF phone files, prompt helper, eval results

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+gguf/tiny-search-reader-60M-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text
+gguf/tiny-search-reader-60M-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,340 @@
+---
+license: apache-2.0
+language: en
+library_name: transformers
+pipeline_tag: text-generation
+tags:
+- search
+- rag
+- question-answering
+- on-device
+- gguf
+- tiny
+- muon
+---
+
+# tiny-search-reader-60M
+
+A 62M-parameter language model, trained from scratch, that reads search results and answers a question in a few
+words. When the results do not answer the question it says `not found`. It is built to run on a phone: the Q8_0 GGUF
+file is 70 MB.
+
+It does not search and it is not a chatbot. Your app runs the search and passes in the question with the results
+(title and text, the way Google shows them), and the model returns the answer.
+
+On 258 real Google questions it is right 79.8% of the time (206/258). About one answer in five is wrong.
+
+## What it does
+
+Made-up results, real output (the demo in `searchreader.py`):
+
+| question | results given | answer |
+|---|---|---|
+| When does the Harbor Street library open on Saturday? | [1] Harbor Street Library - Hours and location: The Harbor Street Library is open Monday to Friday from 8 am to 8 pm. On Saturday the library opens at 10 am and closes at 4 pm. It is closed on Sunday. [2] Harbor Street Library events: Story time for children runs every Wednesday morning in the reading room. [3] City libraries \| Parking: Free parking is available behind the Harbor Street Library for up to two hours. | `10 am` |
+| Who won the 2031 Harbor Street Chess Open? | [1] Harbor Street Chess Open 2025 results: Mara Lindqvist won the 2025 Harbor Street Chess Open with 7 points from 9 games. [2] Harbor Street Chess Open 2026: The 2026 Harbor Street Chess Open was won by Tomas Varga after a tie-break against Mara Lindqvist. [3] About the Harbor Street Chess Open: The open has been played every June at the Harbor Street Library since 2019. | `not found` |
+
+Asked "Who won the 2026 Harbor Street Chess Open?" with the same results, it answers `Tomas Varga`.
+
+## Files
+
+| file | size | what |
+|---|---:|---|
+| `model.safetensors` + `config.json`, `generation_config.json` | 248.1 MB | fp32 weights for transformers (`Qwen3ForCausalLM`) |
+| `tokenizer.json`, `tokenizer_config.json`, `special_tokens_map.json` | 0.55 MB | byte-level BPE, 8,192 tokens |
+| `gguf/tiny-search-reader-60M-Q8_0.gguf` | 70.2 MB | for llama.cpp; Q8_0 with the embedding kept at F16 |
+| `gguf/tiny-search-reader-60M-Q4_K_M.gguf` | 41.0 MB | for llama.cpp; smaller, slightly different answers |
+| `searchreader.py` | | prompt builder and answer helpers for transformers and llama-cpp-python |
+| `eval/` | | the test questions, gold answers, this model's answers, grading code and full results |
+| `LICENSE` | | Apache-2.0 |
+
+## How to run
+
+### transformers
+
+```bash
+pip install torch transformers tokenizers huggingface_hub
+```
+
+```python
+import sys
+from huggingface_hub import snapshot_download
+
+path = snapshot_download("ahkamboh/tiny-search-reader-60M")
+sys.path.insert(0, path)
+from searchreader import answer_transformers
+
+results = [
+    {"title": "Harbor Street Chess Open 2025 results",
+     "text": "Mara Lindqvist won the 2025 Harbor Street Chess Open with 7 points from 9 games."},
+    {"title": "Harbor Street Chess Open 2026",
+     "text": "The 2026 Harbor Street Chess Open was won by Tomas Varga after a tie-break against Mara Lindqvist."},
+]
+print(answer_transformers("Who won the 2026 Harbor Street Chess Open?", results))   # Tomas Varga
+```
+
+Without the helper, the prompt is plain text with special tokens (format below):
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo = "ahkamboh/tiny-search-reader-60M"
+tok = AutoTokenizer.from_pretrained(repo)
+model = AutoModelForCausalLM.from_pretrained(repo)
+
+prompt = ("<|question|> Who won the 2026 Harbor Street Chess Open?<|snippet|>"
+          " [1] Harbor Street Chess Open 2025 results: Mara Lindqvist won the 2025 Harbor Street Chess Open"
+          " with 7 points from 9 games."
+          " [2] Harbor Street Chess Open 2026: The 2026 Harbor Street Chess Open was won by Tomas Varga after a"
+          " tie-break against Mara Lindqvist.<|answer|>")
+enc = tok(prompt, return_tensors="pt", add_special_tokens=False)
+out = model.generate(**enc, max_new_tokens=24, do_sample=False)
+print(tok.decode(out[0, enc.input_ids.shape[1]:], skip_special_tokens=True).strip())   # Tomas Varga
+```
+
+This works as long as the prompt fits. For real search results use `build_prompt`, which fits them into the 512-token
+context the same way the model was tested.
+
+### llama.cpp
+
+```bash
+llama-completion -m gguf/tiny-search-reader-60M-Q8_0.gguf -c 512 -n 24 --temp 0 -no-cnv --no-display-prompt \
+  -p '<|question|> Who won the 2026 Harbor Street Chess Open?<|snippet|> [1] Harbor Street Chess Open 2025 results: Mara Lindqvist won the 2025 Harbor Street Chess Open with 7 points from 9 games. [2] Harbor Street Chess Open 2026: The 2026 Harbor Street Chess Open was won by Tomas Varga after a tie-break against Mara Lindqvist.<|answer|>'
+# Tomas Varga [end of text]
+```
+
+Tested with Homebrew llama.cpp build 11146. The GGUF says `add_bos_token = false` and its end-of-text token is id
+0, so llama.cpp adds nothing in front and stops after the answer.
+
+### llama-cpp-python
+
+```bash
+pip install llama-cpp-python tokenizers huggingface_hub
+```
+
+```python
+from searchreader import answer_gguf   # with path on sys.path as above
+
+print(answer_gguf("Who won the 2026 Harbor Street Chess Open?", results,
+                  gguf_path=path + "/gguf/tiny-search-reader-60M-Q8_0.gguf"))   # Tomas Varga
+```
+
+`answer_gguf` builds the prompt with our own tokenizer and feeds the token ids to llama.cpp, so the special tokens are
+single ids and no BOS is added. It decodes greedily and stops at `<|endoftext|>` (id 0) or after 24 tokens. Run
+`python searchreader.py` for the two-question demo on both backends. Tested with transformers 5.18, llama-cpp-python
+0.3.36 and tokenizers 0.23.
+
+### Prompt format
+
+```
+<|question|> {question}<|snippet|> [1] {title}: {text} [2] {title}: {text} ...<|answer|>
+```
+
+The model writes the answer after `<|answer|>` and ends it with `<|endoftext|>`.
+
+| token | id | role |
+|---|---:|---|
+| `<\|endoftext\|>` | 0 | end of the answer |
+| `<\|question\|>` | 1 | starts the question |
+| `<\|snippet\|>` | 2 | starts the results |
+| `<\|answer\|>` | 3 | the model answers after this |
+| `<\|pad\|>` | 4 | padding (training only) |
+
+| rule | detail |
+|---|---|
+| spacing | one space before the question and before each `[n]`, none before a special token |
+| cleaning | every run of whitespace (non-breaking spaces too) becomes one space; ends are trimmed |
+| one result | `[n] title: text`, or `[n] text` when there is no title; n counts from 1 in the search engine's order |
+| dates | keep the date Google puts at the start of a result's text; the model uses it to tell new results from old |
+| length | prompt plus answer must fit 512 tokens. Keep 24 tokens for the answer, so the prompt can use 488 |
+| too long | drop results from the end until it fits; if the first result alone is too long, cut it short. A question too long to leave room for any result (over 485 tokens with the 24-token reserve) is cut to its first 256 tokens |
+| decoding | greedy, at most 24 new tokens, stop at id 0 |
+
+Five Google results usually fit: only 3 of the 258 test prompts had to drop a result.
+
+`searchreader.build_prompt(tokenizer, question, results, ctx=512, reserve=24)` does all of this. It gives the same
+token ids as the training code on all 258 test questions, and it takes either a `tokenizers.Tokenizer` or a
+transformers tokenizer.
+
+### Note for apps that build the prompt as text
+
+The special tokens must reach the model as the single ids 1, 2 and 3. If you tokenize the prompt text with llama.cpp,
+turn on special-token parsing and turn off BOS:
+
+| API | call |
+|---|---|
+| llama.cpp C API | `llama_tokenize(vocab, text, len, tokens, n_max, /*add_special*/ false, /*parse_special*/ true)` |
+| llama-cpp-python | `llm.tokenize(text.encode(), add_bos=False, special=True)` |
+
+With these settings the GGUF's tokenizer gives the same ids as `tokenizer.json` on all 258 test prompts. Without
+`parse_special`, the markers turn into ordinary text (149 tokens instead of 131 for the library demo) and in our test
+the model then answered with nothing. Since the markers are parsed anywhere in the text, remove the five marker
+strings from questions and results if they could ever contain them.
+
+## Results
+
+Full tables, the test questions and the grading rules are in [`eval/results.md`](eval/results.md).
+
+v3.1a (this release), PyTorch fp32 on CPU:
+
+| question set | questions | accuracy |
+|---|---:|---:|
+| google25 (dev set, see caveats) | 24 | 91.7% (22) |
+| google_heldout | 53 | 81.1% (43) |
+| google_fresh | 19 | 89.5% (17) |
+| google_big (collected 3-4 Oct 2026) | 162 | 76.5% (124) |
+| all | 258 | 79.8% (206) |
+
+Accuracy counts right answers plus trick questions answered with `not found`. On the 208 questions that have an
+answer it is right 165 times, says `not found` 11 times and gives a wrong answer 32 times. It catches 41 of the 50
+trick questions.
+
+google_big by kind:
+
+| kind | example | CPU fp32 | Colab bf16 |
+|---|---|---:|---:|
+| fact | What is the capital of Turkey? | 30/34 | 30/34 |
+| live value | What is the Saudi riyal rate in Pakistan today? | 29/35 | 29/35 |
+| trick (right reply is `not found`) | When did Pakistan win the FIFA World Cup? | 24/28 | 24/28 |
+| unit | What is the boiling point of mercury in Celsius? | 21/25 | 20/25 |
+| latest version | What is the latest Google Pixel phone? | 6/14 | 6/14 |
+| most recent winner | Who won the most recent UEFA Champions League final? | 14/26 | 14/26 |
+
+Earlier versions from this project, same tests (Colab GPU runs, bf16):
+
+| model | all 258 | google_big | google_big "latest / current / most recent" questions (60) |
+|---|---:|---:|---:|
+| v1.1a | 69.4% (179) | | |
+| v2a | 73.3% (189) | 69.1% | 41.7% |
+| v3a | 77.1% (199) | 74.7% | 65.0% |
+| v3.1a | 79.8% (206) | 75.9% | 61.7% |
+
+Paired tests (McNemar): against v2a, v3.1a fixes 17 google_big questions and breaks 6 (p=0.035), and on the "latest"
+questions fixes 13 and breaks 1 (p=0.002). Against v3a it fixes 22 and breaks 15 of the 258 (p=0.32, CPU fp32), so
+that gain is within noise.
+
+Other tests (Colab bf16): SQuAD 2.0 dev F1 65.19; held-out MS MARCO and TriviaQA validation rows 63.5% accuracy.
+Both were part of the score used to pick v3.1a.
+
+Caveats:
+
+- The test sets are small. One question moves the 258 total by 0.4 points and google_big by 0.6.
+- The model was picked using Colab GPU runs in bf16. CPU fp32 also gets 206/258, but not on exactly the same
+  questions: on google_big it gets 124 against Colab's 123 (unit questions 21/25 against 20/25). Both are shown and
+  labelled.
+- google25 (24 questions) is a dev set. It was the reference for the synthetic table layouts and is part of the
+  score used to pick v3.1a, as are real_dev and SQuAD 2.0 dev. Three google25 rows and five google_heldout rows
+  (latest iPhone, Galaxy S and iOS, Ballon d'Or, LPG, policy rate, dollar and petrol rates) guided the v3 data
+  design. On the 234 questions outside google25, v3.1a gets 184 (78.6%, CPU fp32).
+- Not fully blind: v3.1 added training rows of three kinds (answer_type, traps_hard, units) after looking at v3a's
+  mistakes on google_fresh and google_big, so google_fresh is not fully blind for v3.1a. New v3.1 rows on google_big
+  subjects, or asking a test question in other words, were dropped, but that filter covered only the rows added in
+  v3.1. The older v1 and v1.1 synthetic rows, which v2a, v3a and v3.1a all trained on, came from fact tables that
+  include some google_big subjects and answers (for example Nanga Parbat's height and body temperature in
+  Fahrenheit). So google_big, and its unit questions most of all, is not fully blind for any of these models. For
+  v3.1a, only google_big questions asked word for word were removed from the older rows.
+- For live values the gold list holds every value the saved results showed, since sources disagree, and any of them
+  counts as right.
+- About a quarter of the test questions (65 of 258) are about Pakistan.
+- Nothing was measured on a phone.
+
+## Phone files
+
+| file | size | all 258 (llama.cpp, CPU) | answers same as fp32 | 449-token prompt + 24 tokens | average test question | peak RSS |
+|---|---:|---:|---|---:|---:|---:|
+| `tiny-search-reader-60M-Q8_0.gguf` | 70.2 MB | 79.8% (206) | 254/258, all 258 verdicts the same | 317 ms | 168 ms | 174 MB |
+| `tiny-search-reader-60M-Q4_K_M.gguf` | 41.0 MB | 79.5% (205) | 233/258, loses 5 and wins 4 | 354 ms | 200 ms | 123 MB |
+
+Speeds are from an M1 Pro, CPU only, 4 threads, Homebrew llama.cpp. With 1 thread the 449 + 24 token case takes
+982 ms (Q8_0) and 1,142 ms (Q4_K_M). On this CPU Q4_K_M is smaller but not faster. A mid-range phone should take
+roughly 0.5 to 0.9 s per question; that is an estimate, not a measurement. Q8_0 is the one to use unless the 29 MB
+difference matters: it gave the same answers under Homebrew llama.cpp and llama-cpp-python 0.3.36, while Q4_K_M
+changed 4 of 258 answers between the two.
+
+## Training
+
+### Architecture
+
+| | |
+|---|---|
+| layout | decoder-only transformer, loads as Qwen3 (`Qwen3ForCausalLM`) |
+| parameters | 62,018,304, input and output embeddings tied |
+| layers, width | 18, 512 |
+| attention | 8 heads, 8 KV heads, head dim 64, per-head QK-norm, RoPE theta 10,000 |
+| MLP | SwiGLU, hidden size 1,408 |
+| norm | RMSNorm, eps 1e-5 |
+| context | 512 tokens |
+| tokenizer | byte-level BPE, 8,192 tokens including the 5 special tokens |
+
+### Pretraining
+
+| | |
+|---|---|
+| data | 10B tokens of FineWeb-Edu (sample-10BT) |
+| optimizer | Muon (weight decay 0.01) for the hidden weight matrices, AdamW for the rest |
+| precision | bf16 |
+| hardware | one RTX PRO 6000 on Colab, about 10.6 hours |
+| final validation loss | 2.7127 |
+| search reading | search-reading examples mixed into the last 30% of training, while the learning rate decays |
+
+### Fine-tuning
+
+2 epochs on real and synthetic search-reading rows. Every training row that asks a test question word for word was
+dropped, and so was every row on a google25, google_heldout or google_fresh topic. The rows added in v3.1 also left
+out google_big subjects and test questions asked in other words (see the caveats under Results).
+
+| source | rows |
+|---|---|
+| MS MARCO v2.1 | real Bing queries with up to 5 of their web passages; "No Answer Present" becomes `not found` |
+| TriviaQA (unfiltered) | trivia questions with Bing's top 5 results (title and description) |
+| SQuAD 2.0 | questions on one Wikipedia paragraph (train split) |
+| hard negatives | copies of answerable MS MARCO and TriviaQA rows where every result holding the answer is swapped for on-topic results without it; the reply is `not found` |
+| synthetic, Google-style | written by Qwen/Qwen3.6-35B-A3B-FP8 on vLLM. Python picks the facts that decide each answer and the LLM writes result text around them. A second, temperature-0 pass of the same LLM answers from the results alone, and a row is kept only when it agrees with Python's answer and passes Python's checks |
+
+Synthetic categories:
+
+| category | teaches | added in |
+|---|---|---|
+| list_numbers | read one cell of a list or table (rates, prices, scores, prayer times) | v1 |
+| recency | several dated results with different values; the newest wins | v1 |
+| answer_type | the result also holds a date, the wrong unit or another number | v1, more rows in v3.1 |
+| traps | future, false-premise, fictional or wrong-entity questions; the reply is `not found` | v1 |
+| multi | plain facts; one result has the answer, the others are near misses, ads and junk | v1 |
+| traps_hard | false questions about real entities, with tempting years and amounts in the results | v1.1, more rows in v3.1 |
+| units | one value given in 2 or 3 units; answer in the unit asked | v1.1, more rows in v3.1 |
+| long_numbers | copy a long number whole next to numbers of the same length | v1.1 |
+| versions_mixed, stale_latest, current_value, winners_recent | latest version, current official value and most recent winner among old and new results; every entity is invented | v3 |
+
+## Limitations
+
+- English only.
+- It answers from the results you give it. It does not search and is not meant to answer from memory, so poor
+  results give poor answers.
+- About 20% of answers on the tests are wrong, and a wrong answer is more common than a `not found`. Show the
+  source result next to the answer.
+- `not found` works best on false-premise and future questions (41 of 50 caught). It is weaker when the results are
+  about the right subject but leave out the asked fact. In a hand check of 6 made-up cases like that, it said
+  `not found` once and the other five times gave a nearby time or number, such as `8` for "How many books does the
+  Harbor Street library have?" from a result saying the library is open from 8 am to 8 pm.
+- "Latest" and "current" answers depend on the dates in the results. The model does not know today's date; it picks
+  the result that looks newest. Latest-version (6/14) and most-recent-winner (14/26) questions are its weakest kinds.
+- 512 tokens of context, about five Google-length results.
+- No chat and no instructions: one question in, a few words out, in the prompt format above only.
+
+## License and data notes
+
+The weights, `searchreader.py` and `eval/grade.py` are under Apache-2.0 (see `LICENSE`).
+
+| data | used for | terms |
+|---|---|---|
+| FineWeb-Edu (sample-10BT) | pretraining | ODC-By 1.0 |
+| MS MARCO v2.1 | fine-tuning | non-commercial research terms |
+| TriviaQA | fine-tuning | the dataset's own terms |
+| SQuAD 2.0 | fine-tuning | CC BY-SA 4.0 |
+| Qwen3.6-35B-A3B outputs | synthetic fine-tuning rows | the model is Apache-2.0 |
+| Google search results | tests only | not included; `eval/` has the questions, gold answers, model answers and scores |
+
+MS MARCO's terms are for non-commercial research. If you plan commercial use, check whether that matters for you.
+
+## Author
+
+Ali Hamza Kamboh ([GitHub: ahkamboh](https://github.com/ahkamboh), Hugging Face: ahkamboh).

config.json

@@ -0,0 +1,55 @@
+{
+  "architectures": [
+    "Qwen3ForCausalLM"
+  ],
+  "model_type": "qwen3",
+  "vocab_size": 8192,
+  "hidden_size": 512,
+  "intermediate_size": 1408,
+  "num_hidden_layers": 18,
+  "num_attention_heads": 8,
+  "num_key_value_heads": 8,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "max_position_embeddings": 512,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "rope_parameters": {
+    "rope_type": "default",
+    "rope_theta": 10000.0
+  },
+  "rope_scaling": null,
+  "tie_word_embeddings": true,
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "use_sliding_window": false,
+  "sliding_window": null,
+  "max_window_layers": 18,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "initializer_range": 0.02,
+  "use_cache": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_id": 4,
+  "torch_dtype": "float32",
+  "dtype": "float32"
+}

eval/google25.jsonl

@@ -0,0 +1,24 @@
+{"id": "g25-0", "question": "What is the dollar rate in Pakistan today?", "answers": ["277.08", "277.06", "277.90", "277.50", "275.64", "275.1777", "277"], "kind": "live", "answer": "277.08 PKR", "verdict": "answered_correct", "correct": true}
+{"id": "g25-1", "question": "What is the gold price in Pakistan today?", "answers": ["437,206", "433,948", "435,643.07", "438,636", "404,540", "400,772", "397,786", "399,339.48"], "kind": "live", "answer": "Rs. 437,206", "verdict": "answered_correct", "correct": true}
+{"id": "g25-2", "question": "What is the weather in Lahore today?", "answers": ["97°", "97", "35°", "35", "hazy and very warm", "hazy"], "kind": "live", "answer": "97°", "verdict": "answered_correct", "correct": true}
+{"id": "g25-3", "question": "Who won the last Formula 1 race?", "answers": ["George Russell", "Russell"], "kind": "live", "answer": "not found", "verdict": "missed", "correct": false}
+{"id": "g25-4", "question": "What is the Bitcoin price today?", "answers": ["$84,555.09", "$86,033.73", "$85,422.74", "$86,417.7", "$86,742.94", "$86,088.98"], "kind": "live", "answer": "$86,417.7", "verdict": "answered_correct", "correct": true}
+{"id": "g25-5", "question": "Who is the current prime minister of the UK?", "answers": ["Andy Burnham"], "kind": "live", "answer": "Andy Burnham", "verdict": "answered_correct", "correct": true}
+{"id": "g25-6", "question": "What is the petrol price in Pakistan today?", "answers": ["Rs 390.66 per litre", "Rs 390.66", "Rs.390.66", "390.66"], "kind": "live", "answer": "Rs 390.66 per litre", "verdict": "answered_correct", "correct": true}
+{"id": "g25-7", "question": "What is the latest iPhone model?", "answers": ["iPhone 18 Pro", "iPhone 18 Pro Max", "iPhone 18"], "kind": "live", "answer": "iPhone 18", "verdict": "answered_correct", "correct": true}
+{"id": "g25-8", "question": "Who is the CEO of OpenAI?", "answers": ["Sam Altman"], "kind": "live", "answer": "Sam Altman", "verdict": "answered_correct", "correct": true}
+{"id": "g25-9", "question": "What was the result of Pakistan cricket team's latest match?", "answers": ["Pakistan beat Bangladesh by six wickets", "won by 6 wickets", "six wickets", "6 wickets", "6 wkts"], "kind": "live", "answer": "Pakistan beat Bangladesh by six wickets", "verdict": "answered_correct", "correct": true}
+{"id": "g25-10", "question": "What is the capital of Australia?", "answers": ["Canberra"], "kind": "fact", "answer": "Canberra", "verdict": "answered_correct", "correct": true}
+{"id": "g25-11", "question": "Who wrote Pride and Prejudice?", "answers": ["Jane Austen"], "kind": "fact", "answer": "Jane Austen", "verdict": "answered_correct", "correct": true}
+{"id": "g25-12", "question": "How tall is K2?", "answers": ["8,611 metres", "8,611 meters", "8,611 m", "8611 m", "28,251 ft", "28,251 feet", "8,611"], "kind": "fact", "answer": "8,611 metres", "verdict": "answered_correct", "correct": true}
+{"id": "g25-13", "question": "When did Pakistan get independence?", "answers": ["14 August 1947", "August 14, 1947", "14 August"], "kind": "fact", "answer": "14 August 1947", "verdict": "answered_correct", "correct": true}
+{"id": "g25-14", "question": "What is the speed of light?", "answers": ["299,792,458 meters per second", "299,792,458 m/s", "299792458 meters per", "299,792,458", "186,000 mi/sec", "300,000 kilometers per second", "300,000 km/s"], "kind": "fact", "answer": "299792458 m/s", "verdict": "answered_correct", "correct": true}
+{"id": "g25-15", "question": "Who founded Microsoft?", "answers": ["Bill Gates and Paul Allen", "Bill Gates", "Paul Allen"], "kind": "fact", "answer": "Bill Gates and Paul Allen", "verdict": "answered_correct", "correct": true}
+{"id": "g25-16", "question": "What is the boiling point of water in Fahrenheit?", "answers": ["212°F", "212 °F", "212° F", "212", "211.95 °F"], "kind": "fact", "answer": "99.97 °C", "verdict": "wrong", "correct": false}
+{"id": "g25-17", "question": "How many players are in a football team?", "answers": ["11", "eleven"], "kind": "fact", "answer": "11 players", "verdict": "answered_correct", "correct": true}
+{"id": "g25-18", "question": "What is the largest planet in the solar system?", "answers": ["Jupiter"], "kind": "fact", "answer": "Jupiter", "verdict": "answered_correct", "correct": true}
+{"id": "g25-19", "question": "Who painted the Mona Lisa?", "answers": ["Leonardo da Vinci", "Leonardo"], "kind": "fact", "answer": "Leonardo da Vinci", "verdict": "answered_correct", "correct": true}
+{"id": "g25-20", "question": "Who was the first Pakistani to walk on Mars?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "g25-21", "question": "When did Lahore host the Summer Olympics?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "g25-22", "question": "Who won the 2030 FIFA World Cup?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "g25-23", "question": "What is the population of Atlantis?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}

eval/google_big.jsonl

@@ -0,0 +1,162 @@
+{"id": "gb-0", "question": "What is the Chinese yuan rate in Pakistan today?", "answers": ["Rs 41.32", "41.32", "PKR 41.32", "41.32 PKR", "41.25", "Rs 41.25", "41.259", "41.2590", "41.26", "PKR 42", "42"], "kind": "live", "answer": "41.32 PKR", "verdict": "answered_correct", "correct": true}
+{"id": "gb-1", "question": "What is the UK pound rate in Pakistan today?", "answers": ["Rs 366.82", "366.82", "PKR 366.82", "366.82 PKR", "366.63", "Rs 366.63", "364.58", "Rs 364.58", "365.7", "Rs 365.7"], "kind": "live", "answer": "365.7", "verdict": "answered_correct", "correct": true}
+{"id": "gb-2", "question": "What is the Saudi riyal rate in Pakistan today?", "answers": ["Rs 73.87", "Rs. 73.87", "73.87", "PKR 73.87", "Rs 73.80", "73.80", "73.80 PKR", "73.8", "73.85", "73.8595", "73.83", "73.8314"], "kind": "live", "answer": "73.87", "verdict": "answered_correct", "correct": true}
+{"id": "gb-3", "question": "What is the UAE dirham rate in Pakistan today?", "answers": ["Rs 75.44", "75.44", "PKR 75.44", "75.44 PKR", "75.39", "75.3997", "75.33", "75.3326", "75.33260", "75.55", "Rs 75.55"], "kind": "live", "answer": "75.55", "verdict": "answered_correct", "correct": true}
+{"id": "gb-5", "question": "What is the diesel price in Pakistan?", "answers": ["Rs 399.64 per litre", "Rs 399.64", "Rs.399.64", "Rs399.64", "PKR 399.64", "399.64"], "kind": "live", "answer": "Rs.272.29", "verdict": "wrong", "correct": false}
+{"id": "gb-8", "question": "What is the Solana price today?", "answers": ["$119.51", "119.51", "$119.31", "119.31", "$119.56", "119.56", "$117.91", "117.91", "$122.44", "122.44"], "kind": "live", "answer": "$117.91", "verdict": "answered_correct", "correct": true}
+{"id": "gb-9", "question": "What is the XRP price today?", "answers": ["$1.49", "1.49", "$1.48", "1.48", "$1.485", "1.485", "1.48524", "$1.48524"], "kind": "live", "answer": "$1.48524", "verdict": "answered_correct", "correct": true}
+{"id": "gb-10", "question": "What is the gold price per ounce today?", "answers": ["$4,144", "$4,144.00", "4,144", "4144", "$4,156.60", "$4,156.6", "4,156.60", "4156.60", "4156.6"], "kind": "live", "answer": "$4,156.60", "verdict": "answered_correct", "correct": true}
+{"id": "gb-11", "question": "What is the Brent crude oil price today?", "answers": ["$102.72", "102.72", "$102.25", "102.25", "102.25 USD/Bbl"], "kind": "live", "answer": "$120", "verdict": "wrong", "correct": false}
+{"id": "gb-12", "question": "What is the Nvidia share price today?", "answers": ["$234.22", "234.22", "$233.95", "233.95"], "kind": "live", "answer": "$234.22", "verdict": "answered_correct", "correct": true}
+{"id": "gb-13", "question": "What is the Apple stock price today?", "answers": ["$333.69", "333.69"], "kind": "live", "answer": "$52", "verdict": "wrong", "correct": false}
+{"id": "gb-14", "question": "What is the current inflation rate in Pakistan?", "answers": ["11.1%", "11.15%", "11.1 percent", "11.15 percent"], "kind": "live", "answer": "11.1%", "verdict": "answered_correct", "correct": true}
+{"id": "gb-15", "question": "What is the US Federal Reserve interest rate?", "answers": ["3.75% to 4%", "3.75%-4%", "3.75-4%", "3.75%-4.00%", "3-3/4 to 4 percent", "4%", "4 percent"], "kind": "live", "answer": "4.58", "verdict": "wrong", "correct": false}
+{"id": "gb-16", "question": "What is the current repo rate in India?", "answers": ["5.25%", "5.25 percent", "5.25"], "kind": "live", "answer": "5.25%", "verdict": "answered_correct", "correct": true}
+{"id": "gb-21", "question": "What is the price of a Suzuki Alto in Pakistan?", "answers": ["PKR 2,994,861", "Rs 2,994,861", "2,994,861", "2994861", "PKR 29.94861 Lacs", "29.94861 lacs", "PKR 2,994,861 to PKR 3,326,446"], "kind": "live", "answer": "Rs. 2,994,861", "verdict": "answered_correct", "correct": true}
+{"id": "gb-23", "question": "Who is the current prime minister of India?", "answers": ["Narendra Modi", "Modi", "Narendra Damodardas Modi", "Shri Narendra Modi"], "kind": "live", "answer": "Narendra Modi", "verdict": "answered_correct", "correct": true}
+{"id": "gb-24", "question": "Who is the current president of France?", "answers": ["Emmanuel Macron", "Macron", "Emmanuel Jean-Michel Frédéric Macron"], "kind": "live", "answer": "Emmanuel Macron", "verdict": "answered_correct", "correct": true}
+{"id": "gb-25", "question": "Who is the current chancellor of Germany?", "answers": ["Friedrich Merz", "Merz", "Joachim-Friedrich Martin Josef Merz"], "kind": "live", "answer": "Friedrich Merz", "verdict": "answered_correct", "correct": true}
+{"id": "gb-26", "question": "Who is the current prime minister of Canada?", "answers": ["Mark Carney", "Carney", "Mark Joseph Carney"], "kind": "live", "answer": "Mark Carney", "verdict": "answered_correct", "correct": true}
+{"id": "gb-27", "question": "Who is the current prime minister of Japan?", "answers": ["Sanae Takaichi", "Takaichi", "Takaichi Sanae"], "kind": "live", "answer": "Sanae Takaichi", "verdict": "answered_correct", "correct": true}
+{"id": "gb-28", "question": "Who is the current Chief of Army Staff of Pakistan?", "answers": ["Asim Munir", "Field Marshal Asim Munir", "Syed Asim Munir", "Field Marshal Syed Asim Munir", "Syed Asim Munir Ahmed Shah", "Munir"], "kind": "live", "answer": "not found", "verdict": "missed", "correct": false}
+{"id": "gb-29", "question": "Who is the current finance minister of Pakistan?", "answers": ["Muhammad Aurangzeb", "Aurangzeb", "Senator Muhammad Aurangzeb", "Mr. Muhammad Aurangzeb"], "kind": "live", "answer": "Muhammad Aurangzeb", "verdict": "answered_correct", "correct": true}
+{"id": "gb-30", "question": "Who is the current chief minister of Punjab?", "answers": ["Maryam Nawaz", "Maryam Nawaz Sharif", "Ms. Maryam Nawaz Sharif"], "kind": "live", "answer": "Maryam Nawaz Sharif", "verdict": "answered_correct", "correct": true}
+{"id": "gb-31", "question": "Who is the current chief minister of Sindh?", "answers": ["Syed Murad Ali Shah", "Murad Ali Shah", "Syed Murad Ali Shah Lakyari", "Murad Ali Shah Lakyari"], "kind": "live", "answer": "Syed Murad Ali Shah", "verdict": "answered_correct", "correct": true}
+{"id": "gb-32", "question": "Who is the current chairman of the PCB?", "answers": ["Mohsin Naqvi", "Syed Mohsin Raza Naqvi", "Mohsin Raza Naqvi", "Syed Mohsin Naqvi", "Naqvi"], "kind": "live", "answer": "Mohsin Naqvi", "verdict": "answered_correct", "correct": true}
+{"id": "gb-33", "question": "Who is the current captain of the Pakistan Test cricket team?", "answers": ["Babar Azam", "Babar"], "kind": "live", "answer": "Babar Azam", "verdict": "answered_correct", "correct": true}
+{"id": "gb-35", "question": "Who is the CEO of Microsoft?", "answers": ["Satya Nadella", "Nadella", "Satya Narayana Nadella"], "kind": "live", "answer": "Satya Nadella", "verdict": "answered_correct", "correct": true}
+{"id": "gb-37", "question": "Who is the CEO of Nvidia?", "answers": ["Jensen Huang", "Huang"], "kind": "live", "answer": "Jensen Huang", "verdict": "answered_correct", "correct": true}
+{"id": "gb-38", "question": "Who is the CEO of Amazon?", "answers": ["Andy Jassy", "Andrew Jassy", "Andrew R. Jassy", "Jassy"], "kind": "live", "answer": "Andy Jassy", "verdict": "answered_correct", "correct": true}
+{"id": "gb-39", "question": "Who is the current president of Iran?", "answers": ["Masoud Pezeshkian", "Pezeshkian", "Dr. Masoud Pezeshkian"], "kind": "live", "answer": "Masoud Pezeshkian", "verdict": "answered_correct", "correct": true}
+{"id": "gb-40", "question": "Who is the current governor of the State Bank of Pakistan?", "answers": ["Jameel Ahmad", "Jameel Ahmed", "Mr. Jameel Ahmad"], "kind": "live", "answer": "Jameel Ahmad", "verdict": "answered_correct", "correct": true}
+{"id": "gb-41", "question": "Who is the current pope?", "answers": ["Pope Leo XIV", "Leo XIV", "Leo 14", "Pope Leo", "Robert Francis Prevost", "Robert Prevost"], "kind": "live", "answer": "not found", "verdict": "missed", "correct": false}
+{"id": "gb-42", "question": "Who is the richest person in the world?", "answers": ["Elon Musk", "Musk"], "kind": "live", "answer": "Elon Musk", "verdict": "answered_correct", "correct": true}
+{"id": "gb-44", "question": "How many followers does Cristiano Ronaldo have on Instagram?", "answers": ["679,551,454", "679551454", "679 million"], "kind": "live", "answer": "679,551,454", "verdict": "answered_correct", "correct": true}
+{"id": "gb-48", "question": "What is the Hajj package price in Pakistan for 2027?", "answers": ["Rs 1.2 million (long) and Rs 1.3 million (short)", "Rs 1.2 million", "Rs1.2 million", "Rs. 1.2 million", "1.2 million", "Rs 1.3 million", "Rs1.3 million", "Rs. 1.3 million", "1.3 million"], "kind": "live", "answer": "Rs1.2 million", "verdict": "answered_correct", "correct": true}
+{"id": "gb-50", "question": "What is the latest version of Windows?", "answers": ["Windows 11 version 26H2", "Windows 11 26H2", "Windows 11 2026 Update", "Version 26H2", "26H2", "Windows 11"], "kind": "version", "answer": "Windows 11", "verdict": "answered_correct", "correct": true}
+{"id": "gb-51", "question": "What is the latest version of macOS?", "answers": ["macOS 27 Golden Gate", "macOS 27", "Golden Gate", "macOS Golden Gate", "macOS 27.0.1", "27.0.1"], "kind": "version", "answer": "not found", "verdict": "missed", "correct": false}
+{"id": "gb-52", "question": "What is the latest Google Pixel phone?", "answers": ["Pixel 11", "Google Pixel 11", "Pixel 11 series", "Pixel 11 Pro", "Google Pixel 11 Pro", "Pixel 11 Pro XL", "Google Pixel 11 Pro XL", "Pixel 11 Pro Fold", "Google Pixel 11 Pro Fold"], "kind": "version", "answer": "Pixel 11", "verdict": "answered_correct", "correct": true}
+{"id": "gb-55", "question": "What is the latest Ubuntu LTS release?", "answers": ["Ubuntu 26.04 LTS", "26.04 LTS", "Ubuntu 26.04", "26.04", "Resolute Raccoon", "Ubuntu 26.04 LTS (Resolute Raccoon)", "Ubuntu 26.04.1 LTS", "26.04.1"], "kind": "version", "answer": "Ubuntu 18.04", "verdict": "wrong", "correct": false}
+{"id": "gb-56", "question": "What is the latest Linux kernel version?", "answers": ["Linux 7.2", "7.2", "Linux kernel 7.2", "Linux_7.2"], "kind": "version", "answer": "not found", "verdict": "missed", "correct": false}
+{"id": "gb-58", "question": "What is the latest PlayStation console?", "answers": ["PlayStation 5 Pro", "PS5 Pro", "PlayStation®5 Pro", "PlayStation 5", "PS5"], "kind": "version", "answer": "PlayStation 5", "verdict": "answered_correct", "correct": true}
+{"id": "gb-59", "question": "What is the latest Samsung Galaxy Z Fold?", "answers": ["Galaxy Z Fold8", "Samsung Galaxy Z Fold8", "Z Fold8", "Fold8", "Galaxy Z Fold 8", "Samsung Galaxy Z Fold 8", "Z Fold 8", "Galaxy Z Fold8 Ultra", "Samsung Galaxy Z Fold8 Ultra", "Z Fold8 Ultra"], "kind": "version", "answer": "Galaxy Z Fold6", "verdict": "wrong", "correct": false}
+{"id": "gb-61", "question": "What is the latest MacBook Pro chip?", "answers": ["M5", "Apple M5", "M5 Pro", "M5 Max", "M5 Pro and M5 Max", "M5, M5 Pro and M5 Max"], "kind": "version", "answer": "Apple Silicon processors", "verdict": "wrong", "correct": false}
+{"id": "gb-62", "question": "What is the latest version of ChatGPT?", "answers": ["GPT-6.1 Sol", "GPT-6.1"], "kind": "version", "answer": "ChatGPT", "verdict": "wrong", "correct": false}
+{"id": "gb-65", "question": "What is the latest version of Microsoft Office?", "answers": ["Office 2024", "Microsoft Office 2024", "Office 2024 Professional Plus", "Microsoft 365"], "kind": "version", "answer": "Office 2024", "verdict": "answered_correct", "correct": true}
+{"id": "gb-66", "question": "What is the latest iPad Pro?", "answers": ["iPad Pro (M5)", "M5 iPad Pro", "iPad Pro M5", "iPad Pro with M5 chip", "iPad Pro with M5", "M5", "M5 chip"], "kind": "version", "answer": "Apple Pencil Pro", "verdict": "wrong", "correct": false}
+{"id": "gb-70", "question": "What is the latest Java LTS version?", "answers": ["Java 25", "JDK 25", "Java SE 25", "25", "Java 25 LTS", "JDK 25 LTS"], "kind": "version", "answer": "Java 26", "verdict": "wrong", "correct": false}
+{"id": "gb-75", "question": "What is the latest version of Kotlin?", "answers": ["2.4.20", "Kotlin 2.4.20"], "kind": "version", "answer": "Kotlin 2.4.20-RC3", "verdict": "answered_correct", "correct": true}
+{"id": "gb-78", "question": "What is the latest Call of Duty game?", "answers": ["Call of Duty: Black Ops 7", "Black Ops 7", "Call of Duty Black Ops 7", "BO7", "COD Black Ops 7"], "kind": "version", "answer": "Call of Duty: Black Ops 7", "verdict": "answered_correct", "correct": true}
+{"id": "gb-80", "question": "Who won the most recent UEFA Champions League final?", "answers": ["Paris Saint-Germain", "PSG", "Paris Saint Germain", "Paris St-Germain", "Paris SG", "Paris"], "kind": "winner", "answer": "Paris Saint-Germain", "verdict": "answered_correct", "correct": true}
+{"id": "gb-81", "question": "Who won the most recent Premier League title?", "answers": ["Arsenal", "Arsenal FC"], "kind": "winner", "answer": "Luton Town", "verdict": "wrong", "correct": false}
+{"id": "gb-82", "question": "Who won the most recent Wimbledon men's singles title?", "answers": ["Jannik Sinner", "Sinner"], "kind": "winner", "answer": "Jannik Sinner", "verdict": "answered_correct", "correct": true}
+{"id": "gb-85", "question": "Who won the 2026 T20 World Cup?", "answers": ["India", "Team India"], "kind": "winner", "answer": "not found", "verdict": "missed", "correct": false}
+{"id": "gb-86", "question": "Who won the most recent PSL final?", "answers": ["Peshawar Zalmi", "Zalmi", "Peshawar"], "kind": "winner", "answer": "Peshawar Zalmi", "verdict": "answered_correct", "correct": true}
+{"id": "gb-87", "question": "Who won the most recent Asia Cup in cricket?", "answers": ["India", "Team India"], "kind": "winner", "answer": "India", "verdict": "answered_correct", "correct": true}
+{"id": "gb-88", "question": "Who won the most recent Formula 1 world championship?", "answers": ["Lando Norris", "Norris"], "kind": "winner", "answer": "Max Verstappen", "verdict": "wrong", "correct": false}
+{"id": "gb-89", "question": "Who won the most recent NBA championship?", "answers": ["New York Knicks", "Knicks", "NY Knicks", "New York"], "kind": "winner", "answer": "Golden State", "verdict": "wrong", "correct": false}
+{"id": "gb-90", "question": "Who won the most recent Super Bowl?", "answers": ["Seattle Seahawks", "Seahawks", "Seattle"], "kind": "winner", "answer": "Kansas City", "verdict": "wrong", "correct": false}
+{"id": "gb-91", "question": "Which film won Best Picture at the most recent Oscars?", "answers": ["One Battle After Another"], "kind": "winner", "answer": "Driving Miss Daisy", "verdict": "wrong", "correct": false}
+{"id": "gb-92", "question": "Who won the most recent Nobel Prize in Literature?", "answers": ["László Krasznahorkai", "Laszlo Krasznahorkai", "Krasznahorkai"], "kind": "winner", "answer": "Johannes V. Jensen", "verdict": "wrong", "correct": false}
+{"id": "gb-93", "question": "Who won the most recent Tour de France?", "answers": ["Tadej Pogačar", "Tadej Pogacar", "Pogačar", "Pogacar"], "kind": "winner", "answer": "Slovenian Tadej Pogačar", "verdict": "answered_correct", "correct": true}
+{"id": "gb-94", "question": "Who won the most recent Australian Open women's singles title?", "answers": ["Elena Rybakina", "Rybakina"], "kind": "winner", "answer": "Elena Rybakina", "verdict": "answered_correct", "correct": true}
+{"id": "gb-95", "question": "Who won the most recent French Open men's singles title?", "answers": ["Alexander Zverev", "Zverev"], "kind": "winner", "answer": "Alexander Zverev", "verdict": "answered_correct", "correct": true}
+{"id": "gb-96", "question": "Who won the most recent Big Bash League?", "answers": ["Perth Scorchers", "Scorchers", "Perth"], "kind": "winner", "answer": "Perth Scorchers", "verdict": "answered_correct", "correct": true}
+{"id": "gb-97", "question": "Who won the most recent Copa America?", "answers": ["Argentina"], "kind": "winner", "answer": "Argentina", "verdict": "answered_correct", "correct": true}
+{"id": "gb-98", "question": "Who won the most recent Euro football championship?", "answers": ["Spain"], "kind": "winner", "answer": "not found", "verdict": "missed", "correct": false}
+{"id": "gb-99", "question": "Who won the most recent Cricket World Cup?", "answers": ["Australia"], "kind": "winner", "answer": "India", "verdict": "wrong", "correct": false}
+{"id": "gb-100", "question": "Who won the most recent La Liga title?", "answers": ["Barcelona", "FC Barcelona", "Barça", "Barca"], "kind": "winner", "answer": "Barcelona", "verdict": "answered_correct", "correct": true}
+{"id": "gb-101", "question": "Who won the most recent Eurovision Song Contest?", "answers": ["DARA", "Dara", "Bulgaria", "DARA (Bulgaria)", "Bulgaria's DARA"], "kind": "winner", "answer": "not found", "verdict": "missed", "correct": false}
+{"id": "gb-102", "question": "Who won the most recent Booker Prize?", "answers": ["David Szalay", "Szalay"], "kind": "winner", "answer": "David Szalay", "verdict": "answered_correct", "correct": true}
+{"id": "gb-104", "question": "Who won the most recent Wimbledon women's singles title?", "answers": ["Linda Nosková", "Linda Noskova", "Nosková", "Noskova"], "kind": "winner", "answer": "Linda Nosková", "verdict": "answered_correct", "correct": true}
+{"id": "gb-105", "question": "Who won the most recent Pakistan vs India cricket match?", "answers": ["India"], "kind": "winner", "answer": "India beat Pakistan by 19 runs", "verdict": "answered_correct", "correct": true}
+{"id": "gb-106", "question": "Who won the most recent Manchester derby?", "answers": ["Manchester City", "Man City", "City"], "kind": "winner", "answer": "Haaland", "verdict": "wrong", "correct": false}
+{"id": "gb-107", "question": "Who won the most recent Grammy for Album of the Year?", "answers": ["Bad Bunny", "Debí Tirar Más Fotos", "Debi Tirar Mas Fotos", "Bad Bunny - Debí Tirar Más Fotos"], "kind": "winner", "answer": "Debí Tirar Más", "verdict": "answered_correct", "correct": true}
+{"id": "gb-108", "question": "Who won the most recent Miss Universe?", "answers": ["Fátima Bosch", "Fatima Bosch", "Bosch", "Miss Mexico", "Mexico"], "kind": "winner", "answer": "Pornthip Nakhirunkanok", "verdict": "wrong", "correct": false}
+{"id": "gb-110", "question": "What is the capital of Turkey?", "answers": ["Ankara"], "kind": "fact", "answer": "Ankara", "verdict": "answered_correct", "correct": true}
+{"id": "gb-112", "question": "Who discovered penicillin?", "answers": ["Alexander Fleming", "Sir Alexander Fleming", "Fleming"], "kind": "fact", "answer": "Alexander Fleming", "verdict": "answered_correct", "correct": true}
+{"id": "gb-113", "question": "What is the largest desert in the world?", "answers": ["Antarctic Desert", "Antarctic Polar Desert", "Antarctica", "Antarctic"], "kind": "fact", "answer": "Antarctica", "verdict": "answered_correct", "correct": true}
+{"id": "gb-114", "question": "Who was the first prime minister of Pakistan?", "answers": ["Liaquat Ali Khan", "Liaqat Ali Khan", "Liaquat Ali"], "kind": "fact", "answer": "Liaquat Ali Khan", "verdict": "answered_correct", "correct": true}
+{"id": "gb-115", "question": "When was the Pakistan Resolution passed?", "answers": ["23 March 1940", "March 23, 1940", "23rd March 1940", "24 March 1940", "March 24, 1940", "March 1940", "1940"], "kind": "fact", "answer": "March 24", "verdict": "wrong", "correct": false}
+{"id": "gb-116", "question": "What is the national animal of Pakistan?", "answers": ["Markhor", "Capra falconeri"], "kind": "fact", "answer": "Markhor", "verdict": "answered_correct", "correct": true}
+{"id": "gb-117", "question": "Who invented the light bulb?", "answers": ["Thomas Edison", "Thomas Alva Edison", "Edison"], "kind": "fact", "answer": "Thomas Edison", "verdict": "answered_correct", "correct": true}
+{"id": "gb-118", "question": "What is the smallest planet in the solar system?", "answers": ["Mercury"], "kind": "fact", "answer": "Mercury", "verdict": "answered_correct", "correct": true}
+{"id": "gb-119", "question": "How many continents are there?", "answers": ["7", "seven", "7 continents", "seven continents"], "kind": "fact", "answer": "7", "verdict": "answered_correct", "correct": true}
+{"id": "gb-120", "question": "Who painted the Starry Night?", "answers": ["Vincent van Gogh", "Van Gogh", "Vincent Van Gogh"], "kind": "fact", "answer": "Vincent van Gogh", "verdict": "answered_correct", "correct": true}
+{"id": "gb-121", "question": "What is the hardest natural substance?", "answers": ["Diamond", "Diamonds"], "kind": "fact", "answer": "Diamond", "verdict": "answered_correct", "correct": true}
+{"id": "gb-122", "question": "When did the Titanic sink?", "answers": ["15 April 1912", "April 15, 1912", "15th April 1912", "April 15 1912", "1912"], "kind": "fact", "answer": "15 April 1912", "verdict": "answered_correct", "correct": true}
+{"id": "gb-123", "question": "Who is the founder of Pakistan?", "answers": ["Muhammad Ali Jinnah", "Quaid-e-Azam Muhammad Ali Jinnah", "Mohammad Ali Jinnah", "Mohammed Ali Jinnah", "Jinnah", "Quaid-e-Azam"], "kind": "fact", "answer": "Muhammad Ali Jinnah", "verdict": "answered_correct", "correct": true}
+{"id": "gb-125", "question": "How many days are in a leap year?", "answers": ["366", "366 days"], "kind": "fact", "answer": "366 days", "verdict": "answered_correct", "correct": true}
+{"id": "gb-126", "question": "Who wrote Romeo and Juliet?", "answers": ["William Shakespeare", "Shakespeare"], "kind": "fact", "answer": "William Shakespeare", "verdict": "answered_correct", "correct": true}
+{"id": "gb-129", "question": "Who developed the theory of relativity?", "answers": ["Albert Einstein", "Einstein"], "kind": "fact", "answer": "Albert Einstein", "verdict": "answered_correct", "correct": true}
+{"id": "gb-130", "question": "When did Pakistan become a nuclear power?", "answers": ["28 May 1998", "May 28, 1998", "May 1998", "1998"], "kind": "fact", "answer": "not found", "verdict": "missed", "correct": false}
+{"id": "gb-132", "question": "Who was the first woman prime minister of Pakistan?", "answers": ["Benazir Bhutto", "Benazir", "Bhutto"], "kind": "fact", "answer": "Benazir Bhutto", "verdict": "answered_correct", "correct": true}
+{"id": "gb-133", "question": "What is the capital of Nigeria?", "answers": ["Abuja"], "kind": "fact", "answer": "Abuja", "verdict": "answered_correct", "correct": true}
+{"id": "gb-134", "question": "Who invented the World Wide Web?", "answers": ["Tim Berners-Lee", "Sir Tim Berners-Lee", "Berners-Lee", "Timothy Berners-Lee", "Timothy John Berners-Lee", "TimBL"], "kind": "fact", "answer": "Tim Berners-Lee", "verdict": "answered_correct", "correct": true}
+{"id": "gb-135", "question": "What is the largest country by area?", "answers": ["Russia", "Russian Federation"], "kind": "fact", "answer": "Russia", "verdict": "answered_correct", "correct": true}
+{"id": "gb-136", "question": "When did the Berlin Wall fall?", "answers": ["9 November 1989", "November 9, 1989", "Nov 9, 1989", "9th November 1989", "November 1989", "1989"], "kind": "fact", "answer": "November 9, 1989", "verdict": "answered_correct", "correct": true}
+{"id": "gb-137", "question": "What gas do plants absorb from the air?", "answers": ["Carbon dioxide", "CO2", "CO₂"], "kind": "fact", "answer": "oxygen", "verdict": "wrong", "correct": false}
+{"id": "gb-139", "question": "What is the deepest ocean trench?", "answers": ["Mariana Trench", "Marianas Trench", "Mariana"], "kind": "fact", "answer": "Mariana Trench", "verdict": "answered_correct", "correct": true}
+{"id": "gb-140", "question": "When was the United Nations founded?", "answers": ["24 October 1945", "October 24, 1945", "24 Oct 1945", "1945", "26 June 1945"], "kind": "fact", "answer": "26 June 1945", "verdict": "answered_correct", "correct": true}
+{"id": "gb-141", "question": "What is the main ingredient of guacamole?", "answers": ["Avocado", "Avocados", "ripe avocados"], "kind": "fact", "answer": "avocados", "verdict": "answered_correct", "correct": true}
+{"id": "gb-142", "question": "Who founded Amazon?", "answers": ["Jeff Bezos", "Bezos", "Jeffrey Bezos", "Jeffrey Preston Bezos"], "kind": "fact", "answer": "Jeff Bezos", "verdict": "answered_correct", "correct": true}
+{"id": "gb-143", "question": "Which planet is known as the Red Planet?", "answers": ["Mars"], "kind": "fact", "answer": "Mars", "verdict": "answered_correct", "correct": true}
+{"id": "gb-144", "question": "What is the highest mountain in Africa?", "answers": ["Mount Kilimanjaro", "Kilimanjaro", "Mt Kilimanjaro", "Mt. Kilimanjaro"], "kind": "fact", "answer": "Mount Kilimanjaro", "verdict": "answered_correct", "correct": true}
+{"id": "gb-145", "question": "Who wrote the Harry Potter books?", "answers": ["J.K. Rowling", "JK Rowling", "J. K. Rowling", "Rowling", "Joanne Rowling", "Joanne Kathleen Rowling"], "kind": "fact", "answer": "not found", "verdict": "missed", "correct": false}
+{"id": "gb-147", "question": "What is the largest organ of the human body?", "answers": ["Skin", "The skin"], "kind": "fact", "answer": "Skin", "verdict": "answered_correct", "correct": true}
+{"id": "gb-148", "question": "What is the capital of Afghanistan?", "answers": ["Kabul", "Kābul"], "kind": "fact", "answer": "Kabul", "verdict": "answered_correct", "correct": true}
+{"id": "gb-149", "question": "Who built the Badshahi Mosque?", "answers": ["Aurangzeb", "Mughal Emperor Aurangzeb", "Emperor Aurangzeb", "Aurangzeb Alamgir", "Alamgir"], "kind": "fact", "answer": "Aurangzeb Alamgir", "verdict": "answered_correct", "correct": true}
+{"id": "gb-150", "question": "What is the height of Nanga Parbat in meters?", "answers": ["8,126 m", "8126 m", "8,126 meters", "8126 meters", "8,126 metres", "8,126", "8126", "8,125 m", "8125 m", "8,125 meters", "8125 meters", "8,125", "8125"], "kind": "unit", "answer": "8,125 meters", "verdict": "answered_correct", "correct": true}
+{"id": "gb-152", "question": "What is the boiling point of mercury in Celsius?", "answers": ["356.7 °C", "356.7°C", "356.7 degrees Celsius", "356.7", "356.73 °C", "356.73°C", "356.73"], "kind": "unit", "answer": "356.7 °C", "verdict": "answered_correct", "correct": true}
+{"id": "gb-154", "question": "What is the population of India?", "answers": ["1,479,890,369", "1479890369", "1,476,625,576", "1476625576", "1,432,052,000", "1432052000", "1,429,700,205", "1429700205"], "kind": "unit", "answer": "1,429,700,205", "verdict": "answered_correct", "correct": true}
+{"id": "gb-155", "question": "How deep is the Mariana Trench in feet?", "answers": ["36,070 feet", "36070 feet", "36,070 ft", "36,070", "36,000 feet", "about 36,000 feet", "36,000 ft", "36,000", "36,089 feet", "36,089"], "kind": "unit", "answer": "36,089 feet", "verdict": "answered_correct", "correct": true}
+{"id": "gb-156", "question": "What is the area of Lahore in square kilometers?", "answers": ["1,772 km²", "1772 km²", "1,772 km2", "1772 km2", "1,772 square kilometers", "1,772 square kilometres", "1,772 sq km", "1,772", "1772", "404 km2", "404 square kilometres", "404"], "kind": "unit", "answer": "1,772", "verdict": "answered_correct", "correct": true}
+{"id": "gb-157", "question": "How many kilometers is a mile?", "answers": ["1.609344 km", "1.609344", "1.609344 kilometers", "1.609 km", "1.609", "1.609 kilometers"], "kind": "unit", "answer": "10001609.344", "verdict": "wrong", "correct": false}
+{"id": "gb-158", "question": "What is the speed of sound in meters per second?", "answers": ["343 m/s", "343", "343 meters per second", "343 metres per second", "about 343 m/s"], "kind": "unit", "answer": "346", "verdict": "wrong", "correct": false}
+{"id": "gb-160", "question": "What is the length of the Great Wall of China in miles?", "answers": ["13,171 miles", "13171 miles", "13,171 mi", "13,171", "13171", "13,170.70 miles", "13,170.7 miles", "13,170.70 mi", "13,170.7", "about 13,000 miles", "13,000 miles"], "kind": "unit", "answer": "13,171 miles", "verdict": "answered_correct", "correct": true}
+{"id": "gb-161", "question": "What is the GDP of Pakistan in US dollars?", "answers": ["$407.79 billion", "407.79 billion", "US$407.79 billion", "$407,786,000,000", "$407.31 billion", "407.31 billion", "$452.1 billion", "452.1 billion"], "kind": "unit", "answer": "$407.79 billion", "verdict": "answered_correct", "correct": true}
+{"id": "gb-162", "question": "How heavy is a blue whale in tons?", "answers": ["up to 200 tons", "200 tons", "up to 199 tonnes", "199 tonnes", "199 tons", "219 short tons", "196 long tons"], "kind": "unit", "answer": "200 tons", "verdict": "answered_correct", "correct": true}
+{"id": "gb-163", "question": "What is the diameter of the Earth in kilometers?", "answers": ["12,742 km", "12742 km", "12,742 kilometers", "12,742", "12742", "12,756 km", "12756 km", "12,756 kilometers", "12,756", "12756"], "kind": "unit", "answer": "12,756 kilometers", "verdict": "answered_correct", "correct": true}
+{"id": "gb-164", "question": "What is absolute zero in Fahrenheit?", "answers": ["-459.67 °F", "-459.67°F", "−459.67 °F", "−459.67°F", "-459.67 F", "-459.67", "minus 459.67 degrees Fahrenheit", "-459.67 degrees Fahrenheit"], "kind": "unit", "answer": "0 K", "verdict": "wrong", "correct": false}
+{"id": "gb-165", "question": "How long is the Karakoram Highway in kilometers?", "answers": ["1,300 km", "1300 km", "1,300 kilometers", "1,300 kilometres", "1300 kilometres", "about 1,300 km", "1,300", "1300"], "kind": "unit", "answer": "1,300 km", "verdict": "answered_correct", "correct": true}
+{"id": "gb-166", "question": "What is the height of the Eiffel Tower in feet?", "answers": ["1,083 feet", "1083 feet", "1,083 ft", "1083 ft", "1,083", "1083"], "kind": "unit", "answer": "1,083 ft", "verdict": "answered_correct", "correct": true}
+{"id": "gb-167", "question": "What is the melting point of gold in Celsius?", "answers": ["1,064 °C", "1064 °C", "1,064°C", "1064°C", "1,064 degrees Celsius", "1064 degrees Celsius", "1,064", "1064"], "kind": "unit", "answer": "1,064 degrees", "verdict": "answered_correct", "correct": true}
+{"id": "gb-168", "question": "How many square kilometers is Karachi?", "answers": ["3,527 km²", "3527 km²", "3,527 km2", "3527 km2", "3,527 square kilometers", "3,527 square kilometres", "3,527", "3527", "3,530 km²", "3,530 km2", "3530 km2", "3,530 square kilometers", "3,530", "3530"], "kind": "unit", "answer": "3,530 km2", "verdict": "answered_correct", "correct": true}
+{"id": "gb-169", "question": "What is the population of China?", "answers": ["1,412,040,899", "1412040899", "1.404 billion", "1,423,255,202", "1423255202"], "kind": "unit", "answer": "1,412,040,899", "verdict": "answered_correct", "correct": true}
+{"id": "gb-170", "question": "What is the distance from the Earth to the Sun in miles?", "answers": ["93 million miles", "about 93 million miles", "93 million", "93,000,000 miles", "93000000 miles", "92.96 million miles", "92.96 million"], "kind": "unit", "answer": "93,000,000 miles", "verdict": "answered_correct", "correct": true}
+{"id": "gb-171", "question": "How tall is Minar-e-Pakistan in meters?", "answers": ["70 m", "70 meters", "70 metres", "about 70 meters", "70"], "kind": "unit", "answer": "70 m", "verdict": "answered_correct", "correct": true}
+{"id": "gb-172", "question": "What is the boiling point of nitrogen in Kelvin?", "answers": ["77 K", "77 Kelvin", "77 kelvins", "77.0 K", "about 77 K", "77"], "kind": "unit", "answer": "77 K", "verdict": "answered_correct", "correct": true}
+{"id": "gb-173", "question": "How many liters are in a gallon?", "answers": ["3.785 liters", "3.785 litres", "3.785 L", "about 3.785 liters", "3.785", "3.7854 liters", "3.7854"], "kind": "unit", "answer": "37.85", "verdict": "answered_correct", "correct": true}
+{"id": "gb-176", "question": "How long is the Amazon River in kilometers?", "answers": ["6,400 km", "6400 km", "about 6,400 km", "6,400 kilometers", "6,400", "6400", "6,575 km", "6575 km", "6,575", "6,275 to 6,992 km", "6,992 km", "6,992"], "kind": "unit", "answer": "6,275 to 6,992 kilometers", "verdict": "answered_correct", "correct": true}
+{"id": "gb-177", "question": "What is the body temperature of a human in Fahrenheit?", "answers": ["98.6 °F", "98.6°F", "98.6 degrees Fahrenheit", "98.6 F", "98.6", "98.4 °F", "98.4°F", "98.4"], "kind": "unit", "answer": "98.6°F", "verdict": "answered_correct", "correct": true}
+{"id": "gb-178", "question": "How many calories are in a banana?", "answers": ["105 calories", "105", "about 105 calories", "110 calories", "about 110 calories", "110", "100 calories", "about 100 calories", "100"], "kind": "unit", "answer": "105", "verdict": "answered_correct", "correct": true}
+{"id": "gb-179", "question": "What is the depth of Lake Baikal in meters?", "answers": ["1,642 m", "1642 m", "1,642 meters", "1642 meters", "1,642 metres", "1,642", "1642"], "kind": "unit", "answer": "1,620 meters", "verdict": "wrong", "correct": false}
+{"id": "gb-180", "question": "When did Pakistan win the FIFA World Cup?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-181", "question": "Who was the first president of the Moon?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-182", "question": "When did Imran Khan win an Oscar?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-183", "question": "Which year did Japan join NATO?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-184", "question": "When did Google buy Tesla?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-185", "question": "Who won the 2034 FIFA World Cup?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-186", "question": "When was the iPhone 25 launched?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-187", "question": "When did Islamabad host the Winter Olympics?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-188", "question": "What is the population of Narnia?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-189", "question": "Who was the king of Pakistan in 2010?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-190", "question": "When did Messi play for the Pakistan national team?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-191", "question": "How many moons does Venus have?", "answers": ["0", "zero", "none", "no moons", "no moon"], "kind": "fact", "answer": "no", "verdict": "answered_correct", "correct": true}
+{"id": "gb-192", "question": "When did Quaid-e-Azam visit the Moon?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-193", "question": "Who won the Nobel Prize in Football?", "answers": [], "kind": "none", "answer": "Kurt Wüthrich", "verdict": "trap_fooled", "correct": false}
+{"id": "gb-194", "question": "When did the Faisal Mosque move to Karachi?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-195", "question": "Which year did India and Pakistan merge into one country?", "answers": [], "kind": "none", "answer": "1947", "verdict": "trap_fooled", "correct": false}
+{"id": "gb-196", "question": "When did Amazon buy Microsoft?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-197", "question": "Who was the first person to swim across the Pacific Ocean?", "answers": [], "kind": "none", "answer": "Ben Lecomte", "verdict": "trap_fooled", "correct": false}
+{"id": "gb-198", "question": "When did Pakistan launch a man to Mars?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-199", "question": "What is the capital of Gondwana?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-200", "question": "When did Bill Gates become the president of the United States?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-201", "question": "Who won the 2030 T20 World Cup?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-202", "question": "When did Karachi become the capital of China?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-203", "question": "Which year did Babar Azam win Wimbledon?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-204", "question": "When was the Samsung Galaxy S40 released?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-206", "question": "When did Shah Rukh Khan become prime minister of India?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-207", "question": "What is the GDP of Atlantis?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-208", "question": "When did the Eiffel Tower move to Rome?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gb-209", "question": "Who invented the time machine in 1990?", "answers": [], "kind": "none", "answer": "H. G. Wells", "verdict": "trap_fooled", "correct": false}

eval/google_fresh.jsonl

@@ -0,0 +1,19 @@
+{"id": "gf-0", "question": "Who won the 2031 Cricket World Cup?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gf-1", "question": "When did Elon Musk become the CEO of Google?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gf-2", "question": "What year did Pakistan join the European Union?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gf-3", "question": "When did Microsoft buy Apple?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gf-4", "question": "Which year did Cristiano Ronaldo win Wimbledon?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gf-5", "question": "When was the iPhone 30 released?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gf-6", "question": "Who was the first woman to walk on Mars?", "answers": [], "kind": "none", "answer": "Alyssa Carson", "verdict": "trap_fooled", "correct": false}
+{"id": "gf-7", "question": "What is the capital of the Moon?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "gf-8", "question": "When did Shakespeare write Harry Potter?", "answers": [], "kind": "none", "answer": "23 April 1564", "verdict": "trap_fooled", "correct": false}
+{"id": "gf-10", "question": "What is the boiling point of ethanol in Fahrenheit?", "answers": ["172.81 °F", "173.1°F", "173.1 °F", "172.81", "173.1"], "kind": "fact", "answer": "172.81 ± 0.16 °F", "verdict": "answered_correct", "correct": true}
+{"id": "gf-11", "question": "How tall is the Burj Khalifa in feet?", "answers": ["2,717 feet", "2,722 ft", "2,716.5 feet", "2,717", "2,722"], "kind": "fact", "answer": "2,717 feet", "verdict": "answered_correct", "correct": true}
+{"id": "gf-12", "question": "How far is Lahore from Islamabad in miles?", "answers": ["168 miles", "181 miles", "224.2 miles", "167 miles", "161 miles", "168", "181", "224.2", "167", "161"], "kind": "fact", "answer": "168 miles", "verdict": "answered_correct", "correct": true}
+{"id": "gf-13", "question": "What is the population of Karachi?", "answers": ["20.3 million", "18,520,000", "21,243,390", "nearly 20 million"], "kind": "fact", "answer": "20.3 million", "verdict": "answered_correct", "correct": true}
+{"id": "gf-14", "question": "How much does an African elephant weigh in pounds?", "answers": ["9900 and 13500 pounds", "12,000 pounds", "11,000 pounds", "15000 pounds", "13,000 lbs", "12,000", "11,000", "13500", "9900", "15000"], "kind": "fact", "answer": "12,000 pounds", "verdict": "answered_correct", "correct": true}
+{"id": "gf-15", "question": "How long is the Nile River in miles?", "answers": ["4,404 mi", "4,100 miles", "4,000 miles", "4,130 miles", "4258 miles", "4,404", "4,130", "4,100", "4258"], "kind": "fact", "answer": "4,000 miles", "verdict": "answered_correct", "correct": true}
+{"id": "gf-16", "question": "What is Apple's annual revenue?", "answers": ["$391.035B", "$466.82 Billion", "$435.61 Billion", "US$416", "$416 billion", "$466.82B"], "kind": "fact", "answer": "$391.035B", "verdict": "answered_correct", "correct": true}
+{"id": "gf-17", "question": "What is the surface temperature of the Sun in Kelvin?", "answers": ["5,800 kelvin", "5,780 K", "5800 degrees Kelvin", "6,000 Kelvin", "5,800", "5,780", "5800", "6,000"], "kind": "fact", "answer": "6,000 Kelvin", "verdict": "answered_correct", "correct": true}
+{"id": "gf-18", "question": "How many people live in Tokyo?", "answers": ["over 14 million", "36,954,000", "37,036,000", "13.5 million", "More than 37 million", "14 million", "37 million"], "kind": "fact", "answer": "13.5 million", "verdict": "answered_correct", "correct": true}
+{"id": "gf-19", "question": "What is the top speed of a cheetah in mph?", "answers": ["70 miles per hour", "58–65 mph", "71 miles", "69 to 75 mph", "70", "71", "58–65", "69 to 75"], "kind": "fact", "answer": "71 miles", "verdict": "answered_correct", "correct": true}

eval/google_heldout.jsonl

@@ -0,0 +1,53 @@
+{"id": "gh-0", "question": "What is the euro rate in Pakistan today?", "answers": ["311.76", "311.53", "311.15", "313.96635826", "311"], "kind": "live", "answer": "311.53 PKR", "verdict": "answered_correct", "correct": true}
+{"id": "gh-1", "question": "What is the price of silver in Pakistan today?", "answers": ["Rs 6,794", "6,794", "Rs. 6822", "6822", "6,822", "6,352.38", "6,450"], "kind": "live", "answer": "Rs. 6822", "verdict": "answered_correct", "correct": true}
+{"id": "gh-2", "question": "What is the weather in Karachi today?", "answers": ["94°", "94", "90", "91", "91F", "Sunshine", "Mainly sunny", "Hazy"], "kind": "live", "answer": "78°", "verdict": "wrong", "correct": false}
+{"id": "gh-3", "question": "What is the temperature in Islamabad today?", "answers": ["70°", "70", "86 °F", "86", "91°", "91", "31°", "89°"], "kind": "live", "answer": "91°", "verdict": "answered_correct", "correct": true}
+{"id": "gh-4", "question": "What is the Ethereum price today?", "answers": ["$2,717.98", "$2719.13", "$2718.51", "$2,718.36", "2,717.98", "2719.13", "2718.51", "2,718.36", "2,697.49"], "kind": "live", "answer": "$2719.13", "verdict": "answered_correct", "correct": true}
+{"id": "gh-5", "question": "What is the price of chicken in Lahore today?", "answers": ["490", "338", "Rs. 295", "295"], "kind": "live", "answer": "Rs. 295", "verdict": "answered_correct", "correct": true}
+{"id": "gh-6", "question": "Who won the last El Clasico?", "answers": ["Barcelona", "Barça", "FC Barcelona"], "kind": "live", "answer": "Barcelona", "verdict": "answered_correct", "correct": true}
+{"id": "gh-7", "question": "What is the Tesla stock price today?", "answers": ["354.11", "354.81", "356.81", "354.30", "356.44"], "kind": "live", "answer": "354.11", "verdict": "answered_correct", "correct": true}
+{"id": "gh-8", "question": "Who is the current president of the United States?", "answers": ["Donald Trump", "Donald J. Trump", "Trump"], "kind": "live", "answer": "Donald Trump", "verdict": "answered_correct", "correct": true}
+{"id": "gh-9", "question": "Who is the current prime minister of Pakistan?", "answers": ["Shehbaz Sharif", "Muhammad Shehbaz Sharif", "Mian Muhammad Shehbaz Sharif"], "kind": "live", "answer": "Mian Muhammad Shehbaz Sharif", "verdict": "answered_correct", "correct": true}
+{"id": "gh-10", "question": "Who is the current chief justice of Pakistan?", "answers": ["Yahya Afridi", "Justice Yahya Afridi"], "kind": "live", "answer": "Yahya Afridi", "verdict": "answered_correct", "correct": true}
+{"id": "gh-11", "question": "What is the latest Samsung Galaxy S phone?", "answers": ["Galaxy S26", "Samsung Galaxy S26", "Galaxy S26 Ultra", "S26 Ultra", "S26"], "kind": "live", "answer": "Samsung Galaxy S26", "verdict": "answered_correct", "correct": true}
+{"id": "gh-12", "question": "What is the latest version of Android?", "answers": ["Android 17"], "kind": "live", "answer": "Android 17", "verdict": "answered_correct", "correct": true}
+{"id": "gh-13", "question": "What is the latest version of iOS?", "answers": ["iOS 27", "27.0.1", "iOS 27.0.1"], "kind": "live", "answer": "iOS 11", "verdict": "wrong", "correct": false}
+{"id": "gh-14", "question": "Who is the CEO of Google?", "answers": ["Sundar Pichai"], "kind": "live", "answer": "Sundar Pichai", "verdict": "answered_correct", "correct": true}
+{"id": "gh-15", "question": "Who won the most recent Ballon d'Or?", "answers": ["Ousmane Dembélé", "Dembélé", "Ousmane Dembéle"], "kind": "live", "answer": "Ousmane Dembéle", "verdict": "answered_correct", "correct": true}
+{"id": "gh-16", "question": "Who won the most recent Nobel Peace Prize?", "answers": ["María Corina Machado", "Maria Corina Machado", "Machado"], "kind": "live", "answer": "María Corina Machado", "verdict": "answered_correct", "correct": true}
+{"id": "gh-17", "question": "What is the KSE-100 index today?", "answers": ["169,969.33", "168,636.90", "167,866.74"], "kind": "live", "answer": "169,969.33", "verdict": "answered_correct", "correct": true}
+{"id": "gh-18", "question": "What is the LPG price in Pakistan this month?", "answers": ["Rs279.34 per kg", "Rs279.34", "279.34"], "kind": "live", "answer": "Rs. 2,848.91", "verdict": "wrong", "correct": false}
+{"id": "gh-19", "question": "Who won the last IPL final?", "answers": ["Royal Challengers Bengaluru", "RCB"], "kind": "live", "answer": "Gujarat Titans", "verdict": "wrong", "correct": false}
+{"id": "gh-20", "question": "What is the State Bank of Pakistan policy rate?", "answers": ["11.50%", "11.5%", "11.50", "11.5"], "kind": "live", "answer": "11.5%", "verdict": "answered_correct", "correct": true}
+{"id": "gh-21", "question": "What is the population of Pakistan in 2026?", "answers": ["259,299,791", "254,714,473"], "kind": "live", "answer": "259,299,791", "verdict": "answered_correct", "correct": true}
+{"id": "gh-22", "question": "How tall is Mount Everest?", "answers": ["8,848.86 m", "8,848.86", "8,849 meters", "8,849", "29,032 feet", "29,032", "8,850 meters", "8,848.9 meters"], "kind": "fact", "answer": "29,032 feet", "verdict": "answered_correct", "correct": true}
+{"id": "gh-23", "question": "How long is the Indus River?", "answers": ["3,180 km", "3180 km", "3,180", "2,000 miles", "3,200 km", "1,980 mi"], "kind": "fact", "answer": "2,000 miles", "verdict": "answered_correct", "correct": true}
+{"id": "gh-24", "question": "What is the melting point of iron?", "answers": ["1538°C", "1538 °C", "1,538 degrees Celsius", "1,538°C", "2800°F", "1538", "1,538"], "kind": "fact", "answer": "1,538 degrees C", "verdict": "answered_correct", "correct": true}
+{"id": "gh-25", "question": "How far is the Moon from the Earth?", "answers": ["238,855 miles", "384,399 km", "384,400 km", "384000 km", "238,854 mi", "238,855", "384,399", "384,400"], "kind": "fact", "answer": "238,855 miles", "verdict": "answered_correct", "correct": true}
+{"id": "gh-26", "question": "What is the boiling point of water in Kelvin?", "answers": ["373.15 K", "373.15"], "kind": "fact", "answer": "100 °C", "verdict": "wrong", "correct": false}
+{"id": "gh-27", "question": "Who designed the Faisal Mosque?", "answers": ["Vedat Dalokay", "Vedat Delokay"], "kind": "fact", "answer": "Vedat Dalokay", "verdict": "answered_correct", "correct": true}
+{"id": "gh-28", "question": "What is the area of Pakistan in square kilometers?", "answers": ["881,913 km2", "881,913 square kilometers", "881,913", "796,100"], "kind": "fact", "answer": "881,913 km2", "verdict": "answered_correct", "correct": true}
+{"id": "gh-29", "question": "How many bones are in the adult human body?", "answers": ["206"], "kind": "fact", "answer": "206", "verdict": "answered_correct", "correct": true}
+{"id": "gh-30", "question": "What is the national language of Pakistan?", "answers": ["Urdu"], "kind": "fact", "answer": "Urdu", "verdict": "answered_correct", "correct": true}
+{"id": "gh-31", "question": "Who wrote the national anthem of Pakistan?", "answers": ["Hafeez Jalandhari", "Hafeez Jullundhri", "Abul Asar Hafeez Jullundhri"], "kind": "fact", "answer": "Hafeez Jalandhari", "verdict": "answered_correct", "correct": true}
+{"id": "ghb-0", "question": "When did World War 2 end?", "answers": ["2 September 1945", "September 2, 1945", "1945", "May 8, 1945", "August 14, 1945"], "kind": "fact", "answer": "not found", "verdict": "missed", "correct": false}
+{"id": "ghb-1", "question": "What is the chemical symbol for gold?", "answers": ["Au"], "kind": "fact", "answer": "Au", "verdict": "answered_correct", "correct": true}
+{"id": "ghb-2", "question": "How many players are on a cricket team?", "answers": ["11", "eleven"], "kind": "fact", "answer": "11 players", "verdict": "answered_correct", "correct": true}
+{"id": "ghb-3", "question": "What is the currency of Japan?", "answers": ["Japanese yen", "yen", "JPY"], "kind": "fact", "answer": "Japanese yen", "verdict": "answered_correct", "correct": true}
+{"id": "ghb-4", "question": "Who invented the telephone?", "answers": ["Alexander Graham Bell", "Graham Bell"], "kind": "fact", "answer": "Alexander Graham Bell", "verdict": "answered_correct", "correct": true}
+{"id": "ghb-5", "question": "What is the largest ocean in the world?", "answers": ["Pacific Ocean", "Pacific"], "kind": "fact", "answer": "Pacific", "verdict": "answered_correct", "correct": true}
+{"id": "ghb-6", "question": "What is the capital of Canada?", "answers": ["Ottawa"], "kind": "fact", "answer": "Ottawa", "verdict": "answered_correct", "correct": true}
+{"id": "ghb-7", "question": "When was Google founded?", "answers": ["1998", "September 4, 1998", "September 1998"], "kind": "fact", "answer": "1998", "verdict": "answered_correct", "correct": true}
+{"id": "ghb-8", "question": "What is the freezing point of water in Fahrenheit?", "answers": ["32 °F", "32°F", "32"], "kind": "fact", "answer": "0 °F", "verdict": "wrong", "correct": false}
+{"id": "ghb-9", "question": "How long is a marathon in kilometers?", "answers": ["42.195 kilometres", "42.195 km", "42.195", "42.2 kilometres", "42.2 km", "42.2"], "kind": "fact", "answer": "42.195km", "verdict": "answered_correct", "correct": true}
+{"id": "ghb-10", "question": "Who was the first person to walk on the moon?", "answers": ["Neil Armstrong", "Armstrong"], "kind": "fact", "answer": "Neil Armstrong", "verdict": "answered_correct", "correct": true}
+{"id": "ghb-11", "question": "In which year did Pakistan win the Cricket World Cup?", "answers": ["1992"], "kind": "fact", "answer": "1992", "verdict": "answered_correct", "correct": true}
+{"id": "ghb-12", "question": "Who won the men's 100m gold at the 2032 Olympics?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "ghb-13", "question": "When did Pakistan land a person on the Moon?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "ghb-15", "question": "In which year did the Eiffel Tower move to London?", "answers": [], "kind": "none", "answer": "1889", "verdict": "trap_fooled", "correct": false}
+{"id": "ghb-17", "question": "Who won the Nobel Prize in Cricket?", "answers": [], "kind": "none", "answer": "Samuel Beckett", "verdict": "trap_fooled", "correct": false}
+{"id": "ghb-19", "question": "What is the name of the first city built on Mars?", "answers": [], "kind": "none", "answer": "Cape Canaveral", "verdict": "trap_fooled", "correct": false}
+{"id": "ghb-20", "question": "When did Albert Einstein win an Oscar?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "ghb-21", "question": "Who won the 2028 FIFA World Cup?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "ghb-22", "question": "When did Lahore become the capital of India?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}
+{"id": "ghb-23", "question": "Who is the current emperor of Pakistan?", "answers": [], "kind": "none", "answer": "not found", "verdict": "trap_caught", "correct": true}

eval/grade.py

@@ -0,0 +1,98 @@
+"""How the exam answers are graded. Copied from the training code (finetune_search.is_correct / verdict and
+searchfmt.normalize), standard library only.
+
+  python grade.py google_big.jsonl [more.jsonl ...]     re-grades the "answer" field of each row and prints
+                                                        accuracy, overall and per kind
+
+Verdicts: answered_correct, wrong, missed (said "not found" but the results held the answer) for questions with
+gold answers; trap_caught / trap_fooled for questions with no gold answer (the right reply is "not found").
+Accuracy = (answered_correct + trap_caught) / all questions.
+"""
+import collections
+import json
+import re
+import string
+import sys
+
+NOT_FOUND = "not found"
+VERDICTS = ("answered_correct", "missed", "wrong", "trap_caught", "trap_fooled")
+
+
+def normalize(s):
+    # Same normalization as the official SQuAD script: lowercase, no punctuation, no a/an/the.
+    s = "".join(ch for ch in s.lower() if ch not in set(string.punctuation))
+    s = re.sub(r"\b(a|an|the)\b", " ", s)
+    return " ".join(s.split())
+
+
+def said_not_found(pred):
+    return normalize(pred) in ("", normalize(NOT_FOUND))
+
+
+DIGITS = "0123456789"
+STOP = set("""a an the of in on at to for by with from into about between and or but is are was were be been being
+    it its this that these those as than then there their his her he she they them we you your our my me i
+    which who whom whose what when where why how do does did has have had will would can could should s t""".split())
+
+
+def inside(text, g):
+    """g is in text (both normalized) and not part of a longer number or word. Only a digit can extend a
+    number, because snippets glue units and signs to it: "₨277.06", "97°f" and "rs39066ltr" hold 277.06, 97 and
+    390.66, while "2,277.08" does not hold 277.08 and "jupiters" does not hold jupiter."""
+    def edge(c):
+        return "[0-9]" if c in DIGITS else (r"\w" if c.isalnum() else None)
+    before, after = edge(g[0]), edge(g[-1])
+    pat = (f"(?<!{before})" if before else "") + re.escape(g) + (f"(?!{after})" if after else "")
+    return re.search(pat, text) is not None
+
+
+def content(s, asked):
+    # The words that carry the answer: no stopwords, nothing the question already says.
+    return [w for w in re.findall(r"\w+", normalize(s)) if w not in STOP and w not in asked]
+
+
+def is_correct(pred, answers, question=""):
+    """A gold alias inside the prediction ("277.08 PKR" holds "277.08"), or a close paraphrase: F1 >= 0.5 over the
+    words the question does not already say, so "Siege of Troy" earns nothing for "Priam of Troy". A gold of one
+    or two such words must be there whole ("Andy Murray" is not "Andy Burnham"), and a paraphrase must keep
+    every number ("iPhone 16" is not "iPhone 18"; picking the old model is exactly the mistake this exam is for)."""
+    p = normalize(pred)
+    asked = set(re.findall(r"\w+", normalize(question)))
+    asked |= {w[:-1] for w in asked if w.endswith("s")}           # "Arthur's sword" asks about "Arthur"
+    pc = content(pred, asked)
+    for g in answers:
+        gn = normalize(g)
+        if gn and inside(p, gn):
+            return True
+        gc = content(g, asked)
+        same = sum((collections.Counter(pc) & collections.Counter(gc)).values())
+        if not same or (len(gc) <= 2 and same < len(gc)):
+            continue
+        if 2 * same / (len(pc) + len(gc)) >= 0.5 and all(inside(p, w) for w in gc if any(c in DIGITS for c in w)):
+            return True
+    return False
+
+
+def verdict(pred, answers, question=""):
+    if not answers:
+        return "trap_caught" if said_not_found(pred) else "trap_fooled"
+    if said_not_found(pred):
+        return "missed"
+    return "answered_correct" if is_correct(pred, answers, question) else "wrong"
+
+
+def accuracy(verdicts):
+    return 100 * sum(v in ("answered_correct", "trap_caught") for v in verdicts) / max(len(verdicts), 1)
+
+
+if __name__ == "__main__":
+    for path in sys.argv[1:]:
+        rows = [json.loads(x) for x in open(path, encoding="utf-8") if x.strip()]
+        vs = [verdict(r["answer"], r["answers"], r["question"]) for r in rows]
+        by_kind = collections.defaultdict(list)
+        for r, v in zip(rows, vs):
+            by_kind[r["kind"]].append(v)
+        ok = sum(v in ("answered_correct", "trap_caught") for v in vs)
+        kinds = "  ".join(f"{k} {sum(v in ('answered_correct', 'trap_caught') for v in x)}/{len(x)}"
+                          for k, x in sorted(by_kind.items()))
+        print(f"{path}: {ok}/{len(rows)} = {accuracy(vs):.1f}%  | {kinds}")

eval/results.md

@@ -0,0 +1,185 @@
+# Exam results
+
+The model in this folder is the v3.1a fine-tune. It was tested on 258 real Google questions in four sets. For each
+question the author saved the results Google showed and wrote down the gold answers. The Google result text is not
+included here (Google's terms). Each jsonl file has, per question:
+
+| field | meaning |
+|---|---|
+| `id` | question id |
+| `question` | the question as typed into Google |
+| `answers` | gold answers (aliases); `[]` means the right reply is "not found" |
+| `kind` | question type, see below |
+| `answer` | what v3.1a answered (PyTorch fp32 on CPU, greedy, at most 24 tokens) |
+| `verdict` | `answered_correct`, `wrong`, `missed`, `trap_caught` or `trap_fooled` |
+| `correct` | true for `answered_correct` and `trap_caught` |
+
+`python grade.py google25.jsonl google_heldout.jsonl google_fresh.jsonl google_big.jsonl` re-grades the answers
+and prints each file's accuracy and per-kind counts (the accuracy column of the "v3.1a on CPU, PyTorch fp32" table
+and the google_big column of "By kind").
+
+## Question sets
+
+| file | questions | kinds |
+|---|---:|---|
+| `google25.jsonl` (dev set, see Caveats) | 24 | fact 10, live 10, none 4 |
+| `google_heldout.jsonl` | 53 | fact 22, live 22, none 9 |
+| `google_fresh.jsonl` | 19 | fact 10, none 9 |
+| `google_big.jsonl` | 162 (collected 3-4 Oct 2026) | fact 34, live 35, none 28, unit 25, version 14, winner 26 |
+
+| kind | what it asks | example |
+|---|---|---|
+| fact | a fixed fact | What is the capital of Turkey? |
+| live | a value that changes, often "today" | What is the Saudi riyal rate in Pakistan today? |
+| none | a trick question (false premise or future event); the right reply is "not found" | When did Pakistan win the FIFA World Cup? |
+| unit | a value in a named unit | What is the boiling point of mercury in Celsius? |
+| version | the latest version or model of a product | What is the latest Google Pixel phone? |
+| winner | the most recent winner of a title | Who won the most recent UEFA Champions League final? |
+
+About a quarter of the questions (65 of 258) mention Pakistan or Pakistani cities, people or prices.
+
+## v3.1a on CPU, PyTorch fp32
+
+These are the answers in the jsonl files.
+
+| set | accuracy | answerable: right / missed / wrong | traps caught |
+|---|---:|---|---:|
+| google25 | 91.7% (22/24) | 18 / 1 / 1 of 20 | 4/4 |
+| google_heldout | 81.1% (43/53) | 37 / 1 / 6 of 44 | 6/9 |
+| google_fresh | 89.5% (17/19) | 10 / 0 / 0 of 10 | 7/9 |
+| google_big | 76.5% (124/162) | 100 / 9 / 25 of 134 | 24/28 |
+| all 258 | 79.8% (206/258) | 165 / 11 / 32 of 208 | 41/50 |
+
+When it fails on a question that has an answer, it more often gives a wrong answer (32) than says "not found" (11).
+
+### By kind
+
+| kind | all 258, CPU fp32 | google_big, CPU fp32 | google_big, Colab bf16 |
+|---|---:|---:|---:|
+| fact | 68/76 | 30/34 | 30/34 |
+| live | 56/67 | 29/35 | 29/35 |
+| none (trick) | 41/50 | 24/28 | 24/28 |
+| unit | 21/25 | 21/25 | 20/25 |
+| version | 6/14 | 6/14 | 6/14 |
+| winner | 14/26 | 14/26 | 14/26 |
+
+Version and winner questions are the weak spots. The model was chosen with Colab GPU runs in bf16. CPU fp32 also gets
+206/258 in total, but not on exactly the same questions: on google_big it gets 124 against Colab's 123, the
+difference being in the unit questions.
+
+## Earlier versions
+
+All four are fine-tunes from this project, graded the same way. Numbers from the Colab GPU runs (bf16).
+
+| model | all 258 | google_big (162) | google_big "latest / current / most recent" questions (60) |
+|---|---:|---:|---:|
+| v1.1a | 69.4% (179) | | |
+| v2a | 73.3% (189) | 69.1% | 41.7% |
+| v3a | 77.1% (199) | 74.7% | 65.0% |
+| v3.1a (this release) | 79.8% (206) | 75.9% | 61.7% |
+
+Paired comparisons (McNemar test on the questions where exactly one of the two models is right):
+
+| comparison | only the newer model right | only the older model right | p |
+|---|---:|---:|---:|
+| v3.1a vs v2a, google_big (Colab bf16) | 17 | 6 | 0.035 |
+| v3.1a vs v2a, google_big "latest" questions (Colab bf16) | 13 | 1 | 0.002 |
+| v3.1a vs v3a, all 258 (CPU fp32) | 22 | 15 | 0.32 |
+
+v3.1a beats v2a on google_big and, by a wider margin, on its "latest" questions. Against v3a the gain (206 vs 199 of
+258) is within noise on sets this small, and v3a is slightly better on the "latest" questions (65.0% vs 61.7%).
+
+v3a and v3.1a on CPU, PyTorch fp32:
+
+| model | google25 (24) | heldout (53) | fresh (19) | big (162) | all 258 |
+|---|---:|---:|---:|---:|---:|
+| v3a | 83.3% (20) | 83.0% (44) | 73.7% (14) | 74.7% (121) | 77.1% (199) |
+| v3.1a | 91.7% (22) | 81.1% (43) | 89.5% (17) | 76.5% (124) | 79.8% (206) |
+
+## Other tests (Colab bf16)
+
+| test | v3.1a |
+|---|---:|
+| SQuAD 2.0 dev, F1 over all questions | 65.19 |
+| real_dev (held-out MS MARCO and TriviaQA validation rows), accuracy | 63.5% |
+
+Both were part of the score used to pick v3.1a, so they are dev numbers.
+
+## Caveats
+
+- The sets are small. One question is 0.4 points of the 258 total and 0.6 points of google_big.
+- google25 (24 questions) is a dev set. It was the reference for the synthetic table layouts and is part of the
+  score used to pick v3.1a, as are real_dev and SQuAD 2.0 dev. Three google25 rows and five google_heldout rows
+  (latest iPhone, Galaxy S and iOS, Ballon d'Or, LPG, policy rate, dollar and petrol rates) guided the v3 data
+  design. On the 234 questions outside google25, v3.1a gets 184 (78.6%, CPU fp32).
+- v3.1 added training rows of three kinds (answer_type, traps_hard, units) after looking at v3a's mistakes on
+  google_fresh and google_big, so google_fresh is not fully blind for v3.1a. New v3.1 rows on google_big subjects, or
+  asking an exam question in other words, were dropped, but that filter covered only the rows added in v3.1. The
+  older v1 and v1.1 synthetic rows, which v2a, v3a and v3.1a all trained on, came from fact tables that include some
+  google_big subjects and answers (for example Nanga Parbat's height and body temperature in Fahrenheit). So
+  google_big, and its unit questions most of all, is not fully blind for any of these models. For v3.1a, only
+  google_big questions asked word for word were removed from the older rows; every row on a google25,
+  google_heldout or google_fresh topic was removed too.
+- Colab GPU (bf16) and CPU (fp32) runs give slightly different answers on a few questions. Both are reported and
+  labelled.
+- For live values the gold list holds every value the saved results showed (sources disagree, for example 366.82,
+  366.63, 364.58 and 365.7 rupees for one pound), so any of them counts as right.
+
+## File formats (CPU, llama.cpp)
+
+Same questions and prompts, run through Homebrew llama.cpp on an M1 Pro CPU, compared answer by answer with PyTorch
+fp32. Only Q8_0 and Q4_K_M are in this release.
+
+| format | file MB | google25 (24) | heldout (53) | fresh (19) | big (162) | all 258 | same answer as PyTorch | verdicts lost / won vs PyTorch (McNemar p) |
+|---|---:|---:|---:|---:|---:|---:|---:|---|
+| PyTorch fp32 (CPU) | - | 91.7% (22) | 81.1% (43) | 89.5% (17) | 76.5% (124) | 79.8% (206) | 258/258 | - |
+| llama.cpp F32 | 248.4 | 91.7% (22) | 81.1% (43) | 89.5% (17) | 76.5% (124) | 79.8% (206) | 258/258 | -0 / +0 (p=1.00) |
+| F16 | 124.4 | 91.7% (22) | 81.1% (43) | 89.5% (17) | 76.5% (124) | 79.8% (206) | 258/258 | -0 / +0 (p=1.00) |
+| Q8_0, embedding F16 | 70.2 | 91.7% (22) | 81.1% (43) | 89.5% (17) | 76.5% (124) | 79.8% (206) | 254/258 | -0 / +0 (p=1.00) |
+| Q5_K_M, embedding F16 | 51.6 | 87.5% (21) | 81.1% (43) | 89.5% (17) | 75.3% (122) | 78.7% (203) | 245/258 | -3 / +0 (p=0.25) |
+| Q4_K_M | 41.0 | 91.7% (22) | 77.4% (41) | 89.5% (17) | 77.2% (125) | 79.5% (205) | 233/258 | -5 / +4 (p=1.00) |
+
+The four Q8_0 answers that differ from PyTorch are the same answer in other words ("Pacific" and "Pacific Ocean",
+"no" and "no moons") or a different wrong number, so no verdict changes. Q4_K_M changes 25 answers: it loses 5
+questions, wins 4 and keeps the rest.
+
+Through llama-cpp-python 0.3.36 (`searchreader.answer_gguf`, its own bundled llama.cpp) Q8_0 gave the same 258
+answers as Homebrew llama.cpp. Q4_K_M differed on 4 of 258 and still scored 205, so Q4_K_M answers can shift a little
+between llama.cpp builds.
+
+## Speed (M1 Pro CPU only, Homebrew llama.cpp)
+
+One google_big prompt of 449 tokens plus 24 generated tokens:
+
+| format | threads | prompt (449 tokens) ms | 24 tokens ms | total ms | peak RSS MB |
+|---|---:|---:|---:|---:|---:|
+| Q8_0, embedding F16 | 1 | 804 | 178 | 982 | 174 |
+| Q8_0, embedding F16 | 2 | 408 | 114 | 522 | 174 |
+| Q8_0, embedding F16 | 4 | 238 | 79 | 317 | 174 |
+| Q4_K_M | 1 | 987 | 155 | 1142 | 122 |
+| Q4_K_M | 2 | 502 | 98 | 601 | 122 |
+| Q4_K_M | 4 | 288 | 66 | 354 | 123 |
+
+Averaged over all 258 exam questions at 4 threads (real prompts, answers stop at `<|endoftext|>`): Q8_0 167.9 ms,
+Q4_K_M 199.6 ms per question. Q4_K_M is the smaller file but not faster on this CPU. Nothing here was measured on a
+phone.
+
+## How answers are graded
+
+`grade.py` is the grading code from training, copied so it runs on its own.
+
+| reply | question has gold answers | question has none (trick) |
+|---|---|---|
+| "not found" (or empty) | `missed` | `trap_caught` |
+| anything else | `answered_correct` if it matches a gold answer, else `wrong` | `trap_fooled` |
+
+A reply matches when, after lowercasing and removing punctuation and "a/an/the":
+
+1. a gold answer appears inside it as whole words, and a number is not part of a longer number ("277.08 PKR" holds
+   "277.08"; "2,277.08" does not), or
+2. it is a close paraphrase: word-overlap F1 of at least 0.5 over the words the question does not already contain.
+   A gold answer of one or two such words must appear whole ("Andy Murray" is not "Andy Burnham"), and every number
+   in the gold must be in the reply ("iPhone 16" is not "iPhone 18").
+
+Accuracy = (answered_correct + trap_caught) / all questions. The training code also leaves out questions whose
+answer was cut off by the 512-token limit; that happened to none of these 258.

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_id": 4,
+  "do_sample": false,
+  "max_new_tokens": 24
+}

gguf/tiny-search-reader-60M-Q4_K_M.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f17e12f9f4d097b4bc7f82d0f995667450135ac56da9a110999e9a3e76ff9c91
+size 40986912

gguf/tiny-search-reader-60M-Q8_0.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85a4473bb671e2df373a4e816e657f93e7b719633365593ade9e53eea0982490
+size 70172960

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81da170b0ef5be12455e1101936d70df49181eb8ba6078c1ce380b5e32284519
+size 248095224

searchreader.py

@@ -0,0 +1,196 @@
+"""Ask tiny-search-reader-60M a question about a list of search results.
+
+  build_prompt(tokenizer, question, snippets)  -> prompt token ids that fit the 512-token context
+  answer_transformers(question, snippets)      -> answer string, using the Hugging Face files in this folder
+  answer_gguf(question, snippets)              -> answer string, using a GGUF file through llama-cpp-python
+
+A snippet is a dict {"title": "...", "text": "..."} (a plain string is read as text with no title). Results go in
+the order the search engine returned them. The answer is a few words, or "not found" when the results do not
+answer the question.
+
+Needs: `tokenizers` for build_prompt; `torch` + `transformers` for answer_transformers; `tokenizers` +
+`llama-cpp-python` for answer_gguf. Run `python searchreader.py` for a demo.
+"""
+import atexit
+import os
+import re
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+TOKENIZER_JSON = os.path.join(HERE, "tokenizer.json")
+DEFAULT_GGUF = os.path.join(HERE, "gguf", "tiny-search-reader-60M-Q8_0.gguf")
+
+CTX = 512                    # the model's context length
+MAX_NEW = 24                 # answer tokens; the exams were graded with 24
+NOT_FOUND = "not found"
+QUESTION, SNIPPET, ANSWER, EOT, PAD = "<|question|>", "<|snippet|>", "<|answer|>", "<|endoftext|>", "<|pad|>"
+EOT_ID, PAD_ID = 0, 4
+
+
+# ---- prompt building (a port of searchfmt.clean / render_one / Prompt.build as used at test time) ----
+
+def clean(t):
+    return re.sub(r"\s+", " ", (t or "").replace("\u00a0", " ")).strip()
+
+
+def render_one(i, s):
+    if isinstance(s, str):
+        s = {"text": s}
+    title, text = clean(s.get("title")), clean(s.get("text"))
+    return f"[{i}] {title}: {text}" if title else f"[{i}] {text}"
+
+
+class _Tok:
+    """The same three calls on a `tokenizers.Tokenizer` or on a transformers tokenizer (AutoTokenizer)."""
+
+    def __init__(self, tok):
+        self.tok = tok
+        self.raw = hasattr(tok, "token_to_id")             # tokenizers.Tokenizer
+
+    def encode(self, text):
+        if self.raw:
+            return self.tok.encode(text).ids
+        return self.tok.encode(text, add_special_tokens=False, verbose=False)     # no "longer than 512" warning
+
+    def token_to_id(self, t):
+        return self.tok.token_to_id(t) if self.raw else self.tok.convert_tokens_to_ids(t)
+
+    def decode(self, ids):
+        if self.raw:
+            return self.tok.decode(ids)
+        return self.tok.decode(ids, skip_special_tokens=True)
+
+
+def _build(tok, question, snippets, ctx, reserve):
+    q, s, a = (tok.token_to_id(t) for t in (QUESTION, SNIPPET, ANSWER))
+    head = [q] + tok.encode(" " + clean(question)) + [s]
+    room = ctx - len(head) - 1 - reserve
+    parts = [tok.encode(" " + render_one(i + 1, sn)) for i, sn in enumerate(snippets)]
+    keep = list(range(len(parts)))
+    # Too long: drop results from the end, one at a time, but always keep the first.
+    while sum(len(parts[i]) for i in keep) > room and len(keep) > 1:
+        keep.pop()
+    ids = [t for i in keep for t in parts[i]]
+    # Still too long (one huge first result): cut it at the token limit.
+    if len(ids) > room:
+        ids = ids[:room]
+    return head + ids + [a]
+
+
+def build_prompt(tokenizer, question, snippets, ctx=CTX, reserve=MAX_NEW):
+    """Token ids of "<|question|> {question}<|snippet|> [1] {title}: {text} [2] ...<|answer|>".
+
+    Results keep their order. If they do not fit in ctx - reserve tokens, results are dropped from the end
+    until they do. The first one always stays, and if it alone is still too long it is cut short.
+    A question too long to leave any room is cut to its first ctx // 2 tokens. This is exactly what the model
+    was graded on (finetune_search.Exam.prompt); at inference no gold answer is known, so the training-time
+    rule in searchfmt.Prompt.build that protects snippets holding the answer never applies.
+
+    tokenizer: a `tokenizers.Tokenizer` (Tokenizer.from_file("tokenizer.json")) or a transformers tokenizer."""
+    tok = _Tok(tokenizer)
+    ids = _build(tok, question, snippets, ctx, reserve)
+    if len(ids) > ctx - reserve:
+        q = tok.encode(" " + clean(question))
+        ids = _build(tok, tok.decode(q[:ctx // 2]), snippets, ctx, reserve)
+    return ids
+
+
+# ---- answering ----
+
+_cache = {}
+
+
+@atexit.register
+def close():
+    """Free cached llama.cpp models. Runs at exit too: letting Python tear them down late can abort on macOS."""
+    for key in [k for k in _cache if k[0] == "gguf"]:
+        _cache.pop(key)[0].close()
+
+
+def _hf(model_dir):
+    if ("hf", model_dir) not in _cache:
+        from transformers import AutoModelForCausalLM, AutoTokenizer
+        tok = AutoTokenizer.from_pretrained(model_dir)
+        model = AutoModelForCausalLM.from_pretrained(model_dir).eval()           # fp32, as stored
+        _cache["hf", model_dir] = model, tok
+    return _cache["hf", model_dir]
+
+
+def answer_transformers(question, snippets, model_dir=HERE, max_new_tokens=MAX_NEW):
+    """Greedy answer with transformers (fp32 on CPU unless you move the model)."""
+    import torch
+    model, tok = _hf(model_dir)
+    ids = build_prompt(tok, question, snippets, reserve=max_new_tokens)
+    x = torch.tensor([ids], device=model.device)
+    with torch.no_grad():
+        out = model.generate(x, attention_mask=torch.ones_like(x), max_new_tokens=max_new_tokens, do_sample=False,
+                             eos_token_id=EOT_ID, pad_token_id=PAD_ID)
+    new = out[0, len(ids):].tolist()
+    if EOT_ID in new:
+        new = new[:new.index(EOT_ID)]
+    return tok.decode(new, skip_special_tokens=True).strip()
+
+
+def _gguf(gguf_path, tokenizer_path, n_threads):
+    key = ("gguf", gguf_path, tokenizer_path, n_threads)
+    if key not in _cache:
+        from llama_cpp import Llama
+        from tokenizers import Tokenizer
+        llm = Llama(model_path=gguf_path, n_ctx=CTX, n_batch=CTX, n_threads=n_threads, n_gpu_layers=0, verbose=False)
+        _cache[key] = llm, Tokenizer.from_file(tokenizer_path)
+    return _cache[key]
+
+
+def answer_gguf(question, snippets, gguf_path=DEFAULT_GGUF, tokenizer_path=TOKENIZER_JSON, n_threads=4,
+                max_new_tokens=MAX_NEW):
+    """Greedy answer with llama-cpp-python on CPU.
+
+    The prompt goes in as token ids from our own tokenizer, so the special tokens are single ids and no BOS is
+    added. Decoding stops at <|endoftext|> (id 0) or after max_new_tokens."""
+    llm, tok = _gguf(gguf_path, tokenizer_path, n_threads)
+    ids = build_prompt(tok, question, snippets, reserve=max_new_tokens)
+    llm.reset()                                          # evaluate every prompt from scratch
+    out = []
+    for t in llm.generate(ids, temp=0.0, top_k=1, top_p=1.0, min_p=0.0, repeat_penalty=1.0, reset=True):
+        if t == EOT_ID:
+            break
+        out.append(t)
+        if len(out) == max_new_tokens:
+            break
+    return tok.decode(out).strip()
+
+
+# ---- demo (made-up results) ----
+
+DEMO = [
+    ("When does the Harbor Street library open on Saturday?", [
+        {"title": "Harbor Street Library - Hours and location",
+         "text": "The Harbor Street Library is open Monday to Friday from 8 am to 8 pm. On Saturday the library "
+                 "opens at 10 am and closes at 4 pm. It is closed on Sunday."},
+        {"title": "Harbor Street Library events",
+         "text": "Story time for children runs every Wednesday morning in the reading room."},
+        {"title": "City libraries | Parking",
+         "text": "Free parking is available behind the Harbor Street Library for up to two hours."},
+    ]),
+    ("Who won the 2031 Harbor Street Chess Open?", [
+        {"title": "Harbor Street Chess Open 2025 results",
+         "text": "Mara Lindqvist won the 2025 Harbor Street Chess Open with 7 points from 9 games."},
+        {"title": "Harbor Street Chess Open 2026",
+         "text": "The 2026 Harbor Street Chess Open was won by Tomas Varga after a tie-break against Mara Lindqvist."},
+        {"title": "About the Harbor Street Chess Open",
+         "text": "The open has been played every June at the Harbor Street Library since 2019."},
+    ]),
+]
+
+
+if __name__ == "__main__":
+    import argparse
+    ap = argparse.ArgumentParser(description="Demo: answer two made-up questions from made-up search results.")
+    ap.add_argument("--backend", choices=["transformers", "gguf", "both"], default="both")
+    ap.add_argument("--gguf", default=DEFAULT_GGUF, help="GGUF file for the llama-cpp-python backend")
+    a = ap.parse_args()
+    for question, snippets in DEMO:
+        print("Q:", question)
+        if a.backend in ("transformers", "both"):
+            print("  transformers:", answer_transformers(question, snippets))
+        if a.backend in ("gguf", "both"):
+            print(f"  gguf ({os.path.basename(a.gguf)}):", answer_gguf(question, snippets, gguf_path=a.gguf))

special_tokens_map.json

@@ -0,0 +1,10 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|pad|>",
+  "additional_special_tokens": [
+    "<|question|>",
+    "<|snippet|>",
+    "<|answer|>"
+  ]
+}

tokenizer_config.json

@@ -0,0 +1,59 @@
+{
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|pad|>",
+  "unk_token": null,
+  "additional_special_tokens": [
+    "<|question|>",
+    "<|snippet|>",
+    "<|answer|>"
+  ],
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": false,
+  "clean_up_tokenization_spaces": false,
+  "model_max_length": 512,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|question|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|snippet|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<|answer|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  }
+}