Expand model card with sourced comparisons and training figures

Compare published OpenSML-150M and SmolLM2 results, include IFEval, simplify metric tables, and add training details, recorded examples and reproducible figures. Model weights and inference runtime are unchanged.

.gitattributes

@@ -35,3 +35,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 docs/figures/cooldown-detail.png filter=lfs diff=lfs merge=lfs -text
 docs/figures/pretraining-overview.png filter=lfs diff=lfs merge=lfs -text
+docs/figures/benchmark-comparison.png filter=lfs diff=lfs merge=lfs -text
+docs/figures/chat-diagnostics.png filter=lfs diff=lfs merge=lfs -text
+docs/figures/ifeval-comparison.png filter=lfs diff=lfs merge=lfs -text

FILE_MANIFEST.json

@@ -22,8 +22,8 @@
       "sha256": "0e6865f1a81cfd69ca48e6987658854ce0c0d23e91d72ae545393cd5fa032375"
     },
     "README.md": {
-      "bytes": 7699,
-      "sha256": "900df54a78e604f4245328633b77eb3cb606e8c213dc1ca71b41549905ced58e"
+      "bytes": 13155,
+      "sha256": "894e1200eb78c3afd6617410b209bfd389f77c45301c3b6cfb3faf8b9fd380cb"
     },
     "REPOSITORIES.json": {
       "bytes": 716,
@@ -130,8 +130,8 @@
       "sha256": "e0b0c730321360c40f20e53a22890c52cd5294bad2a5777a0a891096557a2f0a"
     },
     "docs/EVALUATION.md": {
-      "bytes": 6613,
-      "sha256": "5522817f8d698f05b43adfc1c5227bab4e917492c898fa4b792d521d42a512d8"
+      "bytes": 8582,
+      "sha256": "79c7190a634b9008502242133bf76e246ab69700814049b5ac39f49ad51c57c3"
     },
     "docs/EXAMPLES.md": {
       "bytes": 1117,
@@ -141,6 +141,22 @@
       "bytes": 13515,
       "sha256": "f98d45b39b9595510fa80d495a3758b211d8c770adc739d381763d3cfb2a2a0b"
     },
+    "docs/figures/benchmark-comparison.png": {
+      "bytes": 125053,
+      "sha256": "37438bfe291e2cc5b7513ec912cab9eaf7ce3d6352543f8e63d44da3a4f7d5b3"
+    },
+    "docs/figures/benchmark-comparison.svg": {
+      "bytes": 89737,
+      "sha256": "64297355167c23c84cc4227bf54eb6be8d66c5837d45e201ed9304594df36d85"
+    },
+    "docs/figures/chat-diagnostics.png": {
+      "bytes": 108673,
+      "sha256": "08388301b65a94ce3bd166de02dd6aa887515f9ed617a8972359b742334cb422"
+    },
+    "docs/figures/chat-diagnostics.svg": {
+      "bytes": 64470,
+      "sha256": "b70ae8d8b730c7f178a41ae9d781d2f7f8fbedf3640f9d4991a77c6c89e39725"
+    },
     "docs/figures/cooldown-detail.png": {
       "bytes": 185666,
       "sha256": "0fa840653d1a5305297ef6f51ba08dc70a4bbfaf8029c411627e39dfc3cdc153"
@@ -149,6 +165,14 @@
       "bytes": 174630,
       "sha256": "1071d21211f7d32d9f9f5874e62f46a2a17306f88f0f355b24aee5b4af4c4e0f"
     },
+    "docs/figures/ifeval-comparison.png": {
+      "bytes": 108050,
+      "sha256": "c5a39cdcbc0b8b2a48dd7e8136c61132b4340e0145fde1cdf1e73a498785e207"
+    },
+    "docs/figures/ifeval-comparison.svg": {
+      "bytes": 69743,
+      "sha256": "9416ed14acd5d52ea3931100e5c2330c6e1c7c7f19724602c389fc71c79a56d8"
+    },
     "docs/figures/pretraining-overview.png": {
       "bytes": 157953,
       "sha256": "c46bba5d4fae1c55701def21829de08fd3756dc9659fee2a54a79f42eb1afcb0"
@@ -369,6 +393,10 @@
       "bytes": 4788,
       "sha256": "2b59f728ebc9bef6deaf378e6908f7b55723b368f60da83f9af5b2b1731d0346"
     },
+    "results/baselines/published_models.json": {
+      "bytes": 2471,
+      "sha256": "2cafe36ca3f2fe3dd00cf9aa63314eac64a73de2c58e2de92cdeee47306bd8ae"
+    },
     "results/package_inference_verification.json": {
       "bytes": 26015,
       "sha256": "fed55313b61008e8f379a7b4bc49535cc501a31a942c7560687de4a4bd1cd47f"
@@ -481,6 +509,10 @@
       "bytes": 1823,
       "sha256": "cac0e352b21365bfb4fef9f5c265aa36426552bb47851be5fdb5ff585648f69a"
     },
+    "scripts/plot_results.py": {
+      "bytes": 6190,
+      "sha256": "d7d137a4ff838c2bd9d1cdd7f8428a9a78101ab97868ebbaab42eb4a561ae648"
+    },
     "scripts/plot_training.py": {
       "bytes": 2877,
       "sha256": "c47f496b10f813dcb61697a0d49ed14043137ee1e2a96911a68fccb31fa9df26"

README.md

@@ -26,103 +26,171 @@ datasets:
 
 # Solstice-1
 
-**An OpenSML 175M-parameter English chat research model trained from scratch on Apple Silicon.**
+**A 175M-parameter English chat model, trained from scratch on five Macs.**
 
-Author: William Zebrowski · License: Apache-2.0 · Status: research preview
+William Zebrowski · OpenSML · Apache-2.0 · Research preview
 
-[Model weights](https://huggingface.co/OpenSML/Solstice-1) · [GitHub source](https://github.com/williamzebrowskI/Solstice-1) · [Run guide](RUN_GUIDE.md) · [Technical report](docs/TECHNICAL_REPORT.md) · [Evaluation](docs/EVALUATION.md) · [Training data](docs/DATA.md) · [Results](results/public/summary.json)
+[Weights](https://huggingface.co/OpenSML/Solstice-1) · [Source](https://github.com/williamzebrowskI/Solstice-1) · [Run guide](RUN_GUIDE.md) · [Technical report](docs/TECHNICAL_REPORT.md) · [Evaluation](docs/EVALUATION.md) · [Data](docs/DATA.md)
 
-Solstice-1 starts a model family under OpenSML. This first release is a 175M chat model with a 2,048-token context. Its selected weights come from **42.001B pretraining tokens and three adapter fine-tuning stages**. The package provides merged FP32 weights, the original tokenizer, a native MLX runner, training-source snapshots, and recorded evaluation evidence.
-
-The model is intended for research into small English language models, short conversations, reading supplied text, and instruction following. It can produce incorrect answers, repeat itself, decline answerable questions, and invent information that was not provided. The measurements below do not establish dependable factual or production behavior.
+Solstice-1 is the first model in the Solstice family. It combines **42.001B pretraining tokens** with three stages of adapter fine-tuning for short conversations, reading comprehension and instruction following. The release includes merged FP32 weights, the original tokenizer, an MLX runner, training records and benchmark outputs.
 
 ## Model specification
 
 | Setting | Value |
 | --- | --- |
-| Public size designation | 175M |
-| Exact parameter count | 174,202,432 |
-| Transformer | 44 layers; width 576; SwiGLU intermediate width 1,536 |
-| Attention | 9 query heads / 3 KV heads; head dimension 64 |
-| Components | RoPE 10,000; RMSNorm; Q/K normalization; tied embeddings |
-| Vocabulary / shared context | 32,000 byte-level BPE tokens / 2,048 tokens |
-| Pretraining exposure | 42,001,367,040 processed token positions |
-| Released weights | Unquantized merged FP32 safetensors |
-| Supported inference | Supplied native MLX runner on Apple Silicon |
-
-This is a different architecture and lineage from OpenSML-150M and OpenSML-JSONLite-175M. Those releases are comparison references, not weight ancestors. The frozen tokenizer is reused from the earlier OpenSML project.
+| Parameters | 175M rounded; 174,202,432 exact |
+| Transformer | 44 layers; hidden width 576; SwiGLU width 1,536 |
+| Attention | 9 query heads, 3 KV heads; head dimension 64 |
+| Components | RoPE (base 10,000), RMSNorm, Q/K normalization, tied embeddings |
+| Tokenizer | 32,000-token byte-level BPE, reused from OpenSML |
+| Context | 2,048 tokens, including conversation history and generated output |
+| Pretraining | 42,001,367,040 processed token positions |
+| Release | Merged FP32 safetensors; native MLX on Apple Silicon |
 
-## Run locally
+Solstice-1 has a separate weight lineage from OpenSML-150M and OpenSML-JSONLite-175M. Reusing the tokenizer does not make those models its weight ancestors.
 
-Download the entire model repository, including its tokenizer and native source. While the repository is private, run `hf auth login` after installing `huggingface_hub`, using an account with access to OpenSML/Solstice-1:
+## Measured results
 
-```bash
-python3.11 -m venv .venv
-source .venv/bin/activate
-python -m pip install huggingface_hub
-hf download OpenSML/Solstice-1 --local-dir Solstice-1
-cd Solstice-1
-python -m pip install -r requirements.txt
-python inference.py --prompt "What is the main function of the heart?" --max-new-tokens 96
-python inference.py --interactive
-```
+### Knowledge and commonsense
 
-The runner uses the model's native structural-token chat format. It checks the weight hash and tokenizer before loading, defaults to greedy FP32 inference, and rejects context overflow without truncation. `/reset` clears an interactive conversation; `/quit` exits. See [recorded package verification](PACKAGING_VERIFICATION.json) for the environment and checks actually completed.
+Zero-shot likelihood scoring on the complete ARC Easy, ARC Challenge, PIQA and HellaSwag evaluation splits. Accuracy ranks the total answer log-likelihood; `acc_norm` ranks it after dividing by answer length in characters. Higher is better.
 
-**Use the supplied runner.** This package does not provide a verified Transformers AutoModel, MLX-LM auto-loader, GGUF, or CUDA implementation. The context limit includes previous turns, role delimiters, the current prompt, and the requested output budget.
+| Benchmark | Metric | Solstice-1 | OpenSML-150M | SmolLM2-135M Instruct |
+| --- | --- | --- | --- | --- |
+| ARC Easy | Accuracy | 62.58% | 56.65% | — |
+| ARC Easy | acc_norm | 62.33% | 55.43% | — |
+| ARC Challenge | Accuracy | 29.35% | 26.02% | — |
+| ARC Challenge | acc_norm | 30.89% | 29.52% | — |
+| PIQA | Accuracy | 66.49% | 64.53% | — |
+| PIQA | acc_norm | 66.21% | 64.09% | 66.3% |
+| HellaSwag | Accuracy | 32.85% | 30.55% | — |
+| HellaSwag | acc_norm | 39.40% | 33.94% | 40.9% |
+| ARC (publisher average) | acc_norm | — | — | 37.3% |
 
-## Measured results
+![Normalized benchmark accuracy for Solstice-1 and published small-model references](docs/figures/benchmark-comparison.png)
 
-Full-split, zero-shot likelihood evaluations use native MLX FP32 scoring:
+### Instruction following
 
-| Benchmark | Examples | Accuracy | Normalized accuracy |
+IFEval checks whether generated responses follow explicit instructions. It measures accuracy, with separate strict and loose scoring at the prompt and instruction levels. **It has no `acc_norm` metric.**
+
+| IFEval accuracy | Solstice-1 | OpenSML-150M | SmolLM2-135M Instruct |
 | --- | --- | --- | --- |
-| ARC Easy | 2376 | 62.58% | 62.33% |
-| ARC Challenge | 1172 | 29.35% | 30.89% |
-| PIQA | 1838 | 66.49% | 66.21% |
-| HellaSwag | 10042 | 32.85% | 39.40% |
+| Strict prompts | 17.93% | 15.16% | — |
+| Strict instructions | 30.10% | 25.30% | — |
+| Loose prompts | 19.22% | 15.71% | — |
+| Loose instructions | 32.49% | 25.78% | — |
+| Publisher prompt/instruction average | — | — | 29.9% |
 
-`acc_norm` divides each candidate's summed log-likelihood by its Unicode-character count. It is distinct from raw accuracy. The evaluator follows pinned task conventions but is not an installed lm-evaluation-harness run.
+![IFEval accuracy compared with OpenSML-150M and the earlier checkpoint 167](docs/figures/ifeval-comparison.png)
 
-| IFEval metric | Score | Passes / checks |
-| --- | --- | --- |
-| Strict prompts | 17.93% | 97/541 |
-| Strict instructions | 30.10% | 251/834 |
-| Loose prompts | 19.22% | 104/541 |
-| Loose instructions | 32.49% | 271/834 |
+Solstice uses greedy generation, no system message or repetition penalty, and a 1,024-token output cap. Of 541 prompts, 130 reach that cap; they remain in the scores. The 150M release used a 1,280-token cap and a different chat format.
+
+Baseline scores come from the published [OpenSML-150M card](https://huggingface.co/OpenSML/OpenSML-150M) and [SmolLM2-135M-Instruct card](https://huggingface.co/HuggingFaceTB/SmolLM2-135M-Instruct), not fresh reruns. “—” means the required breakdown is not published in those cards. SmolLM2 reports ARC and IFEval as averages; no missing sub-scores are inferred.
+
+Solstice uses native MLX FP32 scoring. SmolLM2 uses LightEval; its [evaluation code](https://github.com/huggingface/smollm/tree/438be18dae658a638f3a6976fdf5190bb6d68619/text/evaluation/smollm2) specifies character-normalized choice scores with a leading space excluded and instruct runs with a chat template. OpenSML-150M uses an older native pipeline. Prompt formatting, precision and generation budgets differ, so these are reference comparisons, not a controlled leaderboard. [Exact sources and metric definitions](results/baselines/published_models.json).
+
+### Conversation checks
+
+A separate authored chat panel measures short-answer matching and response behavior. These automatic checks are narrower than human judgments of conversation quality.
+
+| Measure | Solstice-1 |
+| --- | ---: |
+| Answer accuracy, objective cases | 59.44% |
+| Natural stopping, all responses | 99.48% |
+| Repetition flags, all responses | 1.56% |
+
+![Answer accuracy by chat category](docs/figures/chat-diagnostics.png)
+
+The panel contains 180 checked answers and 12 open-ended responses. Repetition flags were reviewed: two were clear loops and one was a milder repeated idea. The full panel and scorer are included in [the saved results](results/public/chat/results.json).
+
+On the separate reserved authored test, Solstice reaches **51.9%**, versus **20.0%** after SFT 1 and **25.9%** after SFT 2. On an older, reused panel, the corresponding scores are **53.84%, 52.13% and 54.55%**: the last stage does not improve every measure. The [evaluation report](docs/EVALUATION.md) explains selection, uncertainty and test exposure.
+
+## Pretraining
+
+Pretraining starts from random weights. Five Apple Silicon workers use BF16 compute with FP32 master weights, gradients and optimizer state. The 2,048-token sequence length and accumulated batch produce **131,072 token positions per update**. AdamW uses betas (0.9, 0.95), weight decay 0.1 and gradient clipping at 1.
+
+### Data mixture
 
-IFEval uses all 541 prompts / 834 instructions, greedy native-chat generation, no system message or repetition penalty, and a 1,024-token output cap. **130/541 responses reach that cap.** All responses are scored as produced. These numbers do not measure general factual accuracy.
+| Source | First 36.801B tokens | Remaining 5.200B tokens |
+| --- | ---: | ---: |
+| FineWeb-Edu | 70% | 56% |
+| DCLM-Edu | 20% | 16% |
+| English FineWiki | 10% | 8% |
+| Python: public Stack-Edu text mirror | — | 15% |
+| SQL: public Stack-Edu text mirror / configured continuation | — | 5% |
 
-On a separate 192-response authored chat check, 107/180 automatically checked answers match the accepted answers; 191/192 responses stop naturally, and three have repetition flags. Automatic answer matching is conservative, and repetition detection is heuristic. These checks are not a broad human preference study.
+These are configured token shares, not measured unique-token counts. Data is streamed with bounded shuffling, quality filters, a recent-document deduplication window and fixed train/validation partitioning. The tokenizer stays frozen. Exact dataset revisions, filtering rules and upstream terms are in [Data provenance](docs/DATA.md).
 
-![Recorded pretraining history](docs/figures/pretraining-overview.png)
+### Loss and learning rate
 
-See the [evaluation report](docs/EVALUATION.md) for baseline comparisons, dataset pins, protocol differences, and private-test scope.
+![Recorded training loss, prose validation loss and learning rate](docs/figures/pretraining-overview.png)
 
-## Training lineage
+The learning rate warms up to 3e-4, then changes through the recorded continuation schedules. At 40.001B tokens, a final 2B-token cooldown lowers it from 7.94e-5 to 1e-5. The selected base finishes at **42.001B tokens**, with prose validation loss **2.8356** on the training-time BF16 evaluation path.
 
-```text
-Fresh Deep44 initialization → streamed prose pretraining
-  → add Python/SQL at 36.801B → 40.001B → 2.000B-token cooldown
-  → 42.001B pretrained base
-  → adapter SFT 1, selected update 2000
-  → adapter SFT 2, selected update 256
-  → adapter SFT 3, selected update 128 → Solstice-1
+![Late pretraining and the final cooldown](docs/figures/cooldown-detail.png)
+
+The shaded region is the cooldown; the dashed line marks the addition of Python and SQL. Training loss uses the changing mixture, while prose validation uses a fixed holdout. Their levels should not be treated as the same measurement. [Curve data](results/pretraining_validation.csv) · [Schedule details](docs/TECHNICAL_REPORT.md#5-distributed-optimization-and-schedules)
+
+## Fine-tuning
+
+All three stages run on one Mac. They train rank-32 LoRA adapters on attention projections, plus the two chat-boundary token rows: **3.786M trainable parameters**. Each stage starts fresh optimizer moments and inherits the previous stage's adapters. The release merges these updates into the original FP32 base weights.
+
+| Stage | Selected updates | Training data and objective |
+| --- | ---: | --- |
+| SFT 1 | 2,000 | Two passes over 48,000 conversations; assistant-only supervision with 20% prose-replay loss |
+| SFT 2 | 256 | Editing, constraints, dialogue and grounding; conversational replay and a parent-distribution penalty |
+| SFT 3 | 128 | Short reasoning, grounding, dialogue and abstention; replay and a lighter parent-distribution penalty |
+
+Public SFT sources include Smol-SmolTalk, SmolTalk everyday conversations, SQuAD v2 and SciQ, alongside authored tasks. Whole conversations are retained; oversized examples and dedicated code, SQL, JSON and tool tasks are excluded from the selected text-chat recipe. The largest stage-1 conversation is 1,385 serialized tokens, within the 2,048-token context.
+
+The final stage was planned for 256 updates; **only its first 128 are in this release**. It was selected using development checks before evaluation on the reserved test. This lineage uses supervised fine-tuning with replay and KL preservation, not DPO or reinforcement learning. Checkpoint 167 is a comparison model, not an ancestor. [Stage recipes, learning rates and selection](docs/TECHNICAL_REPORT.md#6-selected-fine-tuning-lineage) · [Exact lineage](provenance/LINEAGE.json)
+
+## Recorded examples
+
+Greedy FP32 outputs reproduced in the release environment. The incorrect example is included to show the gap between fluent answers and factual reliability.
+
+| Prompt | Model answer | Assessment |
+| --- | --- | --- |
+| What is the capital of Japan? Give a short answer. | Tokyo | Correct |
+| Anika brought a teal notebook. Greta brought a black notebook. What color was Anika’s notebook? | teal | Correct; supplied-text question¹ |
+| Which planet is closest to the Sun? Give a short answer. | venus | Incorrect; the answer is Mercury |
+
+¹ The full prompt also asks the model to use only the supplied information and give a short answer. Full prompts, a conversation-correction example and stop reasons are in [Recorded examples](docs/EXAMPLES.md).
+
+## Run locally
+
+Use Python 3.11 on Apple Silicon. While the repository is private, sign in with `hf auth login` after installing `huggingface_hub`.
+
+```bash
+python3.11 -m venv .venv
+source .venv/bin/activate
+python -m pip install huggingface_hub
+hf download OpenSML/Solstice-1 --local-dir Solstice-1
+cd Solstice-1
+python -m pip install -r requirements.txt
+python inference.py --prompt "What is the main function of the heart?" --max-new-tokens 96
+python inference.py --interactive
 ```
 
-Only updates through 128 of the final 256-update experiment are inherited. Earlier experimental branches and checkpoint 167 are not additional stages in this lineage. [Checkpoint lineage and hashes](provenance/LINEAGE.json) · [Selected data exposure](provenance/SELECTED_PREFIX.json).
+The runner defaults to greedy FP32 generation and uses the native chat tokens. Interactive mode retains previous turns; `/reset` clears history and `/quit` exits. Inputs that exceed the shared context budget are rejected rather than silently truncated. [Sampling and conversation options](RUN_GUIDE.md).
+
+The supplied runner is the supported inference path. Transformers AutoModel, MLX-LM auto-loading, GGUF and CUDA have not been validated for this package. A clean-environment check reproduced 27 saved responses exactly; this verifies the package, not general answer quality. [Verification record](PACKAGING_VERIFICATION.json).
+
+## Limitations
+
+- **Factual errors:** plausible explanations and short answers can be wrong.
+- **Instruction following:** strict IFEval prompt accuracy is 17.93%; many longer outputs reach the token cap.
+- **Answerability:** the model can decline questions whose answers are provided, yet invent answers when information is missing. Results depend strongly on wording.
+- **Repetition:** loops and repeated ideas remain possible, particularly in open-ended responses.
+- **Scope:** reliable coding, SQL, JSON extraction, tool use, multilingual use and long conversations are not established capabilities.
+- **Evaluation coverage:** public and historical tests informed earlier research. The reserved test shares authored task families with training. No comprehensive semantic contamination audit, independent human study or safety assessment is claimed.
 
-## Known limitations
+Solstice-1 is a research release for short English text and chat. It is not a dependable source of facts or a validated system for consequential decisions.
 
-- The model sometimes gives fluent but incorrect factual explanations.
-- On the new authored evaluation, 111/200 answerable extraction questions triggered unnecessary-abstention signals. On a separate set with different wording, it failed all 15 missing-information cases. These are distinct tests, not contradictory measures of the same inputs.
-- Instruction following remains incomplete; only 97/541 IFEval prompts satisfy every strict requirement.
-- Repetition and token-limit stops remain possible, especially on longer requested outputs.
-- JSON extraction, tool use, robust coding, long documents, multilingual use, safety behavior, and prompt-injection resistance are not validated release capabilities.
-- Public and historical benchmarks informed earlier experiments. The newly reserved 1,000-case panel was evaluated after this finalist was selected, but shares authored task families with training. No comprehensive semantic contamination audit or independent human evaluation is claimed.
+## Files, reproducibility and license
 
-## Repository contents and licensing
+[Technical report](docs/TECHNICAL_REPORT.md) · [Data provenance](docs/DATA.md) · [Evaluation and failure analysis](docs/EVALUATION.md) · [Training archive](TRAINING_AND_REPRODUCIBILITY.md) · [Raw benchmark summary](results/public/summary.json)
 
-The weights and author-controlled code are released under [Apache-2.0](https://huggingface.co/OpenSML/Solstice-1/blob/main/LICENSE). Upstream datasets and third-party code retain their own terms; the model license does not relicense those materials. [Source notice](SOURCE_NOTICE.md) · [Third-party notices](THIRD_PARTY_NOTICES.md).
+The repository includes weights on Hugging Face, tokenizer files, inference code, sanitized training snapshots, frozen recipes, selected-checkpoint hashes, curves and saved evaluation outputs. Raw corpora, optimizer binaries and all ancestor weights are not bundled. The training archive documents the run; it is not a verified one-command retraining package.
 
-Technical details, data provenance, exact selected training prefixes, source snapshots, and reproduction prerequisites are included. Full raw corpora, optimizer binaries, and all ancestor weights are not distributed. The historical training archive is not a verified one-command retraining package. [Training and reproducibility](TRAINING_AND_REPRODUCIBILITY.md).
+Weights and author-controlled code use [Apache-2.0](https://huggingface.co/OpenSML/Solstice-1/blob/main/LICENSE). Upstream datasets and third-party code retain their own terms. [Source notice](SOURCE_NOTICE.md) · [Third-party notices](THIRD_PARTY_NOTICES.md) · [Citation](CITATION.cff)

SHA256SUMS

@@ -3,7 +3,7 @@ f09ed498cc42b482dae88d9fb5026bf74e1d8b9187f9ea301aeda0db76d6c859  BACKUP_AND_CLE
 af646a190d135ec25a6ee2c76d1594c3f9ba8005cd3ca6df908606a15023273e  CITATION.cff
 cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30  LICENSE
 0e6865f1a81cfd69ca48e6987658854ce0c0d23e91d72ae545393cd5fa032375  PACKAGING_VERIFICATION.json
-900df54a78e604f4245328633b77eb3cb606e8c213dc1ca71b41549905ced58e  README.md
+894e1200eb78c3afd6617410b209bfd389f77c45301c3b6cfb3faf8b9fd380cb  README.md
 152a87d0098752350d54fdd3cde3861a77eb497c53dc973ee1a8370d12a13ff6  REPOSITORIES.json
 fd05e1595bcdc52942e9425b6efb2c7e89553e6b0ad7d5f93d9075969cd75dba  RUN_GUIDE.md
 87fa33719fe1182d467a05ab417be2a362f50eea59f0ac6496d40e677544e27c  SOURCE_NOTICE.md
@@ -30,11 +30,17 @@ fb204be1d9e72eb1cae140bfdb03dbcadc1843c47b1894a731396ceeedb9a22f  configs/sft1/p
 51d66108416988bad1015d7b52cf63db0571f87f202b2b18c7e98e67a29ec62b  configs/sft2/plan.json
 ae708eca3c09d5a27eab3fddbe4106d0658787beebc9393de57f75234480a284  configs/sft3/plan.json
 e0b0c730321360c40f20e53a22890c52cd5294bad2a5777a0a891096557a2f0a  docs/DATA.md
-5522817f8d698f05b43adfc1c5227bab4e917492c898fa4b792d521d42a512d8  docs/EVALUATION.md
+79c7190a634b9008502242133bf76e246ab69700814049b5ac39f49ad51c57c3  docs/EVALUATION.md
 7a818c365f1c047b349479829396ba5cd841ca8d2f04fa14fc1489083a1fd4b3  docs/EXAMPLES.md
 f98d45b39b9595510fa80d495a3758b211d8c770adc739d381763d3cfb2a2a0b  docs/TECHNICAL_REPORT.md
+37438bfe291e2cc5b7513ec912cab9eaf7ce3d6352543f8e63d44da3a4f7d5b3  docs/figures/benchmark-comparison.png
+64297355167c23c84cc4227bf54eb6be8d66c5837d45e201ed9304594df36d85  docs/figures/benchmark-comparison.svg
+08388301b65a94ce3bd166de02dd6aa887515f9ed617a8972359b742334cb422  docs/figures/chat-diagnostics.png
+b70ae8d8b730c7f178a41ae9d781d2f7f8fbedf3640f9d4991a77c6c89e39725  docs/figures/chat-diagnostics.svg
 0fa840653d1a5305297ef6f51ba08dc70a4bbfaf8029c411627e39dfc3cdc153  docs/figures/cooldown-detail.png
 1071d21211f7d32d9f9f5874e62f46a2a17306f88f0f355b24aee5b4af4c4e0f  docs/figures/cooldown-detail.svg
+c5a39cdcbc0b8b2a48dd7e8136c61132b4340e0145fde1cdf1e73a498785e207  docs/figures/ifeval-comparison.png
+9416ed14acd5d52ea3931100e5c2330c6e1c7c7f19724602c389fc71c79a56d8  docs/figures/ifeval-comparison.svg
 c46bba5d4fae1c55701def21829de08fd3756dc9659fee2a54a79f42eb1afcb0  docs/figures/pretraining-overview.png
 862a5400ecde1ececb915441766f378849c65024288522c84a4883d0c4b31a4e  docs/figures/pretraining-overview.svg
 9e15b1b93bbc1160f3988a7956eee88b2467e5d826f097df35401562c116b90d  evaluation/README.md
@@ -90,6 +96,7 @@ f76237736d1aab8f04517ad9e3336925ca9773399a9ad888f07c15d3ceaa4f87  results/baseli
 187b395945254a4d9359694d5f9ead19a8b95caffcb42477cb9a349cf6ad625e  results/baselines/167/ifeval/results.json
 a5d49e3dc60a6c4dda4799090881cc2aab033b922ac0b5dafd52c0c6c7e2e6d8  results/baselines/attempt5.json
 2b59f728ebc9bef6deaf378e6908f7b55723b368f60da83f9af5b2b1731d0346  results/baselines/parent2000.json
+2cafe36ca3f2fe3dd00cf9aa63314eac64a73de2c58e2de92cdeee47306bd8ae  results/baselines/published_models.json
 fed55313b61008e8f379a7b4bc49535cc501a31a942c7560687de4a4bd1cd47f  results/package_inference_verification.json
 ba2cfb7c8b446dede8e7f33bb78885855e74bd75f861b0c201d6e01e88bc987f  results/pretraining_steps.csv
 c4e3210780814051e3ad1bd99a9c143b9c5dbe910224239a134fddc546e9c9a8  results/pretraining_validation.csv
@@ -118,6 +125,7 @@ da83222ee38d33eea6c7062c28fd310e61da89a9a0315dc62559c4a9181759d9  results/search
 a478bc790345129fa960060490d4c2148351ccfc17ac433ec06364969787a8fe  results/selection/design.json
 0567b916bdf27de7f6eafe8c61da7fb3686a69e08361ac6cb5645ac567cf8f69  results/selection/selection_frozen.json
 cac0e352b21365bfb4fef9f5c265aa36426552bb47851be5fdb5ff585648f69a  scripts/materialize_training_snapshot.py
+d7d137a4ff838c2bd9d1cdd7f8428a9a78101ab97868ebbaab42eb4a561ae648  scripts/plot_results.py
 c47f496b10f813dcb61697a0d49ed14043137ee1e2a96911a68fccb31fa9df26  scripts/plot_training.py
 002b4e552d3d625f96d77d90a0e3195e53e5c4358f64fe3a042792d08b232cd3  scripts/verify_inference.py
 ea7d6388ad1f17fcf244119b2bf15018d981c4e2c127bedf2af73dc74eaa0528  scripts/verify_package.py

docs/EVALUATION.md

@@ -4,26 +4,44 @@ All results identify the same release weight hash, `7b2d8777040f05d8e0849acac4e4
 
 ## Likelihood scoring
 
-| Benchmark | Examples | Accuracy | Normalized accuracy |
-| --- | --- | --- | --- |
-| ARC Easy | 2376 | 62.58% | 62.33% |
-| ARC Challenge | 1172 | 29.35% | 30.89% |
-| PIQA | 1838 | 66.49% | 66.21% |
-| HellaSwag | 10042 | 32.85% | 39.40% |
+| Benchmark | Metric | Solstice-1 | OpenSML-150M | SmolLM2-135M Instruct |
+| --- | --- | --- | --- | --- |
+| ARC Easy | Accuracy | 62.58% | 56.65% | — |
+| ARC Easy | acc_norm | 62.33% | 55.43% | — |
+| ARC Challenge | Accuracy | 29.35% | 26.02% | — |
+| ARC Challenge | acc_norm | 30.89% | 29.52% | — |
+| PIQA | Accuracy | 66.49% | 64.53% | — |
+| PIQA | acc_norm | 66.21% | 64.09% | 66.3% |
+| HellaSwag | Accuracy | 32.85% | 30.55% | — |
+| HellaSwag | acc_norm | 39.40% | 33.94% | 40.9% |
+| ARC (publisher average) | acc_norm | — | — | 37.3% |
 
 Native MLX evaluates complete ARC Easy/Challenge test splits, PIQA validation and HellaSwag validation. It uses zero-shot raw completion prompts, no chat template or inserted BOS/EOS, FP32 weights, vanilla attention, batch size four and chunked scoring. Raw accuracy ranks summed continuation log-likelihood; normalized accuracy divides by choice Unicode-character count. No generated explanation or LLM judge determines the answer. No choices were context-truncated. Dataset revisions and scorer settings are preserved in [the result record](../results/public/choices/results.json).
 
 ## Instruction following
 
-| IFEval metric | Score | Passes / checks |
-| --- | --- | --- |
-| Strict prompts | 17.93% | 97/541 |
-| Strict instructions | 30.10% | 251/834 |
-| Loose prompts | 19.22% | 104/541 |
-| Loose instructions | 32.49% | 271/834 |
+| IFEval accuracy | Solstice-1 | OpenSML-150M | SmolLM2-135M Instruct |
+| --- | --- | --- | --- |
+| Strict prompts | 17.93% | 15.16% | — |
+| Strict instructions | 30.10% | 25.30% | — |
+| Loose prompts | 19.22% | 15.71% | — |
+| Loose instructions | 32.49% | 25.78% | — |
+| Publisher prompt/instruction average | — | — | 29.9% |
 
 Google IFEval is pinned to dataset revision `966cd89545d6b6acfd7638bc708b98261ca58e84` and google-research commit `83f1e7b59e762648c953de751d31f75720683622`. All 541 prompts and 834 instructions use the recorded strict/loose programmatic verifiers. Generation is greedy, native-chat formatted, FP32, without a system message or repetition penalty. Output cap: 1,024 tokens. Token-cap hits: **130/541**; context-cap hits: **0**. All generated responses are scored, including cap hits. The official dataset's split is named `train`; it is used as evaluation input here, not SFT training material.
 
+## Published small-model references
+
+Baseline scores come from the published [OpenSML-150M card](https://huggingface.co/OpenSML/OpenSML-150M) and [SmolLM2-135M-Instruct card](https://huggingface.co/HuggingFaceTB/SmolLM2-135M-Instruct), not fresh reruns. “—” means the required breakdown is not published in those cards. SmolLM2 reports ARC and IFEval as averages; no missing sub-scores are inferred.
+
+Solstice uses native MLX FP32 scoring. SmolLM2 uses LightEval; its [evaluation code](https://github.com/huggingface/smollm/tree/438be18dae658a638f3a6976fdf5190bb6d68619/text/evaluation/smollm2) specifies character-normalized choice scores with a leading space excluded and instruct runs with a chat template. OpenSML-150M uses an older native pipeline. Prompt formatting, precision and generation budgets differ, so these are reference comparisons, not a controlled leaderboard. [Exact sources and metric definitions](../results/baselines/published_models.json).
+
+![Published normalized-accuracy comparison](figures/benchmark-comparison.png)
+
+![IFEval comparison](figures/ifeval-comparison.png)
+
+Figure data is read directly from the saved JSON records by [plot_results.py](../scripts/plot_results.py). Run that script with matplotlib 3.10.8 to regenerate PNG and SVG figures. The SmolLM2 card does not identify how strict versus loose IFEval metrics contribute to its published average, so that average is not assigned to any one of Solstice's four IFEval metrics.
+
 ## Historical comparison
 
 | Metric | Published 150M | 167 | Parent 2000 | Prior attempt 5 | Solstice-1 |
@@ -62,6 +80,8 @@ The new panel flags unnecessary abstention on 111/200 answerable extraction case
 
 Manual inspection of the three chat repetition flags found two clear loops and one milder repeated sentence/idea. Keep the original three-flag result for consistent comparisons; it does not mean all three are equally severe. All 180 objective chat responses stop naturally; the qualitative cases account for the single token-cap hit and the repetition flags.
 
+![Chat category diagnostics](figures/chat-diagnostics.png)
+
 ## Selection and test exposure
 
 The final five-trial search selected one checkpoint using the 300-case new development panel plus historical development, stopping, repetition and prose-retention guards. Finalist identity was frozen before the new 1,000-case test. Small tolerated regressions were part of the declared guards; passing them does not mean every metric improved. The complete comparison and selection record includes the thresholds and reasons.

results/baselines/published_models.json

@@ -0,0 +1,61 @@
+{
+  "checked_on": "2026-11-09",
+  "scope": "Published reference values, not matched fresh runs; percentages rounded as in source cards. Absent metrics remain absent.",
+  "models": {
+    "OpenSML/OpenSML-150M": {
+      "revision": "7708682482f8c877c0793aed1af1064bf56db9ed",
+      "model_card": "https://huggingface.co/OpenSML/OpenSML-150M/blob/7708682482f8c877c0793aed1af1064bf56db9ed/README.md",
+      "readme_sha256": "7edb9a16eaf1ef9232bf5eb3b82dd6a6e5df36fe63108d1525bce00777025bdc",
+      "choices": {
+        "arc_easy": {
+          "acc": 0.5665,
+          "acc_norm": 0.5543
+        },
+        "arc_challenge": {
+          "acc": 0.2602,
+          "acc_norm": 0.2952
+        },
+        "piqa": {
+          "acc": 0.6453,
+          "acc_norm": 0.6409
+        },
+        "hellaswag": {
+          "acc": 0.3055,
+          "acc_norm": 0.3394
+        }
+      },
+      "ifeval": {
+        "strict": {
+          "prompt_accuracy": 0.1516,
+          "instruction_accuracy": 0.253
+        },
+        "loose": {
+          "prompt_accuracy": 0.1571,
+          "instruction_accuracy": 0.2578
+        }
+      }
+    },
+    "HuggingFaceTB/SmolLM2-135M-Instruct": {
+      "revision": "12fd25f77366fa6b3b4b768ec3050bf629380bac",
+      "model_card": "https://huggingface.co/HuggingFaceTB/SmolLM2-135M-Instruct/blob/12fd25f77366fa6b3b4b768ec3050bf629380bac/README.md",
+      "readme_sha256": "4f97533ad95b1b2fea15fbc075c01b94578ebdd7c8138888fa43fa3abd530dc4",
+      "choices": {
+        "piqa": {
+          "acc_norm": 0.663
+        },
+        "hellaswag": {
+          "acc_norm": 0.409
+        }
+      },
+      "arc_publisher_average": 0.373,
+      "ifeval_publisher_average": 0.299,
+      "missing": "Raw accuracy, separate ARC Easy/Challenge scores and separate strict/loose IFEval scores are not reported in the cited model card.",
+      "ifeval_aggregation": "Publisher label: Average prompt/inst. No strict/loose weighting inferred.",
+      "evaluation_code_revision": "438be18dae658a638f3a6976fdf5190bb6d68619",
+      "evaluation_code": "https://github.com/huggingface/smollm/tree/438be18dae658a638f3a6976fdf5190bb6d68619/text/evaluation/smollm2",
+      "lighteval_revision": "ea46419a93fb390e8f694f7c6c64c1e684487c9d",
+      "choice_metric": "loglikelihood_acc_norm_nospace: divide by Unicode-character count excluding an initial space",
+      "protocol": "Published zero-shot LightEval; linked instruct recipe uses a chat template. Not rerun for this release."
+    }
+  }
+}

scripts/plot_results.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""Render the model-card figures from saved scores; matplotlib 3.10.8."""
+import json
+from pathlib import Path
+
+import matplotlib
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+from matplotlib.ticker import PercentFormatter
+import numpy as np
+
+ROOT = Path(__file__).resolve().parents[1]
+OUT = ROOT / 'docs/figures'
+COLORS = {'Solstice-1': '#176b65', 'OpenSML-150M': '#67768a',
+          'SmolLM2-135M Instruct': '#c58a37', 'Checkpoint 167': '#9476a7'}
+
+
+def read(path):
+    return json.loads((ROOT / path).read_text())
+
+
+def save(fig, name):
+    OUT.mkdir(parents=True, exist_ok=True)
+    for ext in ('png', 'svg'):
+        fig.savefig(OUT / f'{name}.{ext}', dpi=180, facecolor='white')
+    plt.close(fig)
+
+
+def main():
+    plt.rcParams.update({'font.family': 'DejaVu Sans', 'font.size': 11,
+                         'axes.spines.top': False, 'axes.spines.right': False,
+                         'axes.spines.left': False, 'axes.titleweight': 'bold'})
+    choice = read('results/public/choices/results.json')['results']
+    public = read('results/baselines/published_models.json')['models']
+    old = public['OpenSML/OpenSML-150M']
+    smol = public['HuggingFaceTB/SmolLM2-135M-Instruct']
+    fig, axes = plt.subplots(2, 2, figsize=(11.5, 7.6))
+    fig.subplots_adjust(left=.2, right=.97, top=.84, bottom=.16, wspace=.85, hspace=.55)
+    fig.suptitle('Solstice-1 | Knowledge and commonsense', x=.08, ha='left', y=.97,
+                 fontsize=19, fontweight='bold')
+    fig.text(.08, .91, 'Zero-shot normalized accuracy · higher is better', fontsize=12)
+    for ax, (key, label) in zip(axes.flat, [('arc_easy', 'ARC Easy'),
+            ('arc_challenge', 'ARC Challenge'), ('piqa', 'PIQA'), ('hellaswag', 'HellaSwag')]):
+        names = ['Solstice-1', 'OpenSML-150M']
+        values = [choice[key]['acc_norm'], old['choices'][key]['acc_norm']]
+        if key in smol['choices']:
+            names.append('SmolLM2-135M Instruct')
+            values.append(smol['choices'][key]['acc_norm'])
+        ax.barh(range(len(names)), values, height=.55, color=[COLORS[n] for n in names])
+        ax.set_yticks(range(len(names)), [n.replace(' Instruct', '\nInstruct') for n in names], fontsize=9)
+        ax.invert_yaxis(); ax.set_xlim(0, .8); ax.set_xticks([0, .2, .4, .6, .8])
+        ax.xaxis.set_major_formatter(PercentFormatter(1, decimals=0))
+        ax.set_title(label, loc='left', pad=10); ax.grid(axis='x', alpha=.15); ax.set_axisbelow(True)
+        ax.tick_params(axis='y', length=0)
+        for y, value in enumerate(values):
+            ax.text(value+.015, y, f'{value:.2%}' if names[y] != 'SmolLM2-135M Instruct' else f'{value:.1%}',
+                    va='center', fontsize=10)
+    fig.text(.08, .06, 'Baseline values are publisher reports; evaluators and prompts differ.\n'
+             'SmolLM2 publishes ARC only as an average (37.3%); its separate ARC scores are not inferred.',
+             fontsize=10, color='#4b5563')
+    save(fig, 'benchmark-comparison')
+
+    own = read('results/public/ifeval/results.json')['metrics']
+    prior = read('results/baselines/167/ifeval/results.json')['metrics']
+    keys = [('strict', 'prompt'), ('strict', 'instruction'), ('loose', 'prompt'), ('loose', 'instruction')]
+    fig, ax = plt.subplots(figsize=(11.5, 5.9))
+    fig.subplots_adjust(left=.08, right=.98, top=.8, bottom=.24)
+    fig.suptitle('Solstice-1 | Instruction following', x=.08, ha='left', y=.97,
+                 fontsize=19, fontweight='bold')
+    fig.text(.08, .9, 'IFEval accuracy · higher is better', fontsize=12)
+    x = np.arange(4)
+    for i, (name, data) in enumerate([('OpenSML-150M', old['ifeval']),
+                                     ('Checkpoint 167', prior), ('Solstice-1', own)]):
+        values = [data[k][u+'_accuracy'] for k, u in keys]
+        bars = ax.bar(x+(i-1)*.25, values, width=.22, label=name, color=COLORS[name])
+        ax.bar_label(bars, labels=[f'{v:.2%}' for v in values], padding=4, fontsize=9)
+    ax.set_xticks(x, ['Strict\nprompts', 'Strict\ninstructions', 'Loose\nprompts', 'Loose\ninstructions'])
+    ax.set_ylim(0, .4); ax.set_yticks([0, .1, .2, .3, .4]); ax.yaxis.set_major_formatter(PercentFormatter(1, decimals=0))
+    ax.grid(axis='y', alpha=.15); ax.set_axisbelow(True)
+    ax.legend(loc='upper left', bbox_to_anchor=(0, -.18), ncols=3, frameon=False)
+    fig.text(.08, .035, '150M used different formatting and a longer output cap. SmolLM2 reports an aggregate (29.9%),\n'
+             'so its strict/loose sub-scores are not plotted. No normalized-accuracy metric applies to IFEval.',
+             fontsize=10, color='#4b5563')
+    save(fig, 'ifeval-comparison')
+
+    chat = read('results/public/chat/results.json')
+    categories = [('reading', 'Reading supplied text'), ('conversation', 'Conversation context'),
+                  ('knowledge', 'Factual knowledge'), ('instructions', 'Instructions'),
+                  ('answerability', 'Answerability'), ('reasoning', 'Short reasoning')]
+    values = [chat['categories'][key]['answer_match']/chat['categories'][key]['count'] for key, _ in categories]
+    fig, ax = plt.subplots(figsize=(11.5, 5.8))
+    fig.subplots_adjust(left=.25, right=.96, top=.79, bottom=.2)
+    fig.suptitle('Solstice-1 | Short-chat diagnostics', x=.08, ha='left', y=.97,
+                 fontsize=19, fontweight='bold')
+    fig.text(.08, .9, 'Conservative automatic answer matching · authored panel', fontsize=12)
+    ax.barh(range(6), values, color=COLORS['Solstice-1'], height=.6)
+    ax.set_yticks(range(6), [label for _, label in categories]); ax.invert_yaxis()
+    ax.set_xlim(0, 1); ax.xaxis.set_major_formatter(PercentFormatter(1, decimals=0))
+    ax.tick_params(axis='y', length=0); ax.grid(axis='x', alpha=.15); ax.set_axisbelow(True)
+    for y, value in enumerate(values): ax.text(value+.015, y, f'{value:.1%}', va='center')
+    fig.text(.08, .05, '30 checked responses per category. This is not a broad human assessment.\n'
+             'Natural stopping is 99.48% across all 192 responses; stopping correctly does not imply a correct answer.',
+             fontsize=10, color='#4b5563')
+    save(fig, 'chat-diagnostics')
+
+
+if __name__ == '__main__':
+    main()