Upload 2 files

data.json

@@ -0,0 +1,505 @@
+{
+  "schema_version": 13,
+  "description": "CogniRank author-grouped model registry with measured TTM, MARC, Procedural Pile, and SmolCUA results.",
+  "evaluation_weights": {
+    "ttm": 0.15789473684210525,
+    "marc": 0.2631578947368421,
+    "procedural_pile": 0.2631578947368421,
+    "smolcua": 0.3157894736842105
+  },
+  "evaluation_protocol": {
+    "version": "cognirank-v9-context-4096",
+    "status": "measured_results_available",
+    "scoring": "(30*SmolCUA + 25*mean(MARC-0..3) + 25*Procedural Pile + 15*TTM)/95; scores on 0–100; no chance correction. Only show CogniScore when all four benchmarks are measured.",
+    "weights": {
+      "ttm": 0.15789473684210525,
+      "marc": 0.2631578947368421,
+      "procedural_pile": 0.2631578947368421,
+      "smolcua": 0.3157894736842105
+    },
+    "marc_aggregation": "Mean of MARC-0..3 (unchanged)",
+    "procedural_pile": {
+      "dataset": "reasoning-core/procedural-pile",
+      "dataset_url": "https://huggingface.co/datasets/reasoning-core/procedural-pile",
+      "status": "scored_in_v9_results",
+      "recommended_source": "Pinned revision of test split or newly generated held-out tasks, not train split",
+      "recommendation": "Select a fixed, public, moderate-size, task-balanced, low-difficulty subset; use verifiable answers and do not evaluate on examples a model trained on.",
+      "warning": "Procedural Pile is a procedural training corpus, not an intrinsically standardized benchmark. The uploaded CogniRank v9 evaluation protocol defines the reported score. Check for training-data overlap."
+    },
+    "prior_benchmark": "bAbI-Core",
+    "prior_protocol": {
+      "version": "cognirank-v7-four-benchmarks",
+      "context_tokens": 4096,
+      "scoring": "weighted mean of four benchmark percentages; remaining weights proportionally normalized from the earlier five-benchmark distribution; no chance correction",
+      "source_file": "cognirank-context-new.json (initial batch); evaluation_results_batch2.json (additional evaluations)",
+      "caveat": "The 4096 context evaluation extrapolates the declared 2048-token context for void.1 and KLM-1.5 without changing weights or RoPE configuration.",
+      "marc_aggregation": "unweighted mean of MARC-0, MARC-1, MARC-2, MARC-3",
+      "derived_from": "cognirank-v6-context-4096; TRIP removed from scoring (near chance performance across all measured checkpoints)",
+      "original_measurements": "evaluation_results.json and evaluation_results_batch2.json preserve source measurements for their respective batches; current four-benchmark scores were recomputed without new inference.",
+      "source_files": [
+        "cognirank-context-new.json",
+        "evaluation_results_batch2.json"
+      ]
+    },
+    "context_tokens": 4096,
+    "new_evaluation_sources": [
+      "cognirank-context-new8.json",
+      "cognirank-context-new6.json"
+    ]
+  },
+  "authors": [
+    {
+      "id": "minima-labs",
+      "name": "MinimaLabs",
+      "link": "https://huggingface.co/MinimaLabs",
+      "color": "#99D9BD",
+      "models": [
+        {
+          "id": "MinimaLabs/KLM-1.5",
+          "name": "KLM-1.5",
+          "parameters": "99.0M",
+          "hf_repo": "MinimaLabs/KLM-1.5",
+          "model_card_url": "https://huggingface.co/MinimaLabs/KLM-1.5",
+          "evaluation_status": "evaluated",
+          "submission_status": "not_confirmed",
+          "scores": {
+            "ttm": 42.55,
+            "marc": 11.5625,
+            "procedural_pile": 0.028325667131668543,
+            "smolcua": 0.0
+          },
+          "date_added": "2026-11-04",
+          "date_submitted": null,
+          "author_note": null,
+          "comment_source": "author",
+          "comment_edited_by_dotlabs": false,
+          "checkpoint_revision": "b2bbc22327c2b85316c23e871a91b1287aefa795",
+          "evaluation": {
+            "version": "cognirank-v9-context-4096",
+            "context_tokens": 4096,
+            "declared_context_tokens": 2048,
+            "context_extrapolated": true,
+            "smolcua_version": "SmolCUA v1.4",
+            "smolcua_protocol": "v2-outer-whitespace-insensitive",
+            "smolcua_format_failure_pct": 100.0,
+            "smolcua_edition": "compact-1200-three-pass",
+            "marc_subtests": {
+              "marc0": 32.5,
+              "marc1": 5.0,
+              "marc2": 2.5,
+              "marc3": 6.25
+            },
+            "evaluation_record": "cognirank-context-new8.json",
+            "evaluation_complete": true
+          },
+          "cogniscore_scoring_version": "cognirank-v9-context-4096",
+          "historical_results": {
+            "babi_core": 0.0,
+            "previous_cogniscore": 9.761184210526315,
+            "previous_scoring_version": "cognirank-v7-four-benchmarks"
+          },
+          "cogniscore_recorded": 9.768638333455701
+        }
+      ]
+    },
+    {
+      "id": "huggingface-tb",
+      "name": "HuggingFaceTB",
+      "link": "https://huggingface.co/HuggingFaceTB",
+      "color": "#F5D18C",
+      "models": [
+        {
+          "id": "HuggingFaceTB/SmolLM2-135M",
+          "name": "SmolLM2-135M",
+          "parameters": "135M",
+          "hf_repo": "HuggingFaceTB/SmolLM2-135M",
+          "model_card_url": "https://huggingface.co/HuggingFaceTB/SmolLM2-135M",
+          "evaluation_status": "evaluated",
+          "submission_status": "not_confirmed",
+          "scores": {
+            "ttm": 46.7,
+            "marc": 5.3125,
+            "procedural_pile": 0.004785877737822812,
+            "smolcua": 0.0
+          },
+          "date_added": "2026-11-04",
+          "date_submitted": null,
+          "author_note": null,
+          "comment_source": "author",
+          "comment_edited_by_dotlabs": false,
+          "checkpoint_revision": "93efa2f097d58c2a74874c7e644dbc9b0cee75a2",
+          "evaluation": {
+            "version": "cognirank-v9-context-4096",
+            "context_tokens": 4096,
+            "declared_context_tokens": 8192,
+            "context_extrapolated": false,
+            "smolcua_version": "SmolCUA v1.4",
+            "smolcua_protocol": "v2-outer-whitespace-insensitive",
+            "smolcua_format_failure_pct": 100.0,
+            "smolcua_edition": "compact-1200-three-pass",
+            "marc_subtests": {
+              "marc0": 12.5,
+              "marc1": 0.0,
+              "marc2": 2.5,
+              "marc3": 6.25
+            },
+            "evaluation_record": "cognirank-context-new6.json",
+            "evaluation_complete": true
+          },
+          "cogniscore_scoring_version": "cognirank-v9-context-4096",
+          "historical_results": {
+            "babi_core": 0.0,
+            "previous_cogniscore": 8.77171052631579,
+            "previous_scoring_version": "cognirank-v7-four-benchmarks"
+          },
+          "cogniscore_recorded": 8.772969967825743
+        }
+      ]
+    },
+    {
+      "id": "axiomic-labs",
+      "name": "AxiomicLabs",
+      "link": "https://huggingface.co/AxiomicLabs",
+      "color": "#A6A0FF",
+      "models": [
+        {
+          "id": "AxiomicLabs/GPT-X2.5-135M",
+          "name": "GPT-X2.5-135M",
+          "parameters": "135M",
+          "hf_repo": "AxiomicLabs/GPT-X2.5-135M",
+          "model_card_url": "https://huggingface.co/AxiomicLabs/GPT-X2.5-135M",
+          "evaluation_status": "evaluated",
+          "submission_status": "not_confirmed",
+          "scores": {
+            "ttm": 43.9,
+            "marc": 6.25,
+            "procedural_pile": 0.0034782608695652162,
+            "smolcua": 0.0
+          },
+          "date_added": "2026-11-04",
+          "date_submitted": null,
+          "author_note": null,
+          "comment_source": "author",
+          "comment_edited_by_dotlabs": false,
+          "checkpoint_revision": "53959ad51244b913365c2d66a49a301a83cac59c",
+          "cogniscore_recorded": 8.577231121281464,
+          "cogniscore_scoring_version": "cognirank-v9-context-4096",
+          "evaluation": {
+            "version": "cognirank-v9-context-4096",
+            "context_tokens": 4096,
+            "declared_context_tokens": 8192,
+            "context_extrapolated": false,
+            "smolcua_version": "SmolCUA v1.4",
+            "smolcua_protocol": "v2-outer-whitespace-insensitive",
+            "smolcua_format_failure_pct": 100.0,
+            "smolcua_edition": "compact-1200-three-pass",
+            "marc_subtests": {
+              "marc0": 11.25,
+              "marc1": 8.75,
+              "marc2": 0.0,
+              "marc3": 5.0
+            },
+            "evaluation_record": "cognirank-context-new8.json",
+            "evaluation_complete": true
+          }
+        }
+      ]
+    },
+    {
+      "id": "dotlabs",
+      "name": "DotLabs",
+      "link": "https://huggingface.co/dotlabs",
+      "color": "#FFFFFF",
+      "models": [
+        {
+          "id": "dotlabs/void.1",
+          "name": "void.1",
+          "parameters": "90.1M",
+          "hf_repo": "dotlabs/void.1",
+          "model_card_url": "https://huggingface.co/dotlabs/void.1",
+          "evaluation_status": "evaluated",
+          "submission_status": "not_confirmed",
+          "scores": {
+            "ttm": 41.65,
+            "marc": 1.875,
+            "procedural_pile": 0.0426930028759097,
+            "smolcua": 0.0
+          },
+          "date_added": "2026-11-04",
+          "date_submitted": null,
+          "author_note": null,
+          "comment_source": "author",
+          "comment_edited_by_dotlabs": false,
+          "checkpoint_revision": "786701ec20edc5f7fee47dfedf4f9bb662ca4bb8",
+          "evaluation": {
+            "version": "cognirank-v9-context-4096",
+            "context_tokens": 4096,
+            "declared_context_tokens": 2048,
+            "context_extrapolated": true,
+            "smolcua_version": "SmolCUA v1.4",
+            "smolcua_protocol": "v2-outer-whitespace-insensitive",
+            "smolcua_format_failure_pct": 100.0,
+            "smolcua_edition": "compact-1200-three-pass",
+            "marc_subtests": {
+              "marc0": 0.0,
+              "marc1": 0.0,
+              "marc2": 1.25,
+              "marc3": 6.25
+            },
+            "evaluation_record": "cognirank-context-new6.json",
+            "evaluation_complete": true
+          },
+          "cogniscore_scoring_version": "cognirank-v9-context-4096",
+          "historical_results": {
+            "babi_core": 0.0,
+            "previous_cogniscore": 7.038157894736842,
+            "previous_scoring_version": "cognirank-v7-four-benchmarks"
+          },
+          "cogniscore_recorded": 7.080971842862081
+        }
+      ]
+    },
+    {
+      "id": "tiiuae",
+      "name": "TII",
+      "link": "https://huggingface.co/tiiuae",
+      "color": "#D5ACFF",
+      "models": [
+        {
+          "id": "tiiuae/Falcon-H1-Tiny-90M-Base",
+          "name": "Falcon-H1-Tiny-90M-Base",
+          "parameters": "90M",
+          "hf_repo": "tiiuae/Falcon-H1-Tiny-90M-Base",
+          "model_card_url": "https://huggingface.co/tiiuae/Falcon-H1-Tiny-90M-Base",
+          "evaluation_status": "evaluated",
+          "submission_status": "not_confirmed",
+          "scores": {
+            "ttm": 41.7,
+            "marc": 7.5,
+            "procedural_pile": 0.018256566209742447,
+            "smolcua": 0.0
+          },
+          "date_added": "2026-11-04",
+          "date_submitted": null,
+          "author_note": null,
+          "comment_source": "author",
+          "comment_edited_by_dotlabs": false,
+          "checkpoint_revision": "7994372e93b62822ae25f8bfb19f653649cea3a3",
+          "cogniscore_recorded": 8.562699096370984,
+          "cogniscore_scoring_version": "cognirank-v9-context-4096",
+          "evaluation": {
+            "version": "cognirank-v9-context-4096",
+            "context_tokens": 4096,
+            "declared_context_tokens": 262144,
+            "context_extrapolated": false,
+            "smolcua_version": "SmolCUA v1.4",
+            "smolcua_protocol": "v2-outer-whitespace-insensitive",
+            "smolcua_format_failure_pct": 100.0,
+            "smolcua_edition": "compact-1200-three-pass",
+            "marc_subtests": {
+              "marc0": 12.5,
+              "marc1": 8.75,
+              "marc2": 2.5,
+              "marc3": 6.25
+            },
+            "evaluation_record": "cognirank-context-new8.json",
+            "evaluation_complete": true
+          }
+        }
+      ]
+    },
+    {
+      "id": "liquidai",
+      "name": "LiquidAI",
+      "link": "https://huggingface.co/LiquidAI",
+      "color": "#9AB5FF",
+      "models": [
+        {
+          "id": "LiquidAI/LFM2.5-230M-Base",
+          "name": "LFM2.5-230M-Base",
+          "parameters": "230M",
+          "hf_repo": "LiquidAI/LFM2.5-230M-Base",
+          "model_card_url": "https://huggingface.co/LiquidAI/LFM2.5-230M-Base",
+          "evaluation_status": "evaluated",
+          "submission_status": "not_confirmed",
+          "scores": {
+            "ttm": 47.4,
+            "marc": 2.1875,
+            "procedural_pile": 2.0767121254978345,
+            "smolcua": 0.0
+          },
+          "date_added": "2026-11-04",
+          "date_submitted": null,
+          "author_note": null,
+          "comment_source": "author",
+          "comment_edited_by_dotlabs": false,
+          "checkpoint_revision": "9d2be5519834990d30996f878b6771cccbd24f2c",
+          "cogniscore_recorded": 8.606371611973113,
+          "cogniscore_scoring_version": "cognirank-v9-context-4096",
+          "evaluation": {
+            "version": "cognirank-v9-context-4096",
+            "context_tokens": 4096,
+            "declared_context_tokens": 128000,
+            "context_extrapolated": false,
+            "smolcua_version": "SmolCUA v1.4",
+            "smolcua_protocol": "v2-outer-whitespace-insensitive",
+            "smolcua_format_failure_pct": 100.0,
+            "smolcua_edition": "compact-1200-three-pass",
+            "marc_subtests": {
+              "marc0": 3.75,
+              "marc1": 0.0,
+              "marc2": 0.0,
+              "marc3": 5.0
+            },
+            "evaluation_record": "cognirank-context-new8.json",
+            "evaluation_complete": true
+          }
+        },
+        {
+          "id": "LiquidAI/LFM2.5-350M",
+          "name": "LFM2.5-350M",
+          "parameters": "350M",
+          "hf_repo": "LiquidAI/LFM2.5-350M",
+          "model_card_url": "https://huggingface.co/LiquidAI/LFM2.5-350M",
+          "evaluation_status": "evaluated",
+          "submission_status": "not_confirmed",
+          "scores": {
+            "ttm": 46.85,
+            "marc": 0.0,
+            "procedural_pile": 11.993131773709633,
+            "smolcua": 14.0053
+          },
+          "date_added": "2026-11-04",
+          "date_submitted": null,
+          "author_note": null,
+          "comment_source": "author",
+          "comment_edited_by_dotlabs": false,
+          "checkpoint_revision": "9921cd4d82c0e3986887aa4b376f9446729ef8a3",
+          "cogniscore_recorded": 14.97618204571306,
+          "cogniscore_scoring_version": "cognirank-v9-context-4096",
+          "evaluation": {
+            "version": "cognirank-v9-context-4096",
+            "context_tokens": 4096,
+            "declared_context_tokens": 128000,
+            "context_extrapolated": false,
+            "smolcua_version": "SmolCUA v1.4",
+            "smolcua_protocol": "v2-outer-whitespace-insensitive",
+            "smolcua_format_failure_pct": 70.3333,
+            "smolcua_edition": "compact-1200-three-pass",
+            "smolcua_reliability": 13.25,
+            "marc_subtests": {
+              "marc0": 0.0,
+              "marc1": 0.0,
+              "marc2": 0.0,
+              "marc3": 0.0
+            },
+            "evaluation_record": "cognirank-context-lfm350.json",
+            "evaluation_complete": true
+          }
+        }
+      ]
+    }
+  ],
+  "reference_models": [
+    {
+      "author": "TII",
+      "author_link": "https://huggingface.co/tiiuae",
+      "author_color": "#D5ACFF",
+      "id": "tiiuae/Falcon-H1-Tiny-90M-Instruct",
+      "name": "Falcon-H1-Tiny-90M-Instruct",
+      "parameters": "90M",
+      "hf_repo": "tiiuae/Falcon-H1-Tiny-90M-Instruct",
+      "model_card_url": "https://huggingface.co/tiiuae/Falcon-H1-Tiny-90M-Instruct",
+      "evaluation_status": "partial",
+      "submission_status": "not_confirmed",
+      "scores": {
+        "ttm": 43.35,
+        "marc": 7.1875,
+        "procedural_pile": null,
+        "smolcua": 0.0
+      },
+      "date_added": null,
+      "date_submitted": null,
+      "author_note": null,
+      "comment_source": "author",
+      "comment_edited_by_dotlabs": false,
+      "checkpoint_revision": "e6389502a0b12cd8da894b395ba5bf7436873b16",
+      "evaluation": {
+        "version": "cognirank-v6-context-4096",
+        "context_tokens": 4096,
+        "declared_context_tokens": 262144,
+        "context_extrapolated": false,
+        "smolcua_version": "SmolCUA v1.4",
+        "smolcua_protocol": "v2-outer-whitespace-insensitive",
+        "smolcua_format_failure_pct": 99.3333,
+        "marc_subtests": {
+          "marc0": 18.75,
+          "marc1": 1.25,
+          "marc2": 1.25,
+          "marc3": 7.5
+        }
+      },
+      "exclusion_reason": "Instruction-tuned checkpoint; CogniRank standard track accepts pretrained base models.",
+      "cogniscore_scoring_version": "cognirank-v8-procedural-pile-pending",
+      "historical_results": {
+        "babi_core": 5.0,
+        "previous_cogniscore": 10.051973684210527,
+        "previous_scoring_version": "cognirank-v7-four-benchmarks"
+      },
+      "cogniscore_recorded": null
+    },
+    {
+      "author": "Liquid AI",
+      "author_link": "https://huggingface.co/LiquidAI",
+      "author_color": "#9AB5FF",
+      "id": "LiquidAI/LFM3-1.7B",
+      "name": "LFM3-1.7B",
+      "parameters": "1.73B",
+      "hf_repo": "LiquidAI/LFM3-1.7B",
+      "model_card_url": "https://huggingface.co/LiquidAI/LFM3-1.7B",
+      "evaluation_status": "partial",
+      "submission_status": "not_confirmed",
+      "scores": {
+        "ttm": 59.6,
+        "marc": 3.125,
+        "procedural_pile": null,
+        "smolcua": 32.1874
+      },
+      "date_added": null,
+      "date_submitted": null,
+      "author_note": null,
+      "comment_source": "author",
+      "comment_edited_by_dotlabs": false,
+      "checkpoint_revision": "0dc4268b561499c648c7ba7c9466bf3d8d01517a",
+      "evaluation": {
+        "version": "cognirank-v6-context-4096",
+        "context_tokens": 4096,
+        "declared_context_tokens": 262144,
+        "context_extrapolated": false,
+        "smolcua_version": "SmolCUA v1.4",
+        "smolcua_protocol": "v2-outer-whitespace-insensitive",
+        "smolcua_format_failure_pct": 58.4167,
+        "marc_subtests": {
+          "marc0": 1.25,
+          "marc1": 6.25,
+          "marc2": 2.5,
+          "marc3": 2.5
+        }
+      },
+      "exclusion_reason": "1.73 billion parameters exceeds the 250M maximum, and this checkpoint is post-trained.",
+      "cogniscore_scoring_version": "cognirank-v8-procedural-pile-pending",
+      "historical_results": {
+        "babi_core": 4.0,
+        "previous_cogniscore": 21.449968421052628,
+        "previous_scoring_version": "cognirank-v7-four-benchmarks"
+      },
+      "cogniscore_recorded": null
+    }
+  ],
+  "eligibility": {
+    "min_parameters_millions": 25,
+    "max_parameters_millions": 350,
+    "below_minimum_exception": "Only approved models below 25M demonstrating generalization across all CogniRank tasks and superior text quality among peers of comparable size",
+    "pretraining_requirement": "Pretrained by the submitting person or team; no fine-tunes, merges, or third-party reuploads"
+  }
+}