| """ |
| Model dataset for the GPT History Explorer. |
| |
| Every field is either a sourced value or ``None``. ``None`` means "not publicly |
| disclosed" or "not found in a source I could check" — it never means zero, and the |
| charts skip it rather than plotting it as a low value. That distinction matters a lot |
| here: OpenAI stopped publishing parameter counts after GPT-3, so most of the modern |
| ``params_b`` column is genuinely unknown rather than small. |
| |
| Prices are USD per million tokens, standard (non-batch, non-cached) API rates at the |
| time of writing. Where a model's price was cut after launch, the current rate is used |
| and the change is noted in ``notes``. |
| |
| To extend: append to ``MODELS``. Nothing else needs editing — the app derives its |
| dropdowns, axes and benchmark list from this file. |
| """ |
|
|
| from __future__ import annotations |
|
|
| import datetime as _dt |
| from typing import Any |
|
|
| import pandas as pd |
|
|
| |
| |
| |
| |
| |
| |
| |
|
|
| BENCHMARKS: dict[str, dict[str, Any]] = { |
| "mmlu": {"label": "MMLU", "group": "Knowledge", "higher_better": True}, |
| "gpqa": {"label": "GPQA Diamond", "group": "Knowledge", "higher_better": True}, |
| "swebench": {"label": "SWE-bench Verified", "group": "Coding", "higher_better": True}, |
| "terminalbench": {"label": "Terminal-Bench", "group": "Coding", "higher_better": True}, |
| "frontiercode": {"label": "FrontierCode 1.1", "group": "Coding", "higher_better": True}, |
| "deepswe": {"label": "DeepSWE v1.1", "group": "Coding", "higher_better": True}, |
| "aime": {"label": "AIME", "group": "Math", "higher_better": True}, |
| "frontiermath": {"label": "FrontierMath T4 v2", "group": "Math", "higher_better": True}, |
| "arcagi": {"label": "ARC-AGI-3", "group": "Reasoning", "higher_better": True}, |
| "osworld": {"label": "OSWorld 2.0", "group": "Agentic", "higher_better": True}, |
| "screenspot": {"label": "ScreenSpot-Pro", "group": "Agentic", "higher_better": True}, |
| "agents_last_exam": {"label": "Agents' Last Exam", "group": "Agentic", "higher_better": True}, |
| "exploitbench": {"label": "ExploitBench", "group": "Security", "higher_better": True}, |
| "srebench": {"label": "SRE-Bench", "group": "Security", "higher_better": True}, |
| "hallucination": {"label": "Hallucination rate", "group": "Safety", "higher_better": False}, |
| "misaligned": {"label": "Misaligned outcomes", "group": "Safety", "higher_better": False}, |
| "cu_safety": {"label": "Computer-use safety failures", "group": "Safety", "higher_better": False}, |
| } |
|
|
| |
| ERAS: list[dict[str, Any]] = [ |
| {"name": "Pre-training era", "start": "2018-01-01", "end": "2022-11-29", |
| "blurb": "Research artefacts. Scale is the whole story: 117M → 175B parameters, " |
| "context 512 → 2,048 tokens. No chat product, no API pricing page."}, |
| {"name": "ChatGPT era", "start": "2022-11-30", "end": "2024-09-11", |
| "blurb": "ChatGPT ships and the model becomes a product. Instruction tuning, RLHF, " |
| "then multimodality. Context grows 4K → 128K; price per token starts falling."}, |
| {"name": "Reasoning era", "start": "2024-09-12", "end": "2025-08-06", |
| "blurb": "o1 introduces inference-time reasoning. A second scaling axis opens: think " |
| "longer rather than train bigger. Output tokens now include hidden reasoning."}, |
| {"name": "Agentic era", "start": "2025-08-07", "end": "2026-12-31", |
| "blurb": "GPT-5 onward. The benchmark that matters shifts from 'what does it know' to " |
| "'can it finish a long task unattended'. Context passes 1M; safety metrics " |
| "become headline numbers."}, |
| ] |
|
|
|
|
| def _m( |
| id: str, name: str, vendor: str, family: str, released: str | None, |
| *, params_b: float | None = None, context: int | None = None, |
| max_output: int | None = None, price_in: float | None = None, |
| price_out: float | None = None, cutoff: str | None = None, |
| modalities: tuple[str, ...] = ("text",), open_weights: bool = False, |
| notes: str = "", bench: dict[str, float] | None = None, |
| source: str = "", |
| ) -> dict[str, Any]: |
| """Build one model record. Keeps the table below readable.""" |
| return { |
| "id": id, "name": name, "vendor": vendor, "family": family, |
| "released": released, "params_b": params_b, "context": context, |
| "max_output": max_output, "price_in": price_in, "price_out": price_out, |
| "cutoff": cutoff, "modalities": list(modalities), |
| "open_weights": open_weights, "notes": notes, |
| "bench": bench or {}, "source": source, |
| } |
|
|
|
|
| |
| |
| |
|
|
| _OPENAI = [ |
| _m("gpt-1", "GPT-1", "OpenAI", "GPT-1", "2018-06-11", |
| params_b=0.117, context=512, |
| notes="'Improving Language Understanding by Generative Pre-Training'. Established the " |
| "recipe — unsupervised pre-training, then supervised fine-tuning per task.", |
| source="OpenAI paper, Jun 2018"), |
|
|
| _m("gpt-2", "GPT-2", "OpenAI", "GPT-2", "2019-02-14", |
| params_b=1.5, context=1024, |
| notes="Staged release over 2019 on misuse concerns — the first time a lab withheld " |
| "weights for safety reasons. Full 1.5B model published Nov 2019.", |
| source="OpenAI, Feb 2019"), |
|
|
| _m("gpt-3", "GPT-3", "OpenAI", "GPT-3", "2020-06-11", |
| params_b=175, context=2048, cutoff="2019-10", |
| bench={"mmlu": 43.9}, |
| notes="117× the parameters of GPT-2. Demonstrated few-shot in-context learning: the " |
| "model adapts from examples in the prompt, without weight updates.", |
| source="'Language Models are Few-Shot Learners'"), |
|
|
| _m("gpt-3.5-turbo", "GPT-3.5 Turbo (ChatGPT)", "OpenAI", "GPT-3.5", "2022-11-30", |
| context=4096, price_in=0.50, price_out=1.50, cutoff="2021-09", |
| bench={"mmlu": 70.0}, |
| notes="The RLHF-tuned chat model. Reached ~100M users in two months — the fastest " |
| "consumer software adoption on record at the time.", |
| source="OpenAI, Nov 2022"), |
|
|
| _m("gpt-4", "GPT-4", "OpenAI", "GPT-4", "2023-03-14", |
| context=8192, max_output=4096, price_in=30.0, price_out=60.0, cutoff="2021-09", |
| modalities=("text", "image"), |
| bench={"mmlu": 86.4, "gpqa": 35.7}, |
| notes="First model where OpenAI declined to publish size, architecture or training " |
| "data. Human-professional-level scores on standardised exams. A 32K-context " |
| "variant sold at $60/$120.", |
| source="GPT-4 Technical Report"), |
|
|
| _m("gpt-4-turbo", "GPT-4 Turbo", "OpenAI", "GPT-4", "2023-11-06", |
| context=128000, max_output=4096, price_in=10.0, price_out=30.0, cutoff="2023-12", |
| modalities=("text", "image"), |
| bench={"mmlu": 86.5}, |
| notes="16× the context at one-third the input price. The point where long-context " |
| "work became economically ordinary rather than exotic.", |
| source="OpenAI DevDay 2023"), |
|
|
| _m("gpt-4o", "GPT-4o", "OpenAI", "GPT-4o", "2024-05-13", |
| context=128000, max_output=16384, price_in=2.50, price_out=10.0, cutoff="2023-10", |
| modalities=("text", "image", "audio"), |
| bench={"mmlu": 88.7, "gpqa": 53.6, "swebench": 33.2}, |
| notes="'o' for omni — one model natively handling text, vision and audio. Launched at " |
| "$5/$15; cut to $2.50/$10 in Aug 2024.", |
| source="OpenAI, May 2024"), |
|
|
| _m("gpt-4o-mini", "GPT-4o mini", "OpenAI", "GPT-4o", "2024-07-18", |
| context=128000, max_output=16384, price_in=0.15, price_out=0.60, cutoff="2023-10", |
| modalities=("text", "image"), |
| bench={"mmlu": 82.0, "gpqa": 40.2}, |
| notes="Replaced GPT-3.5 Turbo as the cheap default at ~1/30th the price of GPT-4o, " |
| "with better benchmarks than the model it replaced.", |
| source="OpenAI, Jul 2024"), |
|
|
| _m("o1-preview", "o1-preview", "OpenAI", "o-series", "2024-09-12", |
| context=128000, max_output=32768, price_in=15.0, price_out=60.0, cutoff="2023-10", |
| bench={"gpqa": 73.3, "aime": 74.4}, |
| notes="The inflection point. Spends tokens 'thinking' before answering, trading " |
| "latency and cost for accuracy — a second scaling axis independent of model size.", |
| source="OpenAI, Sep 2024"), |
|
|
| _m("o1", "o1", "OpenAI", "o-series", "2024-12-05", |
| context=200000, max_output=100000, price_in=15.0, price_out=60.0, cutoff="2023-10", |
| modalities=("text", "image"), |
| bench={"mmlu": 91.8, "gpqa": 78.0, "aime": 83.3, "swebench": 48.9}, |
| notes="General availability of the reasoning line. Hidden reasoning tokens are billed " |
| "as output — the first time the visible answer stopped predicting the bill.", |
| source="OpenAI, Dec 2024"), |
|
|
| _m("o3-mini", "o3-mini", "OpenAI", "o-series", "2025-01-31", |
| context=200000, max_output=100000, price_in=1.10, price_out=4.40, cutoff="2023-10", |
| bench={"gpqa": 79.7, "aime": 87.3}, |
| notes="Reasoning at roughly 1/14th of o1's price. Introduced selectable reasoning " |
| "effort (low/medium/high) as a user-facing control.", |
| source="OpenAI, Jan 2025"), |
|
|
| _m("gpt-4.5", "GPT-4.5", "OpenAI", "GPT-4.5", "2025-02-27", |
| context=128000, max_output=16384, price_in=75.0, price_out=150.0, cutoff="2023-10", |
| modalities=("text", "image"), |
| bench={"mmlu": 90.2, "gpqa": 71.4}, |
| notes="The most expensive OpenAI model ever shipped, and the shortest-lived — " |
| "deprecated from the API within months. The last big bet on pure pre-training " |
| "scale before the line committed to reasoning.", |
| source="OpenAI, Feb 2025"), |
|
|
| _m("gpt-4.1", "GPT-4.1", "OpenAI", "GPT-4.1", "2025-04-14", |
| context=1047576, max_output=32768, price_in=2.00, price_out=8.00, cutoff="2024-06", |
| modalities=("text", "image"), |
| bench={"mmlu": 90.2, "gpqa": 66.3, "swebench": 54.6}, |
| notes="First OpenAI model with a 1M-token context window. API-only — never shipped in " |
| "ChatGPT, which stayed on GPT-4o.", |
| source="OpenAI, Apr 2025"), |
|
|
| _m("o3", "o3", "OpenAI", "o-series", "2025-04-16", |
| context=200000, max_output=100000, price_in=2.00, price_out=8.00, cutoff="2024-06", |
| modalities=("text", "image"), |
| bench={"gpqa": 83.3, "aime": 88.9, "swebench": 69.1}, |
| notes="Reasoning with tool use in the chain of thought. Launched at $10/$40; cut 80% " |
| "to $2/$8 in Jun 2025 — one of the steepest frontier price drops on record.", |
| source="OpenAI, Apr 2025"), |
|
|
| _m("o4-mini", "o4-mini", "OpenAI", "o-series", "2025-04-16", |
| context=200000, max_output=100000, price_in=1.10, price_out=4.40, cutoff="2024-06", |
| modalities=("text", "image"), |
| bench={"gpqa": 81.4, "aime": 92.7, "swebench": 68.1}, |
| notes="Beat the far larger o3 on AIME at half the price — evidence that reasoning " |
| "training, not size, was carrying math performance.", |
| source="OpenAI, Apr 2025"), |
|
|
| _m("gpt-5", "GPT-5", "OpenAI", "GPT-5", "2025-08-07", |
| context=400000, max_output=128000, price_in=1.25, price_out=10.0, cutoff="2024-09", |
| modalities=("text", "image"), |
| bench={"gpqa": 85.7, "aime": 94.6, "swebench": 74.9}, |
| notes="Unified the GPT and o-series lines behind an automatic router that decides " |
| "whether to think. Priced below GPT-4o at launch despite being the flagship.", |
| source="OpenAI, Aug 2025"), |
|
|
| _m("gpt-5.1", "GPT-5.1", "OpenAI", "GPT-5.1", "2025-11-12", |
| context=400000, max_output=128000, price_in=1.25, price_out=10.0, cutoff="2024-09", |
| modalities=("text", "image"), |
| bench={"gpqa": 88.1, "swebench": 76.3}, |
| notes="Added adaptive reasoning — the model decides how long to think per request — " |
| "plus 'none' as a selectable effort level.", |
| source="OpenAI, Nov 2025"), |
|
|
| _m("gpt-5.2", "GPT-5.2", "OpenAI", "GPT-5.2", "2025-12-11", |
| modalities=("text", "image"), |
| notes="Shipped in three modes: instant, thinking (standard and extended), and Pro. " |
| "Context and pricing not confirmed in the sources checked.", |
| source="Release timeline, Dec 2025"), |
|
|
| _m("gpt-5.3-codex", "GPT-5.3-Codex", "OpenAI", "GPT-5.3", "2026-02-05", |
| modalities=("text", "image"), |
| notes="Coding-specialised branch: repository search, terminal execution and debugging " |
| "as first-class abilities rather than tool bolt-ons.", |
| source="Release timeline, Feb 2026"), |
|
|
| _m("gpt-5.4", "GPT-5.4", "OpenAI", "GPT-5.4", "2026-03-05", |
| modalities=("text", "image"), |
| notes="Launched as Thinking and Pro variants; mini and nano followed on 17 Mar 2026.", |
| source="Release timeline, Mar 2026"), |
|
|
| _m("gpt-5.5", "GPT-5.5", "OpenAI", "GPT-5.5", "2026-04-23", |
| modalities=("text", "image"), |
| notes="Codename 'Spud'. Positioned around code, online research, data analysis, and " |
| "producing documents and spreadsheets.", |
| source="OpenAI, Apr 2026"), |
|
|
| _m("gpt-5.6-sol", "GPT-5.6 Sol", "OpenAI", "GPT-5.6", "2026-07-09", |
| context=1050000, max_output=128000, price_in=4.00, price_out=20.0, cutoff="2026-02-16", |
| modalities=("text", "image"), |
| bench={"gpqa": 94.6, "frontiermath": 83.0, "arcagi": 7.8, "osworld": 65.7, |
| "screenspot": 76.9, "terminalbench": 37.3, "deepswe": 70.8, |
| "exploitbench": 78.5, "srebench": 55.9, "hallucination": 12.2, |
| "misaligned": 18.8, "cu_safety": 22.0, "agents_last_exam": 53.6}, |
| notes="Flagship of the three-tier 5.6 family (Sol / Terra / Luna). Preview 26 Jun, " |
| "public 9 Jul 2026. $4/$20 is promotional — held at least to 21 Nov 2026, down " |
| "from $5/$30. Keeps a 'none' reasoning level that Astra drops.", |
| source="OpenAI; OrcaRouter; llm-stats"), |
|
|
| _m("gpt-6-astra", "GPT-6 Astra", "OpenAI", "GPT-6", "2026-09-03", |
| context=1050000, max_output=128000, price_in=10.0, price_out=50.0, cutoff="2026-04-30", |
| modalities=("text", "image"), |
| bench={"gpqa": 96.0, "frontiermath": 97.6, "arcagi": 99.9, "osworld": 72.6, |
| "screenspot": 92.7, "terminalbench": 57.7, "deepswe": 74.1, |
| "frontiercode": 53.3, "exploitbench": 100.0, "srebench": 88.0, |
| "hallucination": 4.2, "misaligned": 3.4, "cu_safety": 2.4, |
| "agents_last_exam": 59.3}, |
| notes="Same context window as Sol at 2.5× the price. Adds asynchronous tool calling " |
| "and mid-turn steering, both requiring the Responses API. The ARC-AGI-3 figure " |
| "is reported under a stateful, expensive harness — not the bare model.", |
| source="OpenAI; The New Stack; DataCamp; ComputingForGeeks"), |
| ] |
|
|
| |
| |
| |
|
|
| _ANTHROPIC = [ |
| _m("claude-2", "Claude 2", "Anthropic", "Claude 2", "2023-07-11", |
| context=100000, price_in=8.00, price_out=24.0, |
| notes="100K context at a time GPT-4 shipped 8K — long context as the differentiator.", |
| source="Anthropic, Jul 2023"), |
| _m("claude-3-opus", "Claude 3 Opus", "Anthropic", "Claude 3", "2024-03-04", |
| context=200000, max_output=4096, price_in=15.0, price_out=75.0, |
| modalities=("text", "image"), bench={"mmlu": 86.8, "gpqa": 50.4}, |
| notes="First model to broadly trade blows with GPT-4 across benchmarks.", |
| source="Anthropic, Mar 2024"), |
| _m("claude-3.5-sonnet", "Claude 3.5 Sonnet", "Anthropic", "Claude 3.5", "2024-06-20", |
| context=200000, max_output=8192, price_in=3.00, price_out=15.0, |
| modalities=("text", "image"), bench={"mmlu": 88.7, "gpqa": 59.4, "swebench": 49.0}, |
| notes="Beat the larger, pricier Opus at 1/5th the cost — mid-size models took the " |
| "frontier for the first time.", |
| source="Anthropic, Jun 2024"), |
| _m("claude-3.7-sonnet", "Claude 3.7 Sonnet", "Anthropic", "Claude 3.7", "2025-02-24", |
| context=200000, max_output=64000, price_in=3.00, price_out=15.0, |
| modalities=("text", "image"), bench={"gpqa": 68.0, "swebench": 62.3}, |
| notes="First hybrid model — instant or extended thinking from one checkpoint.", |
| source="Anthropic, Feb 2025"), |
| _m("claude-opus-4", "Claude Opus 4", "Anthropic", "Claude 4", "2025-05-22", |
| context=200000, max_output=32000, price_in=15.0, price_out=75.0, |
| modalities=("text", "image"), bench={"gpqa": 79.6, "swebench": 72.5}, |
| notes="Aimed squarely at long-running autonomous coding.", |
| source="Anthropic, May 2025"), |
| _m("claude-sonnet-4.5", "Claude Sonnet 4.5", "Anthropic", "Claude 4.5", "2025-09-29", |
| context=200000, max_output=64000, price_in=3.00, price_out=15.0, |
| modalities=("text", "image"), bench={"gpqa": 83.4, "swebench": 77.2}, |
| source="Anthropic, Sep 2025"), |
| _m("claude-opus-4.5", "Claude Opus 4.5", "Anthropic", "Claude 4.5", "2025-11-24", |
| context=200000, max_output=64000, price_in=5.00, price_out=25.0, |
| modalities=("text", "image"), bench={"gpqa": 87.0, "swebench": 80.9}, |
| notes="Opus pricing cut by two-thirds versus Opus 4.", |
| source="Anthropic, Nov 2025"), |
| _m("claude-opus-4.8", "Claude Opus 4.8", "Anthropic", "Claude 4.8", "2026-05-28", |
| context=200000, price_in=5.00, price_out=25.0, modalities=("text", "image"), |
| source="Anthropic, May 2026"), |
| _m("claude-fable-5", "Claude Fable 5", "Anthropic", "Fable 5", "2026-06-09", |
| price_in=10.0, price_out=50.0, modalities=("text", "image"), |
| bench={"frontiercode": 53.5, "screenspot": 87.3, "deepswe": 69.9}, |
| notes="'Mythos-class' model released with safeguards; flagged requests are downgraded " |
| "to Opus rather than answered.", |
| source="Anthropic, Jun 2026"), |
| _m("claude-mythos-5", "Claude Mythos 5", "Anthropic", "Fable 5", "2026-06-09", |
| price_in=10.0, price_out=50.0, modalities=("text", "image"), |
| notes="Restricted-access build of Fable 5 with some safeguards lifted. Identical " |
| "underlying model and pricing.", |
| source="Anthropic, Jun 2026"), |
| _m("claude-opus-5", "Claude Opus 5", "Anthropic", "Opus 5", "2026-07-24", |
| context=1000000, max_output=128000, price_in=5.00, price_out=25.0, |
| modalities=("text", "image"), |
| bench={"gpqa": 93.7, "frontiermath": 73.2, "arcagi": 30.2, "osworld": 70.2, |
| "terminalbench": 52.3, "frontiercode": 53.4, "exploitbench": 70.0, |
| "agents_last_exam": 55.5}, |
| notes="1M context with five-level effort control. Fast mode doubles price for ~2.5× speed.", |
| source="Anthropic, Jul 2026"), |
| _m("claude-fable-5.1", "Claude Fable 5.1", "Anthropic", "Fable 5", "2026-09-01", |
| price_in=10.0, price_out=50.0, modalities=("text", "image"), |
| bench={"frontiermath": 87.8, "terminalbench": 55.8, "cu_safety": 9.5}, |
| notes="Straight to general availability. Cache-read rate cut 75% to $0.25/M.", |
| source="Anthropic, Sep 2026"), |
| ] |
|
|
| _GOOGLE = [ |
| _m("gemini-1.0-ultra", "Gemini 1.0 Ultra", "Google", "Gemini 1", "2023-12-06", |
| context=32768, modalities=("text", "image", "audio", "video"), bench={"mmlu": 90.0}, |
| notes="First model to claim above-human MMLU, though on a non-standard prompting setup.", |
| source="Google, Dec 2023"), |
| _m("gemini-1.5-pro", "Gemini 1.5 Pro", "Google", "Gemini 1.5", "2024-02-15", |
| context=1000000, price_in=1.25, price_out=5.00, |
| modalities=("text", "image", "audio", "video"), bench={"mmlu": 85.9, "gpqa": 46.2}, |
| notes="1M tokens — 8× any competitor at the time — via a mixture-of-experts design. " |
| "A 2M variant followed.", |
| source="Google, Feb 2024"), |
| _m("gemini-2.0-flash", "Gemini 2.0 Flash", "Google", "Gemini 2", "2024-12-11", |
| context=1048576, max_output=8192, price_in=0.10, price_out=0.40, |
| modalities=("text", "image", "audio", "video"), bench={"mmlu": 76.4, "gpqa": 62.1}, |
| notes="1M context at $0.10/M input — long context became nearly free.", |
| source="Google, Dec 2024"), |
| _m("gemini-2.5-pro", "Gemini 2.5 Pro", "Google", "Gemini 2.5", "2025-03-25", |
| context=1048576, max_output=65536, price_in=1.25, price_out=10.0, |
| modalities=("text", "image", "audio", "video"), |
| bench={"gpqa": 84.0, "aime": 92.0, "swebench": 63.8}, |
| notes="Reasoning on by default, with a 1M context window.", |
| source="Google, Mar 2025"), |
| _m("gemini-3-pro", "Gemini 3 Pro", "Google", "Gemini 3", "2025-11-18", |
| context=1048576, price_in=2.00, price_out=12.0, |
| modalities=("text", "image", "audio", "video"), bench={"gpqa": 91.9, "swebench": 76.2}, |
| source="Google, Nov 2025"), |
| _m("gemini-3.8-flash", "Gemini 3.8 Flash", "Google", "Gemini 3.8", None, |
| modalities=("text", "image", "audio", "video"), |
| bench={"gpqa": 95.3, "deepswe": 73.8, "terminalbench": 19.1}, |
| notes="Appears in OpenAI's GPT-6 Astra comparison charts. Release date and pricing " |
| "were not confirmed in the sources checked — plotted only where it has data.", |
| source="Cited in GPT-6 Astra benchmarks"), |
| ] |
|
|
| _OPEN = [ |
| _m("llama-1", "LLaMA 65B", "Meta", "Llama 1", "2023-02-24", |
| params_b=65, context=2048, open_weights=True, bench={"mmlu": 63.4}, |
| notes="Research-licensed, then leaked. Started the open-weight ecosystem in earnest.", |
| source="Meta, Feb 2023"), |
| _m("llama-2", "Llama 2 70B", "Meta", "Llama 2", "2023-07-18", |
| params_b=70, context=4096, open_weights=True, bench={"mmlu": 68.9}, |
| notes="Commercially usable licence — the point open weights became viable in products.", |
| source="Meta, Jul 2023"), |
| _m("llama-3", "Llama 3 70B", "Meta", "Llama 3", "2024-04-18", |
| params_b=70, context=8192, open_weights=True, bench={"mmlu": 82.0}, |
| source="Meta, Apr 2024"), |
| _m("llama-3.1-405b", "Llama 3.1 405B", "Meta", "Llama 3.1", "2024-07-23", |
| params_b=405, context=128000, open_weights=True, bench={"mmlu": 88.6, "gpqa": 51.1}, |
| notes="First open-weight model at genuine frontier scale.", |
| source="Meta, Jul 2024"), |
| _m("llama-4-maverick", "Llama 4 Maverick", "Meta", "Llama 4", "2025-04-05", |
| params_b=400, context=1000000, open_weights=True, |
| modalities=("text", "image"), bench={"mmlu": 85.5, "gpqa": 69.8}, |
| notes="Mixture-of-experts, 17B active parameters. The Scout variant advertised a " |
| "10M-token context window.", |
| source="Meta, Apr 2025"), |
| _m("deepseek-v3", "DeepSeek-V3", "DeepSeek", "DeepSeek V3", "2024-12-26", |
| params_b=671, context=128000, price_in=0.27, price_out=1.10, open_weights=True, |
| bench={"mmlu": 88.5, "gpqa": 59.1}, |
| notes="671B MoE, 37B active, trained for a reported ~$5.6M — a step change in the " |
| "assumed cost of a frontier run.", |
| source="DeepSeek, Dec 2024"), |
| _m("deepseek-r1", "DeepSeek-R1", "DeepSeek", "DeepSeek R1", "2025-01-20", |
| params_b=671, context=128000, price_in=0.55, price_out=2.19, open_weights=True, |
| bench={"mmlu": 90.8, "gpqa": 71.5, "aime": 79.8}, |
| notes="Open-weight reasoning model at roughly o1 level for ~3% of the price. Triggered " |
| "a ~$600B single-day drop in Nvidia's market capitalisation.", |
| source="DeepSeek, Jan 2025"), |
| ] |
|
|
| MODELS: list[dict[str, Any]] = _OPENAI + _ANTHROPIC + _GOOGLE + _OPEN |
|
|
|
|
| |
| |
| |
|
|
| def build_frame() -> pd.DataFrame: |
| """Flatten ``MODELS`` into a DataFrame, one benchmark per column.""" |
| rows = [] |
| for m in MODELS: |
| row = {k: v for k, v in m.items() if k != "bench"} |
| row["released_dt"] = pd.to_datetime(m["released"]) if m["released"] else pd.NaT |
| row["year"] = row["released_dt"].year if pd.notna(row["released_dt"]) else None |
| row["modalities_str"] = ", ".join(m["modalities"]) |
| |
| if m["price_in"] is not None and m["price_out"] is not None: |
| row["price_blended"] = (10 * m["price_in"] + m["price_out"]) / 11 |
| else: |
| row["price_blended"] = None |
| for key in BENCHMARKS: |
| row[key] = m["bench"].get(key) |
| row["n_benchmarks"] = len(m["bench"]) |
| rows.append(row) |
| df = pd.DataFrame(rows) |
| return df.sort_values("released_dt", na_position="last").reset_index(drop=True) |
|
|
|
|
| DF = build_frame() |
|
|
| MODEL_BY_ID = {m["id"]: m for m in MODELS} |
| MODEL_CHOICES = [(m["name"], m["id"]) for m in MODELS] |
| OPENAI_CHOICES = [(m["name"], m["id"]) for m in MODELS if m["vendor"] == "OpenAI"] |
| VENDORS = sorted({m["vendor"] for m in MODELS}) |
|
|
|
|
| def era_for(date: str | None) -> str: |
| """Which era a release date falls in.""" |
| if not date: |
| return "Unknown" |
| d = _dt.date.fromisoformat(date) |
| for era in ERAS: |
| if _dt.date.fromisoformat(era["start"]) <= d <= _dt.date.fromisoformat(era["end"]): |
| return era["name"] |
| return "Unknown" |
|
|
|
|
| DF["era"] = DF["released"].apply(era_for) |
|
|
|
|
| def coverage_note() -> str: |
| """How complete the dataset actually is — shown in the app rather than hidden.""" |
| total = len(DF) |
| have = lambda c: int(DF[c].notna().sum()) |
| return ( |
| f"{total} models · context known for {have('context')} · " |
| f"pricing known for {have('price_in')} · parameters disclosed for {have('params_b')} · " |
| f"{int(DF['n_benchmarks'].gt(0).sum())} carry at least one benchmark" |
| ) |
|
|
