| { |
| "format_version": 1, |
| "status": "development candidate; final topic holdout pending", |
| "selected_name": "fixed_0", |
| "source_comparison": { |
| "path": "mixtures", |
| "sha256": "4e6444d780cb4321d6291e97d79587827e581cb923f4942438948a4f12c275b2" |
| }, |
| "selection": { |
| "kind": "fixed", |
| "weights": [ |
| 0.11395703162765773, |
| 0.8860429683723422 |
| ], |
| "penalty": 0.0 |
| }, |
| "components": { |
| "qwen_chat": { |
| "kind": "expert", |
| "model": "qwen_chat", |
| "calibrator": "components/qwen_chat_calibrator.joblib", |
| "backbone": [ |
| "mlx-community/Qwen3.5-9B-4bit", |
| "8b2b98c00a6b4d291155e4890773ca8f769aee53" |
| ], |
| "engine": "mlx", |
| "prompt_format": "chat_nonthinking", |
| "max_input_tokens": 256 |
| }, |
| "mistral_human_lora_e2": { |
| "kind": "expert", |
| "model": "mistral_human_lora_e2", |
| "calibrator": "components/mistral_human_lora_e2_calibrator.joblib", |
| "backbone": [ |
| "mlx-community/Mistral-7B-v0.3-4bit", |
| "5e8b40756eaa1cb5131715937c77107b4421c3a5" |
| ], |
| "engine": "mlx", |
| "prompt_format": "plain", |
| "adapter": "components/mistral_human_lora_e2_adapter", |
| "max_input_tokens": 256 |
| } |
| }, |
| "artifact_hashes": { |
| "components/qwen_chat_calibrator.joblib": "d52baf2c6e6df9dc02fa92ac9d4dc7fd57d1f02a69650c8fe61fe0b0cd3a312a", |
| "components/mistral_human_lora_e2_calibrator.joblib": "be4db75501ab20eb8167f84d6017b103f57064b972b106d3953903d67bd98a1a", |
| "components/mistral_human_lora_e2_adapter/training.json": "aa71dd6b0099cd810ba0f2fada18c1f07ca801cc7a25def5a8008bbebe63024d", |
| "components/mistral_human_lora_e2_adapter/adapters.safetensors": "31ae766a2a371a1b8c503a00c74c09f3337664a6812fda75548ea5d464b92c89", |
| "components/mistral_human_lora_e2_adapter/adapter_config.json": "891759446a24653c9c59e6cc5713c2ce5bd5dbedda40eef7ebdc966b0a7b1457" |
| }, |
| "dataset": { |
| "path": "/Users/David/Documents/Codex/2026-10-16/set/outputs/PoSSUM/data/improvement-20261018", |
| "manifest_sha256": "44956f833721c180faa612ad197d73841702de2cf091276b6571259d2c557d17" |
| }, |
| "encoder_revisions": { |
| "google-bert/bert-base-uncased": "86b5e0934494bd15c9632b12f734a8a67f723594", |
| "BAAI/bge-small-en-v1.5": "5c38ec7c405ec4b44b94cc5a9bb96e735b38267a" |
| }, |
| "decision_rules": { |
| "human_calibration_prior_right": 0.10294117647058823, |
| "gate_balanced_threshold": 0.21000000000000002 |
| }, |
| "validation": { |
| "all": { |
| "n": 806, |
| "macro_f1": 0.9673660188457008, |
| "balanced_accuracy": 0.9673660188457008, |
| "log_loss": 0.09870895487969024, |
| "brier": 0.023941461804480903, |
| "ece": 0.039024849784404986 |
| }, |
| "sources": { |
| "GPT-4 Political Ideologies": { |
| "n": 398, |
| "macro_f1": 0.9899474641341686, |
| "balanced_accuracy": 0.9898989898989898, |
| "log_loss": 0.09896858466931752, |
| "brier": 0.019562568071896998, |
| "ece": 0.07006204467611839 |
| }, |
| "MITweet": { |
| "n": 408, |
| "macro_f1": 0.8853646353646354, |
| "balanced_accuracy": 0.9016393442622951, |
| "log_loss": 0.0984556885652989, |
| "brier": 0.028213029710285796, |
| "ece": 0.02284640028026002 |
| } |
| }, |
| "topics": { |
| "Abortion": { |
| "n": 210, |
| "macro_f1": 0.8442758542011295, |
| "balanced_accuracy": 0.8508486398511974, |
| "log_loss": 0.14025565586211491, |
| "brier": 0.040778661063792535, |
| "ece": 0.030664147290567313 |
| }, |
| "Black Lives Matter": { |
| "n": 6, |
| "macro_f1": 1.0, |
| "balanced_accuracy": 1.0, |
| "log_loss": 0.024711698764642322, |
| "brier": 0.001974301805566919, |
| "ece": 0.02365485248607744 |
| }, |
| "CHIPS and Science Act": { |
| "n": 2, |
| "single_class": 0, |
| "accuracy": 1.0 |
| }, |
| "Capitol Hill Riot": { |
| "n": 3, |
| "macro_f1": 1.0, |
| "balanced_accuracy": 1.0, |
| "log_loss": 0.07850323308597848, |
| "brier": 0.012923763563718353, |
| "ece": 0.07105263113302518 |
| }, |
| "Covid-19": { |
| "n": 18, |
| "single_class": 0, |
| "accuracy": 1.0 |
| }, |
| "Democracy": { |
| "n": 4, |
| "macro_f1": 1.0, |
| "balanced_accuracy": 1.0, |
| "log_loss": 0.1420151974035414, |
| "brier": 0.03204089683091983, |
| "ece": 0.12214219819988517 |
| }, |
| "Inflation Reduction Act": { |
| "n": 22, |
| "macro_f1": 1.0, |
| "balanced_accuracy": 1.0, |
| "log_loss": 0.037612502024865886, |
| "brier": 0.004689298203689577, |
| "ece": 0.03493301819005371 |
| }, |
| "Mexico\u2013US Border": { |
| "n": 8, |
| "macro_f1": 1.0, |
| "balanced_accuracy": 1.0, |
| "log_loss": 0.052773310749411356, |
| "brier": 0.004125107434660411, |
| "ece": 0.050568977546797456 |
| }, |
| "Russo-Ukrainian War": { |
| "n": 3, |
| "macro_f1": 0.6666666666666666, |
| "balanced_accuracy": 0.75, |
| "log_loss": 0.45891460969118825, |
| "brier": 0.16543452870543476, |
| "ece": 0.31861086242416914 |
| }, |
| "Women\u2019s Right": { |
| "n": 132, |
| "macro_f1": 0.8253968253968254, |
| "balanced_accuracy": 0.8881889763779528, |
| "log_loss": 0.052399952414329605, |
| "brier": 0.01609346186608814, |
| "ece": 0.034008611256927765 |
| }, |
| "synthetic_issue_4": { |
| "n": 398, |
| "macro_f1": 0.9899474641341686, |
| "balanced_accuracy": 0.9898989898989898, |
| "log_loss": 0.09896858466931752, |
| "brier": 0.019562568071896998, |
| "ece": 0.07006204467611839 |
| } |
| } |
| }, |
| "preprocessing": "remove BOM; collapse whitespace", |
| "limitations": [ |
| "Development selection is not a final test.", |
| "Human labels map multiple ideology facets to one binary axis.", |
| "Human validation is strongly left-skewed.", |
| "Author-disjointness is unavailable." |
| ] |
| } |
