| { |
| "reference": "synthetic_ai_reviewed", |
| "split": "test", |
| "dataset": { |
| "dataset_version": "0.1.0", |
| "seed": 42, |
| "protocol": "compositional_groups_v1; fixed before experiments", |
| "counts": { |
| "train": 280, |
| "validation": 60, |
| "test": 60 |
| }, |
| "sha256": { |
| "dataset.jsonl": "054c0a19f2fb886c32018c1d1e8bf9b3c2ee6d32ad8f9acda58619ff5f94a9eb", |
| "splits/train.jsonl": "148466e584ac72e39e5307b6a7ab09dd7a7f2f852c51493eeb1f3bce2b13e4f0", |
| "splits/validation.jsonl": "fab146a33a5ed52e73407b687b2344b516fe1fb099b96b5b554aff00daa86306", |
| "splits/test.jsonl": "67c5244e4633f226f9d5453bac22a39e21d537d59e9b61d0ced823f97e96f176", |
| "source/development.tsv": "05a302da15ff7f13795b1b9df4bfb20928c05820b0f7c83fffeaac7687f791b8", |
| "source/test.tsv": "10d852d417b4b9b80538e807255716f566dffa0fed00dfec4f9b35e6ead72432", |
| "source/prompts.json": "56788c27df679e38ef5930a1345d17b029ca4c7ea3a5a66aa589ccd9727eabe1" |
| } |
| }, |
| "environment": { |
| "python": "3.11.14", |
| "platform": "Linux-7.1.8+deb13-amd64-x86_64-with-glibc2.41", |
| "packages": { |
| "torch": "2.6.0", |
| "transformers": "4.57.1", |
| "numpy": "2.3.4", |
| "scikit-learn": "1.7.2", |
| "pydantic": "2.12.4" |
| }, |
| "source_sha256": { |
| "__init__.py": "07f1e581fc9fb8256b565d0f2f968af61622da5161e441c0487e0d122d71b80e", |
| "__main__.py": "935a1c1166b0c1ea35a82256345000bf2c73ded718d77773bc27a71ecce28f7d", |
| "cli.py": "f9cc4684ec995264cf1223c79a4be5df33d6c5221d797e544f813244db90bd8b", |
| "data.py": "533b9618df7d41c1402a9fa6b02e6cd9f1ac0fdaad549d93cce98784070d52de", |
| "evaluate.py": "386dc2f86c890a3484b714e8311ce95cb771adae8a5cc91ecb266cba4e08712e", |
| "experiment.py": "683db32333171a30d6747980b705791ac17db70f6a90a8af82b6da5186b22c0e", |
| "metrics.py": "a1374b9fa54c1ae278a5b81c52317e06b6b1f97726a726afe5cfc7d924c232e3", |
| "model.py": "697d177b7634b3dbdb29731457774f80f36fd55ac23f2576d8af1a0565ce29b2", |
| "rules.py": "0c2a0dcbbac8220abfd38bc93830a699b52366e781f5b4b977329e5473d046bf", |
| "schema.py": "4afcf1dc0cf75a8eb1392a7cbbf69c6727fb566c44da94ed4e68733024c144d4", |
| "text.py": "67132ac951f1b81972229e75ca311ef8dab8dc810992f8059717528ed79bc43e" |
| } |
| }, |
| "rules_version": "0.1.0", |
| "model_manifest_sha256": "8aec42f49e157ee64bd77fb82656b848d856cd29c0ec0b30fba41d9884f92720", |
| "frozen_sha256": "07bf28b9c3349024949f89af59c3a8e3e0a126527d3de57b9c2b69064228e8a2", |
| "thresholds": { |
| "heuristic": [ |
| 0.5, |
| 0.5, |
| 0.5, |
| 0.5, |
| 0.5 |
| ], |
| "frozen_logistic": [ |
| 0.5, |
| 0.5, |
| 0.5, |
| 0.5, |
| 0.5 |
| ], |
| "bert_finetuned": [ |
| 0.5, |
| 0.5, |
| 0.5, |
| 0.5, |
| 0.5 |
| ] |
| }, |
| "methods": { |
| "heuristic": { |
| "n": 60, |
| "macro_f1": 0.6791489361702128, |
| "micro_f1": 0.7633587786259542, |
| "exact_match": 0.3333333333333333, |
| "per_label": { |
| "action": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "support": 31, |
| "tp": 0, |
| "fp": 0, |
| "fn": 31, |
| "tn": 29 |
| }, |
| "context": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 36, |
| "tp": 24, |
| "fp": 0, |
| "fn": 12, |
| "tn": 24 |
| }, |
| "method": { |
| "precision": 1.0, |
| "recall": 0.42424242424242425, |
| "f1": 0.5957446808510638, |
| "support": 33, |
| "tp": 14, |
| "fp": 0, |
| "fn": 19, |
| "tn": 27 |
| }, |
| "result": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 31, |
| "tp": 31, |
| "fp": 0, |
| "fn": 0, |
| "tn": 29 |
| }, |
| "measurement": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 31, |
| "tp": 31, |
| "fp": 0, |
| "fn": 0, |
| "tn": 29 |
| } |
| }, |
| "zero_division": 0 |
| }, |
| "frozen_logistic": { |
| "n": 60, |
| "macro_f1": 0.9888253968253968, |
| "micro_f1": 0.9878048780487805, |
| "exact_match": 0.9333333333333333, |
| "per_label": { |
| "action": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 31, |
| "tp": 31, |
| "fp": 0, |
| "fn": 0, |
| "tn": 29 |
| }, |
| "context": { |
| "precision": 0.9230769230769231, |
| "recall": 1.0, |
| "f1": 0.96, |
| "support": 36, |
| "tp": 36, |
| "fp": 3, |
| "fn": 0, |
| "tn": 21 |
| }, |
| "method": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 33, |
| "tp": 33, |
| "fp": 0, |
| "fn": 0, |
| "tn": 27 |
| }, |
| "result": { |
| "precision": 0.96875, |
| "recall": 1.0, |
| "f1": 0.9841269841269841, |
| "support": 31, |
| "tp": 31, |
| "fp": 1, |
| "fn": 0, |
| "tn": 28 |
| }, |
| "measurement": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 31, |
| "tp": 31, |
| "fp": 0, |
| "fn": 0, |
| "tn": 29 |
| } |
| }, |
| "zero_division": 0 |
| }, |
| "bert_finetuned": { |
| "n": 60, |
| "macro_f1": 0.9911005961711339, |
| "micro_f1": 0.9908256880733946, |
| "exact_match": 0.95, |
| "per_label": { |
| "action": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 31, |
| "tp": 31, |
| "fp": 0, |
| "fn": 0, |
| "tn": 29 |
| }, |
| "context": { |
| "precision": 0.972972972972973, |
| "recall": 1.0, |
| "f1": 0.9863013698630136, |
| "support": 36, |
| "tp": 36, |
| "fp": 1, |
| "fn": 0, |
| "tn": 23 |
| }, |
| "method": { |
| "precision": 0.9705882352941176, |
| "recall": 1.0, |
| "f1": 0.9850746268656716, |
| "support": 33, |
| "tp": 33, |
| "fp": 1, |
| "fn": 0, |
| "tn": 26 |
| }, |
| "result": { |
| "precision": 0.96875, |
| "recall": 1.0, |
| "f1": 0.9841269841269841, |
| "support": 31, |
| "tp": 31, |
| "fp": 1, |
| "fn": 0, |
| "tn": 28 |
| }, |
| "measurement": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 31, |
| "tp": 31, |
| "fp": 0, |
| "fn": 0, |
| "tn": 29 |
| } |
| }, |
| "zero_division": 0 |
| } |
| } |
| } |
|
|
