Retrain on section_v5b (leak-filtered) with the v4 recipe

README.md

@@ -0,0 +1,56 @@
+---
+library_name: bibr
+base_model: sentence-transformers/all-MiniLM-L6-v2
+language: en
+tags:
+  - text-classification
+  - scholarly-documents
+---
+
+# bibr section classifier
+
+Two-head classifier that bibr uses to type section headings it cannot resolve
+with its alias lookup. It predicts one of 16 canonical section types and
+whether the heading is top level. Encoder: all-MiniLM-L6-v2 (22M parameters),
+mean pooling, input template v3 (heading, position bucket, neighbouring
+headings, start of the body).
+
+This revision retrains the same architecture and template on a larger,
+re-labelled corpus. It is a drop-in replacement for the September 2026
+release: same files, same label order, same runtime cost.
+
+## Training data
+
+- `section_v5b_leakfiltered`: section headings from PMC Open Access JATS
+  papers (English), paper-grouped train/val/test split. Three papers removed
+  by a later leak re-check are excluded.
+- Train subsample of at most 3,000 rows per class: 45,979 rows from 6,946 papers.
+  Epoch chosen on the v5b validation split only.
+- No bibr-validation gold, blind evaluation, or held-out benchmark paper is in
+  the training or validation data (matched by DOI, PMCID and title).
+
+## Evaluation
+
+Canonical-type macro-F1 / accuracy (model-routed rows: macro-F1). "Model-routed" is the subset bibr
+actually sends to this model (headings the alias lookup misses), scored on
+bibr's section gold.
+
+| set | previous release | this revision |
+|---|---|---|
+| v5b test (English JATS, 9,507 headings) | 0.589 / 0.398 | 0.966 / 0.946 |
+| gold, English headings (9,501) | 0.656 / 0.572 | 0.798 / 0.784 |
+| gold, non-English headings (1,524) | 0.213 / 0.295 | 0.265 / 0.344 |
+| gold, model-routed English (3,472) | 0.419 | 0.676 |
+| gold, model-routed non-English (1,240) | 0.168 | 0.217 |
+
+Part of the gain on gold comes from shared labelling conventions: the
+training data and the gold both label subsections by their parent type and
+structured-abstract parts as abstract.
+
+Non-English headings are zero-shot: the training data is English only.
+
+## Files
+
+`model.safetensors`, `tokenizer.json`, `tokenizer_config.json`,
+`label_classes.json`, `inference_config.json` (template version 3), and an
+ONNX Runtime bundle in `onnx/` used by bibr's core install.

classification_report.json

@@ -1,111 +1,111 @@
 {
   "abstract": {
-    "f1-score": 1.0,
-    "precision": 1.0,
-    "recall": 1.0,
-    "support": 388.0
+    "f1-score": 0.9793224813022438,
+    "precision": 0.9780316344463972,
+    "recall": 0.9806167400881057,
+    "support": 1135.0
   },
-  "accuracy": 0.9930157593123209,
+  "accuracy": 0.9469550038789759,
   "acknowledgment": {
-    "f1-score": 0.9958847736625515,
-    "precision": 1.0,
-    "recall": 0.9918032786885246,
-    "support": 244.0
+    "f1-score": 0.9980732177263969,
+    "precision": 0.9961538461538462,
+    "recall": 1.0,
+    "support": 259.0
   },
   "author_contributions": {
-    "f1-score": 0.9710144927536232,
-    "precision": 0.9710144927536232,
-    "recall": 0.9710144927536232,
-    "support": 69.0
+    "f1-score": 0.996268656716418,
+    "precision": 0.9925650557620818,
+    "recall": 1.0,
+    "support": 267.0
   },
   "coi": {
-    "f1-score": 0.971830985915493,
-    "precision": 0.9857142857142858,
-    "recall": 0.9583333333333334,
-    "support": 72.0
+    "f1-score": 1.0,
+    "precision": 1.0,
+    "recall": 1.0,
+    "support": 306.0
   },
   "discussion": {
-    "f1-score": 0.9896519285042333,
-    "precision": 0.9868667917448405,
-    "recall": 0.9924528301886792,
-    "support": 530.0
+    "f1-score": 0.9074598677998111,
+    "precision": 0.8720508166969148,
+    "recall": 0.9458661417322834,
+    "support": 1016.0
   },
   "endnote": {
-    "f1-score": 0.9929446713702191,
-    "precision": 0.9896373056994818,
-    "recall": 0.996274217585693,
-    "support": 1342.0
+    "f1-score": 0.9792694965449161,
+    "precision": 0.9841269841269841,
+    "recall": 0.9744597249508841,
+    "support": 509.0
   },
   "ethics": {
-    "f1-score": 0.9808917197452229,
-    "precision": 1.0,
-    "recall": 0.9625,
-    "support": 160.0
+    "f1-score": 0.9879518072289156,
+    "precision": 0.9787798408488063,
+    "recall": 0.9972972972972973,
+    "support": 370.0
   },
   "funding": {
-    "f1-score": 0.9964664310954063,
-    "precision": 0.9964664310954063,
-    "recall": 0.9964664310954063,
-    "support": 283.0
+    "f1-score": 0.9963369963369964,
+    "precision": 0.9927007299270073,
+    "recall": 1.0,
+    "support": 136.0
   },
   "intro": {
-    "f1-score": 0.9984917043740573,
-    "precision": 0.9969879518072289,
-    "recall": 1.0,
-    "support": 331.0
+    "f1-score": 0.9057971014492754,
+    "precision": 0.8756567425569177,
+    "recall": 0.9380863039399625,
+    "support": 533.0
   },
   "keywords": {
     "f1-score": 1.0,
     "precision": 1.0,
     "recall": 1.0,
-    "support": 286.0
+    "support": 299.0
   },
   "macro avg": {
-    "f1-score": 0.9896559151772297,
-    "precision": 0.990745651057533,
-    "recall": 0.9886625840013419,
-    "support": 5584.0
+    "f1-score": 0.9660878230241505,
+    "precision": 0.9610242032951976,
+    "recall": 0.9717595906879481,
+    "support": 10312.0
   },
   "method": {
-    "f1-score": 0.9873417721518988,
-    "precision": 0.9811320754716981,
-    "recall": 0.9936305732484076,
-    "support": 157.0
+    "f1-score": 0.9330569948186529,
+    "precision": 0.9627887082976904,
+    "recall": 0.9051065540812223,
+    "support": 2487.0
   },
   "open_data": {
-    "f1-score": 0.9726027397260274,
-    "precision": 0.9726027397260274,
-    "recall": 0.9726027397260274,
-    "support": 73.0
+    "f1-score": 0.9982300884955753,
+    "precision": 0.9964664310954063,
+    "recall": 1.0,
+    "support": 282.0
   },
   "references": {
-    "f1-score": 1.0,
-    "precision": 1.0,
+    "f1-score": 0.9987357774968394,
+    "precision": 0.9974747474747475,
     "recall": 1.0,
-    "support": 345.0
+    "support": 395.0
   },
   "results": {
-    "f1-score": 0.9895470383275261,
-    "precision": 0.9793103448275862,
-    "recall": 1.0,
-    "support": 142.0
+    "f1-score": 0.9197598253275109,
+    "precision": 0.9157608695652174,
+    "recall": 0.9237938596491229,
+    "support": 1824.0
   },
   "title": {
-    "f1-score": 0.998661311914324,
-    "precision": 0.9973262032085561,
+    "f1-score": 0.9655172413793104,
+    "precision": 0.9333333333333333,
     "recall": 1.0,
-    "support": 373.0
+    "support": 84.0
   },
   "unknown": {
-    "f1-score": 0.9891650732950924,
-    "precision": 0.9948717948717949,
-    "recall": 0.9835234474017744,
-    "support": 789.0
+    "f1-score": 0.8916256157635468,
+    "precision": 0.900497512437811,
+    "recall": 0.8829268292682927,
+    "support": 410.0
   },
   "weighted avg": {
-    "f1-score": 0.9930047754292499,
-    "precision": 0.9930450267296388,
-    "recall": 0.9930157593123209,
-    "support": 5584.0
+    "f1-score": 0.9470639159236539,
+    "precision": 0.9480511203001257,
+    "recall": 0.9469550038789759,
+    "support": 10312.0
   }
-}
+}

gate.json

@@ -1,46 +1,8 @@
 {
-  "checkpoint": "/tmp/bibr-training-phasec-830a047/checkpoints/section_v4_release",
-  "config": "configs/section_classifier_v4.yaml",
-  "corpus": "/tmp/bibr-training-phasec-830a047/section_v4",
-  "corpus_papers": 2482,
-  "corpus_rows": 36517,
-  "delta": 0.09518694180243625,
-  "label_classes": [
-    "abstract",
-    "acknowledgment",
-    "author_contributions",
-    "coi",
-    "discussion",
-    "endnote",
-    "ethics",
-    "funding",
-    "intro",
-    "keywords",
-    "method",
-    "open_data",
-    "references",
-    "results",
-    "title",
-    "unknown"
-  ],
   "metric": "test_type_macro_f1",
-  "observed": 0.9896559151772297,
-  "pass": true,
-  "reason": "pass",
-  "split_summary": {
-    "test": {
-      "papers": 373,
-      "rows": 5584
-    },
-    "train": {
-      "papers": 1860,
-      "rows": 27294
-    },
-    "val": {
-      "papers": 249,
-      "rows": 3639
-    }
-  },
-  "threshold": 0.8944689733747935,
-  "v3_reference_type_macro_f1": 0.8944689733747935
-}
+  "split": "v5b test, v4-candidate papers removed",
+  "observed": 0.9656228154012606,
+  "previous_release_same_rows": 0.5887,
+  "gold_english_macro_f1": 0.7977,
+  "previous_release_gold_english_macro_f1": 0.656
+}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8827d8ead8603bdff650774f07edbb6d77e56fb8c2486de9fdc59d6ce5b1aea3
-size 90891540
+oid sha256:1c155834e88f6e7e9d3528225079ef57bf32b5108695e7e9a288cd745e0871de
+size 90891508

onnx/bibr_onnx.json

@@ -64,16 +64,16 @@
   "pad_token_id": 0,
   "schema_version": 1,
   "source": {
-    "exported_at": "2026-09-02T20:32:41+00:00",
-    "numpy": "2.3.5",
-    "onnx": "1.22.0",
+    "exported_at": "2026-10-13T21:41:08+00:00",
+    "numpy": "2.5.1",
+    "onnx": "1.23.0",
     "onnxruntime": "1.27.0",
-    "platform": "Linux-7.2.2-1-cachyos-x86_64-with-glibc2.44",
+    "platform": "Linux-7.2.9-2-cachyos-x86_64-with-glibc2.44",
     "python": "3.12.14",
     "repo_id": "scienceverse/bibr-section-classifier",
-    "revision": "969112772a8940d46c5d6b19feee45eeebfd26f9",
-    "torch": "2.11.0+cu130",
-    "transformers": "5.14.1"
+    "revision": null,
+    "torch": "2.13.0+cpu",
+    "transformers": "5.13.1"
   },
   "template_version": 3,
   "tokenizer_file": "tokenizer.json"

onnx/model.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:474de26402d09ddf8db15308b9ebe2e6f8251856d895fccdbf6d18ec5e0ea707
+oid sha256:737950aed5a4b626fe336f30dfc9d8aa7bdaf1f7a6fb76bce87a1e0575f15303
 size 90427110

test_metrics.json

@@ -1,27 +1,64 @@
 {
-  "checkpoint": "/tmp/bibr-training-phasec-830a047/checkpoints/section_v4_release",
-  "config": "configs/section_classifier_v4.yaml",
-  "corpus": "/tmp/bibr-training-phasec-830a047/section_v4",
-  "corpus_papers": 2482,
-  "corpus_rows": 36517,
-  "n_test": 5584,
-  "n_test_papers": 373,
+  "best_epoch": 5,
+  "best_val_acc": 0.9489,
+  "best_val_top_acc": 0.9804,
+  "best_val_type_macro_f1": 0.966,
+  "checkpoint": "this repository revision",
+  "corpus": "section_v5b_leakfiltered minus 3 T1-dropped papers (PMC12679984, PMC12751990, PMC12753070; 20 train rows)",
+  "corpus_papers": 7711,
+  "corpus_rows": 65711,
+  "encoder": "sentence-transformers/all-MiniLM-L6-v2@1110a243fdf4706b3f48f1d95db1a4f5529b4d41",
+  "gold_score_only": {
+    "gold, English headings": {
+      "acc": 0.7836,
+      "macro_f1": 0.7977,
+      "n": 9501
+    },
+    "gold, all": {
+      "acc": 0.7229,
+      "macro_f1": 0.7642,
+      "n": 11025
+    },
+    "gold, non-English headings": {
+      "acc": 0.3445,
+      "macro_f1": 0.2651,
+      "n": 1524
+    }
+  },
+  "n_test": 10312,
+  "n_test_papers": 396,
+  "n_train": 45979,
+  "note": "gold = bibr section-heading gold, scored only (never used for training, epoch or model selection); tier2 = gold rows bibr routes to this model (alias-lookup misses)",
+  "recipe": "eg2_ft ft.py v4 head (mean pool, CE sqrt class weights + 0.5 BCE top-level, AdamW lr 3e-5, cosine, batch 32, max_len 256, bf16, <=6 epochs, patience 2)",
   "split_summary": {
     "test": {
-      "papers": 373,
-      "rows": 5584
+      "papers": 396,
+      "rows": 10312
     },
     "train": {
-      "papers": 1860,
-      "rows": 27294
+      "papers": 6946,
+      "rows": 45979
     },
     "val": {
-      "papers": 249,
-      "rows": 3639
+      "papers": 369,
+      "rows": 9420
     }
   },
   "template_version": 3,
-  "test_top_level_f1": 0.988171459576777,
-  "test_type_macro_f1": 0.9896559151772297,
-  "test_type_micro_f1": 0.9930157593123209
-}
+  "test_top_level_acc": 0.9795384018619084,
+  "test_top_level_f1": 0.9790653834705824,
+  "test_type_macro_f1": 0.9660878230241505,
+  "test_type_micro_f1": 0.9469550038789759,
+  "test_v4_overlap_removed": {
+    "n": 9507,
+    "papers": 368,
+    "top_level_acc": 0.9793836120753129,
+    "top_level_f1": 0.9789969995713673,
+    "type_macro_f1": 0.9656228154012606,
+    "type_micro_f1": 0.945934574524035
+  },
+  "tier2_model_routed_macro_f1_score_only": {
+    "English headings": 0.6761,
+    "non-English headings": 0.2167
+  }
+}