DBBE fetch script, refreshed artifact index, de-anonymized metadata

.pytest_cache/.gitignore

@@ -0,0 +1,2 @@
+# Created by pytest automatically.
+*

.pytest_cache/CACHEDIR.TAG

@@ -0,0 +1,4 @@
+Signature: 8a477f597d28d172789f06886806bc55
+# This file is a cache directory tag created by pytest.
+# For information about cache directory tags, see:
+#	https://bford.info/cachedir/spec.html

.pytest_cache/README.md

@@ -0,0 +1,8 @@
+# pytest cache directory #
+
+This directory contains data from the pytest's cache plugin,
+which provides the `--lf` and `--ff` options, as well as the `cache` fixture.
+
+**Do not** commit this to version control.
+
+See [the docs](https://docs.pytest.org/en/stable/how-to/cache.html) for more information.

.pytest_cache/v/cache/nodeids

@@ -0,0 +1,98 @@
+[
+  "tests/test_data.py::test_conllu_roundtrip",
+  "tests/test_data.py::test_packing_invariants",
+  "tests/test_data.py::test_span_alignment",
+  "tests/test_data_meta.py::test_multitier_loader_yields_region_century_and_survives_windowing",
+  "tests/test_data_meta.py::test_shardset_defaults_unk_when_columns_absent",
+  "tests/test_data_meta.py::test_shardset_reads_region_century_columns",
+  "tests/test_edits.py::test_grave_fold",
+  "tests/test_edits.py::test_inapplicable",
+  "tests/test_edits.py::test_roundtrip[\\u03a3\\u03c9\\u03ba\\u03c1\\u03ac\\u03c4\\u03b7\\u03c2-\\u03a3\\u03c9\\u03ba\\u03c1\\u03ac\\u03c4\\u03b7\\u03c2]",
+  "tests/test_edits.py::test_roundtrip[\\u03b4\\u02bc-\\u03b4\\u03ad]",
+  "tests/test_edits.py::test_roundtrip[\\u03b8\\u03b5\\u03ce-\\u03b8\\u03b5\\u03cc\\u03c2]",
+  "tests/test_edits.py::test_roundtrip[\\u03ba\\u1f00\\u03b3\\u03ce-\\u03ba\\u1f00\\u03b3\\u03ce]",
+  "tests/test_edits.py::test_roundtrip[\\u03bc\\u03b1\\u03b8\\u03b7\\u03c4\\u1f74\\u03c2-\\u03bc\\u03b1\\u03b8\\u03b7\\u03c4\\u03ae\\u03c2]",
+  "tests/test_edits.py::test_roundtrip[\\u03bf\\u1f50\\u03ba-\\u03bf\\u1f50]",
+  "tests/test_edits.py::test_roundtrip[\\u03c4\\u03b9\\u03bb\\u03bb\\u03cc\\u03bc\\u03b5\\u03bd\\u03bf\\u03c2-\\u03c4\\u03af\\u03bb\\u03bb\\u03c9]",
+  "tests/test_edits.py::test_roundtrip[\\u1f00\\u03bd\\u03b8\\u03c1\\u03ce\\u03c0\\u03bf\\u03c5-\\u1f04\\u03bd\\u03b8\\u03c1\\u03c9\\u03c0\\u03bf\\u03c2]",
+  "tests/test_edits.py::test_roundtrip[\\u1f20\\u03bd\\u03b5\\u03af\\u03c7\\u03b5\\u03c4\\u03bf-\\u1f00\\u03bd\\u03ad\\u03c7\\u03c9]",
+  "tests/test_hf_release_smoke.py::test_decode_boundaries_keeps_punctuation",
+  "tests/test_hf_release_smoke.py::test_decode_diacritics_keeps_punctuation_and_boundary",
+  "tests/test_hf_release_smoke.py::test_decode_restoration_fills_masked_gap_and_keeps_punctuation",
+  "tests/test_hf_release_smoke.py::test_decode_restoration_handles_fully_masked_dia_and_boundary",
+  "tests/test_hf_release_smoke.py::test_processor_round_trips_punctuation_with_no_masking",
+  "tests/test_hf_release_smoke.py::test_restore_elastic_sweeps_widths_and_ranks_by_confidence",
+  "tests/test_hf_release_smoke.py::test_tiny_model_forward_pass_shapes",
+  "tests/test_insc_eval_whole.py::test_eval_span_whole_never_overlaps_real_lacuna",
+  "tests/test_insc_eval_whole.py::test_eval_span_whole_runs_end_to_end",
+  "tests/test_insc_eval_whole.py::test_eval_span_whole_skips_records_with_no_valid_position",
+  "tests/test_insc_lacuna.py::test_iphi_cap_plane_captured_not_discarded",
+  "tests/test_insc_lacuna.py::test_iphi_known_length_dash_run_marked_real_lacuna",
+  "tests/test_insc_lacuna.py::test_iphi_no_lacuna_when_no_dash",
+  "tests/test_insc_lacuna.py::test_papyri_dash_run_exact_length",
+  "tests/test_insc_lacuna.py::test_papyri_ellipsis_gets_random_stand_in_width_in_range",
+  "tests/test_insc_lacuna.py::test_papyri_mixed_dash_and_ellipsis_in_one_text",
+  "tests/test_insc_smoke.py::test_restoration_forward_pass_with_masked_gap",
+  "tests/test_marks.py::test_ambiguous_mask_matches_reference",
+  "tests/test_marks.py::test_bracketize_roundtrip",
+  "tests/test_marks.py::test_concat_verses_boundaries",
+  "tests/test_marks.py::test_enforce_circumflex_heavy_leaves_non_circumflex_alone",
+  "tests/test_marks.py::test_enforce_circumflex_heavy_overrides_light",
+  "tests/test_marks.py::test_insert_marks_roundtrip",
+  "tests/test_marks.py::test_macron_roundtrip_against_plain_column",
+  "tests/test_marks.py::test_merge_vowelless_syllables",
+  "tests/test_marks.py::test_merge_vowelless_syllables_leaves_real_syllables_alone",
+  "tests/test_marks.py::test_norma_lines_parse",
+  "tests/test_marks.py::test_parse_macron_combining_marks",
+  "tests/test_marks.py::test_parse_macron_simple",
+  "tests/test_marks.py::test_parse_scan_simple",
+  "tests/test_marks.py::test_scan_corpus_lines_encode",
+  "tests/test_meta_conditioning.py::test_collate_propagates_region_century",
+  "tests/test_meta_conditioning.py::test_collate_unk_constants_match_meta_vocab",
+  "tests/test_meta_conditioning.py::test_data_py_unk_constants_match_meta_vocab",
+  "tests/test_meta_conditioning.py::test_model_ignores_region_century_when_disabled",
+  "tests/test_meta_conditioning.py::test_pack_batch_propagates_region_century_and_defaults_unk",
+  "tests/test_meta_conditioning.py::test_parse_year_rejects_sentinels_and_junk",
+  "tests/test_meta_conditioning.py::test_record_century_id_midpoint_and_fallback",
+  "tests/test_meta_conditioning.py::test_region_to_id_known_and_unknown",
+  "tests/test_meter_hf_release_smoke.py::test_ambiguous_mask_and_mark_insertion",
+  "tests/test_meter_hf_release_smoke.py::test_bracketize_basic",
+  "tests/test_meter_hf_release_smoke.py::test_convert_checkpoint_to_hf_meter_kind",
+  "tests/test_meter_hf_release_smoke.py::test_meter_model_forward_and_decode_shapes",
+  "tests/test_noising.py::test_channel_dropout_frequencies",
+  "tests/test_noising.py::test_channel_masks_are_genuinely_patchy_sometimes",
+  "tests/test_noising.py::test_elastic_preserves_visible_chars_and_lengths",
+  "tests/test_noising.py::test_halfword_pattern_only_masks_within_single_words",
+  "tests/test_noising.py::test_halfword_span_always_within_the_word_and_hits_all_anchors",
+  "tests/test_noising.py::test_iid_rate_matches_t",
+  "tests/test_noising.py::test_is_real_lacuna_none_is_fully_backward_compatible",
+  "tests/test_noising.py::test_labels_only_at_masked_positions_fixed_patterns",
+  "tests/test_noising.py::test_mixture_covers_all_patterns",
+  "tests/test_noising.py::test_real_lacuna_excluded_from_elastic_rebuild_too",
+  "tests/test_noising.py::test_real_lacuna_never_selected_as_synthetic_target_fixed_patterns",
+  "tests/test_noising.py::test_substitute_never_uses_mask_token_and_label_is_true_char",
+  "tests/test_noising.py::test_word_masking_respects_boundaries",
+  "tests/test_normalize.py::test_capital_with_breathing",
+  "tests/test_normalize.py::test_dia_pack_unpack_roundtrip",
+  "tests/test_normalize.py::test_drops_nongreek_record",
+  "tests/test_normalize.py::test_elision_apostrophe_is_word_boundary",
+  "tests/test_normalize.py::test_final_and_lunate_sigma_merge",
+  "tests/test_normalize.py::test_iota_subscript_and_diaeresis",
+  "tests/test_normalize.py::test_kai_ho_logos",
+  "tests/test_normalize.py::test_punct_without_space_is_sentence_final",
+  "tests/test_normalize.py::test_restore_polytonic_simple",
+  "tests/test_normalize.py::test_roundtrip_on_real_records[pristine/pristine-00000.parquet]",
+  "tests/test_normalize.py::test_roundtrip_on_real_records[repaired/repaired-00000.parquet]",
+  "tests/test_packing.py::test_pack_batch_metadata_dropout_default_off_matches_prior_behavior",
+  "tests/test_packing.py::test_pack_batch_metadata_dropout_forces_unk_sometimes",
+  "tests/test_packing.py::test_pack_batch_real_lacuna_never_supervised",
+  "tests/test_packing.py::test_packing_labels_only_at_masked",
+  "tests/test_packing.py::test_packing_no_doc_crossing_and_fill",
+  "tests/test_syntax_smoke.py::test_joint_model_forward_pass",
+  "tests/test_ud_conversion.py::test_auxp_promotes_nominal_to_obl",
+  "tests/test_ud_conversion.py::test_coord_promotes_first_conjunct_with_generic_fallback",
+  "tests/test_ud_conversion.py::test_empty_sentence",
+  "tests/test_ud_conversion.py::test_pnom_copula_reversal",
+  "tests/test_ud_conversion.py::test_simple_sentence_sbj_pred_atr",
+  "tests/test_ud_conversion.py::test_xpos_to_upos_feats_verb_and_noun"
+]

MODEL_CARDS_INDEX.md

@@ -27,18 +27,17 @@ All built on `Stoicheia-doc_clean`.
 | [`Ericu950/Stoicheia-meter`](https://huggingface.co/Ericu950/Stoicheia-meter) | macronization (vowel length) + metrical scansion, trained jointly |
 | [`Ericu950/Stoicheia-macronizer`](https://huggingface.co/Ericu950/Stoicheia-macronizer) | macronization only — the arm the paper's macronization ablation is measured on |
 
-Not released as a model: the authorship-attribution / hermeneutic probes, which are
-designed to be fine-tuned per-task rather than distributed as a general-purpose model
-(see the paper's Discussion section).
-
 ## Datasets
 
 | dataset | what it is |
 |---|---|
-| [`Ericu950/AncientGreek`](https://huggingface.co/datasets/Ericu950/AncientGreek) | main pretraining corpus (gold + silver tiers) |
-| [`Ericu950/SyntheticAncientGreek-CorpusCorporum`](https://huggingface.co/datasets/Ericu950/SyntheticAncientGreek-CorpusCorporum) | bronze synthetic augmentation tier |
+| [`Ericu950/AncientGreek`](https://huggingface.co/datasets/Ericu950/AncientGreek) | pretraining corpus, ~361M words in `pristine` and `repaired` tiers |
+| [`Ericu950/SyntheticAncientGreek-CorpusCorporum`](https://huggingface.co/datasets/Ericu950/SyntheticAncientGreek-CorpusCorporum) | bronze synthetic augmentation tier, machine-translated from Latin |
 | [`Ericu950/Inscriptions_2`](https://huggingface.co/datasets/Ericu950/Inscriptions_2) | PHI inscriptions used for the 10-fold split and restoration fine-tuning |
+| [`Ericu950/Stoicheia-meter-silver`](https://huggingface.co/datasets/Ericu950/Stoicheia-meter-silver) | silver macronization/scansion training data (Hypotactic-derived + constraint-solver mined) |
+| [`Ericu950/norma`](https://huggingface.co/datasets/Ericu950/norma) | mirror of the *Norma Syllabarum Graecarum* benchmark, in the exact split the paper evaluates on |
 
-Not re-released (already public elsewhere, cited not duplicated): the *Norma*
-macronization/scansion benchmark (GitHub), the OGA/AGDT treebank
-(Celano's own repository).
+`AncientGreek` omits the Database of Byzantine Book Epigrams, which is CC BY-NC-SA and so
+cannot travel inside a CC BY-SA compilation; `scripts/fetch_dbbe.py` refetches it under its
+own terms. Not re-released at all: the OGA/AGDT treebank splits, which come from Celano's
+own repository and are cited rather than duplicated.

NOTICE

@@ -1,5 +1,5 @@
 Stoicheia
-Copyright 2026 Anonymous (under review)
+Copyright 2026 Eric Cullhed and Albin Thörn Cleland
 
 This product is licensed under the Apache License, Version 2.0 (see LICENSE).
 

README.md

@@ -18,7 +18,7 @@ macronization and scansion). All model repos are public: weights ship as `model.
 loadable directly through `AutoModel.from_pretrained(..., trust_remote_code=True)`.
 
 
-[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Ericu950/stoicheia/blob/main/Stoicheia_demo.ipynb)
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ericu9500/stoicheia/blob/main/Stoicheia_demo.ipynb)
 
 Run everything in the browser: [`Stoicheia_demo.ipynb`](Stoicheia_demo.ipynb) restores a lacuna of
 unknown width, picks the checkpoint that has provably never read your document, tags and parses a
@@ -75,6 +75,10 @@ decides where the words end.
 - `meter/` — macronization (vowel length) and metrical scansion, including the
   *Norma* benchmark protocol and rule-based silver-data mining pipeline.
 - `tests/` — CPU-only pytest suite.
+- `scripts/fetch_dbbe.py` — refetches the Database of Byzantine Book Epigrams, which the
+  released corpus omits: DBBE is CC BY-NC-SA, whose non-commercial clause a CC BY-SA
+  compilation cannot carry. Run it to reconstruct the pretraining corpus exactly (5,476
+  records, ~0.2M words, 0.1% of the total); what you build then inherits DBBE's terms.
 
 See [`REPRODUCING.md`](REPRODUCING.md) for the full environment setup and end-to-end
 reproduction walkthrough.
@@ -85,9 +89,11 @@ reproduction walkthrough.
 @misc{stoicheia2026,
   title  = {Stoicheia: Character-Level Masked Diffusion for Ancient Greek Textual
             Restoration, Parsing, and Metrical Scansion},
-  author = {Anonymous},
+  author = {Cullhed, Eric and Th\"orn Cleland, Albin},
   year   = {2026},
-  note   = {Under review; citation to be finalized on publication}
+  eprint = {2608.XXXXX},
+  archivePrefix = {arXiv},
+  primaryClass  = {cs.CL}
 }
 ```
 

REPRODUCING.md

@@ -3,7 +3,7 @@
 ## 1. Environment
 
 ```bash
-git clone https://github.com/Ericu950/stoicheia
+git clone https://github.com/ericu9500/stoicheia
 cd stoicheia
 export STOICHEIA_DATA=/path/to/a/writable/data+checkpoints/directory
 source env.sh

Stoicheia_demo.ipynb

@@ -20,7 +20,7 @@
     "4. macronize and scan a line of verse\n",
     "5. score the macronizer against a hand-annotated benchmark\n",
     "\n",
-    "Every model and dataset used below is public. Anonymous release accompanying a paper under review.\n"
+    "Every model and dataset used below is public.\n"
    ]
   },
   {

pyproject.toml

@@ -10,7 +10,8 @@ readme = "README.md"
 requires-python = ">=3.10"
 license = { text = "Apache-2.0" }
 authors = [
-    { name = "Anonymous (under review)" },
+    { name = "Eric Cullhed" },
+    { name = "Albin Thörn Cleland" },
 ]
 
 [tool.setuptools]

scripts/build_notebook.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Build the Stoicheia demo notebook (anonymous, Colab-ready)."""
+"""Build the Stoicheia demo notebook (Colab-ready)."""
 import json, pathlib
 
 ORG = "Ericu950"
@@ -25,7 +25,7 @@ cells = [
     "4. macronize and scan a line of verse",
     "5. score the macronizer against a hand-annotated benchmark",
     "",
-    "Every model and dataset used below is public. Anonymous release accompanying a paper under review."),
+    "Every model and dataset used below is public."),
 
  code("%pip install -q --upgrade transformers huggingface_hub safetensors torch datasets"),
 

scripts/fetch_dbbe.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""Fetch the Database of Byzantine Book Epigrams, which the corpus release omits.
+
+DBBE is distributed under CC BY-NC-SA. The pretraining corpus is a CC BY-SA compilation,
+and a non-commercial clause cannot be honoured inside one, so the 5,476 DBBE records
+(~0.2M words, 0.1% of the corpus) are excluded from the released dataset even though they
+were present in the corpus we pretrained on. This script rebuilds that corpus exactly, on
+the reader's own terms rather than ours.
+
+  python scripts/fetch_dbbe.py --out dbbe.jsonl
+
+Anything you build from the result inherits DBBE's non-commercial and share-alike terms.
+See https://www.dbbe.ugent.be for the licence and citation policy.
+"""
+import argparse
+import json
+import time
+import urllib.request
+from pathlib import Path
+
+API = "https://www.dbbe.ugent.be/api/occurrences"
+UA = {"User-Agent": "stoicheia-corpus-rebuild/1.0 (+https://github.com/ericu9500/stoicheia)"}
+
+
+def fetch_page(page, per_page, retries=3):
+    url = f"{API}?page={page}&limit={per_page}"
+    for attempt in range(retries):
+        try:
+            req = urllib.request.Request(url, headers=UA)
+            with urllib.request.urlopen(req, timeout=60) as r:
+                return json.loads(r.read().decode("utf-8"))
+        except Exception as e:                                   # noqa: BLE001
+            if attempt == retries - 1:
+                raise
+            print(f"  retry {attempt + 1} after {type(e).__name__}", flush=True)
+            time.sleep(5 * (attempt + 1))
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--out", default="dbbe.jsonl")
+    ap.add_argument("--per-page", type=int, default=100)
+    ap.add_argument("--max-pages", type=int, default=0, help="0 = until exhausted")
+    a = ap.parse_args()
+
+    out = Path(a.out)
+    n = page = 0
+    with out.open("w", encoding="utf-8") as fh:
+        while True:
+            page += 1
+            if a.max_pages and page > a.max_pages:
+                break
+            data = fetch_page(page, a.per_page)
+            items = data.get("data") or data.get("items") or []
+            if not items:
+                break
+            for it in items:
+                text = (it.get("text") or "").strip()
+                if not text:
+                    continue
+                fh.write(json.dumps({"source": "dbbe",
+                                     "id": str(it.get("id", "")),
+                                     "license": "CC BY-NC-SA",
+                                     "text": text}, ensure_ascii=False) + "\n")
+                n += 1
+            print(f"  page {page}: {n} records so far", flush=True)
+            time.sleep(1)                                        # be polite to the API
+
+    print(f"wrote {n} records to {out}")
+    print("These records are CC BY-NC-SA: anything you merge them into inherits those terms.")
+
+
+if __name__ == "__main__":
+    main()