File size: 6,278 Bytes
3f92bfe | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 | from pathlib import Path
app_path = Path("app.py")
app = app_path.read_text(encoding="utf-8")
import_marker = "from voynich_lab.segmentation import cross_fit_bpe_scale_curve, discover_bpe_units\n"
import_replacement = (
import_marker
+ "from voynich_lab.unigram_segmentation import segmentation_consensus\n"
)
assert app.count(import_marker) == 1, "segmentation import marker drifted"
app = app.replace(import_marker, import_replacement, 1)
function_marker = "def run_gpu_ui(text: str, window: int, max_symbols: int):\n"
function_block = '''def run_segmentation_consensus_ui(
text: str,
bpe_merges: int,
max_unit_length: int,
min_count: int,
max_vocab: int,
iterations: int,
length_penalty_bits: float,
):
if not text.strip():
raise gr.Error("Load a transcription first.")
try:
result = segmentation_consensus(
text,
bpe_merges=int(bpe_merges),
max_unit_length=int(max_unit_length),
min_count=int(min_count),
max_vocab=int(max_vocab),
iterations=int(iterations),
length_penalty_bits=float(length_penalty_bits),
preview_lines=12,
)
except (ValueError, RuntimeError) as exc:
raise gr.Error(str(exc)) from exc
summary = pd.DataFrame(
[{"metric": key, "value": value} for key, value in result.summary.items()]
)
lines = pd.DataFrame(result.line_rows)
units = pd.DataFrame(result.unit_rows)
note = (
f"Exploratory consensus only: unigram latent model vocabulary={result.model.vocabulary_size}, "
f"EM iterations={result.model.iterations_run}; line-bounded BPE rules={len(result.bpe_rules)}. "
"Both models see the same space-erased glyph lines. Transcription spaces are withheld from fitting "
"and used only afterward as boundary observations. Shared boundaries are candidates for follow-up, "
"not identified linguistic morphemes or words."
)
return summary, lines, units, result.preview, note
'''
assert app.count(function_marker) == 1, "function marker drifted"
app = app.replace(function_marker, function_block + function_marker, 1)
ui_marker = ''' crossfit_btn.click(run_bpe_crossfit_ui, raw_state, [crossfit_table, crossfit_selected])
with gr.Tab("4 · IVTFF audit"):
'''
ui_block = ''' crossfit_btn.click(run_bpe_crossfit_ui, raw_state, [crossfit_table, crossfit_selected])
gr.Markdown(
"#### Independent boundary consensus (exploratory)\\n"
"This path does **not** use spaces as hard word boundaries. A finite unigram latent-unit model "
"and a separate line-bounded BPE are fit to the same space-erased glyph lines; observed spaces "
"are compared to inferred boundaries only after fitting."
)
with gr.Row():
consensus_bpe_merges = gr.Slider(0, 128, value=32, step=1, label="Comparator BPE merges")
consensus_max_len = gr.Slider(2, 12, value=8, step=1, label="Max unigram unit length")
consensus_min_count = gr.Slider(1, 10, value=2, step=1, label="Min recurring substring count")
with gr.Row():
consensus_vocab = gr.Slider(64, 2048, value=512, step=64, label="Unigram candidate vocabulary cap")
consensus_iterations = gr.Slider(2, 30, value=10, step=1, label="EM iterations")
consensus_length_penalty = gr.Slider(0.0, 2.0, value=0.0, step=0.05, label="Long-unit penalty (bits per extra glyph)")
consensus_btn = gr.Button("Run independent segmentation consensus")
consensus_summary = gr.Dataframe(label="Boundary-consensus summary", interactive=False)
consensus_lines = gr.Dataframe(label="Per-line boundary agreement", interactive=False)
consensus_units = gr.Dataframe(label="Top units by method", interactive=False)
consensus_preview = gr.Textbox(label="Segmentation preview", lines=16, interactive=False)
consensus_note = gr.Textbox(label="Interpretation guardrail", lines=5, interactive=False)
consensus_btn.click(
run_segmentation_consensus_ui,
[
text_state,
consensus_bpe_merges,
consensus_max_len,
consensus_min_count,
consensus_vocab,
consensus_iterations,
consensus_length_penalty,
],
[
consensus_summary,
consensus_lines,
consensus_units,
consensus_preview,
consensus_note,
],
)
with gr.Tab("4 · IVTFF audit"):
'''
assert app.count(ui_marker) == 1, "unit-discovery UI marker drifted"
app = app.replace(ui_marker, ui_block, 1)
app_path.write_text(app, encoding="utf-8")
readme_path = Path("README.md")
readme = readme_path.read_text(encoding="utf-8")
old_gap = "2. **Consensus segmentation.** BPE is implemented, including paper-exact held-out BPE, but an independent segmental HSMM and motif-discovery path are not yet implemented. A later consensus analysis should report where independent methods agree rather than treating BPE output as identified linguistic units.\n"
new_gap = "2. **Consensus segmentation.** BPE now has an independent finite-unigram latent segmentation comparator with boundary-consensus diagnostics that withhold transcription spaces from fitting. Remaining work is a third independent family (segmental HSMM and/or motif discovery), quire-held-out stability for the non-BPE model, and explicit certain-vs-uncertain IVTFF boundary stratification in the consensus layer.\n"
assert readme.count(old_gap) == 1, "README segmentation-gap marker drifted"
readme = readme.replace(old_gap, new_gap, 1)
readme_path.write_text(readme, encoding="utf-8")
init_path = Path("src/voynich_lab/__init__.py")
init_text = init_path.read_text(encoding="utf-8")
init_marker = ' "segmentation",\n'
assert init_text.count(init_marker) == 1, "package export marker drifted"
if ' "unigram_segmentation",\n' not in init_text:
init_text = init_text.replace(
init_marker,
init_marker + ' "unigram_segmentation",\n',
1,
)
init_path.write_text(init_text, encoding="utf-8")
|