CuriousDragon's picture
Publish verified Flow Smart Segments curated-category-encoder-ensemble-20261004
8fcf7c9 verified
Raw History Blame Contribute Delete
7.99 kB
from __future__ import annotations
from dataclasses import dataclass
from typing import Sequence
BILOU_LABELS = ("O", "B", "I", "L", "U")
def validate_min_span_chars(value) -> int:
"""Validate a configurable minimum half-open character span length.
``0`` disables the filter and is the legacy default. Only a plain
non-negative ``int`` is accepted; ``bool`` is explicitly rejected so a
caller cannot sneak ``True`` in as ``1``.
"""
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
raise ValueError("min_span_chars must be a non-negative integer")
return value
def filter_min_span_char_spans(spans, min_span_chars, *, start_char, end_char):
"""Keep spans whose exact half-open character width reaches the floor.
The length is ``end_char - start_char`` in real Python string characters
(not bytes and not token count), evaluated on the final spans that reach
the caller. A ``min_span_chars`` of ``0`` returns the spans unchanged, so
legacy behavior is byte-for-byte identical. Original offsets and
confidence are always preserved; no span is extended or re-scored.
"""
minimum = validate_min_span_chars(min_span_chars)
if minimum <= 0:
return list(spans)
return [
span
for span in spans
if int(end_char(span)) - int(start_char(span)) >= minimum
]
@dataclass(frozen=True, slots=True)
class DecodedCategorySpan:
"""One decoded span for a single category, in token-index space."""
category: str
start_index: int
end_index: int
confidence: float
labels: tuple[str, ...]
def softmax_probabilities(logits):
"""Softmax over the last (BILOU) axis of a [sequence, category, label] array."""
import numpy as np
values = np.asarray(logits, dtype="float64")
shifted = values - values.max(axis=-1, keepdims=True)
exponentiated = np.exp(shifted)
return exponentiated / exponentiated.sum(axis=-1, keepdims=True)
def _validate_shape(probabilities, categories: Sequence[str]) -> None:
if probabilities.ndim != 3:
raise ValueError("probabilities must have shape [sequence, category, label]")
if probabilities.shape[1] != len(categories):
raise ValueError(
"category axis does not match the ordered category list: "
f"{probabilities.shape[1]} != {len(categories)}"
)
if probabilities.shape[2] != len(BILOU_LABELS):
raise ValueError(
f"label axis must be {len(BILOU_LABELS)} (BILOU), got {probabilities.shape[2]}"
)
def decode_category(
tag_ids: Sequence[int],
tag_probabilities: Sequence[float],
*,
category: str,
threshold: float,
labels: Sequence[str] = BILOU_LABELS,
) -> list[DecodedCategorySpan]:
"""Greedy BILOU span assembly for one category.
``B`` opens a span, ``I`` continues it, ``L`` closes it, ``U`` is a
single-token span and ``O`` is outside. A span left open at the sequence
end, or a stray ``I``/``L``, still closes so a malformed tag stream cannot
drop text silently.
"""
spans: list[DecodedCategorySpan] = []
start: int | None = None
open_labels: list[str] = []
open_probabilities: list[float] = []
def close(end_index: int) -> None:
nonlocal start, open_labels, open_probabilities
if start is None:
return
confidence = sum(open_probabilities) / len(open_probabilities)
if confidence >= threshold:
spans.append(
DecodedCategorySpan(
category=category,
start_index=start,
end_index=end_index,
confidence=round(confidence, 6),
labels=tuple(open_labels),
)
)
start = None
open_labels = []
open_probabilities = []
for index, tag_id in enumerate(tag_ids):
tag = labels[int(tag_id)]
probability = float(tag_probabilities[index])
if tag == "O":
close(index)
continue
if tag in ("B", "U"):
close(index)
start = index
open_labels = [tag]
open_probabilities = [probability]
if tag == "U":
close(index + 1)
continue
# I or L: continue an open span, or start one if the tag stream is broken.
if start is None:
start = index
open_labels = []
open_probabilities = []
open_labels.append(tag)
open_probabilities.append(probability)
if tag == "L":
close(index + 1)
close(len(tag_ids))
return spans
def decode_multi_head(
probabilities,
*,
categories: Sequence[str],
thresholds: dict[str, float] | None = None,
default_threshold: float = 0.5,
labels: Sequence[str] = BILOU_LABELS,
enabled_categories: Sequence[str] | None = None,
) -> list[DecodedCategorySpan]:
"""Decode every category head independently from one shared encoder pass.
Categories are independent by design: the same token may be positive for
more than one category, and the decoder never forces a single label per
token.
"""
import numpy as np
values = np.asarray(probabilities, dtype="float64")
_validate_shape(values, categories)
configured = thresholds or {}
enabled = set(categories if enabled_categories is None else enabled_categories)
unknown = enabled - set(categories)
if unknown:
raise ValueError(f"enabled decoder categories are unknown: {sorted(unknown)}")
decoded: list[DecodedCategorySpan] = []
for category_index, category in enumerate(categories):
if category not in enabled:
continue
category_probabilities = values[:, category_index, :]
tag_ids = np.argmax(category_probabilities, axis=1)
tag_probabilities = np.take_along_axis(
category_probabilities, tag_ids[:, None], axis=1
)[:, 0]
decoded.extend(
decode_category(
tag_ids.tolist(),
tag_probabilities.tolist(),
category=category,
threshold=float(configured.get(category, default_threshold)),
labels=labels,
)
)
decoded.sort(key=lambda span: (span.start_index, span.end_index, span.category))
return decoded
def decode_text_probabilities(
probabilities, offset_mapping, *, min_span_chars: int = 0, **options
):
"""Decode transcript tokens, forcing special/padded empty offsets to O.
Empty offsets carry no text and were ignored by training. Their logits must
not open, extend, or suppress a transcript span. The input stays unchanged.
``min_span_chars`` (default ``0``) drops assembled spans whose exact
half-open character width ``end_char - start_char`` is below the floor.
This is a pre-stitch, per-category filter applied after BILOU assembly
only; callers that stitch across windows must apply the filter themselves
after their final stitch so a short fragment may still join a valid event.
"""
import numpy as np
minimum = validate_min_span_chars(min_span_chars)
values = np.asarray(probabilities, dtype="float64").copy()
if values.ndim != 3 or len(offset_mapping) != len(values):
raise ValueError("offset mapping must match the probability sequence")
labels = options.get("labels", BILOU_LABELS)
outside = list(labels).index("O")
for index, (start, end) in enumerate(offset_mapping):
if end <= start:
values[index, :, :] = 0.0
values[index, :, outside] = 1.0
spans = decode_multi_head(values, **options)
if minimum <= 0:
return spans
return [
span
for span in spans
if int(offset_mapping[span.end_index - 1][1])
- int(offset_mapping[span.start_index][0])
>= minimum
]