from __future__ import annotations from dataclasses import dataclass from typing import Sequence BILOU_LABELS = ("O", "B", "I", "L", "U") def validate_min_span_chars(value) -> int: """Validate a configurable minimum half-open character span length. ``0`` disables the filter and is the legacy default. Only a plain non-negative ``int`` is accepted; ``bool`` is explicitly rejected so a caller cannot sneak ``True`` in as ``1``. """ if isinstance(value, bool) or not isinstance(value, int) or value < 0: raise ValueError("min_span_chars must be a non-negative integer") return value def filter_min_span_char_spans(spans, min_span_chars, *, start_char, end_char): """Keep spans whose exact half-open character width reaches the floor. The length is ``end_char - start_char`` in real Python string characters (not bytes and not token count), evaluated on the final spans that reach the caller. A ``min_span_chars`` of ``0`` returns the spans unchanged, so legacy behavior is byte-for-byte identical. Original offsets and confidence are always preserved; no span is extended or re-scored. """ minimum = validate_min_span_chars(min_span_chars) if minimum <= 0: return list(spans) return [ span for span in spans if int(end_char(span)) - int(start_char(span)) >= minimum ] @dataclass(frozen=True, slots=True) class DecodedCategorySpan: """One decoded span for a single category, in token-index space.""" category: str start_index: int end_index: int confidence: float labels: tuple[str, ...] def softmax_probabilities(logits): """Softmax over the last (BILOU) axis of a [sequence, category, label] array.""" import numpy as np values = np.asarray(logits, dtype="float64") shifted = values - values.max(axis=-1, keepdims=True) exponentiated = np.exp(shifted) return exponentiated / exponentiated.sum(axis=-1, keepdims=True) def _validate_shape(probabilities, categories: Sequence[str]) -> None: if probabilities.ndim != 3: raise ValueError("probabilities must have shape [sequence, category, label]") if probabilities.shape[1] != len(categories): raise ValueError( "category axis does not match the ordered category list: " f"{probabilities.shape[1]} != {len(categories)}" ) if probabilities.shape[2] != len(BILOU_LABELS): raise ValueError( f"label axis must be {len(BILOU_LABELS)} (BILOU), got {probabilities.shape[2]}" ) def decode_category( tag_ids: Sequence[int], tag_probabilities: Sequence[float], *, category: str, threshold: float, labels: Sequence[str] = BILOU_LABELS, ) -> list[DecodedCategorySpan]: """Greedy BILOU span assembly for one category. ``B`` opens a span, ``I`` continues it, ``L`` closes it, ``U`` is a single-token span and ``O`` is outside. A span left open at the sequence end, or a stray ``I``/``L``, still closes so a malformed tag stream cannot drop text silently. """ spans: list[DecodedCategorySpan] = [] start: int | None = None open_labels: list[str] = [] open_probabilities: list[float] = [] def close(end_index: int) -> None: nonlocal start, open_labels, open_probabilities if start is None: return confidence = sum(open_probabilities) / len(open_probabilities) if confidence >= threshold: spans.append( DecodedCategorySpan( category=category, start_index=start, end_index=end_index, confidence=round(confidence, 6), labels=tuple(open_labels), ) ) start = None open_labels = [] open_probabilities = [] for index, tag_id in enumerate(tag_ids): tag = labels[int(tag_id)] probability = float(tag_probabilities[index]) if tag == "O": close(index) continue if tag in ("B", "U"): close(index) start = index open_labels = [tag] open_probabilities = [probability] if tag == "U": close(index + 1) continue # I or L: continue an open span, or start one if the tag stream is broken. if start is None: start = index open_labels = [] open_probabilities = [] open_labels.append(tag) open_probabilities.append(probability) if tag == "L": close(index + 1) close(len(tag_ids)) return spans def decode_multi_head( probabilities, *, categories: Sequence[str], thresholds: dict[str, float] | None = None, default_threshold: float = 0.5, labels: Sequence[str] = BILOU_LABELS, enabled_categories: Sequence[str] | None = None, ) -> list[DecodedCategorySpan]: """Decode every category head independently from one shared encoder pass. Categories are independent by design: the same token may be positive for more than one category, and the decoder never forces a single label per token. """ import numpy as np values = np.asarray(probabilities, dtype="float64") _validate_shape(values, categories) configured = thresholds or {} enabled = set(categories if enabled_categories is None else enabled_categories) unknown = enabled - set(categories) if unknown: raise ValueError(f"enabled decoder categories are unknown: {sorted(unknown)}") decoded: list[DecodedCategorySpan] = [] for category_index, category in enumerate(categories): if category not in enabled: continue category_probabilities = values[:, category_index, :] tag_ids = np.argmax(category_probabilities, axis=1) tag_probabilities = np.take_along_axis( category_probabilities, tag_ids[:, None], axis=1 )[:, 0] decoded.extend( decode_category( tag_ids.tolist(), tag_probabilities.tolist(), category=category, threshold=float(configured.get(category, default_threshold)), labels=labels, ) ) decoded.sort(key=lambda span: (span.start_index, span.end_index, span.category)) return decoded def decode_text_probabilities( probabilities, offset_mapping, *, min_span_chars: int = 0, **options ): """Decode transcript tokens, forcing special/padded empty offsets to O. Empty offsets carry no text and were ignored by training. Their logits must not open, extend, or suppress a transcript span. The input stays unchanged. ``min_span_chars`` (default ``0``) drops assembled spans whose exact half-open character width ``end_char - start_char`` is below the floor. This is a pre-stitch, per-category filter applied after BILOU assembly only; callers that stitch across windows must apply the filter themselves after their final stitch so a short fragment may still join a valid event. """ import numpy as np minimum = validate_min_span_chars(min_span_chars) values = np.asarray(probabilities, dtype="float64").copy() if values.ndim != 3 or len(offset_mapping) != len(values): raise ValueError("offset mapping must match the probability sequence") labels = options.get("labels", BILOU_LABELS) outside = list(labels).index("O") for index, (start, end) in enumerate(offset_mapping): if end <= start: values[index, :, :] = 0.0 values[index, :, outside] = 1.0 spans = decode_multi_head(values, **options) if minimum <= 0: return spans return [ span for span in spans if int(offset_mapping[span.end_index - 1][1]) - int(offset_mapping[span.start_index][0]) >= minimum ]