File size: 7,991 Bytes
8fcf7c9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 | from __future__ import annotations
from dataclasses import dataclass
from typing import Sequence
BILOU_LABELS = ("O", "B", "I", "L", "U")
def validate_min_span_chars(value) -> int:
"""Validate a configurable minimum half-open character span length.
``0`` disables the filter and is the legacy default. Only a plain
non-negative ``int`` is accepted; ``bool`` is explicitly rejected so a
caller cannot sneak ``True`` in as ``1``.
"""
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
raise ValueError("min_span_chars must be a non-negative integer")
return value
def filter_min_span_char_spans(spans, min_span_chars, *, start_char, end_char):
"""Keep spans whose exact half-open character width reaches the floor.
The length is ``end_char - start_char`` in real Python string characters
(not bytes and not token count), evaluated on the final spans that reach
the caller. A ``min_span_chars`` of ``0`` returns the spans unchanged, so
legacy behavior is byte-for-byte identical. Original offsets and
confidence are always preserved; no span is extended or re-scored.
"""
minimum = validate_min_span_chars(min_span_chars)
if minimum <= 0:
return list(spans)
return [
span
for span in spans
if int(end_char(span)) - int(start_char(span)) >= minimum
]
@dataclass(frozen=True, slots=True)
class DecodedCategorySpan:
"""One decoded span for a single category, in token-index space."""
category: str
start_index: int
end_index: int
confidence: float
labels: tuple[str, ...]
def softmax_probabilities(logits):
"""Softmax over the last (BILOU) axis of a [sequence, category, label] array."""
import numpy as np
values = np.asarray(logits, dtype="float64")
shifted = values - values.max(axis=-1, keepdims=True)
exponentiated = np.exp(shifted)
return exponentiated / exponentiated.sum(axis=-1, keepdims=True)
def _validate_shape(probabilities, categories: Sequence[str]) -> None:
if probabilities.ndim != 3:
raise ValueError("probabilities must have shape [sequence, category, label]")
if probabilities.shape[1] != len(categories):
raise ValueError(
"category axis does not match the ordered category list: "
f"{probabilities.shape[1]} != {len(categories)}"
)
if probabilities.shape[2] != len(BILOU_LABELS):
raise ValueError(
f"label axis must be {len(BILOU_LABELS)} (BILOU), got {probabilities.shape[2]}"
)
def decode_category(
tag_ids: Sequence[int],
tag_probabilities: Sequence[float],
*,
category: str,
threshold: float,
labels: Sequence[str] = BILOU_LABELS,
) -> list[DecodedCategorySpan]:
"""Greedy BILOU span assembly for one category.
``B`` opens a span, ``I`` continues it, ``L`` closes it, ``U`` is a
single-token span and ``O`` is outside. A span left open at the sequence
end, or a stray ``I``/``L``, still closes so a malformed tag stream cannot
drop text silently.
"""
spans: list[DecodedCategorySpan] = []
start: int | None = None
open_labels: list[str] = []
open_probabilities: list[float] = []
def close(end_index: int) -> None:
nonlocal start, open_labels, open_probabilities
if start is None:
return
confidence = sum(open_probabilities) / len(open_probabilities)
if confidence >= threshold:
spans.append(
DecodedCategorySpan(
category=category,
start_index=start,
end_index=end_index,
confidence=round(confidence, 6),
labels=tuple(open_labels),
)
)
start = None
open_labels = []
open_probabilities = []
for index, tag_id in enumerate(tag_ids):
tag = labels[int(tag_id)]
probability = float(tag_probabilities[index])
if tag == "O":
close(index)
continue
if tag in ("B", "U"):
close(index)
start = index
open_labels = [tag]
open_probabilities = [probability]
if tag == "U":
close(index + 1)
continue
# I or L: continue an open span, or start one if the tag stream is broken.
if start is None:
start = index
open_labels = []
open_probabilities = []
open_labels.append(tag)
open_probabilities.append(probability)
if tag == "L":
close(index + 1)
close(len(tag_ids))
return spans
def decode_multi_head(
probabilities,
*,
categories: Sequence[str],
thresholds: dict[str, float] | None = None,
default_threshold: float = 0.5,
labels: Sequence[str] = BILOU_LABELS,
enabled_categories: Sequence[str] | None = None,
) -> list[DecodedCategorySpan]:
"""Decode every category head independently from one shared encoder pass.
Categories are independent by design: the same token may be positive for
more than one category, and the decoder never forces a single label per
token.
"""
import numpy as np
values = np.asarray(probabilities, dtype="float64")
_validate_shape(values, categories)
configured = thresholds or {}
enabled = set(categories if enabled_categories is None else enabled_categories)
unknown = enabled - set(categories)
if unknown:
raise ValueError(f"enabled decoder categories are unknown: {sorted(unknown)}")
decoded: list[DecodedCategorySpan] = []
for category_index, category in enumerate(categories):
if category not in enabled:
continue
category_probabilities = values[:, category_index, :]
tag_ids = np.argmax(category_probabilities, axis=1)
tag_probabilities = np.take_along_axis(
category_probabilities, tag_ids[:, None], axis=1
)[:, 0]
decoded.extend(
decode_category(
tag_ids.tolist(),
tag_probabilities.tolist(),
category=category,
threshold=float(configured.get(category, default_threshold)),
labels=labels,
)
)
decoded.sort(key=lambda span: (span.start_index, span.end_index, span.category))
return decoded
def decode_text_probabilities(
probabilities, offset_mapping, *, min_span_chars: int = 0, **options
):
"""Decode transcript tokens, forcing special/padded empty offsets to O.
Empty offsets carry no text and were ignored by training. Their logits must
not open, extend, or suppress a transcript span. The input stays unchanged.
``min_span_chars`` (default ``0``) drops assembled spans whose exact
half-open character width ``end_char - start_char`` is below the floor.
This is a pre-stitch, per-category filter applied after BILOU assembly
only; callers that stitch across windows must apply the filter themselves
after their final stitch so a short fragment may still join a valid event.
"""
import numpy as np
minimum = validate_min_span_chars(min_span_chars)
values = np.asarray(probabilities, dtype="float64").copy()
if values.ndim != 3 or len(offset_mapping) != len(values):
raise ValueError("offset mapping must match the probability sequence")
labels = options.get("labels", BILOU_LABELS)
outside = list(labels).index("O")
for index, (start, end) in enumerate(offset_mapping):
if end <= start:
values[index, :, :] = 0.0
values[index, :, outside] = 1.0
spans = decode_multi_head(values, **options)
if minimum <= 0:
return spans
return [
span
for span in spans
if int(offset_mapping[span.end_index - 1][1])
- int(offset_mapping[span.start_index][0])
>= minimum
]
|