Download code/src/sponsor_detection/model/smart_segment_decoder.py from CuriousDragon/flow-smart-segments-20261004: direct link, hf CLI and curl.
- Browser
- Download file 7.99 kB
-
https://huggingface.co/CuriousDragon/flow-smart-segments-20261004/resolve/main/code/src/sponsor_detection/model/smart_segment_decoder.py
- Command line
-
hf download hf://CuriousDragon/flow-smart-segments-20261004/code/src/sponsor_detection/model/smart_segment_decoder.py
-
curl -L -o smart_segment_decoder.py https://huggingface.co/CuriousDragon/flow-smart-segments-20261004/resolve/main/code/src/sponsor_detection/model/smart_segment_decoder.py
7.99 kB
| from __future__ import annotations | |
| from dataclasses import dataclass | |
| from typing import Sequence | |
| BILOU_LABELS = ("O", "B", "I", "L", "U") | |
| def validate_min_span_chars(value) -> int: | |
| """Validate a configurable minimum half-open character span length. | |
| ``0`` disables the filter and is the legacy default. Only a plain | |
| non-negative ``int`` is accepted; ``bool`` is explicitly rejected so a | |
| caller cannot sneak ``True`` in as ``1``. | |
| """ | |
| if isinstance(value, bool) or not isinstance(value, int) or value < 0: | |
| raise ValueError("min_span_chars must be a non-negative integer") | |
| return value | |
| def filter_min_span_char_spans(spans, min_span_chars, *, start_char, end_char): | |
| """Keep spans whose exact half-open character width reaches the floor. | |
| The length is ``end_char - start_char`` in real Python string characters | |
| (not bytes and not token count), evaluated on the final spans that reach | |
| the caller. A ``min_span_chars`` of ``0`` returns the spans unchanged, so | |
| legacy behavior is byte-for-byte identical. Original offsets and | |
| confidence are always preserved; no span is extended or re-scored. | |
| """ | |
| minimum = validate_min_span_chars(min_span_chars) | |
| if minimum <= 0: | |
| return list(spans) | |
| return [ | |
| span | |
| for span in spans | |
| if int(end_char(span)) - int(start_char(span)) >= minimum | |
| ] | |
| class DecodedCategorySpan: | |
| """One decoded span for a single category, in token-index space.""" | |
| category: str | |
| start_index: int | |
| end_index: int | |
| confidence: float | |
| labels: tuple[str, ...] | |
| def softmax_probabilities(logits): | |
| """Softmax over the last (BILOU) axis of a [sequence, category, label] array.""" | |
| import numpy as np | |
| values = np.asarray(logits, dtype="float64") | |
| shifted = values - values.max(axis=-1, keepdims=True) | |
| exponentiated = np.exp(shifted) | |
| return exponentiated / exponentiated.sum(axis=-1, keepdims=True) | |
| def _validate_shape(probabilities, categories: Sequence[str]) -> None: | |
| if probabilities.ndim != 3: | |
| raise ValueError("probabilities must have shape [sequence, category, label]") | |
| if probabilities.shape[1] != len(categories): | |
| raise ValueError( | |
| "category axis does not match the ordered category list: " | |
| f"{probabilities.shape[1]} != {len(categories)}" | |
| ) | |
| if probabilities.shape[2] != len(BILOU_LABELS): | |
| raise ValueError( | |
| f"label axis must be {len(BILOU_LABELS)} (BILOU), got {probabilities.shape[2]}" | |
| ) | |
| def decode_category( | |
| tag_ids: Sequence[int], | |
| tag_probabilities: Sequence[float], | |
| *, | |
| category: str, | |
| threshold: float, | |
| labels: Sequence[str] = BILOU_LABELS, | |
| ) -> list[DecodedCategorySpan]: | |
| """Greedy BILOU span assembly for one category. | |
| ``B`` opens a span, ``I`` continues it, ``L`` closes it, ``U`` is a | |
| single-token span and ``O`` is outside. A span left open at the sequence | |
| end, or a stray ``I``/``L``, still closes so a malformed tag stream cannot | |
| drop text silently. | |
| """ | |
| spans: list[DecodedCategorySpan] = [] | |
| start: int | None = None | |
| open_labels: list[str] = [] | |
| open_probabilities: list[float] = [] | |
| def close(end_index: int) -> None: | |
| nonlocal start, open_labels, open_probabilities | |
| if start is None: | |
| return | |
| confidence = sum(open_probabilities) / len(open_probabilities) | |
| if confidence >= threshold: | |
| spans.append( | |
| DecodedCategorySpan( | |
| category=category, | |
| start_index=start, | |
| end_index=end_index, | |
| confidence=round(confidence, 6), | |
| labels=tuple(open_labels), | |
| ) | |
| ) | |
| start = None | |
| open_labels = [] | |
| open_probabilities = [] | |
| for index, tag_id in enumerate(tag_ids): | |
| tag = labels[int(tag_id)] | |
| probability = float(tag_probabilities[index]) | |
| if tag == "O": | |
| close(index) | |
| continue | |
| if tag in ("B", "U"): | |
| close(index) | |
| start = index | |
| open_labels = [tag] | |
| open_probabilities = [probability] | |
| if tag == "U": | |
| close(index + 1) | |
| continue | |
| # I or L: continue an open span, or start one if the tag stream is broken. | |
| if start is None: | |
| start = index | |
| open_labels = [] | |
| open_probabilities = [] | |
| open_labels.append(tag) | |
| open_probabilities.append(probability) | |
| if tag == "L": | |
| close(index + 1) | |
| close(len(tag_ids)) | |
| return spans | |
| def decode_multi_head( | |
| probabilities, | |
| *, | |
| categories: Sequence[str], | |
| thresholds: dict[str, float] | None = None, | |
| default_threshold: float = 0.5, | |
| labels: Sequence[str] = BILOU_LABELS, | |
| enabled_categories: Sequence[str] | None = None, | |
| ) -> list[DecodedCategorySpan]: | |
| """Decode every category head independently from one shared encoder pass. | |
| Categories are independent by design: the same token may be positive for | |
| more than one category, and the decoder never forces a single label per | |
| token. | |
| """ | |
| import numpy as np | |
| values = np.asarray(probabilities, dtype="float64") | |
| _validate_shape(values, categories) | |
| configured = thresholds or {} | |
| enabled = set(categories if enabled_categories is None else enabled_categories) | |
| unknown = enabled - set(categories) | |
| if unknown: | |
| raise ValueError(f"enabled decoder categories are unknown: {sorted(unknown)}") | |
| decoded: list[DecodedCategorySpan] = [] | |
| for category_index, category in enumerate(categories): | |
| if category not in enabled: | |
| continue | |
| category_probabilities = values[:, category_index, :] | |
| tag_ids = np.argmax(category_probabilities, axis=1) | |
| tag_probabilities = np.take_along_axis( | |
| category_probabilities, tag_ids[:, None], axis=1 | |
| )[:, 0] | |
| decoded.extend( | |
| decode_category( | |
| tag_ids.tolist(), | |
| tag_probabilities.tolist(), | |
| category=category, | |
| threshold=float(configured.get(category, default_threshold)), | |
| labels=labels, | |
| ) | |
| ) | |
| decoded.sort(key=lambda span: (span.start_index, span.end_index, span.category)) | |
| return decoded | |
| def decode_text_probabilities( | |
| probabilities, offset_mapping, *, min_span_chars: int = 0, **options | |
| ): | |
| """Decode transcript tokens, forcing special/padded empty offsets to O. | |
| Empty offsets carry no text and were ignored by training. Their logits must | |
| not open, extend, or suppress a transcript span. The input stays unchanged. | |
| ``min_span_chars`` (default ``0``) drops assembled spans whose exact | |
| half-open character width ``end_char - start_char`` is below the floor. | |
| This is a pre-stitch, per-category filter applied after BILOU assembly | |
| only; callers that stitch across windows must apply the filter themselves | |
| after their final stitch so a short fragment may still join a valid event. | |
| """ | |
| import numpy as np | |
| minimum = validate_min_span_chars(min_span_chars) | |
| values = np.asarray(probabilities, dtype="float64").copy() | |
| if values.ndim != 3 or len(offset_mapping) != len(values): | |
| raise ValueError("offset mapping must match the probability sequence") | |
| labels = options.get("labels", BILOU_LABELS) | |
| outside = list(labels).index("O") | |
| for index, (start, end) in enumerate(offset_mapping): | |
| if end <= start: | |
| values[index, :, :] = 0.0 | |
| values[index, :, outside] = 1.0 | |
| spans = decode_multi_head(values, **options) | |
| if minimum <= 0: | |
| return spans | |
| return [ | |
| span | |
| for span in spans | |
| if int(offset_mapping[span.end_index - 1][1]) | |
| - int(offset_mapping[span.start_index][0]) | |
| >= minimum | |
| ] | |