Spaces:
Running on Zero
Running on Zero
| """Text-generation quality metrics shared by training and evaluation.""" | |
| from __future__ import annotations | |
| from typing import Any | |
| def distinct_n(text: str, tokenizer: Any, n: int) -> float: | |
| """Return sliding token-level Distinct-n, excluding special tokens. | |
| Unlike the former repetition metric, adjacent windows overlap: for | |
| ``[A, B, C]`` the bigrams are ``(A, B)`` and ``(B, C)``. | |
| """ | |
| if n < 1: | |
| raise ValueError("n must be at least 1") | |
| special_ids = set(getattr(tokenizer, "all_special_ids", [])) | |
| tokens = [ | |
| token | |
| for token in tokenizer.encode(text, add_special_tokens=False) | |
| if token not in special_ids | |
| ] | |
| grams = [tuple(tokens[index : index + n]) for index in range(len(tokens) - n + 1)] | |
| return float(len(set(grams)) / len(grams)) if grams else 0.0 | |