Download source/tests/span.py from yunusemrejr/Franok: direct link, hf CLI and curl.
- Browser
- Download file 1.9 kB
-
https://huggingface.co/yunusemrejr/Franok/resolve/main/source/tests/span.py
- Command line
-
hf download hf://yunusemrejr/Franok/source/tests/span.py
-
curl -L -o span.py https://huggingface.co/yunusemrejr/Franok/resolve/main/source/tests/span.py
1.9 kB
| """Validate extractable UTF-8 boundaries, question conditioning, padding and abstention.""" | |
| import torch | |
| from franok.span import SpanReadout, labels, pack | |
| from franok.span import boundaries, pieces | |
| from tokenizers import Tokenizer | |
| def main(): | |
| torch.manual_seed(1308) | |
| tokenizer = Tokenizer.from_file("web/tokenizer.json") | |
| text = "IŞIK numarası 73. 😊 Aşı önemlidir." | |
| tok = tokenizer.encode(text).ids | |
| raw, ends, legal = boundaries(text, tok, pieces(tokenizer)) | |
| assert 0 in legal and raw[int(ends[-1]) :] == b"" | |
| head = SpanReadout(16, 12, 9) | |
| prefix = [1, 6, 3, 8, 9, 2, 3] | |
| rows = [ | |
| (prefix + tok + [4], len(prefix), legal, -1, -1), | |
| (prefix + [8, 9, 10, 4], len(prefix), [1, 1, 1, 1], 1, 2), | |
| ] | |
| ids, mask, meta = pack(rows) | |
| states = torch.randn(2, ids.shape[1], 16, requires_grad=True) | |
| scores = head(states, meta[:, 0], meta[:, 1], mask) | |
| gold = labels(meta, scores.shape[1], head.max_span) | |
| assert gold[0] == scores.shape[-1] - 1 and gold[1] == (len(prefix) + 2) * 9 + 1 | |
| assert scores[1, gold[1]] > -1e8 | |
| for j in range(len(tok)): | |
| for delta in range(9): | |
| valid = j >= delta and legal[j - delta] and legal[j + 1] | |
| assert (scores[0, (len(prefix) + j) * 9 + delta].item() > -1e8) == bool( | |
| valid | |
| ) | |
| longer = torch.cat([states, torch.randn(2, 9, 16)], dim=1) | |
| padded = head(longer, meta[:, 0], meta[:, 1], torch.nn.functional.pad(mask, (0, 9))) | |
| torch.testing.assert_close(padded[:, : scores.shape[-1] - 1], scores[:, :-1]) | |
| torch.testing.assert_close(padded[:, -1], scores[:, -1]) | |
| torch.nn.functional.cross_entropy(scores, gold).backward() | |
| assert torch.isfinite(states.grad).all() and states.grad[:, :4].abs().sum() > 0 | |
| print("Unicode spans, row padding, training labels and null score passed.") | |
| if __name__ == "__main__": | |
| main() | |