Fix SyntaxError: sposta from __future__ import annotations in cima al file
Browse files- gptommy_tokenizer.py +2 -2
gptommy_tokenizer.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
|
|
| 1 |
# ============================================================================
|
| 2 |
# LEGAL NOTICE & WATERMARK: R08 NON-COMMERCIAL RESEARCH & ETHICAL LICENSE v2.0
|
| 3 |
# Copyright (c) 2026 Thomas Seri (Sentinel). All Rights Reserved.
|
|
@@ -5,11 +6,11 @@
|
|
| 5 |
# Clean-room reverse engineering, distillation, commercial or military use
|
| 6 |
# is strictly prohibited and constitutes a breach of contract.
|
| 7 |
# ============================================================================
|
|
|
|
| 8 |
__author__ = "Thomas Seri (Sentinel)"
|
| 9 |
__license__ = "R08-Non-Commercial-Ethical-License-v2.0"
|
| 10 |
__jurisdiction__ = "Court of Milan, Italy"
|
| 11 |
|
| 12 |
-
#!/usr/bin/env python3
|
| 13 |
"""gpTommy 32k Tokenizer: Tokenizer nativo ad alta efficienza per la lingua italiana.
|
| 14 |
Autore & Ideatore: Thomas Seri (Sentinel)
|
| 15 |
Ecosistema: Sentinel LifeOS (https://www.sentinelhub.eu)
|
|
@@ -21,7 +22,6 @@ Vocabolario strutturato a schede (SSvvv):
|
|
| 21 |
- 1.925 caratteri singoli, punteggiatura, cifre e simboli matematici
|
| 22 |
- 1.069 token speciali di dialogo AI (<|user|>, <|assistant|>, <|system|>, <|think|>)
|
| 23 |
"""
|
| 24 |
-
from __future__ import annotations
|
| 25 |
|
| 26 |
import unicodedata
|
| 27 |
from pathlib import Path
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
# ============================================================================
|
| 3 |
# LEGAL NOTICE & WATERMARK: R08 NON-COMMERCIAL RESEARCH & ETHICAL LICENSE v2.0
|
| 4 |
# Copyright (c) 2026 Thomas Seri (Sentinel). All Rights Reserved.
|
|
|
|
| 6 |
# Clean-room reverse engineering, distillation, commercial or military use
|
| 7 |
# is strictly prohibited and constitutes a breach of contract.
|
| 8 |
# ============================================================================
|
| 9 |
+
from __future__ import annotations
|
| 10 |
__author__ = "Thomas Seri (Sentinel)"
|
| 11 |
__license__ = "R08-Non-Commercial-Ethical-License-v2.0"
|
| 12 |
__jurisdiction__ = "Court of Milan, Italy"
|
| 13 |
|
|
|
|
| 14 |
"""gpTommy 32k Tokenizer: Tokenizer nativo ad alta efficienza per la lingua italiana.
|
| 15 |
Autore & Ideatore: Thomas Seri (Sentinel)
|
| 16 |
Ecosistema: Sentinel LifeOS (https://www.sentinelhub.eu)
|
|
|
|
| 22 |
- 1.925 caratteri singoli, punteggiatura, cifre e simboli matematici
|
| 23 |
- 1.069 token speciali di dialogo AI (<|user|>, <|assistant|>, <|system|>, <|think|>)
|
| 24 |
"""
|
|
|
|
| 25 |
|
| 26 |
import unicodedata
|
| 27 |
from pathlib import Path
|