| import torch | |
| text = open("all_data.txt", "r", encoding="utf-8").read() | |
| tokens = [ord(c) % 256 for c in text] | |
| torch.save(torch.tensor(tokens, dtype=torch.long), "tokens.pt") | |
| print("Saved", len(tokens), "tokens") | |
| import torch | |
| text = open("all_data.txt", "r", encoding="utf-8").read() | |
| tokens = [ord(c) % 256 for c in text] | |
| torch.save(torch.tensor(tokens, dtype=torch.long), "tokens.pt") | |
| print("Saved", len(tokens), "tokens") | |