jsture commited on
Commit
891e224
·
verified ·
1 Parent(s): b08272b

Add APE SELFIES tokenizer with max length 256

Browse files
Files changed (4) hide show
  1. metadata.json +28 -0
  2. tokenization_ape.py +240 -55
  3. tokenizer_config.json +1 -2
  4. vocab.json +0 -0
metadata.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "ape_source": "modernmolbert.local",
3
+ "created_at_utc": "2026-05-19T16:24:35.971419+00:00",
4
+ "creation_command": "python -m modernmolbert.train_ape_tokenizer",
5
+ "dataset_name": "data/pretrain/chembl36_selfies",
6
+ "extra_vocab_selfies_path": null,
7
+ "extra_vocab_symbols_added": 42,
8
+ "extra_vocab_symbols_path": "tokenizer/extra_symbols/benchmark_missing_selfies_symbols_min10.txt",
9
+ "extra_vocab_symbols_requested": 42,
10
+ "max_merge_pieces": 2,
11
+ "max_vocab_size": 2000,
12
+ "min_freq_for_merge": 3000,
13
+ "representation": "SELFIES",
14
+ "seed": 42,
15
+ "selfies_column": "selfies",
16
+ "shuffle_buffer_size": 100000,
17
+ "special_ids": {
18
+ "bos_token": 0,
19
+ "eos_token": 2,
20
+ "mask_token": 4,
21
+ "pad_token": 1,
22
+ "unk_token": 3
23
+ },
24
+ "tokenizer_path": "tokenizer/chembl36_selfies_2m_ape_max2_min3000.json",
25
+ "tokenizer_sha256": "26ad0e90de9c0a469eb6e3b7aa985e2a2d7cc3f0dd7b2b229ccfce41f639e208",
26
+ "tokenizer_train_size": 2000000,
27
+ "vocab_size": 631
28
+ }
tokenization_ape.py CHANGED
@@ -7,6 +7,7 @@ and loaded by ``AutoTokenizer.from_pretrained(..., trust_remote_code=True)``.
7
  import json
8
  import os
9
  import re
 
10
  from collections import defaultdict
11
  from pathlib import Path
12
  from typing import Any, Literal
@@ -16,7 +17,11 @@ from transformers import PreTrainedTokenizer
16
 
17
  Representation = Literal["SELFIES", "SMILES"]
18
 
19
- VOCAB_FILES_NAMES = {"vocab_file": "vocab.json"}
 
 
 
 
20
  SELFIES_RE = re.compile(r"\[[^\]]+\]")
21
  SMILES_RE = re.compile(
22
  r"(\[[^\]]+\]|Br?|Cl?|Si?|Se?|Li?|Na?|Mg?|Al?|Ca?|Fe?|Zn?|"
@@ -27,6 +32,35 @@ SMILES_RE = re.compile(
27
  )
28
 
29
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
  def _normalize_representation(representation: str) -> Representation:
31
  normalized = representation.upper()
32
  if normalized not in {"SELFIES", "SMILES"}:
@@ -34,6 +68,20 @@ def _normalize_representation(representation: str) -> Representation:
34
  return normalized # type: ignore[return-value]
35
 
36
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
37
  def pre_tokenize_molecule(molecule: str, representation: str) -> list[str]:
38
  active_representation = _normalize_representation(representation)
39
  if active_representation == "SELFIES":
@@ -56,23 +104,35 @@ def ape_tokenize(
56
  vocab: dict[str, int],
57
  representation: str,
58
  unk_token: str = "<unk>",
 
59
  ) -> list[str]:
60
  pieces = pre_tokenize_molecule(text, representation)
61
  if not pieces:
62
  return [unk_token]
63
 
 
 
 
 
64
  tokens: list[str] = []
 
 
 
65
  i = 0
66
- while i < len(pieces):
67
- for j in range(len(pieces), i, -1):
68
- possible_match = "".join(pieces[i:j])
69
- if possible_match in vocab:
70
- tokens.append(possible_match)
 
 
 
71
  i = j
72
  break
73
  else:
74
- tokens.append(unk_token)
75
  i += 1
 
76
  return tokens
77
 
78
 
@@ -85,18 +145,28 @@ class APEPreTrainedTokenizer(PreTrainedTokenizer):
85
  def __init__(
86
  self,
87
  vocab_file: str | os.PathLike[str] | None = None,
88
- vocab: dict[str, int] | None = None,
 
 
89
  representation: str = "SELFIES",
90
  bos_token: str = "<s>",
91
  eos_token: str = "</s>",
92
  unk_token: str = "<unk>",
93
  pad_token: str = "<pad>",
94
  mask_token: str = "<mask>",
95
- model_max_length: int = 512,
96
  **kwargs,
97
  ) -> None:
 
 
 
 
 
 
 
 
98
  if vocab is None:
99
- if vocab_file is None:
100
  vocab = {
101
  bos_token: 0,
102
  pad_token: 1,
@@ -105,18 +175,29 @@ class APEPreTrainedTokenizer(PreTrainedTokenizer):
105
  mask_token: 4,
106
  }
107
  else:
108
- with open(vocab_file, encoding="utf-8") as f:
109
  vocab = json.load(f)
110
 
111
  if vocab is None:
112
  raise ValueError("Loaded vocabulary is None.")
113
 
114
- self.vocab_file = str(vocab_file) if vocab_file is not None else None
115
- self.vocab = {str(token): int(idx) for token, idx in vocab.items()}
 
 
 
 
 
 
 
 
 
 
 
116
  self.ids_to_tokens = {idx: token for token, idx in self.vocab.items()}
117
- self.representation = _normalize_representation(representation)
118
  self.vocabulary_frequency: dict[str, int] = {}
119
- self.pair_counts: dict[tuple[str, str] | str, int] = {}
 
120
 
121
  super().__init__(
122
  bos_token=bos_token,
@@ -140,8 +221,9 @@ class APEPreTrainedTokenizer(PreTrainedTokenizer):
140
 
141
  @vocabulary.setter
142
  def vocabulary(self, value: dict[str, int]) -> None:
143
- self.vocab = {str(token): int(idx) for token, idx in value.items()}
144
  self.update_reverse_vocabulary()
 
145
 
146
  @property
147
  def special_tokens(self) -> dict[str, int]:
@@ -162,7 +244,9 @@ class APEPreTrainedTokenizer(PreTrainedTokenizer):
162
  def special_tokens(self, value: dict[str, int]) -> None:
163
  for token, token_id in value.items():
164
  self.vocab.setdefault(str(token), int(token_id))
 
165
  self.update_reverse_vocabulary()
 
166
 
167
  def get_vocab(self) -> dict[str, int]:
168
  return dict(self.vocab)
@@ -170,17 +254,60 @@ class APEPreTrainedTokenizer(PreTrainedTokenizer):
170
  def update_reverse_vocabulary(self) -> None:
171
  self.ids_to_tokens = {idx: token for token, idx in self.vocab.items()}
172
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
173
  def pre_tokenize(self, molecule: str, representation: str | None = None) -> list[str]:
174
  return pre_tokenize_molecule(molecule, representation or self.representation)
175
 
176
  def _tokenize(self, text: str, **kwargs) -> list[str]:
 
177
  return ape_tokenize(
178
  text,
179
  vocab=self.vocab,
180
  representation=self.representation,
181
  unk_token=str(self.unk_token),
 
182
  )
183
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
184
  def _convert_token_to_id(self, token: str) -> int:
185
  return self.vocab.get(token, self.vocab[str(self.unk_token)])
186
 
@@ -320,6 +447,7 @@ class APEPreTrainedTokenizer(PreTrainedTokenizer):
320
 
321
  if added:
322
  self.update_reverse_vocabulary()
 
323
 
324
  return added
325
 
@@ -343,9 +471,9 @@ class APEPreTrainedTokenizer(PreTrainedTokenizer):
343
  tokenizer_config = json.load(f)
344
  else:
345
  tokenizer_config = {}
 
346
  tokenizer_config.update(
347
  {
348
- "tokenizer_class": self.__class__.__name__,
349
  "representation": self.representation,
350
  "model_max_length": self.model_max_length,
351
  "auto_map": {
@@ -380,8 +508,16 @@ class APEPreTrainedTokenizer(PreTrainedTokenizer):
380
  self.representation = _normalize_representation(representation)
381
  with open(file_path, encoding="utf-8") as f:
382
  vocab = json.load(f)
383
- self.vocab = {str(token): int(idx) for token, idx in vocab.items()}
 
 
 
 
 
 
 
384
  self.ids_to_tokens = {idx: token for token, idx in self.vocab.items()}
 
385
 
386
  def train(
387
  self,
@@ -390,47 +526,92 @@ class APEPreTrainedTokenizer(PreTrainedTokenizer):
390
  representation: str | None = None,
391
  max_vocab_size: int = 5000,
392
  min_freq_for_merge: int = 2000,
 
393
  save_checkpoint: bool = False,
394
  checkpoint_path: str = "checkpoint",
395
  checkpoint_interval: int = 500,
396
  ) -> None:
397
- self.representation = _normalize_representation(representation or type)
398
- text_padding = " " * 80
 
 
 
 
 
 
 
 
 
399
 
400
- print(f"Pretokenizing {self.representation}", end="\r")
401
- tokenized_corpus = [
402
- tokens for sentence in corpus if (tokens := self.pre_tokenize(str(sentence)))
403
- ]
 
404
  vocabulary_frequency: defaultdict[str, int] = defaultdict(int)
405
- for tokens in tokenized_corpus:
 
 
 
 
 
 
406
  for token in tokens:
407
  vocabulary_frequency[token] += 1
 
 
408
  print(
409
  f"Pretokenization complete, found {len(vocabulary_frequency)} tokens",
410
- end="\r",
411
  )
412
 
 
 
 
413
  pre_tokens_counts = len(vocabulary_frequency)
414
  merged_counter = len(vocabulary_frequency) + 1
 
 
 
 
415
  checkpoint_increment = checkpoint_interval
416
  batch = checkpoint_interval + pre_tokens_counts
 
 
 
 
 
 
 
 
417
 
418
  def get_most_common_pair(tokenized):
419
  pair_counts: defaultdict[tuple[str, str], int] = defaultdict(int)
420
  for tokens in tokenized:
421
  for i in range(len(tokens) - 1):
422
- pair_counts[(tokens[i], tokens[i + 1])] += 1
423
 
424
- merged_pair_counts: dict[tuple[str, str] | str, int] = {
425
- pair: count for pair, count in pair_counts.items()
426
- }
427
- self.pair_counts = merged_pair_counts
 
 
 
 
428
  if not pair_counts:
429
  return ("", ""), 0
430
- return max(pair_counts.items(), key=lambda x: x[1], default=(("", ""), 0))
 
 
 
 
 
 
 
431
 
432
  while True:
433
- if save_checkpoint and len(vocabulary_frequency) == batch:
434
  self.vocabulary_frequency = dict(vocabulary_frequency)
435
  self.vocab = {
436
  **{
@@ -449,6 +630,7 @@ class APEPreTrainedTokenizer(PreTrainedTokenizer):
449
  },
450
  }
451
  self.ids_to_tokens = {idx: token for token, idx in self.vocab.items()}
 
452
  checkpoint_dir = Path(checkpoint_path)
453
  checkpoint_dir.mkdir(parents=True, exist_ok=True)
454
  self.save_vocabulary_file(checkpoint_dir / f"checkpoint_{batch}.json")
@@ -457,51 +639,53 @@ class APEPreTrainedTokenizer(PreTrainedTokenizer):
457
  batch += checkpoint_increment
458
 
459
  if len(vocabulary_frequency) >= max_vocab_size:
460
- print("\rMax vocabulary achieved", text_padding)
461
  break
462
 
463
- if all(len(tokens) < 2 for tokens in tokenized_corpus):
464
- print("\rNo more mergeable pairs", text_padding)
465
  break
466
 
467
  most_common_pair, freq = get_most_common_pair(tokenized_corpus)
468
  if freq < min_freq_for_merge:
469
- print("\rNot enough frequency found", text_padding)
470
  break
471
 
472
  if not most_common_pair[0] or not most_common_pair[1]:
473
- print("\rNo valid merge pair found", text_padding)
474
  break
475
 
476
- merged_word = "".join(most_common_pair)
 
477
  if merged_word not in vocabulary_frequency:
478
  print(
479
  f"New merge found: {merged_word} {merged_counter}/{max_vocab_size} "
480
- f"{round(merged_counter / max_vocab_size * 100, 2)}%"
 
481
  )
482
  merged_counter += 1
483
- vocabulary_frequency[merged_word] = vocabulary_frequency.get(merged_word, 0) + freq
484
 
485
  new_tokenized_corpus = []
486
  for tokens in tokenized_corpus:
487
  new_tokens = []
488
- skip_next = False
489
- for i in range(len(tokens)):
490
- if skip_next:
491
- skip_next = False
492
- continue
493
-
494
  if (
495
- i < len(tokens) - 1
496
- and tokens[i] == most_common_pair[0]
497
- and tokens[i + 1] == most_common_pair[1]
498
  ):
499
- new_tokens.append(merged_word)
500
- skip_next = True
501
  else:
502
- new_tokens.append(tokens[i])
 
503
 
504
- new_tokenized_corpus.append(new_tokens)
 
505
 
506
  tokenized_corpus = new_tokenized_corpus
507
 
@@ -514,8 +698,9 @@ class APEPreTrainedTokenizer(PreTrainedTokenizer):
514
  str(self.mask_token): 4,
515
  **{word: idx for idx, word in enumerate(vocabulary_frequency.keys(), start=5)},
516
  }
 
517
  self.ids_to_tokens = {idx: token for token, idx in self.vocab.items()}
518
- print("\nTraining complete.")
519
 
520
  def train_from_iterator(self, iterator, *args, **kwargs) -> None:
521
  raise NotImplementedError("train_from_iterator is not implemented for APE")
 
7
  import json
8
  import os
9
  import re
10
+ from collections.abc import Mapping
11
  from collections import defaultdict
12
  from pathlib import Path
13
  from typing import Any, Literal
 
17
 
18
  Representation = Literal["SELFIES", "SMILES"]
19
 
20
+ VOCAB_FILES_NAMES = {
21
+ "vocab_file": "vocab.json",
22
+ "selfies_vocab_file": "selfies_vocab.json",
23
+ "smiles_vocab_file": "smiles_vocab.json",
24
+ }
25
  SELFIES_RE = re.compile(r"\[[^\]]+\]")
26
  SMILES_RE = re.compile(
27
  r"(\[[^\]]+\]|Br?|Cl?|Si?|Se?|Li?|Na?|Mg?|Al?|Ca?|Fe?|Zn?|"
 
32
  )
33
 
34
 
35
+ def _base_piece_count(token: str, representation: str) -> int:
36
+ """Count primitive molecular pieces in a vocab token."""
37
+ pieces = pre_tokenize_molecule(token, representation)
38
+ return max(1, len(pieces))
39
+
40
+
41
+ def _max_vocab_piece_span(vocab: dict[str, int], representation: str) -> int:
42
+ """Maximum number of primitive pieces covered by any non-special vocab token."""
43
+ max_span = 1
44
+ for token in vocab:
45
+ if token.startswith("<") and token.endswith(">"):
46
+ continue
47
+ max_span = max(max_span, _base_piece_count(token, representation))
48
+ return max_span
49
+
50
+
51
+ def _coerce_vocab(vocab: Mapping[str, Any]) -> dict[str, int]:
52
+ if not isinstance(vocab, Mapping):
53
+ raise ValueError("Vocabulary must be a JSON object mapping token strings to integer IDs.")
54
+ out = {str(token): int(idx) for token, idx in vocab.items()}
55
+ if len(set(out.values())) != len(out):
56
+ raise ValueError("Vocabulary token IDs must be unique.")
57
+ return out
58
+
59
+
60
+ def _token_text(token: Any) -> str:
61
+ return str(getattr(token, "content", token))
62
+
63
+
64
  def _normalize_representation(representation: str) -> Representation:
65
  normalized = representation.upper()
66
  if normalized not in {"SELFIES", "SMILES"}:
 
68
  return normalized # type: ignore[return-value]
69
 
70
 
71
+ def _select_vocab_file(
72
+ *,
73
+ representation: Representation,
74
+ vocab_file: str | os.PathLike[str] | None,
75
+ selfies_vocab_file: str | os.PathLike[str] | None,
76
+ smiles_vocab_file: str | os.PathLike[str] | None,
77
+ ) -> str | os.PathLike[str] | None:
78
+ if representation == "SELFIES" and selfies_vocab_file is not None:
79
+ return selfies_vocab_file
80
+ if representation == "SMILES" and smiles_vocab_file is not None:
81
+ return smiles_vocab_file
82
+ return vocab_file
83
+
84
+
85
  def pre_tokenize_molecule(molecule: str, representation: str) -> list[str]:
86
  active_representation = _normalize_representation(representation)
87
  if active_representation == "SELFIES":
 
104
  vocab: dict[str, int],
105
  representation: str,
106
  unk_token: str = "<unk>",
107
+ max_piece_span: int | None = None,
108
  ) -> list[str]:
109
  pieces = pre_tokenize_molecule(text, representation)
110
  if not pieces:
111
  return [unk_token]
112
 
113
+ if max_piece_span is None:
114
+ max_piece_span = _max_vocab_piece_span(vocab, representation)
115
+
116
+ n = len(pieces)
117
  tokens: list[str] = []
118
+ append_token = tokens.append
119
+ vocab_contains = vocab.__contains__
120
+ join_pieces = "".join
121
  i = 0
122
+
123
+ while i < n:
124
+ upper = min(n, i + max_piece_span)
125
+
126
+ for j in range(upper, i, -1):
127
+ candidate = join_pieces(pieces[i:j])
128
+ if vocab_contains(candidate):
129
+ append_token(candidate)
130
  i = j
131
  break
132
  else:
133
+ append_token(unk_token)
134
  i += 1
135
+
136
  return tokens
137
 
138
 
 
145
  def __init__(
146
  self,
147
  vocab_file: str | os.PathLike[str] | None = None,
148
+ selfies_vocab_file: str | os.PathLike[str] | None = None,
149
+ smiles_vocab_file: str | os.PathLike[str] | None = None,
150
+ vocab: dict[str, Any] | None = None,
151
  representation: str = "SELFIES",
152
  bos_token: str = "<s>",
153
  eos_token: str = "</s>",
154
  unk_token: str = "<unk>",
155
  pad_token: str = "<pad>",
156
  mask_token: str = "<mask>",
157
+ model_max_length: int = 256,
158
  **kwargs,
159
  ) -> None:
160
+ self.representation = _normalize_representation(representation)
161
+ active_vocab_file = _select_vocab_file(
162
+ representation=self.representation,
163
+ vocab_file=vocab_file,
164
+ selfies_vocab_file=selfies_vocab_file,
165
+ smiles_vocab_file=smiles_vocab_file,
166
+ )
167
+
168
  if vocab is None:
169
+ if active_vocab_file is None:
170
  vocab = {
171
  bos_token: 0,
172
  pad_token: 1,
 
175
  mask_token: 4,
176
  }
177
  else:
178
+ with open(active_vocab_file, encoding="utf-8") as f:
179
  vocab = json.load(f)
180
 
181
  if vocab is None:
182
  raise ValueError("Loaded vocabulary is None.")
183
 
184
+ self.vocab_file = str(active_vocab_file) if active_vocab_file is not None else None
185
+ self.selfies_vocab_file = (
186
+ str(selfies_vocab_file) if selfies_vocab_file is not None else None
187
+ )
188
+ self.smiles_vocab_file = str(smiles_vocab_file) if smiles_vocab_file is not None else None
189
+ self.vocab = _coerce_vocab(vocab)
190
+ self._require_special_tokens(
191
+ bos_token=bos_token,
192
+ eos_token=eos_token,
193
+ unk_token=unk_token,
194
+ pad_token=pad_token,
195
+ mask_token=mask_token,
196
+ )
197
  self.ids_to_tokens = {idx: token for token, idx in self.vocab.items()}
 
198
  self.vocabulary_frequency: dict[str, int] = {}
199
+ self.pair_counts: dict[tuple[str, str], int] = {}
200
+ self._max_piece_span = _max_vocab_piece_span(self.vocab, self.representation)
201
 
202
  super().__init__(
203
  bos_token=bos_token,
 
221
 
222
  @vocabulary.setter
223
  def vocabulary(self, value: dict[str, int]) -> None:
224
+ self.vocab = _coerce_vocab(value)
225
  self.update_reverse_vocabulary()
226
+ self._refresh_tokenization_cache()
227
 
228
  @property
229
  def special_tokens(self) -> dict[str, int]:
 
244
  def special_tokens(self, value: dict[str, int]) -> None:
245
  for token, token_id in value.items():
246
  self.vocab.setdefault(str(token), int(token_id))
247
+ self.vocab = _coerce_vocab(self.vocab)
248
  self.update_reverse_vocabulary()
249
+ self._refresh_tokenization_cache()
250
 
251
  def get_vocab(self) -> dict[str, int]:
252
  return dict(self.vocab)
 
254
  def update_reverse_vocabulary(self) -> None:
255
  self.ids_to_tokens = {idx: token for token, idx in self.vocab.items()}
256
 
257
+ def _refresh_tokenization_cache(self) -> None:
258
+ self._max_piece_span = _max_vocab_piece_span(self.vocab, self.representation)
259
+
260
+ def _require_special_tokens(
261
+ self,
262
+ *,
263
+ bos_token: str,
264
+ eos_token: str,
265
+ unk_token: str,
266
+ pad_token: str,
267
+ mask_token: str,
268
+ ) -> None:
269
+ missing = [
270
+ token_text
271
+ for token in [bos_token, eos_token, unk_token, pad_token, mask_token]
272
+ if (token_text := _token_text(token)) not in self.vocab
273
+ ]
274
+ if missing:
275
+ raise ValueError(f"Vocabulary is missing required special tokens: {missing}")
276
+
277
  def pre_tokenize(self, molecule: str, representation: str | None = None) -> list[str]:
278
  return pre_tokenize_molecule(molecule, representation or self.representation)
279
 
280
  def _tokenize(self, text: str, **kwargs) -> list[str]:
281
+
282
  return ape_tokenize(
283
  text,
284
  vocab=self.vocab,
285
  representation=self.representation,
286
  unk_token=str(self.unk_token),
287
+ max_piece_span=self._max_piece_span,
288
  )
289
 
290
+ def encode_molecule(
291
+ self,
292
+ text: str,
293
+ add_special_tokens: bool = True,
294
+ max_length: int | None = None,
295
+ truncation: bool = True,
296
+ ) -> list[int]:
297
+ """Fast molecular encode path avoiding generic Hugging Face tokenizer overhead."""
298
+
299
+ tokens = self._tokenize(text)
300
+
301
+ ids = [self._convert_token_to_id(token) for token in tokens]
302
+
303
+ if add_special_tokens:
304
+ ids = self.build_inputs_with_special_tokens(ids)
305
+
306
+ if max_length is not None and truncation:
307
+ ids = ids[:max_length]
308
+
309
+ return ids
310
+
311
  def _convert_token_to_id(self, token: str) -> int:
312
  return self.vocab.get(token, self.vocab[str(self.unk_token)])
313
 
 
447
 
448
  if added:
449
  self.update_reverse_vocabulary()
450
+ self._refresh_tokenization_cache()
451
 
452
  return added
453
 
 
471
  tokenizer_config = json.load(f)
472
  else:
473
  tokenizer_config = {}
474
+ tokenizer_config.pop("tokenizer_class", None)
475
  tokenizer_config.update(
476
  {
 
477
  "representation": self.representation,
478
  "model_max_length": self.model_max_length,
479
  "auto_map": {
 
508
  self.representation = _normalize_representation(representation)
509
  with open(file_path, encoding="utf-8") as f:
510
  vocab = json.load(f)
511
+ self.vocab = _coerce_vocab(vocab)
512
+ self._require_special_tokens(
513
+ bos_token=str(self.bos_token),
514
+ eos_token=str(self.eos_token),
515
+ unk_token=str(self.unk_token),
516
+ pad_token=str(self.pad_token),
517
+ mask_token=str(self.mask_token),
518
+ )
519
  self.ids_to_tokens = {idx: token for token, idx in self.vocab.items()}
520
+ self._refresh_tokenization_cache()
521
 
522
  def train(
523
  self,
 
526
  representation: str | None = None,
527
  max_vocab_size: int = 5000,
528
  min_freq_for_merge: int = 2000,
529
+ max_merge_pieces: int | None = 8,
530
  save_checkpoint: bool = False,
531
  checkpoint_path: str = "checkpoint",
532
  checkpoint_interval: int = 500,
533
  ) -> None:
534
+ import warnings
535
+
536
+ new_rep = _normalize_representation(representation or type)
537
+ if new_rep != self.representation:
538
+ warnings.warn(
539
+ f"train() representation={new_rep!r} differs from tokenizer "
540
+ f"representation={self.representation!r}. Overwriting.",
541
+ UserWarning,
542
+ stacklevel=2,
543
+ )
544
+ self.representation = new_rep
545
 
546
+ if not corpus:
547
+ raise ValueError("Cannot train APE tokenizer on an empty corpus.")
548
+
549
+ print(f"Pretokenizing {self.representation}...", flush=True)
550
+ tokenized_corpus = []
551
  vocabulary_frequency: defaultdict[str, int] = defaultdict(int)
552
+ saw_tokens = False
553
+
554
+ for sentence in corpus:
555
+ tokens = self.pre_tokenize(str(sentence))
556
+ if not tokens:
557
+ continue
558
+ saw_tokens = True
559
  for token in tokens:
560
  vocabulary_frequency[token] += 1
561
+ if len(tokens) > 1:
562
+ tokenized_corpus.append(tokens)
563
  print(
564
  f"Pretokenization complete, found {len(vocabulary_frequency)} tokens",
565
+ flush=True,
566
  )
567
 
568
+ if not saw_tokens:
569
+ raise ValueError("Cannot train APE tokenizer on an empty corpus.")
570
+
571
  pre_tokens_counts = len(vocabulary_frequency)
572
  merged_counter = len(vocabulary_frequency) + 1
573
+ if save_checkpoint and checkpoint_interval <= 0:
574
+ raise ValueError(
575
+ "checkpoint_interval must be positive when save_checkpoint is enabled."
576
+ )
577
  checkpoint_increment = checkpoint_interval
578
  batch = checkpoint_interval + pre_tokens_counts
579
+ piece_count_cache: dict[str, int] = {}
580
+
581
+ def merged_piece_count(token: str) -> int:
582
+ count = piece_count_cache.get(token)
583
+ if count is None:
584
+ count = _base_piece_count(token, self.representation)
585
+ piece_count_cache[token] = count
586
+ return count
587
 
588
  def get_most_common_pair(tokenized):
589
  pair_counts: defaultdict[tuple[str, str], int] = defaultdict(int)
590
  for tokens in tokenized:
591
  for i in range(len(tokens) - 1):
592
+ pair = (tokens[i], tokens[i + 1])
593
 
594
+ if max_merge_pieces is not None:
595
+ merged_candidate = pair[0] + pair[1]
596
+ if merged_piece_count(merged_candidate) > max_merge_pieces:
597
+ continue
598
+
599
+ pair_counts[pair] += 1
600
+
601
+ self.pair_counts = dict(pair_counts)
602
  if not pair_counts:
603
  return ("", ""), 0
604
+
605
+ most_common_pair = ("", "")
606
+ most_common_frequency = 0
607
+ for pair, count in pair_counts.items():
608
+ if count > most_common_frequency:
609
+ most_common_pair = pair
610
+ most_common_frequency = count
611
+ return most_common_pair, most_common_frequency
612
 
613
  while True:
614
+ if save_checkpoint and len(vocabulary_frequency) >= batch:
615
  self.vocabulary_frequency = dict(vocabulary_frequency)
616
  self.vocab = {
617
  **{
 
630
  },
631
  }
632
  self.ids_to_tokens = {idx: token for token, idx in self.vocab.items()}
633
+ self._refresh_tokenization_cache()
634
  checkpoint_dir = Path(checkpoint_path)
635
  checkpoint_dir.mkdir(parents=True, exist_ok=True)
636
  self.save_vocabulary_file(checkpoint_dir / f"checkpoint_{batch}.json")
 
639
  batch += checkpoint_increment
640
 
641
  if len(vocabulary_frequency) >= max_vocab_size:
642
+ print("Max vocabulary achieved", flush=True)
643
  break
644
 
645
+ if not tokenized_corpus:
646
+ print("No more mergeable pairs", flush=True)
647
  break
648
 
649
  most_common_pair, freq = get_most_common_pair(tokenized_corpus)
650
  if freq < min_freq_for_merge:
651
+ print("Not enough frequency found", flush=True)
652
  break
653
 
654
  if not most_common_pair[0] or not most_common_pair[1]:
655
+ print("No valid merge pair found", flush=True)
656
  break
657
 
658
+ left_token, right_token = most_common_pair
659
+ merged_word = left_token + right_token
660
  if merged_word not in vocabulary_frequency:
661
  print(
662
  f"New merge found: {merged_word} {merged_counter}/{max_vocab_size} "
663
+ f"{round(merged_counter / max_vocab_size * 100, 2)}%",
664
+ flush=True,
665
  )
666
  merged_counter += 1
667
+ vocabulary_frequency[merged_word] += freq
668
 
669
  new_tokenized_corpus = []
670
  for tokens in tokenized_corpus:
671
  new_tokens = []
672
+ append_token = new_tokens.append
673
+ i = 0
674
+ token_count = len(tokens)
675
+ while i < token_count:
 
 
676
  if (
677
+ i < token_count - 1
678
+ and tokens[i] == left_token
679
+ and tokens[i + 1] == right_token
680
  ):
681
+ append_token(merged_word)
682
+ i += 2
683
  else:
684
+ append_token(tokens[i])
685
+ i += 1
686
 
687
+ if len(new_tokens) > 1:
688
+ new_tokenized_corpus.append(new_tokens)
689
 
690
  tokenized_corpus = new_tokenized_corpus
691
 
 
698
  str(self.mask_token): 4,
699
  **{word: idx for idx, word in enumerate(vocabulary_frequency.keys(), start=5)},
700
  }
701
+
702
  self.ids_to_tokens = {idx: token for token, idx in self.vocab.items()}
703
+ self._refresh_tokenization_cache()
704
 
705
  def train_from_iterator(self, iterator, *args, **kwargs) -> None:
706
  raise NotImplementedError("train_from_iterator is not implemented for APE")
tokenizer_config.json CHANGED
@@ -51,9 +51,8 @@
51
  "bos_token": "<s>",
52
  "eos_token": "</s>",
53
  "mask_token": "<mask>",
54
- "model_max_length": 512,
55
  "pad_token": "<pad>",
56
  "representation": "SELFIES",
57
- "tokenizer_class": "APEPreTrainedTokenizer",
58
  "unk_token": "<unk>"
59
  }
 
51
  "bos_token": "<s>",
52
  "eos_token": "</s>",
53
  "mask_token": "<mask>",
54
+ "model_max_length": 256,
55
  "pad_token": "<pad>",
56
  "representation": "SELFIES",
 
57
  "unk_token": "<unk>"
58
  }
vocab.json CHANGED
The diff for this file is too large to render. See raw diff