kdirgul commited on
Commit
6117cbf
·
verified ·
1 Parent(s): 33ad0f9

faz1_01: --target_tokens bilimsel gosterim kabul (5e7/7.5e9)

Browse files
Files changed (1) hide show
  1. code/kod/faz1_01_tokenize_shard.py +1 -1
code/kod/faz1_01_tokenize_shard.py CHANGED
@@ -51,7 +51,7 @@ def text_of(rec):
51
  def main():
52
  ap = argparse.ArgumentParser()
53
  ap.add_argument("--source", required=True, choices=list(SOURCES))
54
- ap.add_argument("--target_tokens", type=int, default=5_000_000)
55
  ap.add_argument("--seq_len", type=int, default=2048)
56
  ap.add_argument("--seqs_per_shard", type=int, default=32768) # ~256MB @2048 int32
57
  ap.add_argument("--out_dir", default="kod/data/shards")
 
51
  def main():
52
  ap = argparse.ArgumentParser()
53
  ap.add_argument("--source", required=True, choices=list(SOURCES))
54
+ ap.add_argument("--target_tokens", type=lambda x: int(float(x)), default=5_000_000) # 5e7/7.5e9 kabul
55
  ap.add_argument("--seq_len", type=int, default=2048)
56
  ap.add_argument("--seqs_per_shard", type=int, default=32768) # ~256MB @2048 int32
57
  ap.add_argument("--out_dir", default="kod/data/shards")