faz1_01: --target_tokens bilimsel gosterim kabul (5e7/7.5e9)
Browse files
code/kod/faz1_01_tokenize_shard.py
CHANGED
|
@@ -51,7 +51,7 @@ def text_of(rec):
|
|
| 51 |
def main():
|
| 52 |
ap = argparse.ArgumentParser()
|
| 53 |
ap.add_argument("--source", required=True, choices=list(SOURCES))
|
| 54 |
-
ap.add_argument("--target_tokens", type=int, default=5_000_000)
|
| 55 |
ap.add_argument("--seq_len", type=int, default=2048)
|
| 56 |
ap.add_argument("--seqs_per_shard", type=int, default=32768) # ~256MB @2048 int32
|
| 57 |
ap.add_argument("--out_dir", default="kod/data/shards")
|
|
|
|
| 51 |
def main():
|
| 52 |
ap = argparse.ArgumentParser()
|
| 53 |
ap.add_argument("--source", required=True, choices=list(SOURCES))
|
| 54 |
+
ap.add_argument("--target_tokens", type=lambda x: int(float(x)), default=5_000_000) # 5e7/7.5e9 kabul
|
| 55 |
ap.add_argument("--seq_len", type=int, default=2048)
|
| 56 |
ap.add_argument("--seqs_per_shard", type=int, default=32768) # ~256MB @2048 int32
|
| 57 |
ap.add_argument("--out_dir", default="kod/data/shards")
|