Upload build/build_mix.py with huggingface_hub
Browse files- build/build_mix.py +9 -0
build/build_mix.py
CHANGED
|
@@ -42,6 +42,15 @@ import struct
|
|
| 42 |
import sys
|
| 43 |
import time
|
| 44 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
MAGIC_FLAGS = 0
|
| 46 |
TOKENS_PER_SHARD = 8_000_000 # 16 MB as uint16; ~140 shards for 1.1B tokens
|
| 47 |
MERGE_BATCH_DOCS = 32 # granularity of the round-robin; ~20k tokens, so a 262k batch spans ~13
|
|
|
|
| 42 |
import sys
|
| 43 |
import time
|
| 44 |
|
| 45 |
+
# Sibling modules (hubsync, ounce100m_credentials) live beside this file, not in any package, and a job
|
| 46 |
+
# started as `python build/build_mix.py` gets only the script's own directory on sys.path -- which is
|
| 47 |
+
# build/, so the credentials import inside hubsync fails and the whole run refuses to be interruption-
|
| 48 |
+
# safe (E-021's lesson, hit again from the other direction). Add both this directory and its parent.
|
| 49 |
+
_HERE = os.path.dirname(os.path.abspath(__file__))
|
| 50 |
+
for _p in (_HERE, os.path.dirname(_HERE)):
|
| 51 |
+
if _p not in sys.path:
|
| 52 |
+
sys.path.insert(0, _p)
|
| 53 |
+
|
| 54 |
MAGIC_FLAGS = 0
|
| 55 |
TOKENS_PER_SHARD = 8_000_000 # 16 MB as uint16; ~140 shards for 1.1B tokens
|
| 56 |
MERGE_BATCH_DOCS = 32 # granularity of the round-robin; ~20k tokens, so a 262k batch spans ~13
|