Cion-lab commited on
Commit
9c74162
·
verified ·
1 Parent(s): 1f792cf

Upload build/build_mix.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. build/build_mix.py +9 -0
build/build_mix.py CHANGED
@@ -42,6 +42,15 @@ import struct
42
  import sys
43
  import time
44
 
 
 
 
 
 
 
 
 
 
45
  MAGIC_FLAGS = 0
46
  TOKENS_PER_SHARD = 8_000_000 # 16 MB as uint16; ~140 shards for 1.1B tokens
47
  MERGE_BATCH_DOCS = 32 # granularity of the round-robin; ~20k tokens, so a 262k batch spans ~13
 
42
  import sys
43
  import time
44
 
45
+ # Sibling modules (hubsync, ounce100m_credentials) live beside this file, not in any package, and a job
46
+ # started as `python build/build_mix.py` gets only the script's own directory on sys.path -- which is
47
+ # build/, so the credentials import inside hubsync fails and the whole run refuses to be interruption-
48
+ # safe (E-021's lesson, hit again from the other direction). Add both this directory and its parent.
49
+ _HERE = os.path.dirname(os.path.abspath(__file__))
50
+ for _p in (_HERE, os.path.dirname(_HERE)):
51
+ if _p not in sys.path:
52
+ sys.path.insert(0, _p)
53
+
54
  MAGIC_FLAGS = 0
55
  TOKENS_PER_SHARD = 8_000_000 # 16 MB as uint16; ~140 shards for 1.1B tokens
56
  MERGE_BATCH_DOCS = 32 # granularity of the round-robin; ~20k tokens, so a 262k batch spans ~13