Neon-tech commited on
Commit
e49cd89
Β·
verified Β·
1 Parent(s): c68dde4

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +91 -174
app.py CHANGED
@@ -1,24 +1,25 @@
1
  import os
2
  import json
3
- import time
4
  import socket
5
  import threading
6
  import gc
7
- import ctypes
8
- import multiprocessing as mp
9
  from pathlib import Path
10
  import numpy as np
11
- import pyarrow.parquet as pq
12
- from tokenizers import Tokenizer
13
-
14
- # ── Config ───────────────────────────────────────────────────────────────────
15
- HF_TOKEN = os.environ.get("HF_TOKEN")
16
- STATE_FILE = "/data/state.json"
17
- RAW_DIR = "/data/raw"
18
- OUT_DIR = "/data/tokenized"
19
- TOK_PATH = "/data/tokenizer.json"
20
- WORKER_ID = socket.gethostname()
21
- POLL_INTERVAL = 15
 
 
 
 
22
 
23
  os.makedirs(OUT_DIR, exist_ok=True)
24
 
@@ -28,173 +29,89 @@ def serve():
28
  s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
29
  s.bind(("0.0.0.0", 7860))
30
  s.listen(5)
31
- print(f"βœ“ [{WORKER_ID}] Listening on port 7860")
32
  while True:
33
  conn, _ = s.accept()
34
  conn.send(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK")
35
  conn.close()
36
 
37
- # ── State ─────────────────────────────────────────────────────────────────────
38
- def load_state():
39
- with open(STATE_FILE) as f:
40
- return json.load(f)
41
-
42
- def save_state(state):
43
- tmp = STATE_FILE + f".tmp.{WORKER_ID}"
44
- with open(tmp, "w") as f:
45
- json.dump(state, f, indent=2)
46
- os.replace(tmp, STATE_FILE)
47
-
48
- # ── Claim a pending shard ─────────────────────────────────────────────────────
49
- def claim_shard(state):
50
- for name, info in state["shards"].items():
51
- if info["status"] == "pending":
52
- raw_path = Path(RAW_DIR) / name
53
- if raw_path.exists():
54
- info["status"] = "claimed"
55
- info["worker"] = WORKER_ID
56
- info["claimed_at"] = time.time()
57
- save_state(state)
58
- return name, raw_path
59
- return None, None
60
-
61
- # ── Tokenizer subprocess ──────────────────────────────────────────────────────
62
- _worker_tokenizer = None
63
-
64
- def init_worker(tok_path):
65
- global _worker_tokenizer
66
- _worker_tokenizer = Tokenizer.from_file(tok_path)
67
-
68
- def tokenize_chunk(texts):
69
- encs = _worker_tokenizer.encode_batch(texts)
70
- return [e.ids for e in encs if len(e.ids) >= 2]
71
-
72
- # ── Process shard ─────────────────────────────────────────────────────────────
73
- def process_shard(name, raw_path, pool):
74
- print(f" [{WORKER_ID}] Processing: {name}")
75
-
76
- out_name = name.replace(".parquet", ".bin")
77
- out_path = Path(OUT_DIR) / out_name
78
- tmp_path = Path(OUT_DIR) / f"{out_name}.tmp"
79
- total_tokens = 0
80
-
81
- try:
82
- pf = pq.ParquetFile(raw_path)
83
- except Exception as e:
84
- raw_path.unlink(missing_ok=True)
85
- return False, f"read_failed: {e}"
86
-
87
- try:
88
- with open(tmp_path, "wb") as f:
89
- for batch in pf.iter_batches(batch_size=5_000, columns=["text"]):
90
- texts = batch.column("text").to_pylist()
91
- mid = len(texts) // 2
92
-
93
- try:
94
- results = pool.map(tokenize_chunk, [texts[:mid], texts[mid:]])
95
- except Exception as e:
96
- tmp_path.unlink(missing_ok=True)
97
- return False, f"tokenize_failed: {e}"
98
-
99
- for ids in results[0] + results[1]:
100
- arr = np.array(ids, dtype=np.uint16)
101
- arr.tofile(f)
102
- total_tokens += len(ids)
103
-
104
- del texts, results
105
- gc.collect()
106
-
107
- except Exception as e:
108
- tmp_path.unlink(missing_ok=True)
109
- return False, f"write_failed: {e}"
110
-
111
- tmp_path.rename(out_path) # ← atomic, only visible when complete
112
- print(f" βœ“ [{WORKER_ID}] {out_name} | {total_tokens:,} tokens")
113
- return True, None
114
-
115
- # ── Force full memory flush ───────────────���───────────────────────────────────
116
- def flush_memory():
117
- gc.collect()
118
- try:
119
- ctypes.CDLL("libc.so.6").malloc_trim(0)
120
- except Exception:
121
- pass
122
-
123
- # ── Worker loop ───────────────────────────────────────────────────────────────
124
- def worker_loop():
125
- os.makedirs(OUT_DIR, exist_ok=True)
126
- print(f"βœ“ [{WORKER_ID}] Loading tokenizer...")
127
- tok = Tokenizer.from_file(TOK_PATH)
128
- print(f"βœ“ [{WORKER_ID}] Tokenizer ready | vocab: {tok.get_vocab_size():,}")
129
- del tok
130
- flush_memory()
131
-
132
- pool = mp.Pool(processes=2, initializer=init_worker, initargs=(TOK_PATH,))
133
- print(f"βœ“ [{WORKER_ID}] Worker pool ready")
134
-
135
- try:
136
- while True:
137
- if not os.path.exists(STATE_FILE):
138
- print(f" [{WORKER_ID}] Waiting for state.json...")
139
- time.sleep(POLL_INTERVAL)
140
- continue
141
-
142
- try:
143
- state = load_state()
144
- except Exception as e:
145
- print(f" [{WORKER_ID}] State read error: {e}")
146
- time.sleep(POLL_INTERVAL)
147
- continue
148
-
149
- total = len(state["shards"]) + len(state.get("queue", []))
150
- done = sum(1 for v in state["shards"].values() if v["status"] == "done")
151
- if total > 0 and done == total:
152
- print(f" [{WORKER_ID}] All done. Sleeping.")
153
- time.sleep(300)
154
- continue
155
-
156
- name, raw_path = claim_shard(state)
157
-
158
- if not name:
159
- print(f" [{WORKER_ID}] Nothing ready β€” polling in {POLL_INTERVAL}s")
160
- time.sleep(POLL_INTERVAL)
161
- continue
162
-
163
- print(f" [{WORKER_ID}] Claimed: {name}")
164
- success, error = process_shard(name, raw_path, pool)
165
-
166
- try:
167
- state = load_state()
168
- except Exception:
169
- pass
170
-
171
- if success:
172
- state["shards"][name]["status"] = "done"
173
- state["shards"][name]["error"] = None
174
- save_state(state)
175
- try:
176
- raw_path.unlink()
177
- print(f" [{WORKER_ID}] Deleted: {raw_path.name}")
178
- except Exception as e:
179
- print(f" [{WORKER_ID}] Delete failed: {e}")
180
- else:
181
- state["shards"][name]["status"] = "pending"
182
- state["shards"][name]["worker"] = None
183
- state["shards"][name]["claimed_at"] = None
184
- state["shards"][name]["error"] = error
185
- save_state(state)
186
- print(f" [{WORKER_ID}] Failed ({error}) β€” left on disk for retry: {name}")
187
-
188
- flush_memory()
189
- time.sleep(5)
190
-
191
- finally:
192
- pool.terminate()
193
- pool.join()
194
 
195
  # ── Entry point ───────────────────────────────────────────────────────────────
196
  if __name__ == "__main__":
197
  threading.Thread(target=serve, daemon=True).start()
198
- threading.Thread(target=worker_loop, daemon=True).start()
199
  while True:
200
- time.sleep(60)
 
1
  import os
2
  import json
 
3
  import socket
4
  import threading
5
  import gc
 
 
6
  from pathlib import Path
7
  import numpy as np
8
+
9
+ # ── Config ────────────────────────────────────────────────────────────────────
10
+ TOK_DIR = "/data/tokenized" # input: cc2025-05_000000.bin etc.
11
+ OUT_DIR = "/data/ready"
12
+ INDEX = "/data/index.json"
13
+
14
+ # Crawl ID β†’ output file mapping
15
+ CRAWL_MAP = {
16
+ "2025-05": "fw-00001-of-00006.bin",
17
+ "2025-08": "fw-00002-of-00006.bin",
18
+ "2025-13": "fw-00003-of-00006.bin",
19
+ "2025-18": "fw-00004-of-00006.bin",
20
+ "2025-21": "fw-00005-of-00006.bin",
21
+ "2025-28": "fw-00006-of-00006.bin",
22
+ }
23
 
24
  os.makedirs(OUT_DIR, exist_ok=True)
25
 
 
29
  s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
30
  s.bind(("0.0.0.0", 7860))
31
  s.listen(5)
32
+ print("βœ“ Listening on port 7860")
33
  while True:
34
  conn, _ = s.accept()
35
  conn.send(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK")
36
  conn.close()
37
 
38
+ # ── Merge ─────────────────────────────────────────────────────────────────────
39
+ def merge():
40
+ # Load existing index if resuming
41
+ if os.path.exists(INDEX):
42
+ with open(INDEX) as f:
43
+ index = json.load(f)
44
+ else:
45
+ index = {}
46
+
47
+ for crawl_id, out_name in CRAWL_MAP.items():
48
+ out_path = Path(OUT_DIR) / out_name
49
+ tmp_path = Path(OUT_DIR) / f"{out_name}.tmp"
50
+
51
+ if out_name in index:
52
+ print(f" SKIP {out_name} β€” already done")
53
+ continue
54
+
55
+ # Find all shard bins for this crawl
56
+ shards = sorted(Path(TOK_DIR).glob(f"cc{crawl_id}_*.bin"))
57
+ if not shards:
58
+ print(f" SKIP {out_name} β€” no shards found for {crawl_id}")
59
+ continue
60
+
61
+ print(f" Merging {len(shards)} shards β†’ {out_name}")
62
+ total_tokens = 0
63
+ total_bytes = 0
64
+
65
+ with open(tmp_path, "wb") as out_f:
66
+ for shard in shards:
67
+ # Stream shard in 256MB chunks to stay under RAM limit
68
+ shard_size = shard.stat().st_size
69
+ chunk_bytes = 256 * 1024 * 1024 # 256MB
70
+ offset = 0
71
+
72
+ with open(shard, "rb") as in_f:
73
+ while offset < shard_size:
74
+ to_read = min(chunk_bytes, shard_size - offset)
75
+ data = in_f.read(to_read)
76
+ out_f.write(data)
77
+ n_toks = len(data) // 2 # uint16 = 2 bytes each
78
+ total_tokens += n_toks
79
+ total_bytes += len(data)
80
+ offset += len(data)
81
+ del data
82
+ gc.collect()
83
+
84
+ print(f" βœ“ {shard.name}")
85
+
86
+ tmp_path.rename(out_path)
87
+
88
+ index[out_name] = {
89
+ "crawl": crawl_id,
90
+ "total_tokens": total_tokens,
91
+ "total_bytes": total_bytes,
92
+ "dtype": "uint16",
93
+ "shards_merged": len(shards),
94
+ }
95
+
96
+ # Save index after each file β€” safe to resume
97
+ with open(INDEX, "w") as f:
98
+ json.dump(index, f, indent=2)
99
+
100
+ print(f" βœ“ {out_name} | {total_tokens:,} tokens")
101
+ gc.collect()
102
+
103
+ # Final summary
104
+ print("\n── Index ────────────────────────────────────────")
105
+ grand_total = 0
106
+ for name, info in index.items():
107
+ print(f" {name}: {info['total_tokens']:,} tokens")
108
+ grand_total += info["total_tokens"]
109
+ print(f" TOTAL: {grand_total:,} tokens")
110
+ print("βœ“ All done")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
111
 
112
  # ── Entry point ───────────────────────────────────────────────────────────────
113
  if __name__ == "__main__":
114
  threading.Thread(target=serve, daemon=True).start()
115
+ merge()
116
  while True:
117
+ import time; time.sleep(60)