Neon-tech commited on
Commit
fcaec83
Β·
verified Β·
1 Parent(s): 6a0d732

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +30 -53
app.py CHANGED
@@ -1,17 +1,12 @@
1
  import os
2
  import json
3
- import socket
4
- import threading
5
  import gc
6
  from pathlib import Path
7
- import numpy as np
8
 
9
- # ── Config ────────────────────────────────────────────────────────────────────
10
- TOK_DIR = "/data/tokenized" # input: cc2025-05_000000.bin etc.
11
  OUT_DIR = "/data/ready"
12
  INDEX = "/data/index.json"
13
 
14
- # Crawl ID β†’ output file mapping
15
  CRAWL_MAP = {
16
  "2025-05": "fw-00001-of-00006.bin",
17
  "2025-08": "fw-00002-of-00006.bin",
@@ -23,50 +18,32 @@ CRAWL_MAP = {
23
 
24
  os.makedirs(OUT_DIR, exist_ok=True)
25
 
26
- # ── Keep-alive ────────────────────────────────────────────────────────────────
27
- #def serve():
28
- s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
29
- s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
30
- s.bind(("0.0.0.0", 7860))
31
- s.listen(5)
32
- print("βœ“ Listening on port 7860")
33
- while True:
34
- conn, _ = s.accept()
35
- conn.send(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK")
36
- conn.close()
37
-
38
- # ── Merge ─────────────────────────────────────────────────────────────────────
39
- def merge():
40
- for crawl_id, out_name in CRAWL_MAP.items():
41
- out_path = Path(OUT_DIR) / out_name
42
- if out_path.exists():
43
- print(f" SKIP {out_name}")
44
- continue
45
-
46
- shards = sorted(Path(TOK_DIR).glob(f"cc{crawl_id}_*.bin"))
47
- if not shards:
48
- print(f" SKIP {out_name} β€” no shards")
49
- continue
50
-
51
- print(f" Merging {len(shards)} shards β†’ {out_name}")
52
- total_tokens = 0
53
-
54
- with open(out_path, "wb") as out_f:
55
- for shard in shards:
56
- with open(shard, "rb") as in_f:
57
- while True:
58
- chunk = in_f.read(4096)
59
- if not chunk:
60
- break
61
- out_f.write(chunk)
62
- total_tokens += len(chunk) // 2
63
- print(f" βœ“ {shard.name}")
64
-
65
- print(f" βœ“ {out_name} | {total_tokens:,} tokens")
66
-
67
- # ── Entry point ───────────────────────────────────────────────────────────────
68
- #if __name__ == "__main__":
69
- #threading.Thread(target=serve, daemon=True).start()
70
- # merge()
71
- # while True:
72
- # import time; time.sleep(60)
 
1
  import os
2
  import json
 
 
3
  import gc
4
  from pathlib import Path
 
5
 
6
+ TOK_DIR = "/data/tokenized"
 
7
  OUT_DIR = "/data/ready"
8
  INDEX = "/data/index.json"
9
 
 
10
  CRAWL_MAP = {
11
  "2025-05": "fw-00001-of-00006.bin",
12
  "2025-08": "fw-00002-of-00006.bin",
 
18
 
19
  os.makedirs(OUT_DIR, exist_ok=True)
20
 
21
+ print("Starting merge...")
22
+ print(f"TOK_DIR contents: {list(Path(TOK_DIR).iterdir())[:5]}")
23
+
24
+ for crawl_id, out_name in CRAWL_MAP.items():
25
+ out_path = Path(OUT_DIR) / out_name
26
+ if out_path.exists():
27
+ print(f" SKIP {out_name}")
28
+ continue
29
+
30
+ shards = sorted(Path(TOK_DIR).glob(f"cc{crawl_id}_*.bin"))
31
+ print(f" {crawl_id}: found {len(shards)} shards")
32
+ if not shards:
33
+ continue
34
+
35
+ total_tokens = 0
36
+ with open(out_path, "wb") as out_f:
37
+ for shard in shards:
38
+ with open(shard, "rb") as in_f:
39
+ while True:
40
+ chunk = in_f.read(4096)
41
+ if not chunk:
42
+ break
43
+ out_f.write(chunk)
44
+ total_tokens += len(chunk) // 2
45
+ print(f" βœ“ {shard.name}")
46
+
47
+ print(f" βœ“ {out_name} | {total_tokens:,} tokens")
48
+
49
+ print("Done")