Neon-tech commited on
Commit
17794fc
Β·
verified Β·
1 Parent(s): e49cd89

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +10 -55
app.py CHANGED
@@ -37,77 +37,32 @@ def serve():
37
 
38
  # ── Merge ─────────────────────────────────────────────────────────────────────
39
  def merge():
40
- # Load existing index if resuming
41
- if os.path.exists(INDEX):
42
- with open(INDEX) as f:
43
- index = json.load(f)
44
- else:
45
- index = {}
46
-
47
  for crawl_id, out_name in CRAWL_MAP.items():
48
  out_path = Path(OUT_DIR) / out_name
49
- tmp_path = Path(OUT_DIR) / f"{out_name}.tmp"
50
-
51
- if out_name in index:
52
- print(f" SKIP {out_name} β€” already done")
53
  continue
54
 
55
- # Find all shard bins for this crawl
56
  shards = sorted(Path(TOK_DIR).glob(f"cc{crawl_id}_*.bin"))
57
  if not shards:
58
- print(f" SKIP {out_name} β€” no shards found for {crawl_id}")
59
  continue
60
 
61
  print(f" Merging {len(shards)} shards β†’ {out_name}")
62
  total_tokens = 0
63
- total_bytes = 0
64
 
65
- with open(tmp_path, "wb") as out_f:
66
  for shard in shards:
67
- # Stream shard in 256MB chunks to stay under RAM limit
68
- shard_size = shard.stat().st_size
69
- chunk_bytes = 256 * 1024 * 1024 # 256MB
70
- offset = 0
71
-
72
  with open(shard, "rb") as in_f:
73
- while offset < shard_size:
74
- to_read = min(chunk_bytes, shard_size - offset)
75
- data = in_f.read(to_read)
76
- out_f.write(data)
77
- n_toks = len(data) // 2 # uint16 = 2 bytes each
78
- total_tokens += n_toks
79
- total_bytes += len(data)
80
- offset += len(data)
81
- del data
82
- gc.collect()
83
-
84
  print(f" βœ“ {shard.name}")
85
 
86
- tmp_path.rename(out_path)
87
-
88
- index[out_name] = {
89
- "crawl": crawl_id,
90
- "total_tokens": total_tokens,
91
- "total_bytes": total_bytes,
92
- "dtype": "uint16",
93
- "shards_merged": len(shards),
94
- }
95
-
96
- # Save index after each file β€” safe to resume
97
- with open(INDEX, "w") as f:
98
- json.dump(index, f, indent=2)
99
-
100
  print(f" βœ“ {out_name} | {total_tokens:,} tokens")
101
- gc.collect()
102
-
103
- # Final summary
104
- print("\n── Index ────────────────────────────────────────")
105
- grand_total = 0
106
- for name, info in index.items():
107
- print(f" {name}: {info['total_tokens']:,} tokens")
108
- grand_total += info["total_tokens"]
109
- print(f" TOTAL: {grand_total:,} tokens")
110
- print("βœ“ All done")
111
 
112
  # ── Entry point ───────────────────────────────────────────────────────────────
113
  if __name__ == "__main__":
 
37
 
38
  # ── Merge ─────────────────────────────────────────────────────────────────────
39
  def merge():
 
 
 
 
 
 
 
40
  for crawl_id, out_name in CRAWL_MAP.items():
41
  out_path = Path(OUT_DIR) / out_name
42
+ if out_path.exists():
43
+ print(f" SKIP {out_name}")
 
 
44
  continue
45
 
 
46
  shards = sorted(Path(TOK_DIR).glob(f"cc{crawl_id}_*.bin"))
47
  if not shards:
48
+ print(f" SKIP {out_name} β€” no shards")
49
  continue
50
 
51
  print(f" Merging {len(shards)} shards β†’ {out_name}")
52
  total_tokens = 0
 
53
 
54
+ with open(out_path, "wb") as out_f:
55
  for shard in shards:
 
 
 
 
 
56
  with open(shard, "rb") as in_f:
57
+ while True:
58
+ chunk = in_f.read(4096)
59
+ if not chunk:
60
+ break
61
+ out_f.write(chunk)
62
+ total_tokens += len(chunk) // 2
 
 
 
 
 
63
  print(f" βœ“ {shard.name}")
64
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
65
  print(f" βœ“ {out_name} | {total_tokens:,} tokens")
 
 
 
 
 
 
 
 
 
 
66
 
67
  # ── Entry point ───────────────────────────────────────────────────────────────
68
  if __name__ == "__main__":