Neon-tech commited on
Commit
977352c
Β·
verified Β·
1 Parent(s): 2d5aeda

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +27 -39
app.py CHANGED
@@ -1,57 +1,45 @@
 
1
  import re
2
- import shutil
3
  from pathlib import Path
 
4
 
5
  TOKENIZED_DIR = "/data/tokenized"
6
 
7
- # ── Group files by month ──────────────────────────────────────────────────────
8
- months = {}
 
9
  for f in sorted(Path(TOKENIZED_DIR).glob("*.bin")):
10
- if "_chunk" in f.name:
11
- continue
12
- m = re.match(r"(cc\d{4}-\d+)_", f.name)
13
  if m:
14
- month = m.group(1)
15
- months.setdefault(month, []).append(f)
16
 
17
- print(f"Found {len(months)} months")
18
 
19
- for month, files in sorted(months.items()):
20
- out_path = Path(TOKENIZED_DIR) / f"{month}.bin"
21
  if out_path.exists():
22
- print(f" βœ“ Already exists, skipping: {month}.bin")
23
- continue
24
-
25
- def sort_key(f):
26
- m_shard = re.search(r"_(\d{6})\.bin$", f.name)
27
- return int(m_shard.group(1)) if m_shard else 999999
28
-
29
- files_sorted = sorted(files, key=sort_key)
30
-
31
- if not files_sorted:
32
- print(f" βœ— No merged files for {month}, skipping")
33
  continue
34
 
35
- print(f"\n {month}.bin β€” {len(files_sorted)} files")
 
36
 
37
- tmp_path = Path(TOKENIZED_DIR) / f"{month}.bin.tmp"
 
38
 
39
- try:
40
- with open(tmp_path, "wb") as out:
41
- for f in files_sorted:
42
- with open(f, "rb") as inp:
43
- shutil.copyfileobj(inp, out, length=64 * 1024 * 1024)
44
- print(f" + {f.name}")
45
- except Exception as e:
46
- print(f" βœ— Failed: {e}")
47
- tmp_path.unlink(missing_ok=True)
48
- continue
49
 
50
  tmp_path.rename(out_path)
51
- print(f" βœ“ {month}.bin complete")
52
 
53
- for f in files_sorted:
54
- f.unlink()
55
- print(f" πŸ—‘ Deleted: {f.name}")
56
 
57
- print("\nDone")
 
1
+ import os
2
  import re
 
3
  from pathlib import Path
4
+ import numpy as np
5
 
6
  TOKENIZED_DIR = "/data/tokenized"
7
 
8
+ # group chunks by base name
9
+ # cc2025-05_000000_chunk000.bin β†’ cc2025-05_000000.bin
10
+ groups = {}
11
  for f in sorted(Path(TOKENIZED_DIR).glob("*.bin")):
12
+ m = re.match(r"(.+)_chunk\d+\.bin$", f.name)
 
 
13
  if m:
14
+ base = m.group(1) + ".bin"
15
+ groups.setdefault(base, []).append(f)
16
 
17
+ print(f"Found {len(groups)} base files to assemble")
18
 
19
+ for base_name, chunks in sorted(groups.items()):
20
+ out_path = Path(TOKENIZED_DIR) / base_name
21
  if out_path.exists():
22
+ print(f" βœ“ Already exists, skipping: {base_name}")
 
 
 
 
 
 
 
 
 
 
23
  continue
24
 
25
+ chunks_sorted = sorted(chunks, key=lambda f: int(re.search(r"_chunk(\d+)\.bin$", f.name).group(1)))
26
+ tmp_path = Path(TOKENIZED_DIR) / f"{base_name}.tmp"
27
 
28
+ print(f" Merging {len(chunks_sorted)} chunks β†’ {base_name}")
29
+ total_tokens = 0
30
 
31
+ with open(tmp_path, "wb") as out:
32
+ for chunk in chunks_sorted:
33
+ arr = np.fromfile(chunk, dtype=np.uint16)
34
+ arr.tofile(out)
35
+ total_tokens += len(arr)
36
+ del arr
 
 
 
 
37
 
38
  tmp_path.rename(out_path)
39
+ print(f" βœ“ {base_name} | {total_tokens:,} tokens")
40
 
41
+ # delete chunks after successful merge
42
+ for chunk in chunks_sorted:
43
+ chunk.unlink()
44
 
45
+ print("Done")