Neon-tech commited on
Commit
a91f4c1
Β·
verified Β·
1 Parent(s): 3594852

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +37 -26
app.py CHANGED
@@ -1,45 +1,56 @@
1
- import os
2
  import re
3
  from pathlib import Path
4
  import numpy as np
5
 
6
  TOKENIZED_DIR = "/data/tokenized"
7
 
8
- # group chunks by base name
9
- # cc2025-05_000000_chunk000.bin β†’ cc2025-05_000000.bin
10
- groups = {}
11
  for f in sorted(Path(TOKENIZED_DIR).glob("*.bin")):
12
- m = re.match(r"(.+)_chunk\d+\.bin$", f.name)
 
 
13
  if m:
14
- base = m.group(1) + ".bin"
15
- groups.setdefault(base, []).append(f)
16
 
17
- print(f"Found {len(groups)} base files to assemble")
18
 
19
- for base_name, chunks in sorted(groups.items()):
20
- out_path = Path(TOKENIZED_DIR) / base_name
21
  if out_path.exists():
22
- print(f" βœ“ Already exists, skipping: {base_name}")
23
  continue
24
 
25
- chunks_sorted = sorted(chunks, key=lambda f: int(re.search(r"_chunk(\d+)\.bin$", f.name).group(1)))
26
- tmp_path = Path(TOKENIZED_DIR) / f"{base_name}.tmp"
 
 
 
 
 
 
 
27
 
28
- print(f" Merging {len(chunks_sorted)} chunks β†’ {base_name}")
 
 
29
  total_tokens = 0
30
 
31
- with open(tmp_path, "wb") as out:
32
- for chunk in chunks_sorted:
33
- arr = np.fromfile(chunk, dtype=np.uint16)
34
- arr.tofile(out)
35
- total_tokens += len(arr)
36
- del arr
 
 
 
 
 
 
37
 
38
  tmp_path.rename(out_path)
39
- print(f" βœ“ {base_name} | {total_tokens:,} tokens")
40
-
41
- # delete chunks after successful merge
42
- for chunk in chunks_sorted:
43
- chunk.unlink()
44
 
45
- print("Done")
 
 
1
  import re
2
  from pathlib import Path
3
  import numpy as np
4
 
5
  TOKENIZED_DIR = "/data/tokenized"
6
 
7
+ # ── Group files by month ──────────────────────────────────────────────────────
8
+ months = {}
 
9
  for f in sorted(Path(TOKENIZED_DIR).glob("*.bin")):
10
+ if "_chunk" in f.name:
11
+ continue
12
+ m = re.match(r"(cc\d{4}-\d+)_", f.name)
13
  if m:
14
+ month = m.group(1)
15
+ months.setdefault(month, []).append(f)
16
 
17
+ print(f"Found {len(months)} months")
18
 
19
+ for month, files in sorted(months.items()):
20
+ out_path = Path(TOKENIZED_DIR) / f"{month}.bin"
21
  if out_path.exists():
22
+ print(f" βœ“ Already exists, skipping: {month}.bin")
23
  continue
24
 
25
+ def sort_key(f):
26
+ m_shard = re.search(r"_(\d{6})\.bin$", f.name)
27
+ return int(m_shard.group(1)) if m_shard else 999999
28
+
29
+ files_sorted = sorted(files, key=sort_key)
30
+
31
+ if not files_sorted:
32
+ print(f" βœ— No merged files for {month}, skipping")
33
+ continue
34
 
35
+ print(f"\n {month}.bin β€” {len(files_sorted)} files")
36
+
37
+ tmp_path = Path(TOKENIZED_DIR) / f"{month}.bin.tmp"
38
  total_tokens = 0
39
 
40
+ try:
41
+ with open(tmp_path, "wb") as out:
42
+ for f in files_sorted:
43
+ arr = np.fromfile(f, dtype=np.uint16)
44
+ arr.tofile(out)
45
+ total_tokens += len(arr)
46
+ del arr
47
+ print(f" + {f.name} ({total_tokens:,} tokens so far)")
48
+ except Exception as e:
49
+ print(f" βœ— Failed: {e}")
50
+ tmp_path.unlink(missing_ok=True)
51
+ continue
52
 
53
  tmp_path.rename(out_path)
54
+ print(f" βœ“ {month}.bin | {total_tokens:,} tokens")
 
 
 
 
55
 
56
+ print("\nDone")