Neon-tech commited on
Commit
2d5aeda
·
verified ·
1 Parent(s): a91f4c1

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +9 -8
app.py CHANGED
@@ -1,6 +1,6 @@
1
  import re
 
2
  from pathlib import Path
3
- import numpy as np
4
 
5
  TOKENIZED_DIR = "/data/tokenized"
6
 
@@ -35,22 +35,23 @@ for month, files in sorted(months.items()):
35
  print(f"\n {month}.bin — {len(files_sorted)} files")
36
 
37
  tmp_path = Path(TOKENIZED_DIR) / f"{month}.bin.tmp"
38
- total_tokens = 0
39
 
40
  try:
41
  with open(tmp_path, "wb") as out:
42
  for f in files_sorted:
43
- arr = np.fromfile(f, dtype=np.uint16)
44
- arr.tofile(out)
45
- total_tokens += len(arr)
46
- del arr
47
- print(f" + {f.name} ({total_tokens:,} tokens so far)")
48
  except Exception as e:
49
  print(f" ✗ Failed: {e}")
50
  tmp_path.unlink(missing_ok=True)
51
  continue
52
 
53
  tmp_path.rename(out_path)
54
- print(f" ✓ {month}.bin | {total_tokens:,} tokens")
 
 
 
 
55
 
56
  print("\nDone")
 
1
  import re
2
+ import shutil
3
  from pathlib import Path
 
4
 
5
  TOKENIZED_DIR = "/data/tokenized"
6
 
 
35
  print(f"\n {month}.bin — {len(files_sorted)} files")
36
 
37
  tmp_path = Path(TOKENIZED_DIR) / f"{month}.bin.tmp"
 
38
 
39
  try:
40
  with open(tmp_path, "wb") as out:
41
  for f in files_sorted:
42
+ with open(f, "rb") as inp:
43
+ shutil.copyfileobj(inp, out, length=64 * 1024 * 1024)
44
+ print(f" + {f.name}")
 
 
45
  except Exception as e:
46
  print(f" ✗ Failed: {e}")
47
  tmp_path.unlink(missing_ok=True)
48
  continue
49
 
50
  tmp_path.rename(out_path)
51
+ print(f" ✓ {month}.bin complete")
52
+
53
+ for f in files_sorted:
54
+ f.unlink()
55
+ print(f" 🗑 Deleted: {f.name}")
56
 
57
  print("\nDone")