Neon-tech commited on
Commit
1207717
·
verified ·
1 Parent(s): 00c1558

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +4 -37
app.py CHANGED
@@ -1,39 +1,6 @@
1
  import os
2
- import json
3
- from pathlib import Path
4
- import pyarrow.parquet as pq
5
 
6
- RAW_DIR = "/data/raw"
7
- STATE_FILE = "/data/state.json"
8
-
9
- deleted = []
10
- kept = []
11
-
12
- for f in Path(RAW_DIR).glob("*.parquet"):
13
- try:
14
- pq.read_schema(f) # lightweight — only reads footer, not full file
15
- kept.append(f.name)
16
- except Exception as e:
17
- print(f" ✗ Corrupt: {f.name} — {e}")
18
- f.unlink()
19
- deleted.append(f.name)
20
-
21
- print(f"\n✓ Kept: {len(kept)} | Deleted: {len(deleted)}")
22
-
23
- # Reset deleted shards back to pending in state.json
24
- if deleted and os.path.exists(STATE_FILE):
25
- with open(STATE_FILE) as f:
26
- state = json.load(f)
27
- reset = 0
28
- for name in deleted:
29
- if name in state["shards"]:
30
- state["shards"][name]["status"] = "pending"
31
- state["shards"][name]["worker"] = None
32
- state["shards"][name]["claimed_at"] = None
33
- state["shards"][name]["error"] = None
34
- reset += 1
35
- tmp = STATE_FILE + ".tmp"
36
- with open(tmp, "w") as f:
37
- json.dump(state, f, indent=2)
38
- os.replace(tmp, STATE_FILE)
39
- print(f"✓ Reset {reset} shards to pending in state.json")
 
1
  import os
2
+ import shutil
 
 
3
 
4
+ shutil.rmtree("/data/raw")
5
+ os.makedirs("/data/raw")
6
+ print("✓ Done")