webxos commited on
Commit
b2070ef
·
verified ·
1 Parent(s): 884683f

Delete to_parquet.py

Browse files
Files changed (1) hide show
  1. to_parquet.py +0 -50
to_parquet.py DELETED
@@ -1,50 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Convert model.json to model.parquet.
3
-
4
- Requires: pandas, pyarrow
5
- pip install pandas pyarrow
6
- """
7
- import json
8
- import os
9
- import sys
10
-
11
- MODEL_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "model.json")
12
- OUT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "model.parquet")
13
-
14
-
15
- def main():
16
- try:
17
- import pandas as pd
18
- except ImportError:
19
- print("Error: pandas required. Install: pip install pandas pyarrow", file=sys.stderr)
20
- sys.exit(1)
21
-
22
- with open(MODEL_PATH, "r", encoding="utf-8") as f:
23
- model = json.load(f)
24
-
25
- rows = []
26
- for state, bucket in model.get("transitions", {}).items():
27
- for nxt, prob in bucket.items():
28
- rows.append({
29
- "state": state,
30
- "next_token": nxt,
31
- "probability": float(prob),
32
- "harmonic_seed": int(model.get("harmonic_seed", 0)),
33
- "format": model.get("format", "unknown"),
34
- "version": model.get("version", "0.0"),
35
- })
36
-
37
- if not rows:
38
- print("Warning: no transitions. Writing metadata-only parquet.")
39
- rows = [{"state": "", "next_token": "", "probability": 0.0,
40
- "harmonic_seed": int(model.get("harmonic_seed", 0)),
41
- "format": model.get("format", "unknown"),
42
- "version": model.get("version", "0.0")}]
43
-
44
- df = pd.DataFrame(rows)
45
- df.to_parquet(OUT_PATH, index=False)
46
- print("Wrote " + str(len(df)) + " rows to " + OUT_PATH)
47
-
48
-
49
- if __name__ == "__main__":
50
- main()