| """Print the columns, row counts and a sample row of every CSV on the essay-data volume.""" |
| import modal |
|
|
| app = modal.App("essay-coach-inspect") |
| vol = modal.Volume.from_name("essay-data") |
| image = modal.Image.debian_slim(python_version="3.11").pip_install("pandas", "openpyxl") |
|
|
|
|
| @app.function(image=image, volumes={"/data": vol}, timeout=900, memory=16384) |
| def inspect(): |
| import glob |
| import os |
| import pandas as pd |
| for p in sorted(glob.glob("/data/**/*.csv", recursive=True)): |
| try: |
| df = pd.read_csv(p, low_memory=False) |
| except Exception as e: |
| print("==", p, "ERR", e); continue |
| print("==", p.replace("/data/", ""), f"{os.path.getsize(p)/1e6:.0f} MB rows {len(df)}") |
| print(" cols:", list(df.columns)[:40]) |
| r = df.iloc[0] |
| print(" row0:", {c: (str(v)[:60] + ("..." if len(str(v)) > 60 else "")) for c, v in r.items()}) |
| for c in df.columns: |
| if df[c].dtype == object and df[c].nunique() < 15: |
| print(f" {c}: {df[c].value_counts().to_dict()}") |
| print(open("/data/ELLIPSE-Corpus/README.md").read()[:1500]) |
| for p in glob.glob("/data/**/*.zip", recursive=True): |
| print("zip:", p, os.path.getsize(p)) |
|
|
|
|
| @app.local_entrypoint() |
| def main(): |
| inspect.remote() |
|
|
