File size: 1,292 Bytes
e1679fa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | """Print the columns, row counts and a sample row of every CSV on the essay-data volume."""
import modal
app = modal.App("essay-coach-inspect")
vol = modal.Volume.from_name("essay-data")
image = modal.Image.debian_slim(python_version="3.11").pip_install("pandas", "openpyxl")
@app.function(image=image, volumes={"/data": vol}, timeout=900, memory=16384)
def inspect():
import glob
import os
import pandas as pd
for p in sorted(glob.glob("/data/**/*.csv", recursive=True)):
try:
df = pd.read_csv(p, low_memory=False)
except Exception as e:
print("==", p, "ERR", e); continue
print("==", p.replace("/data/", ""), f"{os.path.getsize(p)/1e6:.0f} MB rows {len(df)}")
print(" cols:", list(df.columns)[:40])
r = df.iloc[0]
print(" row0:", {c: (str(v)[:60] + ("..." if len(str(v)) > 60 else "")) for c, v in r.items()})
for c in df.columns:
if df[c].dtype == object and df[c].nunique() < 15:
print(f" {c}: {df[c].value_counts().to_dict()}")
print(open("/data/ELLIPSE-Corpus/README.md").read()[:1500])
for p in glob.glob("/data/**/*.zip", recursive=True):
print("zip:", p, os.path.getsize(p))
@app.local_entrypoint()
def main():
inspect.remote()
|