ser-inference / scripts /ge_validate_dataset.py
spacey-cadet's picture
docs: overhaul README and Dockerfile for production pipeline
dddb6a8
Raw History Blame Contribute Delete
2.83 kB
"""
One-time, local Great Expectations run against your training/eval set,
before each retrain — null checks, expected label distribution, expected
duration range. Free, no AWS/warehouse needed. Explicitly NOT continuous
ingestion validation (there's no Gate 1/Gate 2 pipeline here) — this is a
pre-retrain sanity gate you run by hand or in CI right before kicking off
a training job.
Usage:
python scripts/ge_validate_dataset.py --data-csv data/labels.csv
"""
import argparse
import sys
import great_expectations as gx
import pandas as pd
EXPECTED_LABELS = {"angry", "calm", "disgust", "fearful", "happy", "neutral", "sad", "surprised"}
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--data-csv", required=True)
ap.add_argument("--label-col", default="emotion")
ap.add_argument("--duration-col", default="duration_sec")
ap.add_argument("--min-duration", type=float, default=0.3)
ap.add_argument("--max-duration", type=float, default=15.0)
args = ap.parse_args()
df = pd.read_csv(args.data_csv)
context = gx.get_context(mode="ephemeral")
data_source = context.data_sources.add_pandas("training_set")
data_asset = data_source.add_dataframe_asset(name="labels")
batch_def = data_asset.add_batch_definition_whole_dataframe("full")
batch = batch_def.get_batch(batch_parameters={"dataframe": df})
failures = []
result = batch.validate(gx.expectations.ExpectColumnValuesToNotBeNull(column=args.label_col))
if not result.success:
failures.append(f"Nulls found in {args.label_col}")
result = batch.validate(gx.expectations.ExpectColumnDistinctValuesToBeInSet(
column=args.label_col, value_set=list(EXPECTED_LABELS)
))
if not result.success:
failures.append(f"Unexpected label values in {args.label_col} (expected subset of {EXPECTED_LABELS})")
if args.duration_col in df.columns:
result = batch.validate(gx.expectations.ExpectColumnValuesToBeBetween(
column=args.duration_col, min_value=args.min_duration, max_value=args.max_duration
))
if not result.success:
failures.append(f"Durations outside [{args.min_duration}, {args.max_duration}]s")
label_counts = df[args.label_col].value_counts()
print("Label distribution:")
print(label_counts)
imbalance_ratio = label_counts.max() / label_counts.min()
if imbalance_ratio > 5:
print(f"WARNING: class imbalance ratio {imbalance_ratio:.1f}x — expected given "
"the guide's noted Sad/Angry/Calm imbalance, but worth confirming it hasn't worsened.")
if failures:
print("FAILED checks:")
for f in failures:
print(f" - {f}")
sys.exit(1)
print("All Great Expectations checks passed.")
if __name__ == "__main__":
main()