Buckets:

download
raw
15.9 kB
#!/usr/bin/env python3
"""
Demo Data Generator — SDK version.
Same as generate_demo_data.py but uses the yaai Python SDK instead of raw
HTTP calls. Useful for testing the SDK end-to-end and as a usage example.
Datasets:
- california_housing : California house price prediction (regression, 8 features, 20k samples)
- wine : Wine cultivar classification (3 classes, 13 chemical features)
- breast_cancer : Tumor malignancy classification (binary, 30 cell-nucleus features)
- all : Generate all three datasets
Authentication:
Set API_KEY env var to authenticate with an API key (recommended).
Without API_KEY, the client falls back to Google ADC (requires yaai[gcp]).
Usage:
API_KEY=your_api_key python scripts/generate_demo_data_sdk.py --mode full --dataset all
python scripts/generate_demo_data_sdk.py --mode full --dataset wine
python scripts/generate_demo_data_sdk.py --drop-all
"""
import argparse
import asyncio
import os
import sys
from datetime import datetime, timedelta
from pathlib import Path
import numpy as np
from sklearn.datasets import fetch_california_housing, load_breast_cancer, load_wine
from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
from yaai import YaaiClient
from yaai.schemas.model import SchemaFieldCreate
# Load .env file from project root if available
try:
from dotenv import load_dotenv
env_path = Path(__file__).resolve().parent.parent / ".env"
if env_path.exists():
load_dotenv(env_path)
except ImportError:
pass
BASE_URL = "http://localhost:8000/api/v1"
# ---------------------------------------------------------------------------
# Dataset registry
# ---------------------------------------------------------------------------
DATASETS = {
"california_housing": {
"loader": fetch_california_housing,
"model_name": "house-price-predictor",
"description": "Predicts California median house values from census block-group features (income, age, rooms, location).",
"version": "v1.0",
"task": "regression",
"output_name": "predicted_price",
},
"wine": {
"loader": load_wine,
"model_name": "wine-classifier",
"description": "Classifies wines into 3 cultivar classes from 13 chemical analysis measurements (alcohol, flavanoids, color, etc.).",
"version": "v1.0",
"task": "classification",
"output_name": "predicted_cultivar",
},
"breast_cancer": {
"loader": load_breast_cancer,
"model_name": "tumor-classifier",
"description": "Classifies breast tumors as malignant or benign from 30 digitized cell-nucleus measurements.",
"version": "v1.0",
"task": "classification",
"output_name": "predicted_diagnosis",
},
}
def sanitize_feature_name(name: str) -> str:
"""Make feature names API-safe (no spaces or slashes)."""
return name.replace(" ", "_").replace("/", "_").lower()
# ---------------------------------------------------------------------------
# ML helpers
# ---------------------------------------------------------------------------
def load_and_train(dataset_key: str):
"""Load a sklearn dataset and train a quick RandomForest model."""
config = DATASETS[dataset_key]
data = config["loader"]()
X, y = data.data, data.target
feature_names = [sanitize_feature_name(n) for n in data.feature_names]
target_names = [str(n) for n in data.target_names] if hasattr(data, "target_names") else None
if config["task"] == "regression":
model = RandomForestRegressor(n_estimators=50, random_state=42)
else:
model = RandomForestClassifier(n_estimators=50, random_state=42)
model.fit(X, y)
return X, y, feature_names, target_names, model
def apply_drift(X: np.ndarray, drift_factor: float, rng: np.random.Generator) -> np.ndarray:
"""Apply realistic drift: per-feature mean shift + increased noise."""
X_drifted = X.copy()
col_stds = np.std(X_drifted, axis=0)
# Deterministic per-feature drift directions (seeded from column index)
n_features = X_drifted.shape[1]
directions = np.array([1 if i % 3 != 0 else -1 for i in range(n_features)])
# Shift means
shifts = col_stds * drift_factor * 0.5 * directions
X_drifted += shifts
# Add proportional noise
noise_scale = col_stds * drift_factor * 0.15
noise = rng.normal(0, 1, X_drifted.shape) * noise_scale
X_drifted += noise
return X_drifted
# ---------------------------------------------------------------------------
# Record builders
# ---------------------------------------------------------------------------
def build_schema(feature_names: list[str], config: dict) -> list[SchemaFieldCreate]:
"""Build SDK schema objects from feature names and task type."""
schema = []
for name in feature_names:
schema.append(SchemaFieldCreate(field_name=name, direction="input", data_type="numerical"))
if config["task"] == "classification":
schema.append(SchemaFieldCreate(field_name=config["output_name"], direction="output", data_type="categorical"))
schema.append(SchemaFieldCreate(field_name="confidence", direction="output", data_type="numerical"))
else:
schema.append(SchemaFieldCreate(field_name=config["output_name"], direction="output", data_type="numerical"))
return schema
def make_records_batch(
X_batch: np.ndarray,
model,
feature_names: list[str],
config: dict,
target_names: list[str] | None,
) -> list[dict]:
"""Convert a batch of rows into API records with model predictions (one predict call)."""
# Single batched prediction call — avoids per-row overhead
predictions = model.predict(X_batch)
probas = model.predict_proba(X_batch) if config["task"] == "classification" else None
# Round all features at once
X_rounded = np.round(X_batch, 6)
records = []
output_name = config["output_name"]
is_classification = config["task"] == "classification"
for i in range(len(X_batch)):
inputs = {name: float(X_rounded[i, j]) for j, name in enumerate(feature_names)}
if is_classification and probas is not None:
label = target_names[int(predictions[i])] if target_names else str(int(predictions[i]))
outputs = {output_name: label, "confidence": round(float(np.max(probas[i])), 4)}
else:
outputs = {output_name: round(float(predictions[i]), 4)}
records.append({"inputs": inputs, "outputs": outputs})
return records
# ---------------------------------------------------------------------------
# Main generation logic
# ---------------------------------------------------------------------------
def _generate_week_records(
X: np.ndarray,
model,
feature_names: list[str],
config: dict,
target_names: list[str] | None,
week_start: datetime,
drift_factor: float,
records_per_day: int,
rng: np.random.Generator,
) -> list[dict]:
"""Generate one week of inference records with optional drift."""
all_X = []
timestamps = []
for day in range(7):
current_date = week_start + timedelta(days=day)
day_indices = rng.choice(X.shape[0], size=records_per_day, replace=True)
X_day = X[day_indices].copy()
if drift_factor > 0:
X_day = apply_drift(X_day, drift_factor, rng)
all_X.append(X_day)
for _ in range(records_per_day):
timestamps.append(
current_date
+ timedelta(
hours=int(rng.integers(0, 24)),
minutes=int(rng.integers(0, 60)),
seconds=int(rng.integers(0, 60)),
)
)
X_week = np.vstack(all_X)
records = make_records_batch(X_week, model, feature_names, config, target_names)
for rec, ts in zip(records, timestamps, strict=True):
rec["timestamp"] = ts.isoformat()
return records
async def generate_dataset(
client: YaaiClient,
dataset_key: str,
weeks: int = 8,
records_per_day: int = 50,
) -> None:
config = DATASETS[dataset_key]
print(f"\n{'=' * 60}")
print(f" {config['model_name']} ({config['task']})")
print(f" {config['description']}")
print(f"{'=' * 60}")
# ---- 1. Load & train ----
print("\n[1/6] Loading dataset and training model...")
X, y, feature_names, target_names, model = load_and_train(dataset_key)
print(f" Dataset: {X.shape[0]} samples, {X.shape[1]} features")
print(f" Model: {type(model).__name__} (50 trees)")
print(f" Training score: {model.score(X, y):.4f}")
# ---- 2. Create model + version ----
print("\n[2/6] Creating model and version...")
schema = build_schema(feature_names, config)
# Check for existing model with same name
existing_models = await client.list_models()
api_model = None
for m in existing_models:
if m.name == config["model_name"]:
print(f" Found existing model: {m.id}")
api_model = m
break
if api_model is None:
api_model = await client.create_model(config["model_name"], config["description"])
print(f" Created model: {api_model.id}")
version_obj = await client.create_model_version(
model_id=api_model.id,
version=config["version"],
schema_fields=schema,
)
version_id = version_obj.id
print(f" Version: {version_id}")
print(f" Schema: {len(schema)} fields ({len(feature_names)} inputs + {len(schema) - len(feature_names)} outputs)")
rng = np.random.default_rng(42)
# ---- 3. Reference data (no drift, clean baseline) ----
print("\n[3/6] Uploading reference data...")
n_ref = min(500, max(100, X.shape[0] // 5))
ref_indices = rng.choice(X.shape[0], size=n_ref, replace=False)
ref_records = make_records_batch(X[ref_indices], model, feature_names, config, target_names)
ref_result = await client.add_reference_data(api_model.id, version_id, ref_records)
print(f" Uploaded {ref_result.ingested} reference records (clean baseline)")
# ---- 4. Inference data: random weeks with drift ----
print("\n[4/6] Generating inference data with sporadic drift...")
start_date = datetime.now() - timedelta(weeks=weeks)
total_created = 0
# Randomly select ~25% of weeks to have drift
drift_probability = 0.25
max_drift_factor = 0.25 # Cap drift at 25%
# Pre-determine which weeks have drift (for reproducibility)
drift_weeks = set()
for week in range(weeks):
if rng.random() < drift_probability:
drift_weeks.add(week)
# Ensure at least 1 drift week if we have enough weeks
if len(drift_weeks) == 0 and weeks >= 4:
drift_weeks.add(int(rng.integers(weeks // 2, weeks)))
for week in range(weeks):
if week in drift_weeks:
drift_factor = rng.uniform(0.05, max_drift_factor)
else:
drift_factor = 0.0
week_start = start_date + timedelta(weeks=week)
week_records = _generate_week_records(
X,
model,
feature_names,
config,
target_names,
week_start,
drift_factor,
records_per_day,
rng,
)
batch_result = await client.add_inferences(version_id, week_records)
total_created += batch_result.ingested
drift_label = f"drift {drift_factor:.0%}" if drift_factor > 0 else "baseline"
print(f" Week {week + 1:2d}: {batch_result.ingested:5d} records ({drift_label})")
# ---- 5. Backfill drift detection ----
print("\n[5/6] Running drift backfill for historical data...")
job = await client.get_version_job(api_model.id, version_id)
total_runs = 0
if job and job["is_active"]:
result = await client.backfill_job(job["id"])
runs = result["runs_created"]
total_runs += runs
print(f" Job '{job['name']}': {runs} backfill runs")
print(f" Created {total_runs} historical drift runs")
# ---- 6. Summary ----
print("\n[6/6] Done!")
print(f" Reference: {ref_result.ingested} records")
print(f" Inference: {total_created} records across {weeks} weeks")
print(f" Backfill: {total_runs} drift runs")
print(f" Dashboard: http://localhost:3000/models/{api_model.id}/versions/{version_id}/dashboard")
async def drop_all_models(client: YaaiClient) -> int:
models = await client.list_models()
deleted = 0
for m in models:
try:
await client.delete_model(m.id)
print(f" Deleted: {m.name} ({m.id})")
deleted += 1
except Exception as e:
print(f" Failed to delete {m.name}: {e}")
return deleted
async def run(args: argparse.Namespace) -> None:
api_key = os.environ.get("API_KEY")
if api_key:
print(" Using API key from API_KEY environment variable")
else:
print(" No API_KEY set, falling back to Google ADC")
async with YaaiClient(args.api_url, api_key=api_key) as client:
# Test connection
try:
await client.list_models()
except Exception as e:
print(f"Cannot connect to API at {args.api_url}: {e}")
print("Make sure the backend is running (docker compose up)")
sys.exit(1)
if args.drop_all:
print("\nDeleting all existing models...")
deleted = await drop_all_models(client)
print(f"Deleted {deleted} models\n")
if not args.mode:
return
if args.mode == "full":
datasets = list(DATASETS.keys()) if args.dataset == "all" else [args.dataset]
for ds_key in datasets:
await generate_dataset(
client=client,
dataset_key=ds_key,
weeks=args.weeks,
records_per_day=args.records_per_day,
)
print(f"\n{'=' * 60}")
print("All done! View your models at http://localhost:3000/models")
print(f"{'=' * 60}\n")
def main():
parser = argparse.ArgumentParser(
description="Generate demo data using the yaai SDK (same output as generate_demo_data.py).",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Datasets:
california_housing California house price prediction (regression, 8 features)
wine Wine cultivar classification (3 classes, 13 features)
breast_cancer Tumor malignancy classification (binary, 30 features)
all Generate all three datasets
Examples:
API_KEY=... %(prog)s --mode full --dataset all
API_KEY=... %(prog)s --mode full --dataset wine --weeks 12 --records-per-day 80
%(prog)s --drop-all
%(prog)s --drop-all --mode full --dataset all
""",
)
parser.add_argument("--mode", choices=["full"], help="Generation mode")
parser.add_argument("--drop-all", action="store_true", help="Delete all existing models first")
parser.add_argument(
"--dataset",
choices=["all", "california_housing", "wine", "breast_cancer"],
default="all",
help="Which dataset(s) to generate (default: all)",
)
parser.add_argument("--weeks", type=int, default=8, help="Weeks of inference data (default: 8)")
parser.add_argument("--records-per-day", type=int, default=50, help="Records per day (default: 50)")
parser.add_argument("--api-url", type=str, default=BASE_URL, help=f"API base URL (default: {BASE_URL})")
args = parser.parse_args()
if not args.drop_all and not args.mode:
parser.error("Either --mode or --drop-all is required")
asyncio.run(run(args))
if __name__ == "__main__":
main()

Xet Storage Details

Size:
15.9 kB
·
Xet hash:
17facef835ac1da6dbef9a28f7a43515728b7f7b1e82936ec25536d4438a7e42

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.