codegen/generate_toolbox.py

#!/usr/bin/env python3
"""
generate_toolbox.py

Reads registry/*.yaml and regenerates derived artifacts:
  - .env.example (root, derived from providers)
  - directory scaffold (ai-platform/ tree)
  - clients/python + clients/typescript stub stubs
  - mcp/servers manifests stub
  - terraform/datarobot scaffold (kept as committed templates)

Run:  python codegen/generate_toolbox.py   (or: make generate)
"""
from __future__ import annotations

import os
import sys
from pathlib import Path

try:
    import yaml
except ImportError:
    print("PyYAML required:  pip install pyyaml", file=sys.stderr)
    sys.exit(1)

ROOT = Path(__file__).resolve().parent.parent
REGISTRY = ROOT / "registry"

# --- Tree derived from capabilities + providers -------------------------------
BASE_TREE = {
    "00-core": ["inference", "agents", "workflows", "memory", "planning", "evaluation", "orchestration"],
    "10-data": ["ingestion", "connectors", "datasets", "wrangling", "sql", "feature-engineering", "unstructured"],
    "20-predictive-ai": ["projects", "experiments", "autopilot", "time-series", "blueprints", "custom-models"],
    "30-generative-ai": ["llm", "prompts", "rag", "vector-databases", "embeddings", "guardrails"],
    "40-agentic-ai": ["agents", "agent-tools", "mcp", "tracing", "evaluation", "deployment"],
    "50-web-intelligence": ["search", "crawling", "scraping", "extraction", "citations"],
    "60-browser": ["browseruse", "navigation", "interaction", "extraction"],
    "70-tools": ["composio", "custom-tools", "tool-registry"],
    "80-mcp": ["client", "servers", "discovery", "authentication", "permissions"],
    "90-execution": ["sandbox", "containers", "python", "javascript", "distributed-inference"],
    "100-mlops": ["datarobot", "mlflow", "arixe", "langsmith"],
    "110-workloads": ["datarobot"],
    "120-governance": ["authentication", "authorization", "audit", "moderation", "guardrails", "compliance"],
    "130-deployment": ["datarobot", "kubernetes", "docker", "cloud", "ci-cd"],
    "140-apis": ["datarobot", "groq", "openai", "anthropic", "openapi", "mcp"],
    "150-sdks": ["python", "r", "typescript", "go", "rust", "java"],
    "160-cli": ["datarobot"],
    "170-ui": ["gradio", "vercel-ai", "datarobot-apps"],
    "180-voice": ["livekit"],
    "190-hardware": ["echokit"],
    "200-observability": ["opentelemetry", "datadog", "langsmith", "logs", "metrics", "traces"],
    "210-financial-data": ["filings", "companies", "webhooks", "schemas"],
    "220-ai-infrastructure": ["coreweave", "nvidia", "kubernetes", "gpu"],
    "230-geo": ["geocoding", "routing", "geofencing", "gnss", "ins", "sensor-fusion", "maps", "places", "schemas"],
    "900-registry": [],
}


def load_registry(name: str) -> dict:
    path = REGISTRY / name
    if not path.exists():
        return {}
    with open(path) as f:
        return yaml.safe_load(f) or {}


def gen_env(providers: dict) -> str:
    lines = ["# Auto-generated from registry/providers.yaml by codegen/generate_toolbox.py", ""]
    seen = set()
    for name, prov in providers.items():
        apis = prov.get("apis", {})
        vars_ = []
        for api, spec in apis.items():
            if isinstance(spec, dict):
                for key in ("base_env", "token_env", "enabled_flag"):
                    val = spec.get(key)
                    if val and val not in seen:
                        seen.add(val)
                        vars_.append(val)
        if vars_:
            lines.append(f"# ===== {prov.get('label', name)} ===== ")
            for v in vars_:
                lines.append(f'{v}="YOUR_{v}"')
            lines.append("")
    return "\n".join(lines)


def scaffold_tree() -> None:
    base = ROOT / "ai-platform"
    for folder, subs in BASE_TREE.items():
        (base / folder).mkdir(parents=True, exist_ok=True)
        for sub in subs:
            (base / folder / sub).mkdir(parents=True, exist_ok=True)
    # placeholder to keep git-ignorable dirs meaningful
    (base / "900-registry" / ".gitkeep").write_text("")
    print("Scaffolded ai-platform/ tree")


def main() -> int:
    providers = load_registry("providers.yaml").get("providers", {})
    env_path = ROOT / ".env.example"
    env_path.write_text(gen_env(providers))
    print(f"Wrote {env_path}")

    # Keep committed scaffolding in sync
    clients_py = ROOT / "clients/python"
    clients_py.mkdir(parents=True, exist_ok=True)
    for name in providers:
        (clients_py / f"{name}.py").touch(exist_ok=True)

    clients_ts = ROOT / "clients/typescript/src"
    clients_ts.mkdir(parents=True, exist_ok=True)
    for name in providers:
        (clients_ts / f"{name}.ts").touch(exist_ok=True)

    mcp_dir = ROOT / "mcp/servers"
    mcp_dir.mkdir(parents=True, exist_ok=True)

    scaffold_tree()
    print("Done.")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())