Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /kraftaa_transformdash /run_transformations.py
| """ | |
| Run Transformations - Execute the full dbt-style transformation pipeline | |
| Runs Bronze โ Silver โ Gold layers in order | |
| """ | |
| import sys | |
| from pathlib import Path | |
| # Add parent directory to path for imports | |
| sys.path.append(str(Path(__file__).parent)) | |
| from transformations.model_loader import ModelLoader | |
| from orchestration.engine import TransformationEngine | |
| from orchestration.history import RunHistory | |
| from datetime import datetime | |
| def run_pipeline(): | |
| """Execute the full transformation pipeline""" | |
| print("\n" + "=" * 60) | |
| print("๐ RUNNING TRANSFORMATION PIPELINE") | |
| print("=" * 60 + "\n") | |
| # Load models from files | |
| models_dir = Path(__file__).parent / "models" | |
| loader = ModelLoader(models_dir=str(models_dir)) | |
| print("๐ Loading SQL models...") | |
| models = loader.load_all_models() | |
| print(f"โ Loaded {len(models)} models\n") | |
| # Validate dependencies (catches circular dependencies) | |
| print("๐ Validating dependencies...") | |
| try: | |
| from transformations.dag import DAG | |
| dag = DAG(models) | |
| print("โ No circular dependencies detected") | |
| print(f"โ Execution order determined for {len(models)} models\n") | |
| except ValueError as e: | |
| print(f"\nโ DEPENDENCY VALIDATION FAILED") | |
| print("=" * 60) | |
| print(f"Error: {e}") | |
| print("=" * 60) | |
| print("\n๐ก Common causes:") | |
| print(" โข Model A references Model B, and Model B references Model A") | |
| print(" โข Bronze layer referencing Silver/Gold layers") | |
| print(" โข Model referencing itself") | |
| print("\n๐ Fix by ensuring dependencies flow in one direction:") | |
| print(" Bronze (stg_*) โ Silver (int_*) โ Gold (fct_*)") | |
| raise | |
| # Create engine and run | |
| engine = TransformationEngine(models) | |
| context = engine.run(verbose=True) | |
| # Save to history | |
| run_id = datetime.now().strftime("run_%Y%m%d_%H%M%S") | |
| history = RunHistory() | |
| summary = context.get_summary() | |
| history.save_run(run_id, summary, context.logs) | |
| print(f"\n๐พ Run saved to history: {run_id}") | |
| return context | |
| if __name__ == "__main__": | |
| try: | |
| context = run_pipeline() | |
| # Exit with error code if any failures | |
| if context.get_summary()['failures'] > 0: | |
| print("\nโ ๏ธ Pipeline completed with failures") | |
| sys.exit(1) | |
| else: | |
| print("\nโ Pipeline completed successfully!") | |
| sys.exit(0) | |
| except Exception as e: | |
| print(f"\nโ Pipeline failed: {e}") | |
| import traceback | |
| traceback.print_exc() | |
| sys.exit(1) | |
Xet Storage Details
- Size:
- 2.63 kB
- Xet hash:
- 0025bf16fd8c587707af69f2a572aee9957c395b6fdeb8afe6ab0e4ee872aff5
ยท
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.