Buckets:

download
raw
2.63 kB
"""
Run Transformations - Execute the full dbt-style transformation pipeline
Runs Bronze โ†’ Silver โ†’ Gold layers in order
"""
import sys
from pathlib import Path
# Add parent directory to path for imports
sys.path.append(str(Path(__file__).parent))
from transformations.model_loader import ModelLoader
from orchestration.engine import TransformationEngine
from orchestration.history import RunHistory
from datetime import datetime
def run_pipeline():
"""Execute the full transformation pipeline"""
print("\n" + "=" * 60)
print("๐Ÿš€ RUNNING TRANSFORMATION PIPELINE")
print("=" * 60 + "\n")
# Load models from files
models_dir = Path(__file__).parent / "models"
loader = ModelLoader(models_dir=str(models_dir))
print("๐Ÿ“‚ Loading SQL models...")
models = loader.load_all_models()
print(f"โœ“ Loaded {len(models)} models\n")
# Validate dependencies (catches circular dependencies)
print("๐Ÿ” Validating dependencies...")
try:
from transformations.dag import DAG
dag = DAG(models)
print("โœ“ No circular dependencies detected")
print(f"โœ“ Execution order determined for {len(models)} models\n")
except ValueError as e:
print(f"\nโŒ DEPENDENCY VALIDATION FAILED")
print("=" * 60)
print(f"Error: {e}")
print("=" * 60)
print("\n๐Ÿ’ก Common causes:")
print(" โ€ข Model A references Model B, and Model B references Model A")
print(" โ€ข Bronze layer referencing Silver/Gold layers")
print(" โ€ข Model referencing itself")
print("\n๐Ÿ“‹ Fix by ensuring dependencies flow in one direction:")
print(" Bronze (stg_*) โ†’ Silver (int_*) โ†’ Gold (fct_*)")
raise
# Create engine and run
engine = TransformationEngine(models)
context = engine.run(verbose=True)
# Save to history
run_id = datetime.now().strftime("run_%Y%m%d_%H%M%S")
history = RunHistory()
summary = context.get_summary()
history.save_run(run_id, summary, context.logs)
print(f"\n๐Ÿ’พ Run saved to history: {run_id}")
return context
if __name__ == "__main__":
try:
context = run_pipeline()
# Exit with error code if any failures
if context.get_summary()['failures'] > 0:
print("\nโš ๏ธ Pipeline completed with failures")
sys.exit(1)
else:
print("\nโœ… Pipeline completed successfully!")
sys.exit(0)
except Exception as e:
print(f"\nโŒ Pipeline failed: {e}")
import traceback
traceback.print_exc()
sys.exit(1)

Xet Storage Details

Size:
2.63 kB
ยท
Xet hash:
0025bf16fd8c587707af69f2a572aee9957c395b6fdeb8afe6ab0e4ee872aff5

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.