File size: 4,723 Bytes
1fa55f4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
"""
Master retraining script for NutriLoop AI.
Called by GitHub Actions nightly cron or manually.
Runs: Supabase data pull → Prophet training → Cluster training → HF upload.
"""
import os
import sys
from datetime import datetime
from pathlib import Path

# Ensure project root is in path
project_root = Path(__file__).parent.parent
sys.path.insert(0, str(project_root))

from dotenv import load_dotenv
load_dotenv()

from supabase import create_client


def run_retrain():
    """
    Execute the full NutriLoop retraining pipeline.

    Steps:
    1. Pull latest data from Supabase sales_logs
    2. Train Prophet models (train_prophet.py)
    3. Train KMeans clusters (train_clusters.py)
    4. Upload models to Hugging Face (upload_models.py)
    5. Log success/failure to Supabase retrain_log
    """
    print("=" * 60)
    print("[NutriLoop Retrain] Starting full retraining pipeline")
    print("=" * 60)
    start_time = datetime.now()

    supabase_url = os.environ.get("SUPABASE_URL")
    supabase_key = os.environ.get("SUPABASE_KEY")

    if not supabase_url or not supabase_key:
        print("[NutriLoop Retrain] ERROR: SUPABASE_URL and SUPABASE_KEY not set")
        sys.exit(1)

    client = create_client(supabase_url, supabase_key)

    # Step 1: Check data availability
    print("\n[Step 1] Checking Supabase data...")
    try:
        response = client.table("sales_logs").select("id").limit(1).execute()
        print("[Step 1] Supabase connection OK")
    except Exception as e:
        print(f"[Step 1] ERROR: Cannot connect to Supabase: {e}")
        _log_failure(client, str(e))
        sys.exit(1)

    # Step 2: Train Global Multivariate Model
    print("\n[Step 2] Training Global Multivariate Model...")
    try:
        from training.train_global import train_global_model
        success, registry = train_global_model()
        if not success:
            raise Exception("Global model training failed.")
        print(f"[Step 2] Global Model trained successfully")
    except Exception as e:
        print(f"[Step 2] ERROR: Global training failed: {e}")
        _log_failure(client, f"Prophet training: {e}")
        sys.exit(1)

    # Step 3: Train KMeans clusters
    print("\n[Step 3] Training KMeans clusters...")
    try:
        from training.train_clusters import train_clusters
        n_clusters, _ = train_clusters()
        print(f"[Step 3] Clusters: {n_clusters} clusters created")
    except Exception as e:
        print(f"[Step 3] WARNING: Cluster training failed: {e}")
        # Non-fatal - continue
        n_clusters = 0

    # Step 4: Upload to Hugging Face
    print("\n[Step 4] Uploading models to Hugging Face...")
    hf_uploaded = False
    try:
        from training.upload_models import upload_models
        upload_models()
        hf_uploaded = True
        print("[Step 4] HF upload complete")
    except Exception as e:
        print(f"[Step 4] WARNING: HF upload failed: {e}")
        # Non-fatal - continue

    # Step 5: Log success
    elapsed = (datetime.now() - start_time).total_seconds()
    avg_mae = sum(m["mae"] for m in registry.values()) / max(1, len(registry)) if registry else 0.0

    current_iso = datetime.now().isoformat()
    try:
        last_retrain_path = project_root / "models" / "last_retrain.txt"
        last_retrain_path.parent.mkdir(exist_ok=True)
        last_retrain_path.write_text(current_iso)
    except Exception as e:
        print(f"[Step 5] WARNING: Could not write last_retrain.txt: {e}")

    try:
        client.table("retrain_log").insert({
            "model_version": current_iso,
            "rows_used": len(registry),
            "mae_score": round(avg_mae, 4),
            "status": "success",
        }).execute()
    except Exception as e:
        print(f"[Step 5] WARNING: Could not log to retrain_log: {e}")

    # Summary
    hf_status = "Yes" if hf_uploaded else "No"
    print("\n" + "=" * 60)
    print(f"[NutriLoop Retrain] Complete in {elapsed:.1f}s")
    print(f"  Summary:")
    print(f"  Global model trained: {'Yes' if 'success' in locals() and success else 'No'}")
    print(f"  Cluster model trained: Yes")
    print(f"  Clusters: {n_clusters}")
    print(f"  Avg MAE: {avg_mae:.4f}")
    print(f"  Uploaded to HF: {hf_status}")
    print("=" * 60)


def _log_failure(client, error_msg: str) -> None:
    """Log a failed retrain run to Supabase."""
    try:
        client.table("retrain_log").insert({
            "model_version": datetime.now().isoformat(),
            "rows_used": 0,
            "mae_score": 0.0,
            "status": "failed",
            "error_msg": error_msg,
        }).execute()
    except Exception:
        pass


if __name__ == "__main__":
    run_retrain()