Spaces:
Sleeping
Sleeping
updates
Browse files
app.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
import os
|
| 2 |
import httpx
|
| 3 |
-
from fastapi import FastAPI,
|
| 4 |
from fastapi.middleware.cors import CORSMiddleware
|
| 5 |
from huggingface_hub import HfApi
|
| 6 |
from train_lora_full import active_learning_loop
|
|
@@ -10,7 +10,7 @@ from train_lora_full import active_learning_loop
|
|
| 10 |
# ============================================================
|
| 11 |
app = FastAPI(
|
| 12 |
title="LoRA Active Learning Space (B)",
|
| 13 |
-
version="1.
|
| 14 |
description="Pulls embeddings from Space A, performs LoRA fine-tuning, uploads updated model, and notifies Space A."
|
| 15 |
)
|
| 16 |
|
|
@@ -29,6 +29,7 @@ HF_TOKEN = os.getenv("HF_TOKEN")
|
|
| 29 |
MODEL_REPO = os.getenv("MODEL_REPO", "lyi029/codebert-finetuned")
|
| 30 |
EMBEDDING_PATH = "./embeddings_log.jsonl" # 本地暫存檔案
|
| 31 |
|
|
|
|
| 32 |
# ============================================================
|
| 33 |
# 下載 embeddings 函式
|
| 34 |
# ============================================================
|
|
@@ -79,56 +80,55 @@ def health():
|
|
| 79 |
|
| 80 |
|
| 81 |
# ============================================================
|
| 82 |
-
# LoRA 訓練主流程
|
| 83 |
# ============================================================
|
| 84 |
@app.post("/train-lora")
|
| 85 |
-
async def train_lora(
|
| 86 |
-
"""
|
| 87 |
if not HF_TOKEN:
|
| 88 |
raise HTTPException(status_code=400, detail="HF_TOKEN not set in environment variables")
|
| 89 |
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
return {"status": "training started", "detail": "LoRA training loop triggered"}
|
| 132 |
|
| 133 |
|
| 134 |
# ============================================================
|
|
|
|
| 1 |
import os
|
| 2 |
import httpx
|
| 3 |
+
from fastapi import FastAPI, HTTPException
|
| 4 |
from fastapi.middleware.cors import CORSMiddleware
|
| 5 |
from huggingface_hub import HfApi
|
| 6 |
from train_lora_full import active_learning_loop
|
|
|
|
| 10 |
# ============================================================
|
| 11 |
app = FastAPI(
|
| 12 |
title="LoRA Active Learning Space (B)",
|
| 13 |
+
version="1.5",
|
| 14 |
description="Pulls embeddings from Space A, performs LoRA fine-tuning, uploads updated model, and notifies Space A."
|
| 15 |
)
|
| 16 |
|
|
|
|
| 29 |
MODEL_REPO = os.getenv("MODEL_REPO", "lyi029/codebert-finetuned")
|
| 30 |
EMBEDDING_PATH = "./embeddings_log.jsonl" # 本地暫存檔案
|
| 31 |
|
| 32 |
+
|
| 33 |
# ============================================================
|
| 34 |
# 下載 embeddings 函式
|
| 35 |
# ============================================================
|
|
|
|
| 80 |
|
| 81 |
|
| 82 |
# ============================================================
|
| 83 |
+
# LoRA 訓練主流程(同步執行版本)
|
| 84 |
# ============================================================
|
| 85 |
@app.post("/train-lora")
|
| 86 |
+
async def train_lora():
|
| 87 |
+
"""同步執行 LoRA Active Learning 訓練"""
|
| 88 |
if not HF_TOKEN:
|
| 89 |
raise HTTPException(status_code=400, detail="HF_TOKEN not set in environment variables")
|
| 90 |
|
| 91 |
+
print("🚀 Starting LoRA training via active_learning_loop() …")
|
| 92 |
+
|
| 93 |
+
# Step 1: 抓取最新 embeddings
|
| 94 |
+
success = await fetch_embeddings_from_space_a()
|
| 95 |
+
if not success:
|
| 96 |
+
print("⚠️ No embeddings available – skip training.")
|
| 97 |
+
return {"status": "failed", "reason": "no embeddings"}
|
| 98 |
+
|
| 99 |
+
try:
|
| 100 |
+
# Step 2: 執行訓練
|
| 101 |
+
active_learning_loop(
|
| 102 |
+
batch_size=1,
|
| 103 |
+
embedding_path=EMBEDDING_PATH
|
| 104 |
+
)
|
| 105 |
+
print("✅ LoRA training complete!")
|
| 106 |
+
|
| 107 |
+
# Step 3: 上傳更新後模型
|
| 108 |
+
api = HfApi()
|
| 109 |
+
api.upload_folder(
|
| 110 |
+
folder_path="./lora_w_mmd",
|
| 111 |
+
repo_id=MODEL_REPO,
|
| 112 |
+
repo_type="model",
|
| 113 |
+
token=HF_TOKEN,
|
| 114 |
+
)
|
| 115 |
+
print(f"☁️ Model uploaded to {MODEL_REPO} successfully!")
|
| 116 |
+
|
| 117 |
+
# Step 4: 通知 Space A 更新模型
|
| 118 |
+
if SPACE_A_URL:
|
| 119 |
+
try:
|
| 120 |
+
print(f"🔁 Notifying Space A at {SPACE_A_URL}/api/update-model …")
|
| 121 |
+
with httpx.Client(timeout=60.0) as client:
|
| 122 |
+
resp = client.post(f"{SPACE_A_URL}/api/update-model")
|
| 123 |
+
print(f"📨 Space A response: {resp.status_code}")
|
| 124 |
+
except Exception as e:
|
| 125 |
+
print(f"⚠️ Failed to notify Space A: {e}")
|
| 126 |
+
|
| 127 |
+
except Exception as e:
|
| 128 |
+
print(f"❌ Error during training/upload: {e}")
|
| 129 |
+
return {"status": "error", "detail": str(e)}
|
| 130 |
+
|
| 131 |
+
return {"status": "done", "detail": "LoRA training and upload complete"}
|
|
|
|
| 132 |
|
| 133 |
|
| 134 |
# ============================================================
|