Spaces:
Sleeping
Sleeping
Download app.py from lyi029/model_update: direct link, hf CLI and curl.
- Browser
- Download file 8.24 kB
-
https://huggingface.co/spaces/lyi029/model_update/resolve/main/app.py
- Command line
-
hf download hf://spaces/lyi029/model_update/app.py
-
curl -L -o app.py https://huggingface.co/spaces/lyi029/model_update/resolve/main/app.py
8.24 kB
| # ============================================================ | |
| # app_spaceB.py — Active Learning (Entropy Triggered) | |
| # ============================================================ | |
| import os, json, time, torch, numpy as np, pandas as pd | |
| from fastapi import FastAPI, HTTPException, Query | |
| from fastapi.middleware.cors import CORSMiddleware | |
| from datasets import load_dataset | |
| from transformers import PreTrainedTokenizerFast | |
| from huggingface_hub import HfApi | |
| from sklearn.cluster import KMeans | |
| from scipy.spatial.distance import cdist | |
| from typing import List, Optional | |
| from active_loop_full import ( | |
| _sigma_median, _rbf_mmd, _density_uncertainty_sampling, | |
| _train_one_lora, train_from_labeled | |
| ) | |
| # === FastAPI === | |
| app = FastAPI( | |
| title="Malvec Space B - Entropy Triggered AL", | |
| description="Active learning backend driven by uncertainty entropy", | |
| version="2.0" | |
| ) | |
| app.add_middleware( | |
| CORSMiddleware, | |
| allow_origins=["*"], allow_methods=["*"], allow_headers=["*"] | |
| ) | |
| OUTPUT_ROOT = "./lora_entropy_trigger" | |
| os.makedirs(OUTPUT_ROOT, exist_ok=True) | |
| HF_TOKEN = os.getenv("HF_TOKEN", "") | |
| DATASET_REPO = "lyi029/malvec-embeddings-new" | |
| BASE_REPO = os.getenv("MODEL_REPO", "lyi029/codebert-finetuned") | |
| TOKENIZER_REPO = BASE_REPO | |
| # ============================================================ | |
| # Utility: 根據 MMD 選最相似模型分支 | |
| # ============================================================ | |
| def select_best_lora_by_mmd(batch_df, lora_distributions): | |
| emb_cols = [c for c in batch_df.columns if c.startswith("emb_")] | |
| X = torch.tensor(batch_df[emb_cols].to_numpy(), dtype=torch.float32) | |
| best_id, best_mmd = None, float("inf") | |
| for tag, dist_df in lora_distributions.items(): | |
| Y = torch.tensor(dist_df[emb_cols].to_numpy(), dtype=torch.float32) | |
| sigma = _sigma_median(Y.numpy()) | |
| mmd_val = _rbf_mmd(X, Y, sigma) | |
| print(f" ↳ MMD vs {tag}: {mmd_val:.4f}") | |
| if mmd_val < best_mmd: | |
| best_mmd = mmd_val | |
| best_id = tag | |
| print(f"✅ 最相似模型分支: {best_id} (MMD={best_mmd:.4f})") | |
| return best_id | |
| # ============================================================ | |
| # Endpoint 1️⃣: 主動學習迴圈 (Entropy 為觸發條件) | |
| # ============================================================ | |
| def check_new_samples( | |
| batch_size: int = Query(1000), | |
| entropy_threshold: float = Query(1.1) | |
| ): | |
| print("🚀 [Space B] 啟動 Active Learning 流程 (Entropy Trigger 模式) ...") | |
| try: | |
| ds = load_dataset(DATASET_REPO, split="train", token=HF_TOKEN) | |
| df = pd.DataFrame(ds) | |
| print(f"✅ 載入資料集:{len(df)} 筆樣本") | |
| if "embedding" not in df.columns or "label" not in df.columns: | |
| raise ValueError("Dataset 必須包含 'embedding' 和 'label' 欄位") | |
| # === 建立 label_id === | |
| labels_sorted = sorted(df["label"].unique().tolist()) | |
| label2id = {l: i for i, l in enumerate(labels_sorted)} | |
| df["label_id"] = df["label"].map(label2id) | |
| emb_dim = len(df["embedding"].iloc[0]) | |
| emb_cols = [f"emb_{i}" for i in range(emb_dim)] | |
| emb_df = pd.DataFrame(np.stack(df["embedding"].to_numpy()), columns=emb_cols, index=df.index) | |
| df = pd.concat([df.drop(columns=["embedding"]), emb_df], axis=1) | |
| prob_cols: List[str] = [] | |
| if "probs" in df.columns: | |
| prob_dim = len(df["probs"].iloc[0]) | |
| prob_cols = [f"prob_{i}" for i in range(prob_dim)] | |
| prob_df = pd.DataFrame(np.stack(df["probs"].to_numpy()), columns=prob_cols, index=df.index) | |
| df = pd.concat([df.drop(columns=["probs"]), prob_df], axis=1) | |
| tokenizer = PreTrainedTokenizerFast.from_pretrained(TOKENIZER_REPO, token=HF_TOKEN) | |
| tag_stamp = time.strftime("%Y%m%d-%H%M%S") | |
| lora_distributions = {} | |
| last_saved_dir = None | |
| for rnd, idxs in enumerate(range(0, len(df), batch_size), start=1): | |
| batch_df = df.iloc[idxs:idxs + batch_size].copy() | |
| if len(batch_df) == 0: | |
| continue | |
| print(f"\n=== Round {rnd} ===") | |
| # === 4️⃣ 初次 batch 進行 LoRA 微調 | |
| if "text" not in batch_df.columns: | |
| if "text_excerpt" in batch_df.columns: | |
| batch_df["text"] = batch_df["text_excerpt"] | |
| else: | |
| batch_df["text"] = "<asm>" | |
| # === 1️⃣ 計算 entropy === | |
| if len(prob_cols) > 0: | |
| probs = batch_df[prob_cols].to_numpy() | |
| entropies = -np.sum(probs * np.log(probs + 1e-12), axis=1) | |
| mean_entropy = float(np.mean(entropies)) | |
| else: | |
| mean_entropy = 0.0 | |
| print(f"[AL] mean_entropy={mean_entropy:.3f}") | |
| # === 2️⃣ 高熵 → 人工標註 | |
| if mean_entropy > entropy_threshold: | |
| print("⚠️ 高不確定性:輸出人工標註樣本集") | |
| sampled = _density_uncertainty_sampling( | |
| batch_df, emb_cols, prob_cols, num_clusters=10, | |
| total_samples=min(200, len(batch_df)) | |
| ) | |
| label_file = os.path.join(OUTPUT_ROOT, f"selected_for_labeling_round_{rnd}.json") | |
| sampled["entropy"] = entropies[:len(sampled)] | |
| sampled[["source", "label", "entropy"] + emb_cols].to_json( | |
| label_file, orient="records", indent=2, force_ascii=False | |
| ) | |
| print(f"📝 已輸出 {len(sampled)} 筆待人工標註樣本 → {label_file}") | |
| continue | |
| # === 3️⃣ 低熵 → 用 MMD 選分支 | |
| if lora_distributions: | |
| print("🔍 根據 MMD 選擇最相似的 LoRA 分支 ...") | |
| best_branch = select_best_lora_by_mmd(batch_df, lora_distributions) | |
| print(f"✅ 使用分支:{best_branch} 進行推論") | |
| else: | |
| print("ℹ️ 尚無 LoRA 分支,執行初次微調。") | |
| # === 4️⃣ 初次 batch 進行 LoRA 微調 | |
| eval_size = max(1, int(0.2 * len(batch_df))) | |
| eval_df = batch_df.iloc[:eval_size] | |
| train_df = batch_df.iloc[eval_size:] | |
| out_dir = os.path.join(OUTPUT_ROOT, f"round_{rnd}") | |
| os.makedirs(out_dir, exist_ok=True) | |
| _train_one_lora(rnd, BASE_REPO, tokenizer, train_df, eval_df, out_dir, HF_TOKEN) | |
| last_saved_dir = out_dir | |
| lora_distributions[f"round_{rnd}"] = batch_df[emb_cols] | |
| # === 上傳 LoRA | |
| if last_saved_dir: | |
| api = HfApi() | |
| api.upload_folder(folder_path=last_saved_dir, repo_id=BASE_REPO, | |
| repo_type="model", token=HF_TOKEN) | |
| tag_name = f"al-{tag_stamp}" | |
| try: | |
| api.create_tag(repo_id=BASE_REPO, tag=tag_name, repo_type="model") | |
| except Exception as e: | |
| print(f"⚠️ create_tag failed: {e}") | |
| print(f"✅ Uploaded and tagged: {BASE_REPO}:{tag_name}") | |
| return {"status": "ok", "tag": tag_name} | |
| else: | |
| return {"status": "pending_labeling"} | |
| except Exception as e: | |
| print(f"❌ Active Learning 錯誤: {e}") | |
| raise HTTPException(status_code=500, detail=str(e)) | |
| # ============================================================ | |
| # Endpoint 2️⃣: 手動標註後再訓練 | |
| # ============================================================ | |
| def train_labeled_endpoint(label_file: str = Query(...), rnd: int = Query(...)): | |
| try: | |
| out_dir = train_from_labeled(label_file, rnd, BASE_REPO, TOKENIZER_REPO, HF_TOKEN) | |
| return {"status": "ok", "output_dir": out_dir} | |
| except Exception as e: | |
| print(f"❌ train_from_labeled failed: {e}") | |
| raise HTTPException(status_code=500, detail=str(e)) | |
| # ============================================================ | |
| # Endpoint 3️⃣: 健康檢查 | |
| # ============================================================ | |
| def health_check(): | |
| return {"status": "running", "mode": "entropy_trigger"} | |