Spaces:
Runtime error
Runtime error
Download app.py from malvec/codebert_Malvec: direct link, hf CLI and curl.
- Browser
- Download file 16.4 kB
-
https://huggingface.co/spaces/malvec/codebert_Malvec/resolve/main/app.py
- Command line
-
hf download hf://spaces/malvec/codebert_Malvec/app.py
-
curl -L -o app.py https://huggingface.co/spaces/malvec/codebert_Malvec/resolve/main/app.py
16.4 kB
| import os | |
| CACHE_DIR = "/tmp/hf_cache" | |
| os.environ["HF_HOME"] = CACHE_DIR | |
| os.environ["TRANSFORMERS_CACHE"] = CACHE_DIR | |
| os.environ["HUGGINGFACE_HUB_CACHE"] = CACHE_DIR | |
| os.makedirs(CACHE_DIR, exist_ok=True) | |
| import torch | |
| from fastapi import FastAPI, UploadFile, File, HTTPException | |
| from fastapi.middleware.cors import CORSMiddleware | |
| from transformers import AutoTokenizer, AutoModelForSequenceClassification | |
| from typing import List, Optional, Dict | |
| from collections import Counter | |
| import json | |
| from huggingface_hub import hf_hub_download | |
| from som_analyzer import SOMAnalyzer | |
| from datasets import load_dataset | |
| import numpy as np | |
| app = FastAPI() | |
| app.add_middleware( | |
| CORSMiddleware, | |
| allow_origins=["*"], | |
| allow_methods=["*"], | |
| allow_headers=["*"], | |
| ) | |
| # ===== 模型設定 ===== | |
| MODEL_REPO = "raxhel/Malvec_predict" | |
| HF_TOKEN = os.environ.get("hf_token") | |
| DEVICE = torch.device("cpu") | |
| # ===== 載入模型 ===== | |
| print("🤗 Loading model...") | |
| tokenizer = AutoTokenizer.from_pretrained(MODEL_REPO, token=HF_TOKEN) | |
| model = AutoModelForSequenceClassification.from_pretrained( | |
| MODEL_REPO, | |
| token=HF_TOKEN, | |
| output_attentions=True, # ✅ 輸出注意力權重 | |
| output_hidden_states=True # ✅ 關鍵: 輸出 hidden states | |
| ) | |
| model.to(DEVICE) | |
| model.eval() | |
| print("✅ Model loaded successfully") | |
| # ===== 新增:載入 SOM 模型 ===== | |
| print("🧠 Initializing SOM Analyzer...") | |
| SOM_REPO = "malvec/codebert_Malvec" # | |
| SOM_AVAILABLE = False | |
| try: | |
| # | |
| som_weights_path = hf_hub_download( | |
| repo_id=SOM_REPO, | |
| filename="som_weights.npy", | |
| token=HF_TOKEN # | |
| ) | |
| background_data_path = hf_hub_download( | |
| repo_id=SOM_REPO, | |
| filename="som_background_data.pkl", | |
| token=HF_TOKEN # | |
| ) | |
| som_analyzer = SOMAnalyzer( | |
| som_weights_path=som_weights_path, | |
| background_data_path=background_data_path | |
| ) | |
| print("✅ SOM Analyzer initialized successfully!") | |
| SOM_AVAILABLE = True | |
| except Exception as e: | |
| print(f"⚠️ SOM Analyzer initialization failed: {e}") | |
| print(" SOM analysis will be disabled") | |
| # ===== 新增:從 main.py 複製過來的輔助函數 ===== | |
| # (注意:請從你的 main.py 複製最新的程式碼) | |
| async def get_embeddings_from_hf_dataset(filename: str) -> Optional[List[np.ndarray]]: | |
| # ... (請從 main.py 完整複製貼上) ... | |
| try: | |
| print(f"\n📦 Fetching embeddings from HF dataset for: {filename}") | |
| dataset = load_dataset("lyi029/malvec-embeddings", split="train", streaming=True) | |
| embeddings = [] | |
| count = 0 | |
| for sample in dataset: | |
| sample_filename = sample.get('source', '') or sample.get('filename', '') | |
| if filename.lower() in sample_filename.lower(): | |
| emb = sample.get('embedding') or sample.get('emb') | |
| if emb is not None: | |
| embeddings.append(np.array(emb, dtype=np.float32)) | |
| count += 1 | |
| if count >= 1000: break | |
| if embeddings: | |
| print(f"✅ Found {len(embeddings)} embeddings for {filename}") | |
| return embeddings | |
| else: | |
| print(f"⚠️ No embeddings found for {filename} in dataset") | |
| return None | |
| except Exception as e: | |
| print(f"❌ Error fetching from HF dataset: {e}") | |
| return None | |
| def analyze_som_region_probability( | |
| som_analyzer: SOMAnalyzer, | |
| center_row: int, | |
| center_col: int, | |
| embeddings: List[np.ndarray], | |
| radius: int = 2 | |
| ) -> Dict[str, float]: | |
| # (注意:你 main.py 裡的這個函數是同步的(def),不是(async def),這裡保持一致) | |
| print(f" 🔍 Analyzing region around ({center_row}, {center_col}) with radius={radius}") | |
| region_samples = [] | |
| for emb in embeddings: | |
| sample_row, sample_col = som_analyzer.som.winner(emb) # | |
| if abs(sample_row - center_row) <= radius and abs(sample_col - center_col) <= radius: | |
| region_samples.append(emb) | |
| print(f" Found {len(region_samples)} samples in region") | |
| if len(region_samples) == 0: | |
| return {"dropper": 0.0, "APT30": 0.0} | |
| feature_counts = {"dropper": [], "spreader": [], "APT30": [], "Codoso_Gh0st_1": []} | |
| if hasattr(som_analyzer, 'background_embeddings') and hasattr(som_analyzer, 'background_labels'): | |
| for i, bg_emb in enumerate(som_analyzer.background_embeddings): | |
| bg_row, bg_col = som_analyzer.som.winner(bg_emb) # | |
| if abs(bg_row - center_row) <= radius and abs(bg_col - center_col) <= radius: | |
| for feature_name in feature_counts.keys(): | |
| if feature_name in som_analyzer.background_labels: | |
| feature_counts[feature_name].append( | |
| som_analyzer.background_labels[feature_name][i] | |
| ) | |
| probabilities = {} | |
| for feature_name, values in feature_counts.items(): | |
| if len(values) > 0: | |
| prob = sum(values) / len(values) | |
| probabilities[feature_name] = float(prob) | |
| else: | |
| probabilities[feature_name] = 0.0 | |
| return probabilities | |
| async def analyze_with_som(embedding, predicted_family, filename): | |
| # ... (請從 main.py 完整複製貼上) ... | |
| if not SOM_AVAILABLE: | |
| print("⚠️ SOM analysis disabled") | |
| return None | |
| try: | |
| print("\n🗺️ Performing SOM analysis...") | |
| if isinstance(embedding, list): | |
| embedding_array = np.array(embedding, dtype=np.float32) | |
| else: | |
| embedding_array = embedding | |
| winner_row, winner_col = som_analyzer.find_winner_neuron(embedding_array) | |
| print(f" 📍 Winner neuron: ({winner_row}, {winner_col})") | |
| hf_embeddings = await get_embeddings_from_hf_dataset(filename) | |
| if hf_embeddings and len(hf_embeddings) > 0: | |
| print(f"✅ Using {len(hf_embeddings)} embeddings from HF dataset") | |
| features_prob = analyze_som_region_probability( # | |
| som_analyzer, winner_row, winner_col, hf_embeddings, radius=2 | |
| ) | |
| data_source = "hf_dataset" | |
| else: | |
| print(f"⚠️ No HF data found, using background data") | |
| features_dict = som_analyzer.predict_features_from_position( | |
| winner_row, winner_col, radius=1 | |
| ) | |
| features_prob = {k: float(v) for k, v in features_dict.items()} | |
| data_source = "background_data" | |
| features_binary = { | |
| feature: 1 if prob > 0.2 else 0 | |
| for feature, prob in features_prob.items() | |
| } | |
| print(f" ✅ Features detected: {features_binary}") | |
| result = { | |
| "winner_position": {"row": int(winner_row), "col": int(col_winner)}, | |
| "features": features_binary, | |
| "features_probability": features_prob, | |
| "data_source": data_source | |
| } | |
| print(f"✅ SOM analysis complete!") | |
| return result | |
| except Exception as e: | |
| print(f"❌ SOM analysis error: {e}") | |
| return None | |
| def home(): | |
| return { | |
| "message": "Malware Classifier API", | |
| "model": MODEL_REPO, | |
| "endpoints": { | |
| "/predict": "POST - Upload TXT files for prediction", | |
| "/model-info": "GET - Get model information" | |
| } | |
| } | |
| def get_model_info(): | |
| """取得模型資訊""" | |
| # ✅ 檢查 id2label 的 key 類型 | |
| id2label_sample = {} | |
| if hasattr(model.config, 'id2label'): | |
| # 取前 3 個 key 作為範例 | |
| sample_keys = list(model.config.id2label.keys())[:3] | |
| id2label_sample = { | |
| k: { | |
| "type": str(type(k).__name__), | |
| "value": model.config.id2label[k] | |
| } for k in sample_keys | |
| } | |
| return { | |
| "model_name": MODEL_REPO, | |
| "num_labels": model.config.num_labels, | |
| "labels": list(model.config.id2label.values()) if hasattr(model.config, 'id2label') else None, | |
| "device": str(DEVICE), | |
| "max_length": 512, | |
| "id2label_sample": id2label_sample # ✅ 除錯用 | |
| } | |
| async def predict_malware(files: List[UploadFile] = File(...)): | |
| """ | |
| 批次預測惡意軟體家族 | |
| 接收多個 TXT 檔案 (來自同一個執行檔的分段) | |
| 返回: | |
| - final_label: 多數投票的家族標籤 | |
| - embeddings: 最高注意力權重的 embedding | |
| """ | |
| if not files: | |
| raise HTTPException(status_code=400, detail="No files provided") | |
| print(f"\n{'='*60}") | |
| print(f"📥 Received {len(files)} files") | |
| print(f"{'='*60}") | |
| predictions = [] | |
| all_embeddings = [] | |
| try: | |
| with torch.no_grad(): | |
| for idx, file in enumerate(files, 1): | |
| try: | |
| # 讀取檔案內容 | |
| content = await file.read() | |
| asm_code = content.decode('utf-8').strip() | |
| if not asm_code: | |
| print(f"⚠️ Skip empty file: {file.filename}") | |
| continue | |
| print(f"📄 [{idx}/{len(files)}] Processing: {file.filename}") | |
| # Tokenize | |
| inputs = tokenizer( | |
| asm_code, | |
| return_tensors="pt", | |
| max_length=512, | |
| truncation=True, | |
| padding=True | |
| ) | |
| inputs = {k: v.to(DEVICE) for k, v in inputs.items()} | |
| # 模型預測 | |
| outputs = model(**inputs) | |
| # 取得預測結果 | |
| logits = outputs.logits | |
| probs = torch.nn.functional.softmax(logits, dim=-1) | |
| confidence, predicted_class = torch.max(probs, dim=-1) | |
| # ✅ 取得 label (使用整數 key) | |
| predicted_id = predicted_class.item() | |
| if hasattr(model.config, 'id2label') and predicted_id in model.config.id2label: | |
| label_name = model.config.id2label[predicted_id] | |
| else: | |
| label_name = f"class_{predicted_id}" | |
| print(f" 🔍 Predicted ID: {predicted_id}, Label: {label_name}") | |
| # ✅ 取得 embedding (使用 hidden_states) | |
| if hasattr(outputs, 'hidden_states') and outputs.hidden_states is not None: | |
| # 取最後一層的 [CLS] token embedding | |
| last_hidden = outputs.hidden_states[-1] # (batch, seq_len, hidden_size) | |
| cls_embedding = last_hidden[:, 0, :].cpu().numpy()[0] # (768,) | |
| else: | |
| # 備用方案: 使用 logits | |
| cls_embedding = logits.cpu().numpy()[0] | |
| # ✅ 取得注意力權重 | |
| max_attention_score = 0.0 | |
| if hasattr(outputs, 'attentions') and outputs.attentions is not None: | |
| # 取最後一層的注意力,平均所有 head | |
| last_attention = outputs.attentions[-1] # (1, num_heads, seq_len, seq_len) | |
| avg_attention = last_attention.mean(dim=1) # (1, seq_len, seq_len) | |
| # 取 [CLS] token 對所有 token 的注意力 | |
| cls_attention = avg_attention[0, 0, :].cpu().numpy() # (seq_len,) | |
| max_attention_score = float(cls_attention.max()) | |
| # 儲存結果 | |
| predictions.append({ | |
| "filename": file.filename, | |
| "predicted_label": label_name, | |
| "confidence": float(confidence.item()), | |
| "attention_score": max_attention_score | |
| }) | |
| all_embeddings.append({ | |
| "filename": file.filename, | |
| "label": label_name, | |
| "embedding": cls_embedding.tolist(), | |
| "attention_score": max_attention_score | |
| }) | |
| print(f" ✅ Predicted: {label_name} (confidence: {confidence.item():.3f})") | |
| except Exception as e: | |
| print(f" ❌ Error processing {file.filename}: {e}") | |
| continue | |
| if not predictions: | |
| raise HTTPException(status_code=400, detail="No valid files processed") | |
| # ===== 多數投票 (Majority Voting) ===== | |
| label_votes = [p["predicted_label"] for p in predictions] | |
| vote_counts = Counter(label_votes) | |
| final_label = vote_counts.most_common(1)[0][0] | |
| final_count = vote_counts[final_label] | |
| print(f"\n📊 Vote distribution: {dict(vote_counts)}") | |
| print(f"🏆 Final prediction: {final_label} ({final_count}/{len(predictions)} votes)") | |
| # ===== 選擇最高注意力權重的 embedding (屬於最終預測的 label) ===== | |
| # 過濾出預測為 final_label 的 embeddings | |
| same_label_embeddings = [ | |
| emb for emb in all_embeddings | |
| if emb["label"] == final_label | |
| ] | |
| if same_label_embeddings: | |
| # 選擇注意力權重最高的 | |
| best_embedding = max(same_label_embeddings, key=lambda x: x["attention_score"]) | |
| else: | |
| # 如果沒有,就選所有裡面注意力最高的 | |
| best_embedding = max(all_embeddings, key=lambda x: x["attention_score"]) | |
| print(f"🎯 Selected embedding from: {best_embedding['filename']}") | |
| print(f" Attention score: {best_embedding['attention_score']:.4f}") | |
| # ===== 返回結果 ===== | |
| result = { | |
| "final_label": final_label, | |
| "vote_count": final_count, | |
| "total_segments": len(predictions), | |
| "confidence": sum(p["confidence"] for p in predictions) / len(predictions), | |
| "vote_distribution": dict(vote_counts), | |
| "embedding": { | |
| "values": best_embedding["embedding"], | |
| "source_file": best_embedding["filename"], | |
| "attention_score": best_embedding["attention_score"], | |
| "dimension": len(best_embedding["embedding"]) | |
| }, | |
| "segment_predictions": predictions | |
| } | |
| # ===== 新增:執行 SOM 分析 ===== | |
| som_analysis = None | |
| final_embedding_values = result.get("embedding", {}).get("values") | |
| if SOM_AVAILABLE and final_embedding_values: | |
| print("\n🗺️ Performing SOM analysis on HF Space...") | |
| try: | |
| # | |
| som_analysis = await analyze_with_som( | |
| embedding=final_embedding_values, | |
| predicted_family=result.get("final_label", "Unknown"), | |
| filename=files[0].filename # | |
| ) | |
| except Exception as e: | |
| print(f"❌ SOM analysis error on HF Space: {e}") | |
| # | |
| result["som_analysis"] = som_analysis | |
| print(f"\n✅ Prediction with SOM complete!\n") | |
| return result | |
| except Exception as e: | |
| print(f"\n❌ Error in predict_malware: {e}") | |
| import traceback | |
| traceback.print_exc() | |
| raise HTTPException(status_code=500, detail=f"Prediction failed: {str(e)}") | |
| async def predict_batch(files: List[UploadFile] = File(...)): | |
| """ | |
| 簡化版: 只返回 label 和 embedding,不包含詳細資訊 | |
| """ | |
| result = await predict_malware(files) | |
| return { | |
| "final_label": result["final_label"], | |
| "embedding": result["embedding"]["values"] | |
| } | |
| # ✅ HF Space 啟動入口 | |
| if __name__ == "__main__": | |
| import uvicorn | |
| port = int(os.environ.get("PORT", 7860)) | |
| uvicorn.run(app, host="0.0.0.0", port=port) |