codebert_Malvec / app.py
111306047
add tsne
8dcc174
Raw History Blame
11.8 kB
import os
CACHE_DIR = "/tmp/hf_cache"
os.environ["HF_HOME"] = CACHE_DIR
os.environ["TRANSFORMERS_CACHE"] = CACHE_DIR
os.environ["HUGGINGFACE_HUB_CACHE"] = CACHE_DIR
os.makedirs(CACHE_DIR, exist_ok=True)
import torch
from fastapi import FastAPI, UploadFile, File, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from transformers import AutoTokenizer, AutoModelForSequenceClassification
from typing import List
from collections import Counter
import json
app = FastAPI()
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_methods=["*"],
allow_headers=["*"],
)
# ===== 模型設定 =====
MODEL_REPO = "raxhel/Malvec_predict"
HF_TOKEN = os.environ.get("hf_token")
DEVICE = torch.device("cpu")
# ===== 載入模型 =====
print("🤗 Loading model...")
tokenizer = AutoTokenizer.from_pretrained(MODEL_REPO, token=HF_TOKEN)
model = AutoModelForSequenceClassification.from_pretrained(
MODEL_REPO,
token=HF_TOKEN,
output_attentions=True, # ✅ 輸出注意力權重
output_hidden_states=True # ✅ 關鍵: 輸出 hidden states
)
model.to(DEVICE)
model.eval()
print("✅ Model loaded successfully")
@app.get("/")
def home():
return {
"message": "Malware Classifier API",
"model": MODEL_REPO,
"endpoints": {
"/predict": "POST - Upload TXT files for prediction",
"/model-info": "GET - Get model information"
}
}
@app.get("/model-info")
def get_model_info():
"""取得模型資訊"""
# ✅ 檢查 id2label 的 key 類型
id2label_sample = {}
if hasattr(model.config, 'id2label'):
# 取前 3 個 key 作為範例
sample_keys = list(model.config.id2label.keys())[:3]
id2label_sample = {
k: {
"type": str(type(k).__name__),
"value": model.config.id2label[k]
} for k in sample_keys
}
return {
"model_name": MODEL_REPO,
"num_labels": model.config.num_labels,
"labels": list(model.config.id2label.values()) if hasattr(model.config, 'id2label') else None,
"device": str(DEVICE),
"max_length": 512,
"id2label_sample": id2label_sample # ✅ 除錯用
}
# ===== 載入預先降維的資料 =====
# 在模型載入時執行一次
print("📊 Loading pre-computed t-SNE embeddings...")
df_tsne = pd.read_json("tsne_points.json") # 你上傳的 JSON
tsne_coords = df_tsne[['x', 'y']].values # (N, 2)
# 載入原始 embeddings(用於 kNN 投影)
df_original = pd.read_csv("cobert_finetune_embedding_checkpoint-126000.csv")
dim_cols = [f'emb_{i}' for i in range(768)]
original_embeddings = df_original[dim_cols].to_numpy() # (N, 768)
# 標準化(必須和訓練時一致)
scaler = StandardScaler()
original_embeddings_scaled = scaler.fit_transform(original_embeddings)
# 建立 kNN 模型(用於投影新點)
knn_model = NearestNeighbors(n_neighbors=10, metric='euclidean')
knn_model.fit(original_embeddings_scaled)
print(f"✅ Loaded {len(df_tsne)} pre-computed t-SNE points")
def project_new_embedding_to_tsne(new_embedding_768d, k=10):
"""
將新的 768 維 embedding 投影到已有的 t-SNE 空間
方法:找到 k 個最近鄰,用距離加權平均它們的 t-SNE 座標
Args:
new_embedding_768d: array of shape (768,)
k: 使用幾個最近鄰
Returns:
dict: {'x': float, 'y': float}
"""
# 1. 標準化新 embedding(使用相同的 scaler)
new_emb_scaled = scaler.transform([new_embedding_768d]) # (1, 768)
# 2. 找到 k 個最近鄰
distances, indices = knn_model.kneighbors(new_emb_scaled, n_neighbors=k)
distances = distances[0] # (k,)
indices = indices[0] # (k,)
# 3. 距離加權平均(距離越近權重越大)
epsilon = 1e-8
weights = 1 / (distances + epsilon)
weights = weights / weights.sum()
# 4. 計算加權平均的 t-SNE 座標
neighbor_coords = tsne_coords[indices] # (k, 2)
projected_x = np.dot(weights, neighbor_coords[:, 0])
projected_y = np.dot(weights, neighbor_coords[:, 1])
return {
'x': float(projected_x),
'y': float(projected_y),
'method': 'kNN_projection',
'k': k,
'nearest_distance': float(distances[0])
}
@app.post("/predict")
async def predict_malware(files: List[UploadFile] = File(...)):
"""
批次預測惡意軟體家族
接收多個 TXT 檔案 (來自同一個執行檔的分段)
返回:
- final_label: 多數投票的家族標籤
- embeddings: 最高注意力權重的 embedding
"""
if not files:
raise HTTPException(status_code=400, detail="No files provided")
print(f"\n{'='*60}")
print(f"📥 Received {len(files)} files")
print(f"{'='*60}")
predictions = []
all_embeddings = []
try:
with torch.no_grad():
for idx, file in enumerate(files, 1):
try:
# 讀取檔案內容
content = await file.read()
asm_code = content.decode('utf-8').strip()
if not asm_code:
print(f"⚠️ Skip empty file: {file.filename}")
continue
print(f"📄 [{idx}/{len(files)}] Processing: {file.filename}")
# Tokenize
inputs = tokenizer(
asm_code,
return_tensors="pt",
max_length=512,
truncation=True,
padding=True
)
inputs = {k: v.to(DEVICE) for k, v in inputs.items()}
# 模型預測
outputs = model(**inputs)
# 取得預測結果
logits = outputs.logits
probs = torch.nn.functional.softmax(logits, dim=-1)
confidence, predicted_class = torch.max(probs, dim=-1)
# ✅ 取得 label (使用整數 key)
predicted_id = predicted_class.item()
if hasattr(model.config, 'id2label') and predicted_id in model.config.id2label:
label_name = model.config.id2label[predicted_id]
else:
label_name = f"class_{predicted_id}"
print(f" 🔍 Predicted ID: {predicted_id}, Label: {label_name}")
# ✅ 取得 embedding (使用 hidden_states)
if hasattr(outputs, 'hidden_states') and outputs.hidden_states is not None:
# 取最後一層的 [CLS] token embedding
last_hidden = outputs.hidden_states[-1] # (batch, seq_len, hidden_size)
cls_embedding = last_hidden[:, 0, :].cpu().numpy()[0] # (768,)
else:
# 備用方案: 使用 logits
cls_embedding = logits.cpu().numpy()[0]
# ✅ 取得注意力權重
max_attention_score = 0.0
if hasattr(outputs, 'attentions') and outputs.attentions is not None:
# 取最後一層的注意力,平均所有 head
last_attention = outputs.attentions[-1] # (1, num_heads, seq_len, seq_len)
avg_attention = last_attention.mean(dim=1) # (1, seq_len, seq_len)
# 取 [CLS] token 對所有 token 的注意力
cls_attention = avg_attention[0, 0, :].cpu().numpy() # (seq_len,)
max_attention_score = float(cls_attention.max())
# 儲存結果
predictions.append({
"filename": file.filename,
"predicted_label": label_name,
"confidence": float(confidence.item()),
"attention_score": max_attention_score
})
all_embeddings.append({
"filename": file.filename,
"label": label_name,
"embedding": cls_embedding.tolist(),
"attention_score": max_attention_score
})
print(f" ✅ Predicted: {label_name} (confidence: {confidence.item():.3f})")
except Exception as e:
print(f" ❌ Error processing {file.filename}: {e}")
continue
if not predictions:
raise HTTPException(status_code=400, detail="No valid files processed")
# ===== 多數投票 (Majority Voting) =====
label_votes = [p["predicted_label"] for p in predictions]
vote_counts = Counter(label_votes)
final_label = vote_counts.most_common(1)[0][0]
final_count = vote_counts[final_label]
print(f"\n📊 Vote distribution: {dict(vote_counts)}")
print(f"🏆 Final prediction: {final_label} ({final_count}/{len(predictions)} votes)")
# ===== 選擇最高注意力權重的 embedding (屬於最終預測的 label) =====
# 過濾出預測為 final_label 的 embeddings
same_label_embeddings = [
emb for emb in all_embeddings
if emb["label"] == final_label
]
if same_label_embeddings:
# 選擇注意力權重最高的
best_embedding = max(same_label_embeddings, key=lambda x: x["attention_score"])
else:
# 如果沒有,就選所有裡面注意力最高的
best_embedding = max(all_embeddings, key=lambda x: x["attention_score"])
print(f"🎯 Selected embedding from: {best_embedding['filename']}")
print(f" Attention score: {best_embedding['attention_score']:.4f}")
# ===== 返回結果 =====
result = {
"final_label": final_label,
"vote_count": final_count,
"total_segments": len(predictions),
"confidence": sum(p["confidence"] for p in predictions) / len(predictions),
"vote_distribution": dict(vote_counts),
"embedding": {
"values": best_embedding["embedding"],
"source_file": best_embedding["filename"],
"attention_score": best_embedding["attention_score"],
"dimension": len(best_embedding["embedding"])
},
"segment_predictions": predictions
}
print(f"\n✅ Prediction complete!\n")
return result
except Exception as e:
print(f"\n❌ Error in predict_malware: {e}")
import traceback
traceback.print_exc()
raise HTTPException(status_code=500, detail=f"Prediction failed: {str(e)}")
@app.post("/predict-batch")
async def predict_batch(files: List[UploadFile] = File(...)):
"""
簡化版: 只返回 label 和 embedding,不包含詳細資訊
"""
result = await predict_malware(files)
return {
"final_label": result["final_label"],
"embedding": result["embedding"]["values"]
}
# ✅ HF Space 啟動入口
if __name__ == "__main__":
import uvicorn
port = int(os.environ.get("PORT", 7860))
uvicorn.run(app, host="0.0.0.0", port=port)