111306047 commited on
Commit
8a33636
·
1 Parent(s): 4ecb6e9

upload som to hf space

Browse files
Files changed (2) hide show
  1. app.py +169 -2
  2. requirements.txt +4 -1
app.py CHANGED
@@ -9,9 +9,13 @@ import torch
9
  from fastapi import FastAPI, UploadFile, File, HTTPException
10
  from fastapi.middleware.cors import CORSMiddleware
11
  from transformers import AutoTokenizer, AutoModelForSequenceClassification
12
- from typing import List
13
  from collections import Counter
14
  import json
 
 
 
 
15
 
16
  app = FastAPI()
17
  app.add_middleware(
@@ -39,6 +43,150 @@ model.to(DEVICE)
39
  model.eval()
40
  print("✅ Model loaded successfully")
41
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
  @app.get("/")
43
  def home():
44
  return {
@@ -224,7 +372,26 @@ async def predict_malware(files: List[UploadFile] = File(...)):
224
  "segment_predictions": predictions
225
  }
226
 
227
- print(f"\n✅ Prediction complete!\n")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
228
 
229
  return result
230
 
 
9
  from fastapi import FastAPI, UploadFile, File, HTTPException
10
  from fastapi.middleware.cors import CORSMiddleware
11
  from transformers import AutoTokenizer, AutoModelForSequenceClassification
12
+ from typing import List, Optional, Dict
13
  from collections import Counter
14
  import json
15
+ from huggingface_hub import hf_hub_download
16
+ from som_analyzer import SOMAnalyzer
17
+ from datasets import load_dataset
18
+ import numpy as np
19
 
20
  app = FastAPI()
21
  app.add_middleware(
 
43
  model.eval()
44
  print("✅ Model loaded successfully")
45
 
46
+ # ===== 新增:載入 SOM 模型 =====
47
+ print("🧠 Initializing SOM Analyzer...")
48
+ SOM_REPO = "malvec/codebert_Malvec" #
49
+ SOM_AVAILABLE = False
50
+ try:
51
+ #
52
+ som_weights_path = hf_hub_download(
53
+ repo_id=SOM_REPO,
54
+ filename="som_weights.npy",
55
+ token=HF_TOKEN #
56
+ )
57
+ background_data_path = hf_hub_download(
58
+ repo_id=SOM_REPO,
59
+ filename="som_background_data.pkl",
60
+ token=HF_TOKEN #
61
+ )
62
+
63
+ som_analyzer = SOMAnalyzer(
64
+ som_weights_path=som_weights_path,
65
+ background_data_path=background_data_path
66
+ )
67
+ print("✅ SOM Analyzer initialized successfully!")
68
+ SOM_AVAILABLE = True
69
+ except Exception as e:
70
+ print(f"⚠️ SOM Analyzer initialization failed: {e}")
71
+ print(" SOM analysis will be disabled")
72
+
73
+
74
+ # ===== 新增:從 main.py 複製過來的輔助函數 =====
75
+ # (注意:請從你的 main.py 複製最新的程式碼)
76
+
77
+ async def get_embeddings_from_hf_dataset(filename: str) -> Optional[List[np.ndarray]]:
78
+ # ... (請從 main.py 完整複製貼上) ...
79
+ try:
80
+ print(f"\n📦 Fetching embeddings from HF dataset for: {filename}")
81
+ dataset = load_dataset("lyi029/malvec-embeddings", split="train", streaming=True)
82
+ embeddings = []
83
+ count = 0
84
+ for sample in dataset:
85
+ sample_filename = sample.get('source', '') or sample.get('filename', '')
86
+ if filename.lower() in sample_filename.lower():
87
+ emb = sample.get('embedding') or sample.get('emb')
88
+ if emb is not None:
89
+ embeddings.append(np.array(emb, dtype=np.float32))
90
+ count += 1
91
+ if count >= 1000: break
92
+ if embeddings:
93
+ print(f"✅ Found {len(embeddings)} embeddings for {filename}")
94
+ return embeddings
95
+ else:
96
+ print(f"⚠️ No embeddings found for {filename} in dataset")
97
+ return None
98
+ except Exception as e:
99
+ print(f"❌ Error fetching from HF dataset: {e}")
100
+ return None
101
+
102
+ def analyze_som_region_probability(
103
+ som_analyzer: SOMAnalyzer,
104
+ center_row: int,
105
+ center_col: int,
106
+ embeddings: List[np.ndarray],
107
+ radius: int = 2
108
+ ) -> Dict[str, float]:
109
+
110
+ # (注意:你 main.py 裡的這個函數是同步的(def),不是(async def),這裡保持一致)
111
+ print(f" 🔍 Analyzing region around ({center_row}, {center_col}) with radius={radius}")
112
+ region_samples = []
113
+ for emb in embeddings:
114
+ sample_row, sample_col = som_analyzer.som.winner(emb) #
115
+ if abs(sample_row - center_row) <= radius and abs(sample_col - center_col) <= radius:
116
+ region_samples.append(emb)
117
+ print(f" Found {len(region_samples)} samples in region")
118
+ if len(region_samples) == 0:
119
+ return {"dropper": 0.0, "APT30": 0.0}
120
+
121
+ feature_counts = {"dropper": [], "spreader": [], "APT30": [], "Codoso_Gh0st_1": []}
122
+ if hasattr(som_analyzer, 'background_embeddings') and hasattr(som_analyzer, 'background_labels'):
123
+ for i, bg_emb in enumerate(som_analyzer.background_embeddings):
124
+ bg_row, bg_col = som_analyzer.som.winner(bg_emb) #
125
+ if abs(bg_row - center_row) <= radius and abs(bg_col - center_col) <= radius:
126
+ for feature_name in feature_counts.keys():
127
+ if feature_name in som_analyzer.background_labels:
128
+ feature_counts[feature_name].append(
129
+ som_analyzer.background_labels[feature_name][i]
130
+ )
131
+ probabilities = {}
132
+ for feature_name, values in feature_counts.items():
133
+ if len(values) > 0:
134
+ prob = sum(values) / len(values)
135
+ probabilities[feature_name] = float(prob)
136
+ else:
137
+ probabilities[feature_name] = 0.0
138
+ return probabilities
139
+
140
+
141
+ async def analyze_with_som(embedding, predicted_family, filename):
142
+ # ... (請從 main.py 完整複製貼上) ...
143
+ if not SOM_AVAILABLE:
144
+ print("⚠️ SOM analysis disabled")
145
+ return None
146
+ try:
147
+ print("\n🗺️ Performing SOM analysis...")
148
+ if isinstance(embedding, list):
149
+ embedding_array = np.array(embedding, dtype=np.float32)
150
+ else:
151
+ embedding_array = embedding
152
+
153
+ winner_row, winner_col = som_analyzer.find_winner_neuron(embedding_array)
154
+ print(f" 📍 Winner neuron: ({winner_row}, {winner_col})")
155
+
156
+ hf_embeddings = await get_embeddings_from_hf_dataset(filename)
157
+
158
+ if hf_embeddings and len(hf_embeddings) > 0:
159
+ print(f"✅ Using {len(hf_embeddings)} embeddings from HF dataset")
160
+ features_prob = analyze_som_region_probability( #
161
+ som_analyzer, winner_row, winner_col, hf_embeddings, radius=2
162
+ )
163
+ data_source = "hf_dataset"
164
+ else:
165
+ print(f"⚠️ No HF data found, using background data")
166
+ features_dict = som_analyzer.predict_features_from_position(
167
+ winner_row, winner_col, radius=1
168
+ )
169
+ features_prob = {k: float(v) for k, v in features_dict.items()}
170
+ data_source = "background_data"
171
+
172
+ features_binary = {
173
+ feature: 1 if prob > 0.2 else 0
174
+ for feature, prob in features_prob.items()
175
+ }
176
+ print(f" ✅ Features detected: {features_binary}")
177
+
178
+ result = {
179
+ "winner_position": {"row": int(winner_row), "col": int(col_winner)},
180
+ "features": features_binary,
181
+ "features_probability": features_prob,
182
+ "data_source": data_source
183
+ }
184
+ print(f"✅ SOM analysis complete!")
185
+ return result
186
+ except Exception as e:
187
+ print(f"❌ SOM analysis error: {e}")
188
+ return None
189
+
190
  @app.get("/")
191
  def home():
192
  return {
 
372
  "segment_predictions": predictions
373
  }
374
 
375
+ # ===== 新增:執行 SOM 分析 =====
376
+ som_analysis = None
377
+ final_embedding_values = result.get("embedding", {}).get("values")
378
+
379
+ if SOM_AVAILABLE and final_embedding_values:
380
+ print("\n🗺️ Performing SOM analysis on HF Space...")
381
+ try:
382
+ #
383
+ som_analysis = await analyze_with_som(
384
+ embedding=final_embedding_values,
385
+ predicted_family=result.get("final_label", "Unknown"),
386
+ filename=files[0].filename #
387
+ )
388
+ except Exception as e:
389
+ print(f"❌ SOM analysis error on HF Space: {e}")
390
+
391
+ #
392
+ result["som_analysis"] = som_analysis
393
+
394
+ print(f"\n✅ Prediction with SOM complete!\n")
395
 
396
  return result
397
 
requirements.txt CHANGED
@@ -7,4 +7,7 @@ plotly
7
  numpy
8
  pandas
9
  python-multipart
10
- httpx
 
 
 
 
7
  numpy
8
  pandas
9
  python-multipart
10
+ httpx
11
+ datasets
12
+ minisom
13
+ huggingface_hub