File size: 7,544 Bytes
6d92c0d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 | import numpy as np
import json
import pickle
from minisom import MiniSom
class SOMAnalyzer:
"""SOM 分析器 - 載入預訓練的 SOM 並分析新檔案"""
def __init__(self, som_weights_path, background_data_path):
"""
初始化 SOM 分析器
Args:
som_weights_path: SOM 權重檔案路徑 (.npy)
background_data_path: 背景資料檔案路徑 (.pkl),包含所有訓練資料的 embeddings 和標籤
"""
# 載入 SOM 權重
self.som_weights = np.load(som_weights_path)
self.som_size = self.som_weights.shape[0]
self.input_dim = self.som_weights.shape[2]
# 建立 SOM 模型
self.som = MiniSom(
self.som_size,
self.som_size,
self.input_dim,
sigma=0.5,
learning_rate=0.5,
random_seed=77
)
self.som._weights = self.som_weights.copy()
# 載入背景資料
with open(background_data_path, 'rb') as f:
background = pickle.load(f)
self.background_embeddings = background['embeddings']
self.background_labels = background['labels']
self.id2label = background['id2label']
def find_winner_neuron(self, embedding):
"""
找到 embedding 對應的 winner neuron
Args:
embedding: 768 維的 embedding vector
Returns:
(row, col): winner neuron 的座標
"""
return self.som.winner(embedding)
def predict_features_from_position(self, row, col, radius=1):
"""
✅ 根據 SOM 位置智能預測特徵值
分析該位置及其周圍區域的背景樣本,統計特徵的比例來預測
Args:
row: winner neuron 的 row
col: winner neuron 的 col
radius: 搜尋半徑(0=只看該格,1=3x3區域,2=5x5區域)
Returns:
dict: 特徵預測結果,例如 {'dropper': 1, 'APT30': 0}
"""
print(f" 🔍 Analyzing region around ({row}, {col}) with radius={radius}")
# 收集該區域的所有樣本
region_samples = {'dropper': [], 'APT30': []}
for i, emb in enumerate(self.background_embeddings):
sample_row, sample_col = self.som.winner(emb)
# 檢查是否在指定半徑內
if abs(sample_row - row) <= radius and abs(sample_col - col) <= radius:
# 收集該樣本的特徵值
if 'dropper' in self.background_labels:
region_samples['dropper'].append(self.background_labels['dropper'][i])
if 'APT30' in self.background_labels:
region_samples['APT30'].append(self.background_labels['APT30'][i])
# 根據該區域的統計來預測
predictions = {}
for feature_name, values in region_samples.items():
if len(values) == 0:
# 沒有樣本,預設為 0
predictions[feature_name] = 0
print(f" {feature_name}: No samples → 0")
else:
# 計算該特徵為 1 的比例
positive_count = sum(values)
total_count = len(values)
positive_ratio = positive_count / total_count
# 如果超過 5%,預測為 1
predictions[feature_name] = 1 if positive_ratio > 0.05 else 0
print(f" {feature_name}: {positive_count}/{total_count} samples = {positive_ratio:.1%} → {predictions[feature_name]}")
return predictions
def generate_som_json_with_marker(self, new_embedding, feature_name, feature_value,
filename="unknown", predicted_family=None):
"""
生成包含新檔案標記的 SOM JSON
Args:
new_embedding: 新檔案的 768 維 embedding
feature_name: 特徵名稱 (如 'dropper', 'APT30')
feature_value: 該特徵的值 (0 或 1)
filename: 檔案名稱
predicted_family: 預測的惡意軟體家族
Returns:
dict: JSON 格式的 SOM 資料
"""
# 找到新檔案的位置
new_row, new_col = self.find_winner_neuron(new_embedding)
# 計算背景資料的 neuron 統計
neuron_stats = {}
for i, emb in enumerate(self.background_embeddings):
row, col = self.som.winner(emb)
key = f"{row},{col}"
if key not in neuron_stats:
neuron_stats[key] = {}
# 根據 feature_name 取得對應的標籤
label = self.background_labels[feature_name][i]
neuron_stats[key][label] = neuron_stats[key].get(label, 0) + 1
# 建立 JSON 結構
som_json = {
"title": f"SOM neuron composition by {feature_name}",
"som_size": self.som_size,
"neurons": [],
"new_file": {
"filename": filename,
"row": int(new_row),
"col": int(new_col),
"feature_name": feature_name,
"feature_value": int(feature_value),
"predicted_family": predicted_family
}
}
# 轉換每個 neuron 的統計資料
for key, counts in neuron_stats.items():
row, col = map(int, key.split(","))
total = sum(counts.values())
proportions = {
self.id2label.get(lbl, str(lbl)): cnt/total
for lbl, cnt in counts.items()
}
som_json["neurons"].append({
"row": row,
"col": col,
"counts": counts,
"proportions": proportions
})
return som_json
def prepare_background_data(df_merged, output_path):
"""
準備背景資料(只需執行一次)
Args:
df_merged: 合併後的 DataFrame,包含 embeddings 和所有標籤
output_path: 輸出檔案路徑
"""
# 提取 embeddings
emb_cols = [f"emb_{i}" for i in range(768)]
embeddings = df_merged[emb_cols].values.astype(float)
# 提取所有標籤
labels = {
'true_label_id': df_merged['true_label_id'].fillna(0).astype(int).values,
'dropper': df_merged['dropper'].fillna(0).astype(int).values,
'spreader': df_merged['spreader'].fillna(0).astype(int).values,
'APT30': df_merged['APT30'].fillna(0).astype(int).values,
'Codoso_Gh0st_1': df_merged['Codoso_Gh0st_1'].fillna(0).astype(int).values,
}
# 建立 id2label 映射
label_list = sorted(df_merged['true_label'].unique())
label2id = {l: i for i, l in enumerate(label_list)}
id2label = {i: l for l, i in label2id.items()}
# 保存
background_data = {
'embeddings': embeddings,
'labels': labels,
'id2label': id2label
}
with open(output_path, 'wb') as f:
pickle.dump(background_data, f)
print(f"✅ 背景資料已保存至: {output_path}")
print(f" - 資料筆數: {len(embeddings)}")
print(f" - Embedding 維度: {embeddings.shape[1]}")
print(f" - 特徵數量: {len(labels)}") |