test / som_analyzer.py
raxhel's picture
Upload 3 files
6d92c0d verified
Raw History Blame Contribute Delete
7.54 kB
import numpy as np
import json
import pickle
from minisom import MiniSom
class SOMAnalyzer:
"""SOM 分析器 - 載入預訓練的 SOM 並分析新檔案"""
def __init__(self, som_weights_path, background_data_path):
"""
初始化 SOM 分析器
Args:
som_weights_path: SOM 權重檔案路徑 (.npy)
background_data_path: 背景資料檔案路徑 (.pkl),包含所有訓練資料的 embeddings 和標籤
"""
# 載入 SOM 權重
self.som_weights = np.load(som_weights_path)
self.som_size = self.som_weights.shape[0]
self.input_dim = self.som_weights.shape[2]
# 建立 SOM 模型
self.som = MiniSom(
self.som_size,
self.som_size,
self.input_dim,
sigma=0.5,
learning_rate=0.5,
random_seed=77
)
self.som._weights = self.som_weights.copy()
# 載入背景資料
with open(background_data_path, 'rb') as f:
background = pickle.load(f)
self.background_embeddings = background['embeddings']
self.background_labels = background['labels']
self.id2label = background['id2label']
def find_winner_neuron(self, embedding):
"""
找到 embedding 對應的 winner neuron
Args:
embedding: 768 維的 embedding vector
Returns:
(row, col): winner neuron 的座標
"""
return self.som.winner(embedding)
def predict_features_from_position(self, row, col, radius=1):
"""
✅ 根據 SOM 位置智能預測特徵值
分析該位置及其周圍區域的背景樣本,統計特徵的比例來預測
Args:
row: winner neuron 的 row
col: winner neuron 的 col
radius: 搜尋半徑(0=只看該格,1=3x3區域,2=5x5區域)
Returns:
dict: 特徵預測結果,例如 {'dropper': 1, 'APT30': 0}
"""
print(f" 🔍 Analyzing region around ({row}, {col}) with radius={radius}")
# 收集該區域的所有樣本
region_samples = {'dropper': [], 'APT30': []}
for i, emb in enumerate(self.background_embeddings):
sample_row, sample_col = self.som.winner(emb)
# 檢查是否在指定半徑內
if abs(sample_row - row) <= radius and abs(sample_col - col) <= radius:
# 收集該樣本的特徵值
if 'dropper' in self.background_labels:
region_samples['dropper'].append(self.background_labels['dropper'][i])
if 'APT30' in self.background_labels:
region_samples['APT30'].append(self.background_labels['APT30'][i])
# 根據該區域的統計來預測
predictions = {}
for feature_name, values in region_samples.items():
if len(values) == 0:
# 沒有樣本,預設為 0
predictions[feature_name] = 0
print(f" {feature_name}: No samples → 0")
else:
# 計算該特徵為 1 的比例
positive_count = sum(values)
total_count = len(values)
positive_ratio = positive_count / total_count
# 如果超過 5%,預測為 1
predictions[feature_name] = 1 if positive_ratio > 0.05 else 0
print(f" {feature_name}: {positive_count}/{total_count} samples = {positive_ratio:.1%} → {predictions[feature_name]}")
return predictions
def generate_som_json_with_marker(self, new_embedding, feature_name, feature_value,
filename="unknown", predicted_family=None):
"""
生成包含新檔案標記的 SOM JSON
Args:
new_embedding: 新檔案的 768 維 embedding
feature_name: 特徵名稱 (如 'dropper', 'APT30')
feature_value: 該特徵的值 (0 或 1)
filename: 檔案名稱
predicted_family: 預測的惡意軟體家族
Returns:
dict: JSON 格式的 SOM 資料
"""
# 找到新檔案的位置
new_row, new_col = self.find_winner_neuron(new_embedding)
# 計算背景資料的 neuron 統計
neuron_stats = {}
for i, emb in enumerate(self.background_embeddings):
row, col = self.som.winner(emb)
key = f"{row},{col}"
if key not in neuron_stats:
neuron_stats[key] = {}
# 根據 feature_name 取得對應的標籤
label = self.background_labels[feature_name][i]
neuron_stats[key][label] = neuron_stats[key].get(label, 0) + 1
# 建立 JSON 結構
som_json = {
"title": f"SOM neuron composition by {feature_name}",
"som_size": self.som_size,
"neurons": [],
"new_file": {
"filename": filename,
"row": int(new_row),
"col": int(new_col),
"feature_name": feature_name,
"feature_value": int(feature_value),
"predicted_family": predicted_family
}
}
# 轉換每個 neuron 的統計資料
for key, counts in neuron_stats.items():
row, col = map(int, key.split(","))
total = sum(counts.values())
proportions = {
self.id2label.get(lbl, str(lbl)): cnt/total
for lbl, cnt in counts.items()
}
som_json["neurons"].append({
"row": row,
"col": col,
"counts": counts,
"proportions": proportions
})
return som_json
def prepare_background_data(df_merged, output_path):
"""
準備背景資料(只需執行一次)
Args:
df_merged: 合併後的 DataFrame,包含 embeddings 和所有標籤
output_path: 輸出檔案路徑
"""
# 提取 embeddings
emb_cols = [f"emb_{i}" for i in range(768)]
embeddings = df_merged[emb_cols].values.astype(float)
# 提取所有標籤
labels = {
'true_label_id': df_merged['true_label_id'].fillna(0).astype(int).values,
'dropper': df_merged['dropper'].fillna(0).astype(int).values,
'spreader': df_merged['spreader'].fillna(0).astype(int).values,
'APT30': df_merged['APT30'].fillna(0).astype(int).values,
'Codoso_Gh0st_1': df_merged['Codoso_Gh0st_1'].fillna(0).astype(int).values,
}
# 建立 id2label 映射
label_list = sorted(df_merged['true_label'].unique())
label2id = {l: i for i, l in enumerate(label_list)}
id2label = {i: l for l, i in label2id.items()}
# 保存
background_data = {
'embeddings': embeddings,
'labels': labels,
'id2label': id2label
}
with open(output_path, 'wb') as f:
pickle.dump(background_data, f)
print(f"✅ 背景資料已保存至: {output_path}")
print(f" - 資料筆數: {len(embeddings)}")
print(f" - Embedding 維度: {embeddings.shape[1]}")
print(f" - 特徵數量: {len(labels)}")