File size: 7,544 Bytes
c0e1a78
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
import numpy as np
import json
import pickle
from minisom import MiniSom


class SOMAnalyzer:
    """SOM 分析器 - 載入預訓練的 SOM 並分析新檔案"""
    
    def __init__(self, som_weights_path, background_data_path):
        """
        初始化 SOM 分析器
        
        Args:
            som_weights_path: SOM 權重檔案路徑 (.npy)
            background_data_path: 背景資料檔案路徑 (.pkl),包含所有訓練資料的 embeddings 和標籤
        """
        # 載入 SOM 權重
        self.som_weights = np.load(som_weights_path)
        self.som_size = self.som_weights.shape[0]
        self.input_dim = self.som_weights.shape[2]
        
        # 建立 SOM 模型
        self.som = MiniSom(
            self.som_size, 
            self.som_size, 
            self.input_dim,
            sigma=0.5, 
            learning_rate=0.5, 
            random_seed=77
        )
        self.som._weights = self.som_weights.copy()
        
        # 載入背景資料
        with open(background_data_path, 'rb') as f:
            background = pickle.load(f)
            self.background_embeddings = background['embeddings']
            self.background_labels = background['labels']
            self.id2label = background['id2label']
    
    def find_winner_neuron(self, embedding):
        """
        找到 embedding 對應的 winner neuron
        
        Args:
            embedding: 768 維的 embedding vector
            
        Returns:
            (row, col): winner neuron 的座標
        """
        return self.som.winner(embedding)
    
    def predict_features_from_position(self, row, col, radius=1):
        """
        ✅ 根據 SOM 位置智能預測特徵值
        
        分析該位置及其周圍區域的背景樣本,統計特徵的比例來預測
        
        Args:
            row: winner neuron 的 row
            col: winner neuron 的 col
            radius: 搜尋半徑(0=只看該格,1=3x3區域,2=5x5區域)
            
        Returns:
            dict: 特徵預測結果,例如 {'dropper': 1, 'APT30': 0}
        """
        print(f"   🔍 Analyzing region around ({row}, {col}) with radius={radius}")
        
        # 收集該區域的所有樣本
        region_samples = {'dropper': [], 'APT30': []}
        
        for i, emb in enumerate(self.background_embeddings):
            sample_row, sample_col = self.som.winner(emb)
            
            # 檢查是否在指定半徑內
            if abs(sample_row - row) <= radius and abs(sample_col - col) <= radius:
                # 收集該樣本的特徵值
                if 'dropper' in self.background_labels:
                    region_samples['dropper'].append(self.background_labels['dropper'][i])
                if 'APT30' in self.background_labels:
                    region_samples['APT30'].append(self.background_labels['APT30'][i])
        
        # 根據該區域的統計來預測
        predictions = {}
        
        for feature_name, values in region_samples.items():
            if len(values) == 0:
                # 沒有樣本,預設為 0
                predictions[feature_name] = 0
                print(f"      {feature_name}: No samples → 0")
            else:
                # 計算該特徵為 1 的比例
                positive_count = sum(values)
                total_count = len(values)
                positive_ratio = positive_count / total_count
                
                # 如果超過 5%,預測為 1
                predictions[feature_name] = 1 if positive_ratio > 0.05 else 0
                
                print(f"      {feature_name}: {positive_count}/{total_count} samples = {positive_ratio:.1%} → {predictions[feature_name]}")
        
        return predictions
    
    def generate_som_json_with_marker(self, new_embedding, feature_name, feature_value, 
                                       filename="unknown", predicted_family=None):
        """
        生成包含新檔案標記的 SOM JSON
        
        Args:
            new_embedding: 新檔案的 768 維 embedding
            feature_name: 特徵名稱 (如 'dropper', 'APT30')
            feature_value: 該特徵的值 (0 或 1)
            filename: 檔案名稱
            predicted_family: 預測的惡意軟體家族
            
        Returns:
            dict: JSON 格式的 SOM 資料
        """
        # 找到新檔案的位置
        new_row, new_col = self.find_winner_neuron(new_embedding)
        
        # 計算背景資料的 neuron 統計
        neuron_stats = {}
        for i, emb in enumerate(self.background_embeddings):
            row, col = self.som.winner(emb)
            key = f"{row},{col}"
            
            if key not in neuron_stats:
                neuron_stats[key] = {}
            
            # 根據 feature_name 取得對應的標籤
            label = self.background_labels[feature_name][i]
            neuron_stats[key][label] = neuron_stats[key].get(label, 0) + 1
        
        # 建立 JSON 結構
        som_json = {
            "title": f"SOM neuron composition by {feature_name}",
            "som_size": self.som_size,
            "neurons": [],
            "new_file": {
                "filename": filename,
                "row": int(new_row),
                "col": int(new_col),
                "feature_name": feature_name,
                "feature_value": int(feature_value),
                "predicted_family": predicted_family
            }
        }
        
        # 轉換每個 neuron 的統計資料
        for key, counts in neuron_stats.items():
            row, col = map(int, key.split(","))
            total = sum(counts.values())
            proportions = {
                self.id2label.get(lbl, str(lbl)): cnt/total 
                for lbl, cnt in counts.items()
            }
            som_json["neurons"].append({
                "row": row,
                "col": col,
                "counts": counts,
                "proportions": proportions
            })
        
        return som_json


def prepare_background_data(df_merged, output_path):
    """
    準備背景資料(只需執行一次)
    
    Args:
        df_merged: 合併後的 DataFrame,包含 embeddings 和所有標籤
        output_path: 輸出檔案路徑
    """
    # 提取 embeddings
    emb_cols = [f"emb_{i}" for i in range(768)]
    embeddings = df_merged[emb_cols].values.astype(float)
    
    # 提取所有標籤
    labels = {
        'true_label_id': df_merged['true_label_id'].fillna(0).astype(int).values,
        'dropper': df_merged['dropper'].fillna(0).astype(int).values,
        'spreader': df_merged['spreader'].fillna(0).astype(int).values,
        'APT30': df_merged['APT30'].fillna(0).astype(int).values,
        'Codoso_Gh0st_1': df_merged['Codoso_Gh0st_1'].fillna(0).astype(int).values,
    }
    
    # 建立 id2label 映射
    label_list = sorted(df_merged['true_label'].unique())
    label2id = {l: i for i, l in enumerate(label_list)}
    id2label = {i: l for l, i in label2id.items()}
    
    # 保存
    background_data = {
        'embeddings': embeddings,
        'labels': labels,
        'id2label': id2label
    }
    
    with open(output_path, 'wb') as f:
        pickle.dump(background_data, f)
    
    print(f"✅ 背景資料已保存至: {output_path}")
    print(f"   - 資料筆數: {len(embeddings)}")
    print(f"   - Embedding 維度: {embeddings.shape[1]}")
    print(f"   - 特徵數量: {len(labels)}")