Spaces:
Runtime error
Runtime error
Download som_analyzer.py from malvec/codebert_Malvec: direct link, hf CLI and curl.
- Browser
- Download file 7.54 kB
-
https://huggingface.co/spaces/malvec/codebert_Malvec/resolve/main/som_analyzer.py
- Command line
-
hf download hf://spaces/malvec/codebert_Malvec/som_analyzer.py
-
curl -L -o som_analyzer.py https://huggingface.co/spaces/malvec/codebert_Malvec/resolve/main/som_analyzer.py
7.54 kB
| import numpy as np | |
| import json | |
| import pickle | |
| from minisom import MiniSom | |
| class SOMAnalyzer: | |
| """SOM 分析器 - 載入預訓練的 SOM 並分析新檔案""" | |
| def __init__(self, som_weights_path, background_data_path): | |
| """ | |
| 初始化 SOM 分析器 | |
| Args: | |
| som_weights_path: SOM 權重檔案路徑 (.npy) | |
| background_data_path: 背景資料檔案路徑 (.pkl),包含所有訓練資料的 embeddings 和標籤 | |
| """ | |
| # 載入 SOM 權重 | |
| self.som_weights = np.load(som_weights_path) | |
| self.som_size = self.som_weights.shape[0] | |
| self.input_dim = self.som_weights.shape[2] | |
| # 建立 SOM 模型 | |
| self.som = MiniSom( | |
| self.som_size, | |
| self.som_size, | |
| self.input_dim, | |
| sigma=0.5, | |
| learning_rate=0.5, | |
| random_seed=77 | |
| ) | |
| self.som._weights = self.som_weights.copy() | |
| # 載入背景資料 | |
| with open(background_data_path, 'rb') as f: | |
| background = pickle.load(f) | |
| self.background_embeddings = background['embeddings'] | |
| self.background_labels = background['labels'] | |
| self.id2label = background['id2label'] | |
| def find_winner_neuron(self, embedding): | |
| """ | |
| 找到 embedding 對應的 winner neuron | |
| Args: | |
| embedding: 768 維的 embedding vector | |
| Returns: | |
| (row, col): winner neuron 的座標 | |
| """ | |
| return self.som.winner(embedding) | |
| def predict_features_from_position(self, row, col, radius=1): | |
| """ | |
| ✅ 根據 SOM 位置智能預測特徵值 | |
| 分析該位置及其周圍區域的背景樣本,統計特徵的比例來預測 | |
| Args: | |
| row: winner neuron 的 row | |
| col: winner neuron 的 col | |
| radius: 搜尋半徑(0=只看該格,1=3x3區域,2=5x5區域) | |
| Returns: | |
| dict: 特徵預測結果,例如 {'dropper': 1, 'APT30': 0} | |
| """ | |
| print(f" 🔍 Analyzing region around ({row}, {col}) with radius={radius}") | |
| # 收集該區域的所有樣本 | |
| region_samples = {'dropper': [], 'APT30': []} | |
| for i, emb in enumerate(self.background_embeddings): | |
| sample_row, sample_col = self.som.winner(emb) | |
| # 檢查是否在指定半徑內 | |
| if abs(sample_row - row) <= radius and abs(sample_col - col) <= radius: | |
| # 收集該樣本的特徵值 | |
| if 'dropper' in self.background_labels: | |
| region_samples['dropper'].append(self.background_labels['dropper'][i]) | |
| if 'APT30' in self.background_labels: | |
| region_samples['APT30'].append(self.background_labels['APT30'][i]) | |
| # 根據該區域的統計來預測 | |
| predictions = {} | |
| for feature_name, values in region_samples.items(): | |
| if len(values) == 0: | |
| # 沒有樣本,預設為 0 | |
| predictions[feature_name] = 0 | |
| print(f" {feature_name}: No samples → 0") | |
| else: | |
| # 計算該特徵為 1 的比例 | |
| positive_count = sum(values) | |
| total_count = len(values) | |
| positive_ratio = positive_count / total_count | |
| # 如果超過 5%,預測為 1 | |
| predictions[feature_name] = 1 if positive_ratio > 0.05 else 0 | |
| print(f" {feature_name}: {positive_count}/{total_count} samples = {positive_ratio:.1%} → {predictions[feature_name]}") | |
| return predictions | |
| def generate_som_json_with_marker(self, new_embedding, feature_name, feature_value, | |
| filename="unknown", predicted_family=None): | |
| """ | |
| 生成包含新檔案標記的 SOM JSON | |
| Args: | |
| new_embedding: 新檔案的 768 維 embedding | |
| feature_name: 特徵名稱 (如 'dropper', 'APT30') | |
| feature_value: 該特徵的值 (0 或 1) | |
| filename: 檔案名稱 | |
| predicted_family: 預測的惡意軟體家族 | |
| Returns: | |
| dict: JSON 格式的 SOM 資料 | |
| """ | |
| # 找到新檔案的位置 | |
| new_row, new_col = self.find_winner_neuron(new_embedding) | |
| # 計算背景資料的 neuron 統計 | |
| neuron_stats = {} | |
| for i, emb in enumerate(self.background_embeddings): | |
| row, col = self.som.winner(emb) | |
| key = f"{row},{col}" | |
| if key not in neuron_stats: | |
| neuron_stats[key] = {} | |
| # 根據 feature_name 取得對應的標籤 | |
| label = self.background_labels[feature_name][i] | |
| neuron_stats[key][label] = neuron_stats[key].get(label, 0) + 1 | |
| # 建立 JSON 結構 | |
| som_json = { | |
| "title": f"SOM neuron composition by {feature_name}", | |
| "som_size": self.som_size, | |
| "neurons": [], | |
| "new_file": { | |
| "filename": filename, | |
| "row": int(new_row), | |
| "col": int(new_col), | |
| "feature_name": feature_name, | |
| "feature_value": int(feature_value), | |
| "predicted_family": predicted_family | |
| } | |
| } | |
| # 轉換每個 neuron 的統計資料 | |
| for key, counts in neuron_stats.items(): | |
| row, col = map(int, key.split(",")) | |
| total = sum(counts.values()) | |
| proportions = { | |
| self.id2label.get(lbl, str(lbl)): cnt/total | |
| for lbl, cnt in counts.items() | |
| } | |
| som_json["neurons"].append({ | |
| "row": row, | |
| "col": col, | |
| "counts": counts, | |
| "proportions": proportions | |
| }) | |
| return som_json | |
| def prepare_background_data(df_merged, output_path): | |
| """ | |
| 準備背景資料(只需執行一次) | |
| Args: | |
| df_merged: 合併後的 DataFrame,包含 embeddings 和所有標籤 | |
| output_path: 輸出檔案路徑 | |
| """ | |
| # 提取 embeddings | |
| emb_cols = [f"emb_{i}" for i in range(768)] | |
| embeddings = df_merged[emb_cols].values.astype(float) | |
| # 提取所有標籤 | |
| labels = { | |
| 'true_label_id': df_merged['true_label_id'].fillna(0).astype(int).values, | |
| 'dropper': df_merged['dropper'].fillna(0).astype(int).values, | |
| 'spreader': df_merged['spreader'].fillna(0).astype(int).values, | |
| 'APT30': df_merged['APT30'].fillna(0).astype(int).values, | |
| 'Codoso_Gh0st_1': df_merged['Codoso_Gh0st_1'].fillna(0).astype(int).values, | |
| } | |
| # 建立 id2label 映射 | |
| label_list = sorted(df_merged['true_label'].unique()) | |
| label2id = {l: i for i, l in enumerate(label_list)} | |
| id2label = {i: l for l, i in label2id.items()} | |
| # 保存 | |
| background_data = { | |
| 'embeddings': embeddings, | |
| 'labels': labels, | |
| 'id2label': id2label | |
| } | |
| with open(output_path, 'wb') as f: | |
| pickle.dump(background_data, f) | |
| print(f"✅ 背景資料已保存至: {output_path}") | |
| print(f" - 資料筆數: {len(embeddings)}") | |
| print(f" - Embedding 維度: {embeddings.shape[1]}") | |
| print(f" - 特徵數量: {len(labels)}") |