lyi029 commited on
Commit
90970fd
·
1 Parent(s): a549c99
Files changed (1) hide show
  1. app.py +215 -63
app.py CHANGED
@@ -8,7 +8,7 @@ from transformers import AutoTokenizer, AutoModelForSequenceClassification
8
  from typing import List
9
  from collections import Counter
10
  from fastapi.staticfiles import StaticFiles
11
- from huggingface_hub import HfApi
12
  import datetime
13
 
14
  CURRENT_TAG = os.getenv("MODEL_TAG", "main")
@@ -150,14 +150,120 @@ def upload_embeddings_to_hub(local_path: str):
150
  return False
151
 
152
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
153
  @app.post("/predict")
154
  async def predict_malware(files: List[UploadFile] = File(...)):
155
- """批次預測惡意軟體家族"""
156
  if not files:
157
  raise HTTPException(status_code=400, detail="No files provided")
158
 
159
  predictions, all_embeddings = [], []
160
-
161
  print(f"\n📥 Received {len(files)} files")
162
 
163
  try:
@@ -171,92 +277,138 @@ async def predict_malware(files: List[UploadFile] = File(...)):
171
 
172
  print(f"📄 [{idx}/{len(files)}] {file.filename}")
173
 
174
- inputs = tokenizer(
175
- asm_code,
176
- return_tensors="pt",
177
- max_length=512,
178
- truncation=True,
179
- padding=True
180
- ).to(DEVICE)
181
-
182
- outputs = model(**inputs)
183
- logits = outputs.logits
184
- probs = torch.nn.functional.softmax(logits, dim=-1)
185
- confidence, predicted_class = torch.max(probs, dim=-1)
186
-
187
- predicted_id = predicted_class.item()
188
- label_name = model.config.id2label.get(predicted_id, f"class_{predicted_id}")
189
-
190
- # embedding
191
- if outputs.hidden_states:
192
- cls_embedding = outputs.hidden_states[-1][:, 0, :].cpu().numpy()[0]
193
- else:
194
- cls_embedding = logits.cpu().numpy()[0]
195
-
196
- # attention
197
- max_attention_score = 0.0
198
- if outputs.attentions:
199
- last_attention = outputs.attentions[-1].mean(dim=1)
200
- cls_attention = last_attention[0, 0, :].cpu().numpy()
201
- max_attention_score = float(cls_attention.max())
202
-
203
- predictions.append({
204
- "filename": file.filename,
205
- "predicted_label": label_name,
206
- "confidence": float(confidence.item()),
207
- "attention_score": max_attention_score
208
- })
209
- all_embeddings.append({
210
- "filename": file.filename,
211
- "label": label_name,
212
- "embedding": cls_embedding.tolist(),
213
- "attention_score": max_attention_score
214
- })
 
 
 
 
 
 
 
 
 
215
  except Exception as e:
216
  print(f"❌ {file.filename} error:", e)
217
 
218
  if not predictions:
219
  raise HTTPException(status_code=400, detail="No valid files processed")
220
 
221
- # 多數投票
222
  votes = [p["predicted_label"] for p in predictions]
223
  final_label, count = Counter(votes).most_common(1)[0]
224
- best_embedding = max(all_embeddings, key=lambda x: x["attention_score"])
225
 
226
  result = {
227
  "final_label": final_label,
228
  "vote_count": count,
229
  "total_segments": len(predictions),
230
  "confidence": sum(p["confidence"] for p in predictions) / len(predictions),
231
- "embedding": {
232
- "values": best_embedding["embedding"],
233
- "source_file": best_embedding["filename"],
234
- "attention_score": best_embedding["attention_score"]
235
- },
236
  "segment_predictions": predictions
237
  }
238
 
239
- # 寫入本地 log
240
  log_path = os.path.join("results", "embeddings_log.jsonl")
241
  with open(log_path, "a") as f:
242
- f.write(json.dumps({
243
- "label": final_label,
244
- "text": asm_code,
245
- "embedding": best_embedding["embedding"],
246
- "source": best_embedding["filename"]
247
- }) + "\n")
 
 
 
 
248
 
249
- print(f"✅ Prediction complete ({final_label}), embedding logged.")
 
 
250
 
251
- # ☁️ 自動上傳到 HF Dataset
252
- upload_embeddings_to_hub(log_path)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
253
 
 
254
  return result
255
 
256
  except Exception as e:
257
  raise HTTPException(status_code=500, detail=f"Prediction failed: {e}")
258
-
259
-
260
  # ----------------------------------------------------------
261
  # 手動觸發 Space B 訓練
262
  # ----------------------------------------------------------
 
8
  from typing import List
9
  from collections import Counter
10
  from fastapi.staticfiles import StaticFiles
11
+ from huggingface_hub import HfApi,hf_hub_download
12
  import datetime
13
 
14
  CURRENT_TAG = os.getenv("MODEL_TAG", "main")
 
150
  return False
151
 
152
 
153
+ # @app.post("/predict")
154
+ # async def predict_malware(files: List[UploadFile] = File(...)):
155
+ # """批次預測惡意軟體家族"""
156
+ # if not files:
157
+ # raise HTTPException(status_code=400, detail="No files provided")
158
+
159
+ # predictions, all_embeddings = [], []
160
+
161
+ # print(f"\n📥 Received {len(files)} files")
162
+
163
+ # try:
164
+ # with torch.no_grad():
165
+ # for idx, file in enumerate(files, 1):
166
+ # try:
167
+ # content = await file.read()
168
+ # asm_code = content.decode("utf-8").strip()
169
+ # if not asm_code:
170
+ # continue
171
+
172
+ # print(f"📄 [{idx}/{len(files)}] {file.filename}")
173
+
174
+ # inputs = tokenizer(
175
+ # asm_code,
176
+ # return_tensors="pt",
177
+ # max_length=512,
178
+ # truncation=True,
179
+ # padding=True
180
+ # ).to(DEVICE)
181
+
182
+ # outputs = model(**inputs)
183
+ # logits = outputs.logits
184
+ # probs = torch.nn.functional.softmax(logits, dim=-1)
185
+ # confidence, predicted_class = torch.max(probs, dim=-1)
186
+
187
+ # predicted_id = predicted_class.item()
188
+ # label_name = model.config.id2label.get(predicted_id, f"class_{predicted_id}")
189
+
190
+ # # embedding
191
+ # if outputs.hidden_states:
192
+ # cls_embedding = outputs.hidden_states[-1][:, 0, :].cpu().numpy()[0]
193
+ # else:
194
+ # cls_embedding = logits.cpu().numpy()[0]
195
+
196
+ # # attention
197
+ # max_attention_score = 0.0
198
+ # if outputs.attentions:
199
+ # last_attention = outputs.attentions[-1].mean(dim=1)
200
+ # cls_attention = last_attention[0, 0, :].cpu().numpy()
201
+ # max_attention_score = float(cls_attention.max())
202
+
203
+ # predictions.append({
204
+ # "filename": file.filename,
205
+ # "predicted_label": label_name,
206
+ # "confidence": float(confidence.item()),
207
+ # "attention_score": max_attention_score
208
+ # })
209
+ # all_embeddings.append({
210
+ # "filename": file.filename,
211
+ # "label": label_name,
212
+ # "embedding": cls_embedding.tolist(),
213
+ # "attention_score": max_attention_score
214
+ # })
215
+ # except Exception as e:
216
+ # print(f"❌ {file.filename} error:", e)
217
+
218
+ # if not predictions:
219
+ # raise HTTPException(status_code=400, detail="No valid files processed")
220
+
221
+ # # 多數投票
222
+ # votes = [p["predicted_label"] for p in predictions]
223
+ # final_label, count = Counter(votes).most_common(1)[0]
224
+ # best_embedding = max(all_embeddings, key=lambda x: x["attention_score"])
225
+
226
+ # result = {
227
+ # "final_label": final_label,
228
+ # "vote_count": count,
229
+ # "total_segments": len(predictions),
230
+ # "confidence": sum(p["confidence"] for p in predictions) / len(predictions),
231
+ # "embedding": {
232
+ # "values": best_embedding["embedding"],
233
+ # "source_file": best_embedding["filename"],
234
+ # "attention_score": best_embedding["attention_score"]
235
+ # },
236
+ # "segment_predictions": predictions
237
+ # }
238
+
239
+ # # 寫入本地 log
240
+ # log_path = os.path.join("results", "embeddings_log.jsonl")
241
+ # with open(log_path, "a") as f:
242
+ # f.write(json.dumps({
243
+ # "label": final_label,
244
+ # "text": asm_code,
245
+ # "embedding": best_embedding["embedding"],
246
+ # "source": best_embedding["filename"]
247
+ # }) + "\n")
248
+
249
+ # print(f"✅ Prediction complete ({final_label}), embedding logged.")
250
+
251
+ # # ☁️ 自動上傳到 HF Dataset
252
+ # upload_embeddings_to_hub(log_path)
253
+
254
+ # return result
255
+
256
+ # except Exception as e:
257
+ # raise HTTPException(status_code=500, detail=f"Prediction failed: {e}")
258
+
259
+
260
  @app.post("/predict")
261
  async def predict_malware(files: List[UploadFile] = File(...)):
262
+ """批次預測惡意軟體家族 + 自動上傳 embeddings 到 Dataset"""
263
  if not files:
264
  raise HTTPException(status_code=400, detail="No files provided")
265
 
266
  predictions, all_embeddings = [], []
 
267
  print(f"\n📥 Received {len(files)} files")
268
 
269
  try:
 
277
 
278
  print(f"📄 [{idx}/{len(files)}] {file.filename}")
279
 
280
+ # === 分段處理 ===
281
+ segments = [seg.strip() for seg in asm_code.splitlines() if seg.strip()]
282
+ for seg_idx, segment in enumerate(segments, 1):
283
+ inputs = tokenizer(
284
+ segment,
285
+ return_tensors="pt",
286
+ max_length=512,
287
+ truncation=True,
288
+ padding=True
289
+ ).to(DEVICE)
290
+
291
+ outputs = model(**inputs)
292
+ logits = outputs.logits
293
+ probs = torch.nn.functional.softmax(logits, dim=-1)
294
+ confidence, predicted_class = torch.max(probs, dim=-1)
295
+
296
+ predicted_id = predicted_class.item()
297
+ label_name = model.config.id2label.get(predicted_id, f"class_{predicted_id}")
298
+
299
+ # === embedding ===
300
+ if outputs.hidden_states:
301
+ cls_embedding = outputs.hidden_states[-1][:, 0, :].cpu().numpy()[0]
302
+ else:
303
+ cls_embedding = logits.cpu().numpy()[0]
304
+
305
+ # === attention ===
306
+ max_attention_score = 0.0
307
+ if outputs.attentions:
308
+ last_attention = outputs.attentions[-1].mean(dim=1)
309
+ cls_attention = last_attention[0, 0, :].cpu().numpy()
310
+ max_attention_score = float(cls_attention.max())
311
+
312
+ record = {
313
+ "filename": file.filename,
314
+ "segment_index": seg_idx,
315
+ "text": segment,
316
+ "predicted_label": label_name,
317
+ "confidence": float(confidence.item()),
318
+ "attention_score": max_attention_score,
319
+ "embedding": cls_embedding.tolist()
320
+ }
321
+ all_embeddings.append(record)
322
+
323
+ predictions.append({
324
+ "filename": file.filename,
325
+ "predicted_label": label_name,
326
+ "confidence": float(confidence.item()),
327
+ "attention_score": max_attention_score
328
+ })
329
+
330
  except Exception as e:
331
  print(f"❌ {file.filename} error:", e)
332
 
333
  if not predictions:
334
  raise HTTPException(status_code=400, detail="No valid files processed")
335
 
336
+ # === 多數投票 ===
337
  votes = [p["predicted_label"] for p in predictions]
338
  final_label, count = Counter(votes).most_common(1)[0]
 
339
 
340
  result = {
341
  "final_label": final_label,
342
  "vote_count": count,
343
  "total_segments": len(predictions),
344
  "confidence": sum(p["confidence"] for p in predictions) / len(predictions),
 
 
 
 
 
345
  "segment_predictions": predictions
346
  }
347
 
348
+ # === 寫入本地 embeddings_log.jsonl ===
349
  log_path = os.path.join("results", "embeddings_log.jsonl")
350
  with open(log_path, "a") as f:
351
+ for rec in all_embeddings:
352
+ f.write(json.dumps({
353
+ "label": rec["predicted_label"],
354
+ "text": rec["text"],
355
+ "embedding": rec["embedding"],
356
+ "source": rec["filename"],
357
+ "segment_index": rec["segment_index"]
358
+ }) + "\n")
359
+
360
+ print(f"✅ Logged {len(all_embeddings)} embeddings locally.")
361
 
362
+ # === 合併上傳到 Hugging Face Dataset ===
363
+ repo_id = "lyi029/malvec-embeddings"
364
+ api = HfApi()
365
 
366
+ try:
367
+ # 下載雲端舊資料
368
+ old_data = []
369
+ try:
370
+ old_path = hf_hub_download(
371
+ repo_id=repo_id,
372
+ filename="embeddings_log.jsonl",
373
+ repo_type="dataset",
374
+ token=HF_TOKEN
375
+ )
376
+ with open(old_path, "r") as f:
377
+ old_data = [json.loads(l) for l in f if l.strip()]
378
+ except Exception:
379
+ print("ℹ️ No previous embeddings found in dataset.")
380
+
381
+ # 新增本地新資料
382
+ with open(log_path, "r") as f:
383
+ new_data = [json.loads(l) for l in f if l.strip()]
384
+
385
+ # 合併去重(依 source + segment_index 為唯一鍵)
386
+ merged_dict = {(d["source"], d.get("segment_index", 0)): d for d in (old_data + new_data)}
387
+ merged = list(merged_dict.values())
388
+
389
+ # 寫回暫存檔
390
+ with open(log_path, "w") as f:
391
+ for item in merged:
392
+ f.write(json.dumps(item) + "\n")
393
+
394
+ # 上傳 dataset
395
+ api.upload_file(
396
+ path_or_fileobj=log_path,
397
+ path_in_repo="embeddings_log.jsonl",
398
+ repo_id=repo_id,
399
+ repo_type="dataset",
400
+ token=HF_TOKEN,
401
+ )
402
+ print(f"☁️ Uploaded merged {len(merged)} embeddings to dataset: {repo_id}")
403
+
404
+ except Exception as e:
405
+ print(f"⚠️ Dataset upload failed: {e}")
406
 
407
+ print(f"✅ Prediction complete ({final_label})")
408
  return result
409
 
410
  except Exception as e:
411
  raise HTTPException(status_code=500, detail=f"Prediction failed: {e}")
 
 
412
  # ----------------------------------------------------------
413
  # 手動觸發 Space B 訓練
414
  # ----------------------------------------------------------