lyi029 commited on
Commit
c1c083d
·
1 Parent(s): a178790
Files changed (1) hide show
  1. app.py +70 -21
app.py CHANGED
@@ -186,25 +186,90 @@ def get_model_info():
186
  # "segment_predictions": predictions
187
  # }
188
 
189
- # # 寫入本地 logfrom huggingface_hub import HfApi
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
190
 
191
  HF_DATASET_REPO = "lyi029/malvec-embeddings"
192
 
193
  def upload_embeddings_to_hub(local_path: str):
194
- """將 embeddings_log.jsonl 上傳到 Hugging Face Dataset"""
195
  api = HfApi()
 
 
 
196
  try:
 
 
 
 
 
 
 
 
 
 
 
 
 
197
  api.upload_file(
198
  path_or_fileobj=local_path,
199
- path_in_repo="logs/embeddings_log.jsonl",
200
  repo_id=HF_DATASET_REPO,
201
  repo_type="dataset",
202
  token=HF_TOKEN,
203
  )
204
- print(f"☁️ Uploaded embeddings to dataset: {HF_DATASET_REPO}/logs/embeddings_log.jsonl")
 
 
 
 
 
205
  return True
 
 
 
 
206
  except Exception as e:
207
- print(f"⚠️ Failed to upload embeddings: {e}")
 
 
208
  return False
209
 
210
 
@@ -314,22 +379,6 @@ async def predict_malware(files: List[UploadFile] = File(...)):
314
  except Exception as e:
315
  raise HTTPException(status_code=500, detail=f"Prediction failed: {e}")
316
 
317
- # log_path = os.path.join("results", "embeddings_log.jsonl")
318
- # with open(log_path, "a") as f:
319
- # f.write(json.dumps({
320
- # "label": final_label,
321
- # "text" : asm_code,
322
- # "embedding": best_embedding["embedding"],
323
- # "source": best_embedding["filename"]
324
- # }) + "\n")
325
-
326
- # print(f"✅ Prediction complete ({final_label}), embedding logged.")
327
- # return result
328
-
329
- # except Exception as e:
330
- # raise HTTPException(status_code=500, detail=f"Prediction failed: {e}")
331
-
332
-
333
 
334
  # ----------------------------------------------------------
335
  # 手動觸發 Space B 訓練
 
186
  # "segment_predictions": predictions
187
  # }
188
 
189
+ # # 寫入本地 log
190
+ # log_path = os.path.join("results", "embeddings_log.jsonl")
191
+ # with open(log_path, "a") as f:
192
+ # f.write(json.dumps({
193
+ # "label": final_label,
194
+ # "text" : asm_code,
195
+ # "embedding": best_embedding["embedding"],
196
+ # "source": best_embedding["filename"]
197
+ # }) + "\n")
198
+
199
+ # print(f"✅ Prediction complete ({final_label}), embedding logged.")
200
+ # return result
201
+
202
+ # except Exception as e:
203
+ # raise HTTPException(status_code=500, detail=f"Prediction failed: {e}")
204
+
205
+ # from huggingface_hub import HfApi
206
+
207
+ # HF_DATASET_REPO = "lyi029/malvec-embeddings"
208
+
209
+ # def upload_embeddings_to_hub(local_path: str):
210
+ # """將 embeddings_log.jsonl 上傳到 Hugging Face Dataset"""
211
+ # api = HfApi()
212
+ # try:
213
+ # api.upload_file(
214
+ # path_or_fileobj=local_path,
215
+ # path_in_repo="logs/embeddings_log.jsonl",
216
+ # repo_id=HF_DATASET_REPO,
217
+ # repo_type="dataset",
218
+ # token=HF_TOKEN,
219
+ # )
220
+ # print(f"☁️ Uploaded embeddings to dataset: {HF_DATASET_REPO}/logs/embeddings_log.jsonl")
221
+ # return True
222
+ # except Exception as e:
223
+ # print(f"⚠️ Failed to upload embeddings: {e}")
224
+ # return False
225
+
226
+ from huggingface_hub import HfApi, HfHubHTTPError
227
+ import datetime
228
 
229
  HF_DATASET_REPO = "lyi029/malvec-embeddings"
230
 
231
  def upload_embeddings_to_hub(local_path: str):
232
+ """自動建立 Dataset 並分批上傳 embeddings_log.jsonl"""
233
  api = HfApi()
234
+ timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
235
+ remote_path = f"logs/embeddings_{timestamp}.jsonl"
236
+
237
  try:
238
+ # 確認 Dataset 存在
239
+ try:
240
+ api.dataset_info(repo_id=HF_DATASET_REPO, token=HF_TOKEN)
241
+ print(f"📂 Dataset {HF_DATASET_REPO} 已存在。")
242
+ except HfHubHTTPError as e:
243
+ if e.response is not None and e.response.status_code == 404:
244
+ print(f"🆕 Dataset 不存在,建立中...")
245
+ api.create_repo(repo_id=HF_DATASET_REPO, repo_type="dataset", token=HF_TOKEN)
246
+ print(f"✅ 已建立 Dataset: {HF_DATASET_REPO}")
247
+ else:
248
+ raise
249
+
250
+ print(f"🚀 上傳 {local_path} → {HF_DATASET_REPO}/{remote_path}")
251
  api.upload_file(
252
  path_or_fileobj=local_path,
253
+ path_in_repo=remote_path,
254
  repo_id=HF_DATASET_REPO,
255
  repo_type="dataset",
256
  token=HF_TOKEN,
257
  )
258
+ print(f"☁️ 已上傳 embeddings 至 {HF_DATASET_REPO}/{remote_path}")
259
+
260
+ # 上傳後清空本地檔案(避免爆 container)
261
+ open(local_path, "w").close()
262
+ print(f"🧹 已清空本地 {local_path}")
263
+
264
  return True
265
+
266
+ except HfHubHTTPError as e:
267
+ print(f"❌ HF API HTTP error: {e.response.status_code} - {e.response.text}")
268
+ return False
269
  except Exception as e:
270
+ import traceback
271
+ print("❌ Upload failed:", e)
272
+ traceback.print_exc()
273
  return False
274
 
275
 
 
379
  except Exception as e:
380
  raise HTTPException(status_code=500, detail=f"Prediction failed: {e}")
381
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
382
 
383
  # ----------------------------------------------------------
384
  # 手動觸發 Space B 訓練