updates
Browse files
app.py
CHANGED
|
@@ -186,25 +186,90 @@ def get_model_info():
|
|
| 186 |
# "segment_predictions": predictions
|
| 187 |
# }
|
| 188 |
|
| 189 |
-
# # 寫入本地
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 190 |
|
| 191 |
HF_DATASET_REPO = "lyi029/malvec-embeddings"
|
| 192 |
|
| 193 |
def upload_embeddings_to_hub(local_path: str):
|
| 194 |
-
"""
|
| 195 |
api = HfApi()
|
|
|
|
|
|
|
|
|
|
| 196 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 197 |
api.upload_file(
|
| 198 |
path_or_fileobj=local_path,
|
| 199 |
-
path_in_repo=
|
| 200 |
repo_id=HF_DATASET_REPO,
|
| 201 |
repo_type="dataset",
|
| 202 |
token=HF_TOKEN,
|
| 203 |
)
|
| 204 |
-
print(f"☁️
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 205 |
return True
|
|
|
|
|
|
|
|
|
|
|
|
|
| 206 |
except Exception as e:
|
| 207 |
-
|
|
|
|
|
|
|
| 208 |
return False
|
| 209 |
|
| 210 |
|
|
@@ -314,22 +379,6 @@ async def predict_malware(files: List[UploadFile] = File(...)):
|
|
| 314 |
except Exception as e:
|
| 315 |
raise HTTPException(status_code=500, detail=f"Prediction failed: {e}")
|
| 316 |
|
| 317 |
-
# log_path = os.path.join("results", "embeddings_log.jsonl")
|
| 318 |
-
# with open(log_path, "a") as f:
|
| 319 |
-
# f.write(json.dumps({
|
| 320 |
-
# "label": final_label,
|
| 321 |
-
# "text" : asm_code,
|
| 322 |
-
# "embedding": best_embedding["embedding"],
|
| 323 |
-
# "source": best_embedding["filename"]
|
| 324 |
-
# }) + "\n")
|
| 325 |
-
|
| 326 |
-
# print(f"✅ Prediction complete ({final_label}), embedding logged.")
|
| 327 |
-
# return result
|
| 328 |
-
|
| 329 |
-
# except Exception as e:
|
| 330 |
-
# raise HTTPException(status_code=500, detail=f"Prediction failed: {e}")
|
| 331 |
-
|
| 332 |
-
|
| 333 |
|
| 334 |
# ----------------------------------------------------------
|
| 335 |
# 手動觸發 Space B 訓練
|
|
|
|
| 186 |
# "segment_predictions": predictions
|
| 187 |
# }
|
| 188 |
|
| 189 |
+
# # 寫入本地 log
|
| 190 |
+
# log_path = os.path.join("results", "embeddings_log.jsonl")
|
| 191 |
+
# with open(log_path, "a") as f:
|
| 192 |
+
# f.write(json.dumps({
|
| 193 |
+
# "label": final_label,
|
| 194 |
+
# "text" : asm_code,
|
| 195 |
+
# "embedding": best_embedding["embedding"],
|
| 196 |
+
# "source": best_embedding["filename"]
|
| 197 |
+
# }) + "\n")
|
| 198 |
+
|
| 199 |
+
# print(f"✅ Prediction complete ({final_label}), embedding logged.")
|
| 200 |
+
# return result
|
| 201 |
+
|
| 202 |
+
# except Exception as e:
|
| 203 |
+
# raise HTTPException(status_code=500, detail=f"Prediction failed: {e}")
|
| 204 |
+
|
| 205 |
+
# from huggingface_hub import HfApi
|
| 206 |
+
|
| 207 |
+
# HF_DATASET_REPO = "lyi029/malvec-embeddings"
|
| 208 |
+
|
| 209 |
+
# def upload_embeddings_to_hub(local_path: str):
|
| 210 |
+
# """將 embeddings_log.jsonl 上傳到 Hugging Face Dataset"""
|
| 211 |
+
# api = HfApi()
|
| 212 |
+
# try:
|
| 213 |
+
# api.upload_file(
|
| 214 |
+
# path_or_fileobj=local_path,
|
| 215 |
+
# path_in_repo="logs/embeddings_log.jsonl",
|
| 216 |
+
# repo_id=HF_DATASET_REPO,
|
| 217 |
+
# repo_type="dataset",
|
| 218 |
+
# token=HF_TOKEN,
|
| 219 |
+
# )
|
| 220 |
+
# print(f"☁️ Uploaded embeddings to dataset: {HF_DATASET_REPO}/logs/embeddings_log.jsonl")
|
| 221 |
+
# return True
|
| 222 |
+
# except Exception as e:
|
| 223 |
+
# print(f"⚠️ Failed to upload embeddings: {e}")
|
| 224 |
+
# return False
|
| 225 |
+
|
| 226 |
+
from huggingface_hub import HfApi, HfHubHTTPError
|
| 227 |
+
import datetime
|
| 228 |
|
| 229 |
HF_DATASET_REPO = "lyi029/malvec-embeddings"
|
| 230 |
|
| 231 |
def upload_embeddings_to_hub(local_path: str):
|
| 232 |
+
"""自動建立 Dataset 並分批上傳 embeddings_log.jsonl"""
|
| 233 |
api = HfApi()
|
| 234 |
+
timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
|
| 235 |
+
remote_path = f"logs/embeddings_{timestamp}.jsonl"
|
| 236 |
+
|
| 237 |
try:
|
| 238 |
+
# 確認 Dataset 存在
|
| 239 |
+
try:
|
| 240 |
+
api.dataset_info(repo_id=HF_DATASET_REPO, token=HF_TOKEN)
|
| 241 |
+
print(f"📂 Dataset {HF_DATASET_REPO} 已存在。")
|
| 242 |
+
except HfHubHTTPError as e:
|
| 243 |
+
if e.response is not None and e.response.status_code == 404:
|
| 244 |
+
print(f"🆕 Dataset 不存在,建立中...")
|
| 245 |
+
api.create_repo(repo_id=HF_DATASET_REPO, repo_type="dataset", token=HF_TOKEN)
|
| 246 |
+
print(f"✅ 已建立 Dataset: {HF_DATASET_REPO}")
|
| 247 |
+
else:
|
| 248 |
+
raise
|
| 249 |
+
|
| 250 |
+
print(f"🚀 上傳 {local_path} → {HF_DATASET_REPO}/{remote_path}")
|
| 251 |
api.upload_file(
|
| 252 |
path_or_fileobj=local_path,
|
| 253 |
+
path_in_repo=remote_path,
|
| 254 |
repo_id=HF_DATASET_REPO,
|
| 255 |
repo_type="dataset",
|
| 256 |
token=HF_TOKEN,
|
| 257 |
)
|
| 258 |
+
print(f"☁️ 已上傳 embeddings 至 {HF_DATASET_REPO}/{remote_path}")
|
| 259 |
+
|
| 260 |
+
# 上傳後清空本地檔案(避免爆 container)
|
| 261 |
+
open(local_path, "w").close()
|
| 262 |
+
print(f"🧹 已清空本地 {local_path}")
|
| 263 |
+
|
| 264 |
return True
|
| 265 |
+
|
| 266 |
+
except HfHubHTTPError as e:
|
| 267 |
+
print(f"❌ HF API HTTP error: {e.response.status_code} - {e.response.text}")
|
| 268 |
+
return False
|
| 269 |
except Exception as e:
|
| 270 |
+
import traceback
|
| 271 |
+
print("❌ Upload failed:", e)
|
| 272 |
+
traceback.print_exc()
|
| 273 |
return False
|
| 274 |
|
| 275 |
|
|
|
|
| 379 |
except Exception as e:
|
| 380 |
raise HTTPException(status_code=500, detail=f"Prediction failed: {e}")
|
| 381 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 382 |
|
| 383 |
# ----------------------------------------------------------
|
| 384 |
# 手動觸發 Space B 訓練
|