diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..4155b6d5cef9df8d7e2d037de450708737929c71 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,11 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +lightgbm/lightgbm_APK.tl filter=lfs diff=lfs merge=lfs -text +lightgbm/lightgbm_ELF.tl filter=lfs diff=lfs merge=lfs -text +lightgbm/lightgbm_PDF.tl filter=lfs diff=lfs merge=lfs -text +lightgbm/lightgbm_PE.tl filter=lfs diff=lfs merge=lfs -text +lightgbm/lightgbm_Win32.tl filter=lfs diff=lfs merge=lfs -text +lightgbm/lightgbm_Win64.tl filter=lfs diff=lfs merge=lfs -text +lightgbm/lightgbm_all.tl filter=lfs diff=lfs merge=lfs -text +lightgbm/lightgbm_dotnet.tl filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..34736a3e397fa9df313abc4ef8d26efef8c5707a --- /dev/null +++ b/README.md @@ -0,0 +1,359 @@ +--- +license: apache-2.0 +datasets: + - joyce8/EMBER2024 +language: + - en +tags: + - malware-detection + - cybersecurity + - onnxruntime + - lightgbm + - pytorch + - tabnet + - binary-classification +pipeline_tag: text-classification +library_name: onnxruntime +--- + +# EMBER2024 Malware Detection Models + +4종 모델 아키텍처(DNN, TabNet, Hybrid GBDT2NN, LightGBM)를 [EMBER2024](https://huggingface.co/datasets/joyce8/EMBER2024) 데이터셋의 8개 파일 타입 subset 전체에 대해 학습·평가하고 배포 가능한 포맷으로 변환한 모델 컬렉션입니다. + +> **학습 환경**: NVIDIA DGX Spark (GB10 Grace Blackwell, 128 GB 통합 메모리, CUDA 13) +> **코드**: [github.com/evan0416/ember-ml](https://github.com/evan0416/ember-ml) +> **데이터셋 논문**: [Joyce et al., KDD 2025 (arXiv:2506.05074)](https://arxiv.org/abs/2506.05074) + +--- + +## 모델 구성 + +| 디렉토리 | 아키텍처 | 배포 포맷 | 파라미터 | +|---------|---------|---------|---------| +| `dnn/` | Feed-Forward DNN (PReLU + Dropout) | ONNX (INT8 Static / FP32) | 13.2 M (PE) / 0.73 M (non-PE) | +| `tabnet/` | TabNet ([Arik & Pfister, 2021](https://arxiv.org/abs/1908.07442)) | ONNX FP32 | ~3 M | +| `hybrid/` | GBDT2NN ([DeepGBM, KDD 2019](https://www.microsoft.com/en-us/research/publication/deepgbm-a-deep-learning-framework-distilled-by-gbdt-for-online-prediction-tasks/)) | ONNX (nn_part) + LightGBM booster | ~1 M NN | +| `lightgbm/` | LightGBM (사전학습, [joyce8/EMBER2024-benchmark-models](https://huggingface.co/joyce8/EMBER2024-benchmark-models)) | Treelite `.tl` | — | + +### Subset 목록 + +| Subset | 대상 파일 타입 | 입력 차원 | +|--------|-------------|---------| +| `PE` | PE 바이너리 전체 (Win32 + Win64 + .NET) | 2,568 | +| `Win32` | Windows 32-bit PE | 2,568 | +| `Win64` | Windows 64-bit PE | 2,568 | +| `.NET` | .NET 어셈블리 | 2,568 | +| `APK` | Android APK | 696 | +| `ELF` | Linux ELF | 696 | +| `PDF` | PDF 문서 | 696 | +| `all` | 전체 파일 타입 혼합 | 2,568 | + +--- + +## 디렉토리 구조 + +파일명 규칙: `{모델}_{subset}[_suffix].{ext}` +`.NET` subset은 파일명에서 `dotnet`으로 변환됩니다. + +``` +dnn/ +├── dnn_PE.onnx # INT8 Static (배포용, PE/Win32/Win64/dotnet/all) +├── dnn_PE_fp32.onnx # FP32 ONNX (참조용, INT8 계열만 추가 포함) +├── dnn_PE.pt # PyTorch 체크포인트 +├── dnn_PE_metrics.json # 평가 결과 (AUC, TPR@1%FPR) +├── dnn_PE_benchmark.json # 크기·레이턴시 +├── dnn_APK.onnx # FP32 (non-PE — INT8 AUC 손실 과다) +├── dnn_APK.pt +└── ... + +tabnet/ +├── tabnet_PE.onnx # FP32 ONNX (134 MB — sparsemax 언폴딩) +├── tabnet_PE.zip # pytorch-tabnet 네이티브 (7 MB, 경량) +└── ... + +hybrid/ +├── hybrid_PE_nnpart.onnx # GBDT2NN nn_part ONNX (5.1 MB) +├── hybrid_PE_lgbm.model # LightGBM booster (3.6 MB) +├── hybrid_PE.pt # PyTorch 체크포인트 +└── ... + +lightgbm/ +├── lightgbm_PE.tl # Treelite 직렬화 (플랫폼 독립, 재컴파일 필요) +└── ... +``` + +--- + +## 성능 결과 (EMBER2024 test set) + +> 평가 기준: ROC-AUC, TPR @ 1% FPR (논문 §4.1), challenge set detection rate @ FPR=1% 임계값 +> Challenge set: 6,315 evasive malware (양성 only, Win32 3,225 / .NET 829 / Win64 814 / PDF 805 / ELF 386 / APK 256) + +### DNN + +| Subset | ROC-AUC | TPR@1%FPR | 배포 포맷 | 크기 | +|--------|---------|----------|---------|------| +| PE | 0.9969 | 0.9539 | INT8 Static ONNX | 13.3 MB | +| Win32 | 0.9966 | 0.9468 | INT8 Static ONNX | 13.3 MB | +| Win64 | 0.9969 | 0.9656 | INT8 Static ONNX | 13.3 MB | +| .NET | 0.9939 | 0.8976 | INT8 Static ONNX | 13.3 MB | +| all | 0.9942 | 0.9103 | INT8 Static ONNX | 13.3 MB | +| APK | 0.9761 | 0.7682 | FP32 ONNX | 3.9 MB | +| ELF | 0.9840 | 0.8103 | FP32 ONNX | 3.9 MB | +| PDF | 0.9795 | 0.8902 | FP32 ONNX | 3.9 MB | + +> non-PE(APK/ELF/PDF)는 입력 696-dim, 파라미터 수 부족으로 INT8 AUC 손실이 크므로 FP32 유지. + +### TabNet + +| Subset | ROC-AUC | TPR@1%FPR | 배포 포맷 | 크기 | +|--------|---------|----------|---------|------| +| PE | 0.9951 | 0.9212 | FP32 ONNX | 134 MB | +| Win32 | 0.9949 | 0.9322 | FP32 ONNX | 134 MB | +| Win64 | 0.9946 | 0.9323 | FP32 ONNX | 134 MB | +| .NET | 0.9925 | 0.8702 | FP32 ONNX | 134 MB | +| all | 0.9922 | 0.8912 | FP32 ONNX | 134 MB | +| APK | 0.9741 | 0.7028 | FP32 ONNX | 13.5 MB | +| ELF | 0.9793 | 0.5460 | FP32 ONNX | 13.5 MB | +| PDF | 0.9810 | 0.8597 | FP32 ONNX | 13.5 MB | + +> PE 계열 ONNX 크기 134 MB: sparsemax attention loop이 ONNX 그래프로 언폴딩되는 구조적 특성. 크기 우선이면 `tabnet_model.zip`(7 MB) 직접 사용 권장. + +### Hybrid (GBDT2NN) + +| Subset | ROC-AUC | TPR@1%FPR | 배포 포맷 | 크기 | +|--------|---------|----------|---------|------| +| PE | 0.9982 | 0.9752 | nn_part ONNX + LightGBM booster | 5.1 + 3.6 MB | +| Win32 | 0.9981 | 0.9747 | nn_part ONNX + LightGBM booster | 5.1 + 3.6 MB | +| Win64 | 0.9983 | 0.9812 | nn_part ONNX + LightGBM booster | 5.1 + 3.6 MB | +| .NET | 0.9961 | 0.9466 | nn_part ONNX + LightGBM booster | 5.1 + 3.5 MB | +| all | 0.9970 | 0.9528 | nn_part ONNX + LightGBM booster | 5.1 + 3.6 MB | +| APK | 0.9821 | 0.8003 | nn_part ONNX + LightGBM booster | 5.1 + 3.5 MB | +| ELF | 0.9899 | 0.8827 | nn_part ONNX + LightGBM booster | 5.1 + 3.6 MB | +| PDF | 0.9879 | 0.9283 | nn_part ONNX + LightGBM booster | 5.1 + 3.6 MB | + +### LightGBM (Treelite 컴파일) + +| Subset | ROC-AUC | TPR@1%FPR | 크기 (.tl) | 크기 (원본 .model) | +|--------|---------|----------|-----------|-----------------| +| PE | 0.9983 | 0.9707 | 5.3 MB | 3.6 MB | +| Win32 | 0.9984 | 0.9736 | 5.3 MB | 3.6 MB | +| Win64 | 0.9989 | 0.9831 | 5.3 MB | 3.6 MB | +| .NET | 0.9980 | 0.9566 | 5.3 MB | 3.5 MB | +| all | 0.9968 | 0.9440 | 5.3 MB | 3.6 MB | +| APK | 0.9861 | 0.8157 | 5.3 MB | 3.5 MB | +| ELF | 0.9929 | 0.9140 | 5.3 MB | 3.6 MB | +| PDF | 0.9913 | 0.9275 | 5.3 MB | 3.6 MB | + +> 원본 LightGBM 모델: [joyce8/EMBER2024-benchmark-models](https://huggingface.co/joyce8/EMBER2024-benchmark-models). `.tl`은 Treelite 3.9.1로 직렬화된 플랫폼 독립 파일 — 각 플랫폼에서 재컴파일 필요. + +### Challenge Set Detection Rate + +> Challenge set: 6,315 evasive malware (전부 양성). test set FPR=1% 임계값 적용. + +| Subset | DNN | TabNet | Hybrid | LightGBM | +|--------|-----|--------|--------|----------| +| `.NET` | 58.6% | 70.0% | 80.6% | 79.6% | +| `APK` | 27.3% | 29.3% | 34.4% | 33.6% | +| `ELF` | 11.7% | 4.4% | 23.8% | 30.3% | +| `PDF` | 41.5% | 40.1% | 56.9% | 57.1% | +| `PE` | 38.5% | 36.9% | 58.2% | 58.8% | +| `Win32`| 36.6% | 45.3% | 58.4% | 69.9% | +| `Win64`| 46.3% | 44.1% | 59.5% | 59.7% | +| `all` | 35.3% | 42.3% | 54.1% | 48.4% | + +--- + +## 추론 성능 (Apple M1, darwin-arm64) + +> `warm_batch1` 레이턴시: 배치 크기=1, 캐시 웜업 후 측정. 배포 환경(x86_64 Linux)과 다를 수 있음. + +### 레이턴시 (ms, warm batch=1) + +| Subset | DNN | TabNet | Hybrid | LightGBM | +|--------|-----|--------|--------|----------| +| `.NET` | 0.248 | 5.465 | 0.151 | 0.050 | +| `APK` | 0.035 | 0.846 | 0.145 | 0.031 | +| `ELF` | 0.039 | 0.505 | 0.160 | 0.036 | +| `PDF` | 0.036 | 2.230 | 0.172 | 0.048 | +| `PE` | 0.290 | 4.402 | 0.138 | 0.028 | +| `Win32`| 0.288 | 4.693 | 0.141 | 0.044 | +| `Win64`| 0.220 | 5.621 | 0.422 | 0.039 | +| `all` | 0.254 | 4.788 | 0.147 | 0.068 | + +> TabNet 레이턴시 높음: sparsemax attention이 ONNX 그래프로 언폴딩되는 구조적 특성. +> Hybrid = nn_part ONNX 추론만 측정 (LightGBM leaf extraction 제외). +> LightGBM 레이턴시 = 컴파일 `.dylib` 기준; 업로드 파일은 `.tl` (재컴파일 필요). + +### 모델 파일 크기 (배포 포맷) + +| Subset | DNN | TabNet `.onnx` | TabNet `.zip` | Hybrid (nn+lgbm) | LightGBM `.tl` | +|--------|-----|----------------|---------------|-------------------|----------------| +| PE 계열 | 13.3 MB (INT8) | 140.2 MB | 7.4 MB | 5.3 + 3.8 MB | 5.3 MB | +| non-PE | 3.9 MB (FP32) | 13.5 MB | 3.2 MB | 5.3 + 3.7 MB | 5.3 MB | + +--- + +## 사용 방법 + +### 의존성 설치 + +```bash +pip install onnxruntime>=1.20 numpy +# LightGBM / Hybrid 추론 시 +pip install "treelite==3.9.1" "treelite_runtime==3.9.1" lightgbm>=4.6 +# TabNet 체크포인트 직접 사용 시 +pip install pytorch-tabnet>=4.1 +``` + +### DNN 추론 (ONNX Runtime) + +```python +import numpy as np +import onnxruntime as ort +from huggingface_hub import hf_hub_download + +# PE subset — INT8 Static +model_path = hf_hub_download( + repo_id="cycloevan/ember-model", + filename="dnn/dnn_PE.onnx", +) +sess = ort.InferenceSession(model_path, providers=["CPUExecutionProvider"]) + +# X: np.ndarray shape (N, 2568), dtype float32 +X = np.random.randn(1, 2568).astype(np.float32) +logit = sess.run(["logit"], {"features": X})[0] # shape (N, 1) +prob = 1 / (1 + np.exp(-logit.ravel())) # sigmoid → [0, 1] +print(f"malware probability: {prob[0]:.4f}") +``` + +```python +# APK subset — FP32 +model_path = hf_hub_download( + repo_id="cycloevan/ember-model", + filename="dnn/dnn_APK.onnx", +) +sess = ort.InferenceSession(model_path, providers=["CPUExecutionProvider"]) +X = np.random.randn(1, 696).astype(np.float32) # non-PE: dim=696 +prob = 1 / (1 + np.exp(-sess.run(["logit"], {"features": X})[0].ravel())) +``` + +### TabNet 추론 (ONNX Runtime) + +```python +import numpy as np +import onnxruntime as ort +from huggingface_hub import hf_hub_download + +model_path = hf_hub_download( + repo_id="cycloevan/ember-model", + filename="tabnet/tabnet_PE.onnx", +) +sess = ort.InferenceSession(model_path, providers=["CPUExecutionProvider"]) +X = np.random.randn(1, 2568).astype(np.float32) +# 출력: logit (sigmoid 전) +logit = sess.run(["logit"], {"features": X})[0] +prob = 1 / (1 + np.exp(-logit.ravel())) +``` + +### Hybrid 추론 (ONNX + LightGBM) + +```python +import numpy as np +import lightgbm as lgb +import onnxruntime as ort +from huggingface_hub import hf_hub_download + +# 1. LightGBM booster로 leaf indices 추출 +booster = lgb.Booster(model_file=hf_hub_download( + repo_id="cycloevan/ember-model", + filename="hybrid/hybrid_PE_lgbm.model", +)) +X_raw = np.random.randn(1, 2568).astype(np.float64) +leaf_indices = booster.predict(X_raw, pred_leaf=True).astype(np.int64) # (N, n_trees) + +# 2. GBDT2NN ONNX로 최종 분류 +nn_sess = ort.InferenceSession(hf_hub_download( + repo_id="cycloevan/ember-model", + filename="hybrid/hybrid_PE_nnpart.onnx", +), providers=["CPUExecutionProvider"]) +logit = nn_sess.run(["logit"], {"leaf_indices": leaf_indices})[0] +prob = 1 / (1 + np.exp(-logit.ravel())) +print(f"malware probability: {prob[0]:.4f}") +``` + +### LightGBM 추론 (Treelite 컴파일 — 빠른 추론) + +```python +# 1. Treelite .tl → 플랫폼별 공유 라이브러리 컴파일 (최초 1회) +import treelite, treelite_runtime, sys, numpy as np +from huggingface_hub import hf_hub_download + +tl_path = hf_hub_download( + repo_id="cycloevan/ember-model", + filename="lightgbm/lightgbm_PE.tl", +) +tl_model = treelite.Model.deserialize(tl_path) +lib_ext = ".dylib" if sys.platform == "darwin" else ".so" +lib_path = tl_path.replace(".tl", lib_ext) +tl_model.export_lib( + toolchain="clang" if sys.platform == "darwin" else "gcc", + libpath=lib_path, + verbose=False, +) + +# 2. 추론 +predictor = treelite_runtime.Predictor(lib_path, verbose=False) +X = np.random.randn(1, 2568).astype(np.float32) +prob = predictor.predict(treelite_runtime.DMatrix(X)) +print(f"malware probability: {prob[0]:.4f}") +``` + +> **주의**: `treelite==3.9.1` + `treelite_runtime==3.9.1` 필요. 4.x는 `export_lib()` 미지원. + +--- + +## 학습 및 평가 환경 + +| 항목 | 내용 | +|------|------| +| 데이터셋 | [EMBER2024](https://huggingface.co/datasets/joyce8/EMBER2024) — train 52주(2.6 M), test 12주(606 K), challenge 6,315 | +| Feature 차원 | PE 2,568 (v3) / non-PE 696 (유효 prefix) | +| Split 정책 | 시간적 순서 고정 (temporal split), 임의 셔플 없음 | +| 학습 환경 | DGX Spark (GB10 Grace Blackwell, 128 GB, CUDA 13) | +| 프레임워크 | PyTorch 2.11.0, pytorch-tabnet 4.1, LightGBM 4.6 | +| 재현 시드 | 42 | +| DNN 아키텍처 | Linear(2568→2568→1024→512→1) + PReLU + Dropout(0.5) | +| Hybrid | LightGBM leaf extraction → Linear(n_trees→512→256→1) + PReLU | +| 평가 지표 | ROC-AUC, PR-AUC, **TPR @ 1% FPR** (논문 §4.1) | + +--- + +## 알려진 한계 + +- **TabNet ONNX 크기**: sparsemax attention loop 언폴딩으로 PE 계열 ONNX가 134 MB로 팽창. 원본 `tabnet_model.zip`(7 MB)이 경량. +- **Treelite `.dylib`**: Mac ARM64 전용 사전 컴파일 파일. 다른 플랫폼은 `.tl`에서 재컴파일 필요. +- **DNN non-PE INT8**: 696-dim 모델은 양자화 AUC 손실이 크므로 FP32 유지. +- **Hybrid 추론**: 단일 ONNX 파일이 아님 — LightGBM leaf extraction + nn_part ONNX 2단계. +- **challenge detection rate**: test set에서 FPR=1% 임계값으로 측정. subset별 분포 차이로 값이 상이할 수 있음. + +--- + +## 인용 + +```bibtex +@inproceedings{joyce2025ember2024, + title = {EMBER2024: An Open Dataset for Training Behavioral Malware Detection Models}, + author = {Joyce, Ruby and Rudd, Ethan M. and others}, + booktitle = {Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining}, + year = {2025}, + url = {https://arxiv.org/abs/2506.05074} +} +``` + +--- + +## 라이선스 + +코드 및 모델 가중치: Apache 2.0 +LightGBM 원본 모델(`hybrid/hybrid_*_lgbm.model`): [joyce8/EMBER2024-benchmark-models](https://huggingface.co/joyce8/EMBER2024-benchmark-models) 라이선스 준수 diff --git a/dnn/dnn_APK.onnx b/dnn/dnn_APK.onnx new file mode 100644 index 0000000000000000000000000000000000000000..6371b382c1d3913955eec75b95bd8882b9135f24 --- /dev/null +++ b/dnn/dnn_APK.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7fc471fef4a3ab3b1deaeccba5473b81a62a8b4d0c1f435c4238150765cba64 +size 3902529 diff --git a/dnn/dnn_APK.pt b/dnn/dnn_APK.pt new file mode 100644 index 0000000000000000000000000000000000000000..396e3a8e2315cc9ee8569189113f516ee92f4788 --- /dev/null +++ b/dnn/dnn_APK.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82128b485920e8532d9ccc8148b52254e61ab091bacb99978f377aa64e54af3d +size 3929735 diff --git a/dnn/dnn_APK_benchmark.json b/dnn/dnn_APK_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..0f44c52f48424940c8ba997a08196f0e8b01f725 --- /dev/null +++ b/dnn/dnn_APK_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "dnn", + "subset": "APK", + "quant": "fp32", + "model_size_mb": 3.903, + "original_size_mb": 3.93, + "size_reduction_pct": 0.7, + "latency_ms": { + "cold_batch1": 0.061, + "warm_batch1": 0.035, + "warm_batch16": 0.215, + "warm_batch256": 2.326, + "throughput_batch1": 28571.4, + "ms_per_sample_batch1": 0.035, + "throughput_batch16": 74418.6, + "ms_per_sample_batch16": 0.0134, + "throughput_batch256": 110060.2, + "ms_per_sample_batch256": 0.0091 + }, + "original_latency_ms": { + "cold_batch1": 0.196, + "warm_batch1": 0.094, + "warm_batch16": 0.528, + "warm_batch256": 1.149, + "throughput_batch1": 10638.3, + "ms_per_sample_batch1": 0.094, + "throughput_batch16": 30303.0, + "ms_per_sample_batch16": 0.033, + "throughput_batch256": 222802.4, + "ms_per_sample_batch256": 0.0045 + }, + "test_metrics": { + "roc_auc": 0.9760728567708332, + "pr_auc": 0.9782617133148918, + "tpr_at_1pct_fpr": 0.7681666666666667, + "n_samples": 96000, + "n_positive": 48000, + "auc_delta_vs_original": -0.0 + }, + "peak_rss_delta_mb": 0.0, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/dnn/dnn_APK_metrics.json b/dnn/dnn_APK_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dc40e4e9a9df679dbb44b4c596d66dc9c7058c --- /dev/null +++ b/dnn/dnn_APK_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "dnn", + "subset": "APK", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9760728567708332, + "pr_auc": 0.9782617133148918, + "tpr_at_1pct_fpr": 0.7681666666666667, + "n_samples": 96000, + "n_positive": 48000, + "auc_delta_vs_original": -0.0 + }, + "challenge_metrics": { + "detection_rate": 0.2734375, + "threshold": 0.893254017132744 + }, + "auc_gate_failed": false, + "platform": "macOS-15.7.7-arm64-arm-64bit" +} \ No newline at end of file diff --git a/dnn/dnn_ELF.onnx b/dnn/dnn_ELF.onnx new file mode 100644 index 0000000000000000000000000000000000000000..36ece2a2b068c69421de13eb95899e4688376f04 --- /dev/null +++ b/dnn/dnn_ELF.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a3ee26c129cfcef77ab60de38f4d5296fd0cff1d7f99d3ca4bbd3bdf2d5e568 +size 3902529 diff --git a/dnn/dnn_ELF.pt b/dnn/dnn_ELF.pt new file mode 100644 index 0000000000000000000000000000000000000000..861948d8c55118b64c12fb5588e79ba9dbb95a65 --- /dev/null +++ b/dnn/dnn_ELF.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f875a91e09738ee9371ccde5693f61367f41a7a2cb623096d3e3854da5c32e9c +size 3929735 diff --git a/dnn/dnn_ELF_benchmark.json b/dnn/dnn_ELF_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..bed3a944164c739273676de629e0b18233c15eb2 --- /dev/null +++ b/dnn/dnn_ELF_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "dnn", + "subset": "ELF", + "quant": "fp32", + "model_size_mb": 3.903, + "original_size_mb": 3.93, + "size_reduction_pct": 0.7, + "latency_ms": { + "cold_batch1": 0.055, + "warm_batch1": 0.039, + "ms_per_sample_batch1": 0.0387, + "throughput_batch1": 25864.6, + "warm_batch16": 0.183, + "ms_per_sample_batch16": 0.0114, + "throughput_batch16": 87489.4, + "warm_batch256": 2.135, + "ms_per_sample_batch256": 0.0083, + "throughput_batch256": 119926.0 + }, + "original_latency_ms": { + "cold_batch1": 0.135, + "warm_batch1": 0.094, + "ms_per_sample_batch1": 0.0942, + "throughput_batch1": 10612.8, + "warm_batch16": 0.522, + "ms_per_sample_batch16": 0.0326, + "throughput_batch16": 30663.9, + "warm_batch256": 1.07, + "ms_per_sample_batch256": 0.0042, + "throughput_batch256": 239209.3 + }, + "test_metrics": { + "roc_auc": 0.9839878888888888, + "pr_auc": 0.9845206683196877, + "tpr_at_1pct_fpr": 0.8103333333333333, + "n_samples": 12000, + "n_positive": 6000, + "auc_delta_vs_original": 6e-06 + }, + "peak_rss_delta_mb": 2.2, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/dnn/dnn_ELF_metrics.json b/dnn/dnn_ELF_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..4cc6880afbc914934f7c7b83103f6640f9784af0 --- /dev/null +++ b/dnn/dnn_ELF_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "dnn", + "subset": "ELF", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9839878888888888, + "pr_auc": 0.9845206683196877, + "tpr_at_1pct_fpr": 0.8103333333333333, + "n_samples": 12000, + "n_positive": 6000, + "auc_delta_vs_original": 6e-06 + }, + "challenge_metrics": { + "detection_rate": 0.11658031088082901, + "threshold": 0.9921941479742924 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/dnn/dnn_PDF.onnx b/dnn/dnn_PDF.onnx new file mode 100644 index 0000000000000000000000000000000000000000..be9f54a1dac195dfa8c3419fce2e445ba8373908 --- /dev/null +++ b/dnn/dnn_PDF.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9817bd278a5614376a7c1e1267e4e098cab64a9684936fc847df8b3bae0beac +size 3902529 diff --git a/dnn/dnn_PDF.pt b/dnn/dnn_PDF.pt new file mode 100644 index 0000000000000000000000000000000000000000..e829b66f43355c00306851f2ef1646c5540398c7 --- /dev/null +++ b/dnn/dnn_PDF.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d742551ed4f103eabda97a63ce9fba3be032039b97dd5f8fb3c66a328da7bfa9 +size 3929735 diff --git a/dnn/dnn_PDF_benchmark.json b/dnn/dnn_PDF_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..9a1ee222e18be1410e3ee59cb32987c56e3e05c3 --- /dev/null +++ b/dnn/dnn_PDF_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "dnn", + "subset": "PDF", + "quant": "fp32", + "model_size_mb": 3.903, + "original_size_mb": 3.93, + "size_reduction_pct": 0.7, + "latency_ms": { + "cold_batch1": 0.05, + "warm_batch1": 0.036, + "warm_batch16": 0.201, + "warm_batch256": 2.391, + "throughput_batch1": 27777.8, + "ms_per_sample_batch1": 0.036, + "throughput_batch16": 79602.0, + "ms_per_sample_batch16": 0.0126, + "throughput_batch256": 107068.2, + "ms_per_sample_batch256": 0.0093 + }, + "original_latency_ms": { + "cold_batch1": 0.177, + "warm_batch1": 0.102, + "warm_batch16": 0.502, + "warm_batch256": 1.122, + "throughput_batch1": 9803.9, + "ms_per_sample_batch1": 0.102, + "throughput_batch16": 31872.5, + "ms_per_sample_batch16": 0.0314, + "throughput_batch256": 228164.0, + "ms_per_sample_batch256": 0.0044 + }, + "test_metrics": { + "roc_auc": 0.9795445, + "pr_auc": 0.9848222884024884, + "tpr_at_1pct_fpr": 0.8901666666666667, + "n_samples": 24000, + "n_positive": 12000, + "auc_delta_vs_original": 0.0 + }, + "peak_rss_delta_mb": 2.1, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/dnn/dnn_PDF_metrics.json b/dnn/dnn_PDF_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..561f717488877253f95ff90b6919cbdd4c09e007 --- /dev/null +++ b/dnn/dnn_PDF_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "dnn", + "subset": "PDF", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9795445, + "pr_auc": 0.9848222884024884, + "tpr_at_1pct_fpr": 0.8901666666666667, + "n_samples": 24000, + "n_positive": 12000, + "auc_delta_vs_original": 0.0 + }, + "challenge_metrics": { + "detection_rate": 0.41490683229813663, + "threshold": 0.7553592072806331 + }, + "auc_gate_failed": false, + "platform": "macOS-15.7.7-arm64-arm-64bit" +} \ No newline at end of file diff --git a/dnn/dnn_PE.onnx b/dnn/dnn_PE.onnx new file mode 100644 index 0000000000000000000000000000000000000000..c669ba28994dc2d16a3cc29de11d7a3f964f20d6 --- /dev/null +++ b/dnn/dnn_PE.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9fbe6a00e369366c42b64aeec8e61bf70d96060fd78575a01354e0027459be7 +size 13303487 diff --git a/dnn/dnn_PE.pt b/dnn/dnn_PE.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3e97170c0f8d8d3b1f53b8592e337ed8c27ada2 --- /dev/null +++ b/dnn/dnn_PE.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:382d174985ddd146df8a9e0d1bae262339734aed7841b35a5000b6b7c4fcdf43 +size 52938695 diff --git a/dnn/dnn_PE_benchmark.json b/dnn/dnn_PE_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..81294c30dbe2f78a2f8b631e7930e04afca8042b --- /dev/null +++ b/dnn/dnn_PE_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "dnn", + "subset": "PE", + "quant": "int8-static", + "model_size_mb": 13.303, + "original_size_mb": 52.939, + "size_reduction_pct": 74.9, + "latency_ms": { + "cold_batch1": 0.778, + "warm_batch1": 0.29, + "warm_batch16": 1.036, + "warm_batch256": 12.935, + "throughput_batch1": 3448.3, + "ms_per_sample_batch1": 0.29, + "throughput_batch16": 15444.0, + "ms_per_sample_batch16": 0.0648, + "throughput_batch256": 19791.3, + "ms_per_sample_batch256": 0.0505 + }, + "original_latency_ms": { + "cold_batch1": 2.916, + "warm_batch1": 2.466, + "warm_batch16": 4.939, + "warm_batch256": 11.88, + "throughput_batch1": 405.5, + "ms_per_sample_batch1": 2.466, + "throughput_batch16": 3239.5, + "ms_per_sample_batch16": 0.3087, + "throughput_batch256": 21548.8, + "ms_per_sample_batch256": 0.0464 + }, + "test_metrics": { + "roc_auc": 0.9968856578492659, + "pr_auc": 0.9969432521647822, + "tpr_at_1pct_fpr": 0.9471575053488233, + "n_samples": 100000, + "n_positive": 50011, + "auc_delta_vs_original": -6.2e-05 + }, + "peak_rss_delta_mb": 0.0, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/dnn/dnn_PE_fp32.onnx b/dnn/dnn_PE_fp32.onnx new file mode 100644 index 0000000000000000000000000000000000000000..f0881fcfbea3561f77c4ba9bb170d8fe6844ef0f --- /dev/null +++ b/dnn/dnn_PE_fp32.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df67ff72e0ee7cfc6381dbec0f8f5945c3db77dadb46af930b3f6c4670a5e150 +size 52851589 diff --git a/dnn/dnn_PE_metrics.json b/dnn/dnn_PE_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..da82ef1bd5a5c342c29dc806e5ce7927d7787b68 --- /dev/null +++ b/dnn/dnn_PE_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "dnn", + "subset": "PE", + "quant": "int8-static", + "test_metrics": { + "roc_auc": 0.9968856578492659, + "pr_auc": 0.9969432521647822, + "tpr_at_1pct_fpr": 0.9471575053488233, + "n_samples": 100000, + "n_positive": 50011, + "auc_delta_vs_original": -6.2e-05 + }, + "challenge_metrics": { + "detection_rate": 0.3849962862094578, + "threshold": 0.9285938677767953 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/dnn/dnn_Win32.onnx b/dnn/dnn_Win32.onnx new file mode 100644 index 0000000000000000000000000000000000000000..14b9120ad16a4d1c4e9da9f05353e28f52ff8463 --- /dev/null +++ b/dnn/dnn_Win32.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:500f4ed1d1317a20b95cc1147f68eecd10abcf43527ad8234d1cba6aa4fa7a28 +size 13303486 diff --git a/dnn/dnn_Win32.pt b/dnn/dnn_Win32.pt new file mode 100644 index 0000000000000000000000000000000000000000..1298abd0457396157b866497fda92f822ed4093b --- /dev/null +++ b/dnn/dnn_Win32.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f0f67ab9635cdc54f253a3bc591181c0c40092ddd7613ae48cdad9cf5b633be +size 52938695 diff --git a/dnn/dnn_Win32_benchmark.json b/dnn/dnn_Win32_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..aa6969f25cf8c35af89598aef09f4d647daf1611 --- /dev/null +++ b/dnn/dnn_Win32_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "dnn", + "subset": "Win32", + "quant": "int8-static", + "model_size_mb": 13.303, + "original_size_mb": 52.939, + "size_reduction_pct": 74.9, + "latency_ms": { + "cold_batch1": 0.487, + "warm_batch1": 0.288, + "warm_batch16": 0.99, + "warm_batch256": 8.912, + "throughput_batch1": 3472.2, + "ms_per_sample_batch1": 0.288, + "throughput_batch16": 16161.6, + "ms_per_sample_batch16": 0.0619, + "throughput_batch256": 28725.3, + "ms_per_sample_batch256": 0.0348 + }, + "original_latency_ms": { + "cold_batch1": 2.393, + "warm_batch1": 2.343, + "warm_batch16": 4.234, + "warm_batch256": 11.15, + "throughput_batch1": 426.8, + "ms_per_sample_batch1": 2.343, + "throughput_batch16": 3778.9, + "ms_per_sample_batch16": 0.2646, + "throughput_batch256": 22959.6, + "ms_per_sample_batch256": 0.0436 + }, + "test_metrics": { + "roc_auc": 0.9965419976367095, + "pr_auc": 0.9966226288532287, + "tpr_at_1pct_fpr": 0.9478530944531651, + "n_samples": 100000, + "n_positive": 50204, + "auc_delta_vs_original": -8e-05 + }, + "peak_rss_delta_mb": 5.3, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/dnn/dnn_Win32_fp32.onnx b/dnn/dnn_Win32_fp32.onnx new file mode 100644 index 0000000000000000000000000000000000000000..cb9fbff6c737901870389e31e126d13ac79dadca --- /dev/null +++ b/dnn/dnn_Win32_fp32.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:359d7e4e4cc4222bc8756d179415e93affb84f2602b0587e4f063dbb8047945e +size 52851589 diff --git a/dnn/dnn_Win32_metrics.json b/dnn/dnn_Win32_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..7642d4ab8b18fec3618569de019bfd9e660f3fe3 --- /dev/null +++ b/dnn/dnn_Win32_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "dnn", + "subset": "Win32", + "quant": "int8-static", + "test_metrics": { + "roc_auc": 0.9965419976367095, + "pr_auc": 0.9966226288532287, + "tpr_at_1pct_fpr": 0.9478530944531651, + "n_samples": 100000, + "n_positive": 50204, + "auc_delta_vs_original": -8e-05 + }, + "challenge_metrics": { + "detection_rate": 0.3655813953488372, + "threshold": 0.7420705254159128 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/dnn/dnn_Win64.onnx b/dnn/dnn_Win64.onnx new file mode 100644 index 0000000000000000000000000000000000000000..8251f24d10cb79a95b999dba6ce68dbe807806f9 --- /dev/null +++ b/dnn/dnn_Win64.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24bac8430ecb1b15cf0e28a097eff6fe525025d115b9f1bca9fcf75a31882167 +size 13303486 diff --git a/dnn/dnn_Win64.pt b/dnn/dnn_Win64.pt new file mode 100644 index 0000000000000000000000000000000000000000..00bd28f73909d7b5124c6d675d521d6d033345f7 --- /dev/null +++ b/dnn/dnn_Win64.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39d22c9b0123afb6c0a6c11d17a147905c35b35214bbf318a54acdfff1d892a5 +size 52938695 diff --git a/dnn/dnn_Win64_benchmark.json b/dnn/dnn_Win64_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..2dea599ec87553a3c04d663fa307750adf041cf6 --- /dev/null +++ b/dnn/dnn_Win64_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "dnn", + "subset": "Win64", + "quant": "int8-static", + "model_size_mb": 13.303, + "original_size_mb": 52.939, + "size_reduction_pct": 74.9, + "latency_ms": { + "cold_batch1": 0.426, + "warm_batch1": 0.22, + "warm_batch16": 0.941, + "warm_batch256": 8.919, + "throughput_batch1": 4545.5, + "ms_per_sample_batch1": 0.22, + "throughput_batch16": 17003.2, + "ms_per_sample_batch16": 0.0588, + "throughput_batch256": 28702.8, + "ms_per_sample_batch256": 0.0348 + }, + "original_latency_ms": { + "cold_batch1": 2.789, + "warm_batch1": 2.389, + "warm_batch16": 4.246, + "warm_batch256": 11.135, + "throughput_batch1": 418.6, + "ms_per_sample_batch1": 2.389, + "throughput_batch16": 3768.3, + "ms_per_sample_batch16": 0.2654, + "throughput_batch256": 22990.6, + "ms_per_sample_batch256": 0.0435 + }, + "test_metrics": { + "roc_auc": 0.9968797740122215, + "pr_auc": 0.9970818991078805, + "tpr_at_1pct_fpr": 0.9616965868794326, + "n_samples": 100000, + "n_positive": 49920, + "auc_delta_vs_original": -3.3e-05 + }, + "peak_rss_delta_mb": 0.0, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/dnn/dnn_Win64_fp32.onnx b/dnn/dnn_Win64_fp32.onnx new file mode 100644 index 0000000000000000000000000000000000000000..2e016802451b5d145706d232a53e2c8068178da4 --- /dev/null +++ b/dnn/dnn_Win64_fp32.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc072a860523d3e1557fcb98837758d5f064d5edb57cec2e98931b5d51928bb6 +size 52851589 diff --git a/dnn/dnn_Win64_metrics.json b/dnn/dnn_Win64_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..5c4f5a89628b872d2bc58d6148745b035eb07816 --- /dev/null +++ b/dnn/dnn_Win64_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "dnn", + "subset": "Win64", + "quant": "int8-static", + "test_metrics": { + "roc_auc": 0.9968797740122215, + "pr_auc": 0.9970818991078805, + "tpr_at_1pct_fpr": 0.9616965868794326, + "n_samples": 100000, + "n_positive": 49920, + "auc_delta_vs_original": -3.3e-05 + }, + "challenge_metrics": { + "detection_rate": 0.4631449631449631, + "threshold": 0.5563582119497066 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/dnn/dnn_all.onnx b/dnn/dnn_all.onnx new file mode 100644 index 0000000000000000000000000000000000000000..ff9bcf52574de28f3b473c02db50d8bb7dcfa200 --- /dev/null +++ b/dnn/dnn_all.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55ffa1127b72aa5f593b4963f769697d6a696beb60995ac771d1a6ebbb34a1db +size 13303487 diff --git a/dnn/dnn_all.pt b/dnn/dnn_all.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f881110de261af7ebb344a2e12d84e49dea2a27 --- /dev/null +++ b/dnn/dnn_all.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5563545cd2037339668863a623d0b6f84e90cec5053d85abbc9a4e3faa8ad3e +size 52938695 diff --git a/dnn/dnn_all_benchmark.json b/dnn/dnn_all_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..2428eda368e81b620f4532a33345d18b3a15b88c --- /dev/null +++ b/dnn/dnn_all_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "dnn", + "subset": "all", + "quant": "int8-static", + "model_size_mb": 13.303, + "original_size_mb": 52.939, + "size_reduction_pct": 74.9, + "latency_ms": { + "cold_batch1": 0.429, + "warm_batch1": 0.254, + "warm_batch16": 0.93, + "warm_batch256": 8.955, + "throughput_batch1": 3937.0, + "ms_per_sample_batch1": 0.254, + "throughput_batch16": 17204.3, + "ms_per_sample_batch16": 0.0581, + "throughput_batch256": 28587.4, + "ms_per_sample_batch256": 0.035 + }, + "original_latency_ms": { + "cold_batch1": 2.658, + "warm_batch1": 2.394, + "warm_batch16": 4.271, + "warm_batch256": 11.371, + "throughput_batch1": 417.7, + "ms_per_sample_batch1": 2.394, + "throughput_batch16": 3746.2, + "ms_per_sample_batch16": 0.2669, + "throughput_batch256": 22513.4, + "ms_per_sample_batch256": 0.0444 + }, + "test_metrics": { + "roc_auc": 0.9938331270533947, + "pr_auc": 0.9937970701852039, + "tpr_at_1pct_fpr": 0.8870437241462001, + "n_samples": 100000, + "n_positive": 50041, + "auc_delta_vs_original": -0.000375 + }, + "peak_rss_delta_mb": 0.0, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/dnn/dnn_all_fp32.onnx b/dnn/dnn_all_fp32.onnx new file mode 100644 index 0000000000000000000000000000000000000000..eff3f1ac2eead691e027fe7bc8f09e037b4d3719 --- /dev/null +++ b/dnn/dnn_all_fp32.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c97fe488d5dedac6e88793a271b47018301dd7c7fb59c1cdbd23dad489e86f5b +size 52851589 diff --git a/dnn/dnn_all_metrics.json b/dnn/dnn_all_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..29e238df40995976c3aa2326268cec3fc1f5d414 --- /dev/null +++ b/dnn/dnn_all_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "dnn", + "subset": "all", + "quant": "int8-static", + "test_metrics": { + "roc_auc": 0.9938331270533947, + "pr_auc": 0.9937970701852039, + "tpr_at_1pct_fpr": 0.8870437241462001, + "n_samples": 100000, + "n_positive": 50041, + "auc_delta_vs_original": -0.000375 + }, + "challenge_metrics": { + "detection_rate": 0.3534441805225653, + "threshold": 0.9838834411565777 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/dnn/dnn_dotnet.onnx b/dnn/dnn_dotnet.onnx new file mode 100644 index 0000000000000000000000000000000000000000..ed18d686fb590384fea920db029461b1042a877d --- /dev/null +++ b/dnn/dnn_dotnet.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dff033cb3a75fc3af64c11c6557207974480d7286fea7d50225d5f41c0f871cb +size 13303487 diff --git a/dnn/dnn_dotnet.pt b/dnn/dnn_dotnet.pt new file mode 100644 index 0000000000000000000000000000000000000000..b83c190daf3634c69973226bb199971163bf39c2 --- /dev/null +++ b/dnn/dnn_dotnet.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05c84608a67b7a3de08102f0d8b75c2fbd9ae3b5a97224d854717f258dab8abf +size 52938695 diff --git a/dnn/dnn_dotnet_benchmark.json b/dnn/dnn_dotnet_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..478dc9108bc847516d5c3af615bfb3fea8f2014b --- /dev/null +++ b/dnn/dnn_dotnet_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "dnn", + "subset": ".NET", + "quant": "int8-static", + "model_size_mb": 13.303, + "original_size_mb": 52.939, + "size_reduction_pct": 74.9, + "latency_ms": { + "cold_batch1": 0.429, + "warm_batch1": 0.248, + "warm_batch16": 0.909, + "warm_batch256": 8.858, + "throughput_batch1": 4032.3, + "ms_per_sample_batch1": 0.248, + "throughput_batch16": 17601.8, + "ms_per_sample_batch16": 0.0568, + "throughput_batch256": 28900.4, + "ms_per_sample_batch256": 0.0346 + }, + "original_latency_ms": { + "cold_batch1": 2.44, + "warm_batch1": 2.368, + "warm_batch16": 4.208, + "warm_batch256": 11.103, + "throughput_batch1": 422.3, + "ms_per_sample_batch1": 2.368, + "throughput_batch16": 3802.3, + "ms_per_sample_batch16": 0.263, + "throughput_batch256": 23056.8, + "ms_per_sample_batch256": 0.0434 + }, + "test_metrics": { + "roc_auc": 0.9919714812355357, + "pr_auc": 0.9900265259892996, + "tpr_at_1pct_fpr": 0.8444236421489054, + "n_samples": 100000, + "n_positive": 49946, + "auc_delta_vs_original": -0.001909 + }, + "peak_rss_delta_mb": 0.0, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/dnn/dnn_dotnet_fp32.onnx b/dnn/dnn_dotnet_fp32.onnx new file mode 100644 index 0000000000000000000000000000000000000000..d621b675a6529b386a12a74b874aecf0476188e4 --- /dev/null +++ b/dnn/dnn_dotnet_fp32.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e182e7c89766c2625d318c55e854a428b7a1bccdf5e71b52d805eb9fc59aa056 +size 52851589 diff --git a/dnn/dnn_dotnet_metrics.json b/dnn/dnn_dotnet_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..1f67b87f6dffc8910abffa3caf8490ed86b1522b --- /dev/null +++ b/dnn/dnn_dotnet_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "dnn", + "subset": ".NET", + "quant": "int8-static", + "test_metrics": { + "roc_auc": 0.9919714812355357, + "pr_auc": 0.9900265259892996, + "tpr_at_1pct_fpr": 0.8444236421489054, + "n_samples": 100000, + "n_positive": 49946, + "auc_delta_vs_original": -0.001909 + }, + "challenge_metrics": { + "detection_rate": 0.586248492159228, + "threshold": 0.9723500243107374 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/hybrid/hybrid_APK.pt b/hybrid/hybrid_APK.pt new file mode 100644 index 0000000000000000000000000000000000000000..642995315d0e0e9d9ff0edb83e83f916cb89ad24 --- /dev/null +++ b/hybrid/hybrid_APK.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b65ff8cb5da0cf77b0b542785967c4bf63d580a1ff8be1513b3517cf834b6653 +size 5331649 diff --git a/hybrid/hybrid_APK_benchmark.json b/hybrid/hybrid_APK_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..d101e48d179de08af8dc1831f625476c08fa22b9 --- /dev/null +++ b/hybrid/hybrid_APK_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "hybrid", + "subset": "APK", + "quant": "fp32", + "model_size_mb": 5.326, + "original_size_mb": 5.332, + "size_reduction_pct": 0.1, + "latency_ms": { + "cold_batch1": 0.455, + "warm_batch1": 0.145, + "warm_batch16": 0.805, + "warm_batch256": 11.495, + "throughput_batch1": 6896.6, + "ms_per_sample_batch1": 0.145, + "throughput_batch16": 19875.8, + "ms_per_sample_batch16": 0.0503, + "throughput_batch256": 22270.6, + "ms_per_sample_batch256": 0.0449 + }, + "original_latency_ms": { + "cold_batch1": 0.466, + "warm_batch1": 0.211, + "warm_batch16": 0.908, + "warm_batch256": 6.452, + "throughput_batch1": 4739.3, + "ms_per_sample_batch1": 0.211, + "throughput_batch16": 17621.1, + "ms_per_sample_batch16": 0.0568, + "throughput_batch256": 39677.6, + "ms_per_sample_batch256": 0.0252 + }, + "test_metrics": { + "roc_auc": 0.9827782760416667, + "pr_auc": 0.9845011360409881, + "tpr_at_1pct_fpr": 0.8002916666666666, + "n_samples": 96000, + "n_positive": 48000, + "auc_delta_vs_original": 0.000631 + }, + "peak_rss_delta_mb": 0.0, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/hybrid/hybrid_APK_lgbm.model b/hybrid/hybrid_APK_lgbm.model new file mode 100644 index 0000000000000000000000000000000000000000..1e80103aa062c4907d39ca0d03abe018785e3f3c --- /dev/null +++ b/hybrid/hybrid_APK_lgbm.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a13274463201e7d3ddec74b098350051266fcf0812aff302617397f1765cbc90 +size 3679987 diff --git a/hybrid/hybrid_APK_metrics.json b/hybrid/hybrid_APK_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..f72da29952c33f1dfb26d682f8fcf48755cd1d8a --- /dev/null +++ b/hybrid/hybrid_APK_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "hybrid", + "subset": "APK", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9827782760416667, + "pr_auc": 0.9845011360409881, + "tpr_at_1pct_fpr": 0.8002916666666666, + "n_samples": 96000, + "n_positive": 48000, + "auc_delta_vs_original": 0.000631 + }, + "challenge_metrics": { + "detection_rate": 0.34375, + "threshold": 0.9999923489401177 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/hybrid/hybrid_APK_nnpart.onnx b/hybrid/hybrid_APK_nnpart.onnx new file mode 100644 index 0000000000000000000000000000000000000000..d08d6f5bafca0af8e89447081b5f10f309d60198 --- /dev/null +++ b/hybrid/hybrid_APK_nnpart.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7052c06fe40e9b2ab700f004f5d5e95f80e78d644073b13caa64e393ec2ae8e9 +size 5325521 diff --git a/hybrid/hybrid_ELF.pt b/hybrid/hybrid_ELF.pt new file mode 100644 index 0000000000000000000000000000000000000000..7fa71f7389269b78a58b90db9637513227212b08 --- /dev/null +++ b/hybrid/hybrid_ELF.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a1e2daf9a192da640e68210bd3a6ecf9031f938e32508cd23365481b1cdc560 +size 5331649 diff --git a/hybrid/hybrid_ELF_benchmark.json b/hybrid/hybrid_ELF_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..5dc314433f659d8e306b739922f49b52775c563c --- /dev/null +++ b/hybrid/hybrid_ELF_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "hybrid", + "subset": "ELF", + "quant": "fp32", + "model_size_mb": 5.326, + "original_size_mb": 5.332, + "size_reduction_pct": 0.1, + "latency_ms": { + "cold_batch1": 0.483, + "warm_batch1": 0.16, + "warm_batch16": 1.058, + "warm_batch256": 12.195, + "throughput_batch1": 6250.0, + "ms_per_sample_batch1": 0.16, + "throughput_batch16": 15122.9, + "ms_per_sample_batch16": 0.0661, + "throughput_batch256": 20992.2, + "ms_per_sample_batch256": 0.0476 + }, + "original_latency_ms": { + "cold_batch1": 0.487, + "warm_batch1": 0.211, + "warm_batch16": 0.912, + "warm_batch256": 6.459, + "throughput_batch1": 4739.3, + "ms_per_sample_batch1": 0.211, + "throughput_batch16": 17543.9, + "ms_per_sample_batch16": 0.057, + "throughput_batch256": 39634.6, + "ms_per_sample_batch256": 0.0252 + }, + "test_metrics": { + "roc_auc": 0.9899468888888888, + "pr_auc": 0.9863592706579489, + "tpr_at_1pct_fpr": 0.8826666666666667, + "n_samples": 12000, + "n_positive": 6000, + "auc_delta_vs_original": 0.0 + }, + "peak_rss_delta_mb": 0.0, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/hybrid/hybrid_ELF_lgbm.model b/hybrid/hybrid_ELF_lgbm.model new file mode 100644 index 0000000000000000000000000000000000000000..d3ebc95b6c8d684ad09112943e1bd362bed45f54 --- /dev/null +++ b/hybrid/hybrid_ELF_lgbm.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e4072938955ac6020f80bca6cff7c12e97865102d578b3099e2011048eb9c43 +size 3764582 diff --git a/hybrid/hybrid_ELF_metrics.json b/hybrid/hybrid_ELF_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..08957839f122b343d586db1587bae55e4522f37c --- /dev/null +++ b/hybrid/hybrid_ELF_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "hybrid", + "subset": "ELF", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9899468888888888, + "pr_auc": 0.9863592706579489, + "tpr_at_1pct_fpr": 0.8826666666666667, + "n_samples": 12000, + "n_positive": 6000, + "auc_delta_vs_original": 0.0 + }, + "challenge_metrics": { + "detection_rate": 0.23834196891191708, + "threshold": 0.7018789316233345 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/hybrid/hybrid_ELF_nnpart.onnx b/hybrid/hybrid_ELF_nnpart.onnx new file mode 100644 index 0000000000000000000000000000000000000000..d4673f0a8a3c740851ab5cafa2fd5aede752735f --- /dev/null +++ b/hybrid/hybrid_ELF_nnpart.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:182798a23e2a8f7c478d561abd8103c32dcb9e8feb6decf0728261ab9d0893c6 +size 5325521 diff --git a/hybrid/hybrid_PDF.pt b/hybrid/hybrid_PDF.pt new file mode 100644 index 0000000000000000000000000000000000000000..b974a4d9ccc903d03d3a92e0b235ed2b8e736856 --- /dev/null +++ b/hybrid/hybrid_PDF.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5785b87d151d9f4fefd2184df92444655697fe77fc9db74fa52e787dd8139ff7 +size 5331649 diff --git a/hybrid/hybrid_PDF_benchmark.json b/hybrid/hybrid_PDF_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..c51262a93948f87a9727210913ca1c4d8709158b --- /dev/null +++ b/hybrid/hybrid_PDF_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "hybrid", + "subset": "PDF", + "quant": "fp32", + "model_size_mb": 5.326, + "original_size_mb": 5.332, + "size_reduction_pct": 0.1, + "latency_ms": { + "cold_batch1": 0.316, + "warm_batch1": 0.172, + "warm_batch16": 0.907, + "warm_batch256": 12.104, + "throughput_batch1": 5814.0, + "ms_per_sample_batch1": 0.172, + "throughput_batch16": 17640.6, + "ms_per_sample_batch16": 0.0567, + "throughput_batch256": 21150.0, + "ms_per_sample_batch256": 0.0473 + }, + "original_latency_ms": { + "cold_batch1": 0.405, + "warm_batch1": 0.185, + "warm_batch16": 0.861, + "warm_batch256": 6.762, + "throughput_batch1": 5405.4, + "ms_per_sample_batch1": 0.185, + "throughput_batch16": 18583.0, + "ms_per_sample_batch16": 0.0538, + "throughput_batch256": 37858.6, + "ms_per_sample_batch256": 0.0264 + }, + "test_metrics": { + "roc_auc": 0.9879419722222221, + "pr_auc": 0.9911053956151543, + "tpr_at_1pct_fpr": 0.9283333333333333, + "n_samples": 24000, + "n_positive": 12000, + "auc_delta_vs_original": -0.0 + }, + "peak_rss_delta_mb": 5.3, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/hybrid/hybrid_PDF_lgbm.model b/hybrid/hybrid_PDF_lgbm.model new file mode 100644 index 0000000000000000000000000000000000000000..6d72e2cd69dfbe88b9483489f6f2448b5c0ca4c2 --- /dev/null +++ b/hybrid/hybrid_PDF_lgbm.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7e30eb883f8918cf28aaa98b6124cf2eb25a98ea3d58f3053a4fb49e24c3161 +size 3749426 diff --git a/hybrid/hybrid_PDF_metrics.json b/hybrid/hybrid_PDF_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..2c09e6d83babbd57b0bfc95dfc209c60ed47ee32 --- /dev/null +++ b/hybrid/hybrid_PDF_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "hybrid", + "subset": "PDF", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9879419722222221, + "pr_auc": 0.9911053956151543, + "tpr_at_1pct_fpr": 0.9283333333333333, + "n_samples": 24000, + "n_positive": 12000, + "auc_delta_vs_original": -0.0 + }, + "challenge_metrics": { + "detection_rate": 0.568944099378882, + "threshold": 0.6341729126337898 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/hybrid/hybrid_PDF_nnpart.onnx b/hybrid/hybrid_PDF_nnpart.onnx new file mode 100644 index 0000000000000000000000000000000000000000..fa3ac4a737e3812234e0157db5d85f2db6385b65 --- /dev/null +++ b/hybrid/hybrid_PDF_nnpart.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74761f00107f5e7dfc7faa7e1426aa88d81219a040edf7974024b4c88d0e46b5 +size 5325521 diff --git a/hybrid/hybrid_PE.pt b/hybrid/hybrid_PE.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd35858fffec045117bfc0aa70d1d2313ced9ed5 --- /dev/null +++ b/hybrid/hybrid_PE.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:563c5ea02f4772b4543e6595d0cea1a1ee1db1e13465a8c14447bafbc6cd3558 +size 5331649 diff --git a/hybrid/hybrid_PE_benchmark.json b/hybrid/hybrid_PE_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..3da65b2a59458c1ccd76a24bc2eb5a547585583f --- /dev/null +++ b/hybrid/hybrid_PE_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "hybrid", + "subset": "PE", + "quant": "fp32", + "model_size_mb": 5.326, + "original_size_mb": 5.332, + "size_reduction_pct": 0.1, + "latency_ms": { + "cold_batch1": 0.415, + "warm_batch1": 0.138, + "warm_batch16": 0.973, + "warm_batch256": 12.139, + "throughput_batch1": 7246.4, + "ms_per_sample_batch1": 0.138, + "throughput_batch16": 16444.0, + "ms_per_sample_batch16": 0.0608, + "throughput_batch256": 21089.1, + "ms_per_sample_batch256": 0.0474 + }, + "original_latency_ms": { + "cold_batch1": 0.513, + "warm_batch1": 0.2, + "warm_batch16": 0.831, + "warm_batch256": 7.127, + "throughput_batch1": 5000.0, + "ms_per_sample_batch1": 0.2, + "throughput_batch16": 19253.9, + "ms_per_sample_batch16": 0.0519, + "throughput_batch256": 35919.7, + "ms_per_sample_batch256": 0.0278 + }, + "test_metrics": { + "roc_auc": 0.9981821473120159, + "pr_auc": 0.9983690915561053, + "tpr_at_1pct_fpr": 0.9736058067225211, + "n_samples": 100000, + "n_positive": 50011, + "auc_delta_vs_original": -2.7e-05 + }, + "peak_rss_delta_mb": 2.7, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/hybrid/hybrid_PE_lgbm.model b/hybrid/hybrid_PE_lgbm.model new file mode 100644 index 0000000000000000000000000000000000000000..694cc031d93af5f22a9022ac66ebec1d9630e588 --- /dev/null +++ b/hybrid/hybrid_PE_lgbm.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4252027863492ac138785c8c18576f43dad77d00faddc14e8c0072e8db419f99 +size 3756042 diff --git a/hybrid/hybrid_PE_metrics.json b/hybrid/hybrid_PE_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..011d046b6439484288ac4379ffb60b9744c3abbd --- /dev/null +++ b/hybrid/hybrid_PE_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "hybrid", + "subset": "PE", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9981821473120159, + "pr_auc": 0.9983690915561053, + "tpr_at_1pct_fpr": 0.9736058067225211, + "n_samples": 100000, + "n_positive": 50011, + "auc_delta_vs_original": -2.7e-05 + }, + "challenge_metrics": { + "detection_rate": 0.5820747709829166, + "threshold": 0.5705497667488678 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/hybrid/hybrid_PE_nnpart.onnx b/hybrid/hybrid_PE_nnpart.onnx new file mode 100644 index 0000000000000000000000000000000000000000..f3541ff82d975e4ff98fdef55912e913b2e975c5 --- /dev/null +++ b/hybrid/hybrid_PE_nnpart.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ee857f44249ca2483babcae8f62bc795b2278bb7f718ef097d7058af33a7ef7 +size 5325521 diff --git a/hybrid/hybrid_Win32.pt b/hybrid/hybrid_Win32.pt new file mode 100644 index 0000000000000000000000000000000000000000..3d32b50cbcc8372c11005e91e54586f99b1d8829 --- /dev/null +++ b/hybrid/hybrid_Win32.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37f1ba17d7dac074c154d111ec9564e01cfc92278f8f8e1a4e6be2b73f87c566 +size 5331649 diff --git a/hybrid/hybrid_Win32_benchmark.json b/hybrid/hybrid_Win32_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..adf916b1f8464740f74bdaf04ed8ae3afe952864 --- /dev/null +++ b/hybrid/hybrid_Win32_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "hybrid", + "subset": "Win32", + "quant": "fp32", + "model_size_mb": 5.326, + "original_size_mb": 5.332, + "size_reduction_pct": 0.1, + "latency_ms": { + "cold_batch1": 0.424, + "warm_batch1": 0.141, + "warm_batch16": 0.926, + "warm_batch256": 11.671, + "throughput_batch1": 7092.2, + "ms_per_sample_batch1": 0.141, + "throughput_batch16": 17278.6, + "ms_per_sample_batch16": 0.0579, + "throughput_batch256": 21934.7, + "ms_per_sample_batch256": 0.0456 + }, + "original_latency_ms": { + "cold_batch1": 0.498, + "warm_batch1": 0.193, + "warm_batch16": 1.041, + "warm_batch256": 6.808, + "throughput_batch1": 5181.3, + "ms_per_sample_batch1": 0.193, + "throughput_batch16": 15369.8, + "ms_per_sample_batch16": 0.0651, + "throughput_batch256": 37602.8, + "ms_per_sample_batch256": 0.0266 + }, + "test_metrics": { + "roc_auc": 0.9981537496665784, + "pr_auc": 0.9983845191446993, + "tpr_at_1pct_fpr": 0.973448330810294, + "n_samples": 100000, + "n_positive": 50204, + "auc_delta_vs_original": 4.3e-05 + }, + "peak_rss_delta_mb": 0.0, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/hybrid/hybrid_Win32_lgbm.model b/hybrid/hybrid_Win32_lgbm.model new file mode 100644 index 0000000000000000000000000000000000000000..aafbc2de6745a55e9b8259fb8d66824a4903cb56 --- /dev/null +++ b/hybrid/hybrid_Win32_lgbm.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1e9fc174e4fcc6c0aba3ff29eb6d96ee9e240057f76940a8a1d6009ac0a4267 +size 3741926 diff --git a/hybrid/hybrid_Win32_metrics.json b/hybrid/hybrid_Win32_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..570355a3f344dd7b76efd72b12fc925250746229 --- /dev/null +++ b/hybrid/hybrid_Win32_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "hybrid", + "subset": "Win32", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9981537496665784, + "pr_auc": 0.9983845191446993, + "tpr_at_1pct_fpr": 0.973448330810294, + "n_samples": 100000, + "n_positive": 50204, + "auc_delta_vs_original": 4.3e-05 + }, + "challenge_metrics": { + "detection_rate": 0.5835658914728682, + "threshold": 0.3378524600070836 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/hybrid/hybrid_Win32_nnpart.onnx b/hybrid/hybrid_Win32_nnpart.onnx new file mode 100644 index 0000000000000000000000000000000000000000..625fda0a99297b2762d6a47e045556bfb6982b62 --- /dev/null +++ b/hybrid/hybrid_Win32_nnpart.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00cb615952676b24004be6a4bb78a1c1a973c6596f414a148b80563b724df72e +size 5325521 diff --git a/hybrid/hybrid_Win64.pt b/hybrid/hybrid_Win64.pt new file mode 100644 index 0000000000000000000000000000000000000000..3722c12d9c4e26ded79ee598bc265e44db035362 --- /dev/null +++ b/hybrid/hybrid_Win64.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dd4de8b8c5b8f11880ec759dca836931a53914ffcab71337d13db4f68541ac2 +size 5331649 diff --git a/hybrid/hybrid_Win64_benchmark.json b/hybrid/hybrid_Win64_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..3f4f4f676a4bcfc8e22bea7c00ff488b3df4daa9 --- /dev/null +++ b/hybrid/hybrid_Win64_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "hybrid", + "subset": "Win64", + "quant": "fp32", + "model_size_mb": 5.326, + "original_size_mb": 5.332, + "size_reduction_pct": 0.1, + "latency_ms": { + "cold_batch1": 0.447, + "warm_batch1": 0.422, + "warm_batch16": 2.156, + "warm_batch256": 13.837, + "throughput_batch1": 2369.7, + "ms_per_sample_batch1": 0.422, + "throughput_batch16": 7421.2, + "ms_per_sample_batch16": 0.1348, + "throughput_batch256": 18501.1, + "ms_per_sample_batch256": 0.0541 + }, + "original_latency_ms": { + "cold_batch1": 0.929, + "warm_batch1": 0.246, + "warm_batch16": 1.035, + "warm_batch256": 9.283, + "throughput_batch1": 4065.0, + "ms_per_sample_batch1": 0.246, + "throughput_batch16": 15458.9, + "ms_per_sample_batch16": 0.0647, + "throughput_batch256": 27577.3, + "ms_per_sample_batch256": 0.0363 + }, + "test_metrics": { + "roc_auc": 0.9982428399016703, + "pr_auc": 0.9985168980151862, + "tpr_at_1pct_fpr": 0.9811498397435897, + "n_samples": 100000, + "n_positive": 49920, + "auc_delta_vs_original": -4.6e-05 + }, + "peak_rss_delta_mb": 0.0, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/hybrid/hybrid_Win64_lgbm.model b/hybrid/hybrid_Win64_lgbm.model new file mode 100644 index 0000000000000000000000000000000000000000..a7995889d4659ce4b75e85af740041250f278b17 --- /dev/null +++ b/hybrid/hybrid_Win64_lgbm.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eddddc26eb346d74810a0dfcc672342eca5709ece4da259934d6d3c77ca971b +size 3729014 diff --git a/hybrid/hybrid_Win64_metrics.json b/hybrid/hybrid_Win64_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..5b3d30ff1eec7570a3cde0fe9644bef2919a538f --- /dev/null +++ b/hybrid/hybrid_Win64_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "hybrid", + "subset": "Win64", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9982428399016703, + "pr_auc": 0.9985168980151862, + "tpr_at_1pct_fpr": 0.9811498397435897, + "n_samples": 100000, + "n_positive": 49920, + "auc_delta_vs_original": -4.6e-05 + }, + "challenge_metrics": { + "detection_rate": 0.5945945945945946, + "threshold": 0.44910266935932824 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/hybrid/hybrid_Win64_nnpart.onnx b/hybrid/hybrid_Win64_nnpart.onnx new file mode 100644 index 0000000000000000000000000000000000000000..df4a1f2832f1658a6a3474cd071e0566d40a651c --- /dev/null +++ b/hybrid/hybrid_Win64_nnpart.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa27767654df42378a55a8cd2e1bbbdf4e4a6786fef2dd9e9a5293f7f8dcff9e +size 5325521 diff --git a/hybrid/hybrid_all.pt b/hybrid/hybrid_all.pt new file mode 100644 index 0000000000000000000000000000000000000000..724103bda23f9513c25a6c90b26b33aa157a9586 --- /dev/null +++ b/hybrid/hybrid_all.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de23de572c3c93127e8701a47a2b9ddb1319285cc540aab64963576563bc6514 +size 5331649 diff --git a/hybrid/hybrid_all_benchmark.json b/hybrid/hybrid_all_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..bb882c85e755e27fd426da420845bda7e83fe289 --- /dev/null +++ b/hybrid/hybrid_all_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "hybrid", + "subset": "all", + "quant": "fp32", + "model_size_mb": 5.326, + "original_size_mb": 5.332, + "size_reduction_pct": 0.1, + "latency_ms": { + "cold_batch1": 0.45, + "warm_batch1": 0.147, + "warm_batch16": 0.885, + "warm_batch256": 12.452, + "throughput_batch1": 6802.7, + "ms_per_sample_batch1": 0.147, + "throughput_batch16": 18079.1, + "ms_per_sample_batch16": 0.0553, + "throughput_batch256": 20558.9, + "ms_per_sample_batch256": 0.0486 + }, + "original_latency_ms": { + "cold_batch1": 0.461, + "warm_batch1": 0.188, + "warm_batch16": 0.962, + "warm_batch256": 7.493, + "throughput_batch1": 5319.1, + "ms_per_sample_batch1": 0.188, + "throughput_batch16": 16632.0, + "ms_per_sample_batch16": 0.0601, + "throughput_batch256": 34165.2, + "ms_per_sample_batch256": 0.0293 + }, + "test_metrics": { + "roc_auc": 0.9971621806918503, + "pr_auc": 0.9973390129212234, + "tpr_at_1pct_fpr": 0.9512599668272017, + "n_samples": 100000, + "n_positive": 50041, + "auc_delta_vs_original": 0.000152 + }, + "peak_rss_delta_mb": 2.6, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/hybrid/hybrid_all_lgbm.model b/hybrid/hybrid_all_lgbm.model new file mode 100644 index 0000000000000000000000000000000000000000..854292a7272b2a2322f5bf85678e61a7726e66a2 --- /dev/null +++ b/hybrid/hybrid_all_lgbm.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af4ec038685797c586142d177965fe451cac96f424f631aeb66f8d116c161d07 +size 3759761 diff --git a/hybrid/hybrid_all_metrics.json b/hybrid/hybrid_all_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..ada2418dad2e926a5fef5ef22f2515cc482c3d2b --- /dev/null +++ b/hybrid/hybrid_all_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "hybrid", + "subset": "all", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9971621806918503, + "pr_auc": 0.9973390129212234, + "tpr_at_1pct_fpr": 0.9512599668272017, + "n_samples": 100000, + "n_positive": 50041, + "auc_delta_vs_original": 0.000152 + }, + "challenge_metrics": { + "detection_rate": 0.5412509897070467, + "threshold": 0.8706207157788477 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/hybrid/hybrid_all_nnpart.onnx b/hybrid/hybrid_all_nnpart.onnx new file mode 100644 index 0000000000000000000000000000000000000000..69bbfcf355c5318b30e84816093222370de1e042 --- /dev/null +++ b/hybrid/hybrid_all_nnpart.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95c04fddcd08ba9d303e96be3ec703d8374d8fb921314cc3779f56b8a161e484 +size 5325521 diff --git a/hybrid/hybrid_dotnet.pt b/hybrid/hybrid_dotnet.pt new file mode 100644 index 0000000000000000000000000000000000000000..47ac4d18394839c86a8b5b682163a98295a501cc --- /dev/null +++ b/hybrid/hybrid_dotnet.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1163e622b38f562aaa078aa4f2b43c4c0f3c82208394bf4cd0550a76671bf0c4 +size 5331649 diff --git a/hybrid/hybrid_dotnet_benchmark.json b/hybrid/hybrid_dotnet_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..b55ab115aac47b06bcbedf1647bca42c9a3c3713 --- /dev/null +++ b/hybrid/hybrid_dotnet_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "hybrid", + "subset": ".NET", + "quant": "fp32", + "model_size_mb": 5.326, + "original_size_mb": 5.332, + "size_reduction_pct": 0.1, + "latency_ms": { + "cold_batch1": 0.474, + "warm_batch1": 0.151, + "warm_batch16": 0.789, + "warm_batch256": 12.65, + "throughput_batch1": 6622.5, + "ms_per_sample_batch1": 0.151, + "throughput_batch16": 20278.8, + "ms_per_sample_batch16": 0.0493, + "throughput_batch256": 20237.2, + "ms_per_sample_batch256": 0.0494 + }, + "original_latency_ms": { + "cold_batch1": 0.537, + "warm_batch1": 0.191, + "warm_batch16": 0.874, + "warm_batch256": 6.482, + "throughput_batch1": 5235.6, + "ms_per_sample_batch1": 0.191, + "throughput_batch16": 18306.6, + "ms_per_sample_batch16": 0.0546, + "throughput_batch256": 39494.0, + "ms_per_sample_batch256": 0.0253 + }, + "test_metrics": { + "roc_auc": 0.9960508297936879, + "pr_auc": 0.9960429797610809, + "tpr_at_1pct_fpr": 0.9466423737636648, + "n_samples": 100000, + "n_positive": 49946, + "auc_delta_vs_original": -3.8e-05 + }, + "peak_rss_delta_mb": 2.6, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/hybrid/hybrid_dotnet_lgbm.model b/hybrid/hybrid_dotnet_lgbm.model new file mode 100644 index 0000000000000000000000000000000000000000..50dae72b77bb274fbb3d4e94535e4e37ae3568eb --- /dev/null +++ b/hybrid/hybrid_dotnet_lgbm.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbd91b4a823d5dac3f56f38640a5599836501ccbd4fcce273d28c9aed0afc24c +size 3697641 diff --git a/hybrid/hybrid_dotnet_metrics.json b/hybrid/hybrid_dotnet_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..3b80496c281a36ad244c34ba8f334a89ae61d175 --- /dev/null +++ b/hybrid/hybrid_dotnet_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "hybrid", + "subset": ".NET", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9960508297936879, + "pr_auc": 0.9960429797610809, + "tpr_at_1pct_fpr": 0.9466423737636648, + "n_samples": 100000, + "n_positive": 49946, + "auc_delta_vs_original": -3.8e-05 + }, + "challenge_metrics": { + "detection_rate": 0.8057901085645356, + "threshold": 0.9990034294799655 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/hybrid/hybrid_dotnet_nnpart.onnx b/hybrid/hybrid_dotnet_nnpart.onnx new file mode 100644 index 0000000000000000000000000000000000000000..60d569bef78c399e4f156be014f29989d78acdee --- /dev/null +++ b/hybrid/hybrid_dotnet_nnpart.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19cccd135dd6ecb3a131fc51c2b47ed8f74d89ad83fece789baadf54b894ce05 +size 5325521 diff --git a/lightgbm/lightgbm_APK.tl b/lightgbm/lightgbm_APK.tl new file mode 100644 index 0000000000000000000000000000000000000000..ce72add341a0e0a06ec55a821ea7f881606c9e43 --- /dev/null +++ b/lightgbm/lightgbm_APK.tl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fde556199271e5f14de1f1bf557744ea8289539316f32b30a7471329e1993571 +size 5301629 diff --git a/lightgbm/lightgbm_APK_benchmark.json b/lightgbm/lightgbm_APK_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..d0bc3af8c6fa3811f3570b720686e81bf77e74cd --- /dev/null +++ b/lightgbm/lightgbm_APK_benchmark.json @@ -0,0 +1,42 @@ +{ + "model": "lgbm", + "subset": "APK", + "quant": "int8-static", + "model_size_mb": 1.85, + "original_size_mb": 3.68, + "size_reduction_pct": 49.7, + "latency_ms": { + "cold_batch1": 0.054, + "warm_batch1": 0.031, + "ms_per_sample_batch1": 0.0311, + "throughput_batch1": 32150.9, + "warm_batch16": 0.165, + "ms_per_sample_batch16": 0.0103, + "throughput_batch16": 96865.7, + "warm_batch256": 1.698, + "ms_per_sample_batch256": 0.0066, + "throughput_batch256": 150776.3 + }, + "original_latency_ms": { + "cold_batch1": 0.141, + "warm_batch1": 0.075, + "ms_per_sample_batch1": 0.0749, + "throughput_batch1": 13357.2, + "warm_batch16": 0.365, + "ms_per_sample_batch16": 0.0228, + "throughput_batch16": 43838.5, + "warm_batch256": 3.293, + "ms_per_sample_batch256": 0.0129, + "throughput_batch256": 77737.6 + }, + "test_metrics": { + "roc_auc": 0.9861369791666666, + "pr_auc": 0.9870864941779258, + "tpr_at_1pct_fpr": 0.8157083333333334, + "n_samples": 96000, + "n_positive": 48000 + }, + "peak_rss_delta_mb": 7.9, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/lightgbm/lightgbm_APK_metrics.json b/lightgbm/lightgbm_APK_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..44c0b9f2600722c249efec2a676a7d5f1e0d9b45 --- /dev/null +++ b/lightgbm/lightgbm_APK_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "lgbm", + "subset": "APK", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9861369791666666, + "pr_auc": 0.9870864941779258, + "tpr_at_1pct_fpr": 0.8157083333333334, + "n_samples": 96000, + "n_positive": 48000, + "auc_delta_vs_original": 0.0 + }, + "challenge_metrics": { + "detection_rate": 0.3359375, + "threshold": 0.8551083540363797 + }, + "auc_gate_failed": false, + "platform": "macOS-15.7.7-arm64-arm-64bit" +} \ No newline at end of file diff --git a/lightgbm/lightgbm_ELF.tl b/lightgbm/lightgbm_ELF.tl new file mode 100644 index 0000000000000000000000000000000000000000..ea9362b6f0c349d24ea274e96904a0701d1aa168 --- /dev/null +++ b/lightgbm/lightgbm_ELF.tl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:972d6f7e45848c66a1838ca10a9d4d5fae2198bfc50db85fa02f87aa9e520add +size 5301629 diff --git a/lightgbm/lightgbm_ELF_benchmark.json b/lightgbm/lightgbm_ELF_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..12de9bfd086b20842668582eb296da17cac738f3 --- /dev/null +++ b/lightgbm/lightgbm_ELF_benchmark.json @@ -0,0 +1,42 @@ +{ + "model": "lgbm", + "subset": "ELF", + "quant": "int8-static", + "model_size_mb": 1.834, + "original_size_mb": 3.765, + "size_reduction_pct": 51.3, + "latency_ms": { + "cold_batch1": 0.062, + "warm_batch1": 0.036, + "ms_per_sample_batch1": 0.0357, + "throughput_batch1": 28039.7, + "warm_batch16": 0.129, + "ms_per_sample_batch16": 0.008, + "throughput_batch16": 124320.3, + "warm_batch256": 1.831, + "ms_per_sample_batch256": 0.0072, + "throughput_batch256": 139777.9 + }, + "original_latency_ms": { + "cold_batch1": 0.103, + "warm_batch1": 0.066, + "ms_per_sample_batch1": 0.0663, + "throughput_batch1": 15087.8, + "warm_batch16": 0.245, + "ms_per_sample_batch16": 0.0153, + "throughput_batch16": 65399.9, + "warm_batch256": 4.332, + "ms_per_sample_batch256": 0.0169, + "throughput_batch256": 59091.1 + }, + "test_metrics": { + "roc_auc": 0.9929004444444444, + "pr_auc": 0.9929616648340884, + "tpr_at_1pct_fpr": 0.914, + "n_samples": 12000, + "n_positive": 6000 + }, + "peak_rss_delta_mb": 0.1, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/lightgbm/lightgbm_ELF_metrics.json b/lightgbm/lightgbm_ELF_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..5f57ff5a49942809c2c4ece9345b068f3c717fd8 --- /dev/null +++ b/lightgbm/lightgbm_ELF_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "lgbm", + "subset": "ELF", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9929004444444444, + "pr_auc": 0.9929616648340884, + "tpr_at_1pct_fpr": 0.914, + "n_samples": 12000, + "n_positive": 6000, + "auc_delta_vs_original": 0.0 + }, + "challenge_metrics": { + "detection_rate": 0.30310880829015546, + "threshold": 0.9957756021784342 + }, + "auc_gate_failed": false, + "platform": "macOS-15.7.7-arm64-arm-64bit" +} \ No newline at end of file diff --git a/lightgbm/lightgbm_PDF.tl b/lightgbm/lightgbm_PDF.tl new file mode 100644 index 0000000000000000000000000000000000000000..eb86bc999b546c35c90976403f4e420a17c48244 --- /dev/null +++ b/lightgbm/lightgbm_PDF.tl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:478c8f8a6df498917ed368369e140b839ad56f794c14e087928e24dd6796d869 +size 5292285 diff --git a/lightgbm/lightgbm_PDF_benchmark.json b/lightgbm/lightgbm_PDF_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..ddc4ac825f6cfa7bb6a1161f0b15eeaee06b5890 --- /dev/null +++ b/lightgbm/lightgbm_PDF_benchmark.json @@ -0,0 +1,42 @@ +{ + "model": "lgbm", + "subset": "PDF", + "quant": "int8-static", + "model_size_mb": 1.834, + "original_size_mb": 3.749, + "size_reduction_pct": 51.1, + "latency_ms": { + "cold_batch1": 0.129, + "warm_batch1": 0.048, + "ms_per_sample_batch1": 0.048, + "throughput_batch1": 20825.4, + "warm_batch16": 0.113, + "ms_per_sample_batch16": 0.0071, + "throughput_batch16": 141742.4, + "warm_batch256": 1.732, + "ms_per_sample_batch256": 0.0068, + "throughput_batch256": 147804.8 + }, + "original_latency_ms": { + "cold_batch1": 0.092, + "warm_batch1": 0.068, + "ms_per_sample_batch1": 0.0679, + "throughput_batch1": 14717.5, + "warm_batch16": 0.24, + "ms_per_sample_batch16": 0.015, + "throughput_batch16": 66542.1, + "warm_batch256": 2.708, + "ms_per_sample_batch256": 0.0106, + "throughput_batch256": 94542.2 + }, + "test_metrics": { + "roc_auc": 0.9912552777777778, + "pr_auc": 0.9932672115475838, + "tpr_at_1pct_fpr": 0.9275, + "n_samples": 24000, + "n_positive": 12000 + }, + "peak_rss_delta_mb": 7.9, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/lightgbm/lightgbm_PDF_metrics.json b/lightgbm/lightgbm_PDF_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..23ad3b6e80e59a2eee449e00f4af32047a024744 --- /dev/null +++ b/lightgbm/lightgbm_PDF_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "lgbm", + "subset": "PDF", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9912552777777778, + "pr_auc": 0.9932672115475838, + "tpr_at_1pct_fpr": 0.9275, + "n_samples": 24000, + "n_positive": 12000, + "auc_delta_vs_original": 0.0 + }, + "challenge_metrics": { + "detection_rate": 0.5714285714285714, + "threshold": 0.41298968182723267 + }, + "auc_gate_failed": false, + "platform": "macOS-15.7.7-arm64-arm-64bit" +} \ No newline at end of file diff --git a/lightgbm/lightgbm_PE.tl b/lightgbm/lightgbm_PE.tl new file mode 100644 index 0000000000000000000000000000000000000000..2bb645b55c486cfe8cf87fdafa30b18b2edc09a5 --- /dev/null +++ b/lightgbm/lightgbm_PE.tl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3477f7458f2d1f5fda8f20e93d1de589fa96825c5de1de4cfeed41e91b74a84 +size 5305953 diff --git a/lightgbm/lightgbm_PE_benchmark.json b/lightgbm/lightgbm_PE_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..9a6dad9c2e51bbf0c0e77096e39147d4aea59331 --- /dev/null +++ b/lightgbm/lightgbm_PE_benchmark.json @@ -0,0 +1,42 @@ +{ + "model": "lgbm", + "subset": "PE", + "quant": "int8-static", + "model_size_mb": 1.768, + "original_size_mb": 3.756, + "size_reduction_pct": 52.9, + "latency_ms": { + "cold_batch1": 0.06, + "warm_batch1": 0.028, + "ms_per_sample_batch1": 0.028, + "throughput_batch1": 35660.7, + "warm_batch16": 0.144, + "ms_per_sample_batch16": 0.009, + "throughput_batch16": 110776.0, + "warm_batch256": 1.644, + "ms_per_sample_batch256": 0.0064, + "throughput_batch256": 155717.5 + }, + "original_latency_ms": { + "cold_batch1": 0.6, + "warm_batch1": 0.342, + "ms_per_sample_batch1": 0.3419, + "throughput_batch1": 2924.6, + "warm_batch16": 0.716, + "ms_per_sample_batch16": 0.0447, + "throughput_batch16": 22351.0, + "warm_batch256": 6.121, + "ms_per_sample_batch256": 0.0239, + "throughput_batch256": 41821.3 + }, + "test_metrics": { + "roc_auc": 0.998270741516304, + "pr_auc": 0.9984000207279566, + "tpr_at_1pct_fpr": 0.9686069064805742, + "n_samples": 100000, + "n_positive": 50011 + }, + "peak_rss_delta_mb": 7.9, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/lightgbm/lightgbm_PE_metrics.json b/lightgbm/lightgbm_PE_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..0f0126cae62bc4e783853c6a85d34b671e0d70db --- /dev/null +++ b/lightgbm/lightgbm_PE_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "lgbm", + "subset": "PE", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.998270741516304, + "pr_auc": 0.9984000207279566, + "tpr_at_1pct_fpr": 0.9686069064805742, + "n_samples": 100000, + "n_positive": 50011, + "auc_delta_vs_original": 0.0 + }, + "challenge_metrics": { + "detection_rate": 0.5877692498143104, + "threshold": 0.5877238439568709 + }, + "auc_gate_failed": false, + "platform": "macOS-15.7.7-arm64-arm-64bit" +} \ No newline at end of file diff --git a/lightgbm/lightgbm_Win32.tl b/lightgbm/lightgbm_Win32.tl new file mode 100644 index 0000000000000000000000000000000000000000..a6d45d922da43ba27b3246a5bb1dd5edc6fa86c8 --- /dev/null +++ b/lightgbm/lightgbm_Win32.tl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d45b3259750ec9c667614ee9fe0602c806314249d7689e38d97b99988e0c7640 +size 5304301 diff --git a/lightgbm/lightgbm_Win32_benchmark.json b/lightgbm/lightgbm_Win32_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..184bfba1d2c830f9de87ff03df4882439d851d0f --- /dev/null +++ b/lightgbm/lightgbm_Win32_benchmark.json @@ -0,0 +1,42 @@ +{ + "model": "lgbm", + "subset": "Win32", + "quant": "int8-static", + "model_size_mb": 1.768, + "original_size_mb": 3.742, + "size_reduction_pct": 52.8, + "latency_ms": { + "cold_batch1": 0.084, + "warm_batch1": 0.044, + "ms_per_sample_batch1": 0.0437, + "throughput_batch1": 22884.3, + "warm_batch16": 0.103, + "ms_per_sample_batch16": 0.0064, + "throughput_batch16": 155738.0, + "warm_batch256": 1.537, + "ms_per_sample_batch256": 0.006, + "throughput_batch256": 166603.9 + }, + "original_latency_ms": { + "cold_batch1": 0.107, + "warm_batch1": 0.052, + "ms_per_sample_batch1": 0.0515, + "throughput_batch1": 19411.3, + "warm_batch16": 0.306, + "ms_per_sample_batch16": 0.0191, + "throughput_batch16": 52351.3, + "warm_batch256": 3.65, + "ms_per_sample_batch256": 0.0143, + "throughput_batch256": 70144.9 + }, + "test_metrics": { + "roc_auc": 0.9984861748002601, + "pr_auc": 0.9986221219958593, + "tpr_at_1pct_fpr": 0.9722133694526333, + "n_samples": 100000, + "n_positive": 50204 + }, + "peak_rss_delta_mb": 7.9, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/lightgbm/lightgbm_Win32_metrics.json b/lightgbm/lightgbm_Win32_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..4cbedf76d5c17f16c9339de0fbe91df74d16afce --- /dev/null +++ b/lightgbm/lightgbm_Win32_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "lgbm", + "subset": "Win32", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9984861748002601, + "pr_auc": 0.9986221219958593, + "tpr_at_1pct_fpr": 0.9722133694526333, + "n_samples": 100000, + "n_positive": 50204, + "auc_delta_vs_original": 0.0 + }, + "challenge_metrics": { + "detection_rate": 0.6989147286821705, + "threshold": 0.5587557836736478 + }, + "auc_gate_failed": false, + "platform": "macOS-15.7.7-arm64-arm-64bit" +} \ No newline at end of file diff --git a/lightgbm/lightgbm_Win64.tl b/lightgbm/lightgbm_Win64.tl new file mode 100644 index 0000000000000000000000000000000000000000..d27559037d9b874b3252420dcff641a68a60dc93 --- /dev/null +++ b/lightgbm/lightgbm_Win64.tl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8c6b8738b5ce0b9380eb80952066e82f2c1a53792e8ad09bd9324551311a50b +size 5302925 diff --git a/lightgbm/lightgbm_Win64_benchmark.json b/lightgbm/lightgbm_Win64_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..653ee69014446337bbcb1235fdc169e5c438aff9 --- /dev/null +++ b/lightgbm/lightgbm_Win64_benchmark.json @@ -0,0 +1,42 @@ +{ + "model": "lgbm", + "subset": "Win64", + "quant": "int8-static", + "model_size_mb": 1.801, + "original_size_mb": 3.729, + "size_reduction_pct": 51.7, + "latency_ms": { + "cold_batch1": 0.057, + "warm_batch1": 0.039, + "ms_per_sample_batch1": 0.0391, + "throughput_batch1": 25566.9, + "warm_batch16": 0.103, + "ms_per_sample_batch16": 0.0064, + "throughput_batch16": 155970.8, + "warm_batch256": 1.571, + "ms_per_sample_batch256": 0.0061, + "throughput_batch256": 162965.3 + }, + "original_latency_ms": { + "cold_batch1": 0.113, + "warm_batch1": 0.069, + "ms_per_sample_batch1": 0.069, + "throughput_batch1": 14502.2, + "warm_batch16": 0.271, + "ms_per_sample_batch16": 0.017, + "throughput_batch16": 58985.3, + "warm_batch256": 4.381, + "ms_per_sample_batch256": 0.0171, + "throughput_batch256": 58432.4 + }, + "test_metrics": { + "roc_auc": 0.9988180677742535, + "pr_auc": 0.9989467405297935, + "tpr_at_1pct_fpr": 0.9829527243589744, + "n_samples": 100000, + "n_positive": 49920 + }, + "peak_rss_delta_mb": 5.3, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/lightgbm/lightgbm_Win64_metrics.json b/lightgbm/lightgbm_Win64_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..9c3f6c43a27cb80d8dcccc250e1757c014cdcc41 --- /dev/null +++ b/lightgbm/lightgbm_Win64_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "lgbm", + "subset": "Win64", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9988180677742535, + "pr_auc": 0.9989467405297935, + "tpr_at_1pct_fpr": 0.9829527243589744, + "n_samples": 100000, + "n_positive": 49920, + "auc_delta_vs_original": 0.0 + }, + "challenge_metrics": { + "detection_rate": 0.597051597051597, + "threshold": 0.560516395174044 + }, + "auc_gate_failed": false, + "platform": "macOS-15.7.7-arm64-arm-64bit" +} \ No newline at end of file diff --git a/lightgbm/lightgbm_all.tl b/lightgbm/lightgbm_all.tl new file mode 100644 index 0000000000000000000000000000000000000000..dff399f2c065395f87d1df63400748f7d04b2aeb --- /dev/null +++ b/lightgbm/lightgbm_all.tl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ba8a0b894c7cf30b44b699e9fc164e42247a6ff8b4619d35de8f23c93922814 +size 5308009 diff --git a/lightgbm/lightgbm_all_benchmark.json b/lightgbm/lightgbm_all_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..7ff2bdac9fcf0523e57e29d00f453061ef751af9 --- /dev/null +++ b/lightgbm/lightgbm_all_benchmark.json @@ -0,0 +1,42 @@ +{ + "model": "lgbm", + "subset": "all", + "quant": "int8-static", + "model_size_mb": 1.768, + "original_size_mb": 3.76, + "size_reduction_pct": 53.0, + "latency_ms": { + "cold_batch1": 0.065, + "warm_batch1": 0.068, + "ms_per_sample_batch1": 0.068, + "throughput_batch1": 14713.2, + "warm_batch16": 0.095, + "ms_per_sample_batch16": 0.0059, + "throughput_batch16": 168503.5, + "warm_batch256": 1.394, + "ms_per_sample_batch256": 0.0054, + "throughput_batch256": 183638.2 + }, + "original_latency_ms": { + "cold_batch1": 0.107, + "warm_batch1": 0.068, + "ms_per_sample_batch1": 0.0679, + "throughput_batch1": 14722.9, + "warm_batch16": 0.295, + "ms_per_sample_batch16": 0.0184, + "throughput_batch16": 54228.4, + "warm_batch256": 3.277, + "ms_per_sample_batch256": 0.0128, + "throughput_batch256": 78118.4 + }, + "test_metrics": { + "roc_auc": 0.9969703959628942, + "pr_auc": 0.9971498778446223, + "tpr_at_1pct_fpr": 0.9450250794348634, + "n_samples": 100000, + "n_positive": 50041 + }, + "peak_rss_delta_mb": 7.9, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/lightgbm/lightgbm_all_metrics.json b/lightgbm/lightgbm_all_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..55c65e972da48b364faf0c192eaaaa48b99a0cde --- /dev/null +++ b/lightgbm/lightgbm_all_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "lgbm", + "subset": "all", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9969703959628942, + "pr_auc": 0.9971498778446223, + "tpr_at_1pct_fpr": 0.9450250794348634, + "n_samples": 100000, + "n_positive": 50041, + "auc_delta_vs_original": 0.0 + }, + "challenge_metrics": { + "detection_rate": 0.48392715756136184, + "threshold": 0.7111641781630391 + }, + "auc_gate_failed": false, + "platform": "macOS-15.7.7-arm64-arm-64bit" +} \ No newline at end of file diff --git a/lightgbm/lightgbm_dotnet.tl b/lightgbm/lightgbm_dotnet.tl new file mode 100644 index 0000000000000000000000000000000000000000..0ff215d994414705223c188f3a2e853a135b9056 --- /dev/null +++ b/lightgbm/lightgbm_dotnet.tl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa367ab77433c960341111cfd3e6c90d114a7c0b25f8ed0cf66e2b73c705137c +size 5301869 diff --git a/lightgbm/lightgbm_dotnet_benchmark.json b/lightgbm/lightgbm_dotnet_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..9c29f8fabd7a16161a0e361089c369fd8109a6a8 --- /dev/null +++ b/lightgbm/lightgbm_dotnet_benchmark.json @@ -0,0 +1,42 @@ +{ + "model": "lgbm", + "subset": ".NET", + "quant": "int8-static", + "model_size_mb": 1.817, + "original_size_mb": 3.698, + "size_reduction_pct": 50.9, + "latency_ms": { + "cold_batch1": 0.056, + "warm_batch1": 0.05, + "ms_per_sample_batch1": 0.0503, + "throughput_batch1": 19869.9, + "warm_batch16": 0.142, + "ms_per_sample_batch16": 0.0089, + "throughput_batch16": 112664.0, + "warm_batch256": 1.641, + "ms_per_sample_batch256": 0.0064, + "throughput_batch256": 155963.2 + }, + "original_latency_ms": { + "cold_batch1": 0.142, + "warm_batch1": 0.067, + "ms_per_sample_batch1": 0.0671, + "throughput_batch1": 14911.4, + "warm_batch16": 0.251, + "ms_per_sample_batch16": 0.0157, + "throughput_batch16": 63732.9, + "warm_batch256": 5.093, + "ms_per_sample_batch256": 0.0199, + "throughput_batch256": 50269.9 + }, + "test_metrics": { + "roc_auc": 0.9979601608207316, + "pr_auc": 0.9980288248731665, + "tpr_at_1pct_fpr": 0.9561326232330918, + "n_samples": 100000, + "n_positive": 49946 + }, + "peak_rss_delta_mb": 7.9, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/lightgbm/lightgbm_dotnet_metrics.json b/lightgbm/lightgbm_dotnet_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..ef7780eb8b82a4943bf6853e8e753af8586d98c5 --- /dev/null +++ b/lightgbm/lightgbm_dotnet_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "lgbm", + "subset": ".NET", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9979601608207316, + "pr_auc": 0.9980288248731665, + "tpr_at_1pct_fpr": 0.9561326232330918, + "n_samples": 100000, + "n_positive": 49946, + "auc_delta_vs_original": 0.0 + }, + "challenge_metrics": { + "detection_rate": 0.7961399276236429, + "threshold": 0.9365163168402125 + }, + "auc_gate_failed": false, + "platform": "macOS-15.7.7-arm64-arm-64bit" +} \ No newline at end of file diff --git a/metrics_summary.json b/metrics_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ecc8687a12592c6700f35ff3c4b498a398281325 --- /dev/null +++ b/metrics_summary.json @@ -0,0 +1,333 @@ +{ + "generated": "2026-06-23", + "models": { + "dnn": { + ".NET": { + "quant": "int8-static", + "roc_auc": 0.9919714812355357, + "tpr_at_1pct_fpr": 0.8444236421489054, + "auc_delta": -0.001909, + "challenge_dr": 0.586248492159228, + "model_size_mb": 13.303, + "latency_warm_ms": 0.248, + "throughput_batch1": 4032.3 + }, + "APK": { + "quant": "fp32", + "roc_auc": 0.9760728567708332, + "tpr_at_1pct_fpr": 0.7681666666666667, + "auc_delta": -0.0, + "challenge_dr": 0.2734375, + "model_size_mb": 3.903, + "latency_warm_ms": 0.035, + "throughput_batch1": 28571.4 + }, + "ELF": { + "quant": "fp32", + "roc_auc": 0.9839878888888888, + "tpr_at_1pct_fpr": 0.8103333333333333, + "auc_delta": 6e-06, + "challenge_dr": 0.11658031088082901, + "model_size_mb": 3.903, + "latency_warm_ms": 0.039, + "throughput_batch1": 25864.6 + }, + "PDF": { + "quant": "fp32", + "roc_auc": 0.9795445, + "tpr_at_1pct_fpr": 0.8901666666666667, + "auc_delta": 0.0, + "challenge_dr": 0.41490683229813663, + "model_size_mb": 3.903, + "latency_warm_ms": 0.036, + "throughput_batch1": 27777.8 + }, + "PE": { + "quant": "int8-static", + "roc_auc": 0.9968856578492659, + "tpr_at_1pct_fpr": 0.9471575053488233, + "auc_delta": -6.2e-05, + "challenge_dr": 0.3849962862094578, + "model_size_mb": 13.303, + "latency_warm_ms": 0.29, + "throughput_batch1": 3448.3 + }, + "Win32": { + "quant": "int8-static", + "roc_auc": 0.9965419976367095, + "tpr_at_1pct_fpr": 0.9478530944531651, + "auc_delta": -8e-05, + "challenge_dr": 0.3655813953488372, + "model_size_mb": 13.303, + "latency_warm_ms": 0.288, + "throughput_batch1": 3472.2 + }, + "Win64": { + "quant": "int8-static", + "roc_auc": 0.9968797740122215, + "tpr_at_1pct_fpr": 0.9616965868794326, + "auc_delta": -3.3e-05, + "challenge_dr": 0.4631449631449631, + "model_size_mb": 13.303, + "latency_warm_ms": 0.22, + "throughput_batch1": 4545.5 + }, + "all": { + "quant": "int8-static", + "roc_auc": 0.9938331270533947, + "tpr_at_1pct_fpr": 0.8870437241462001, + "auc_delta": -0.000375, + "challenge_dr": 0.3534441805225653, + "model_size_mb": 13.303, + "latency_warm_ms": 0.254, + "throughput_batch1": 3937.0 + } + }, + "tabnet": { + ".NET": { + "quant": "fp32", + "roc_auc": 0.9923499432769739, + "tpr_at_1pct_fpr": 0.8700196211908862, + "auc_delta": -0.000126, + "challenge_dr": 0.6996381182147166, + "model_size_mb": 140.154, + "latency_warm_ms": 5.465, + "throughput_batch1": 183.0 + }, + "APK": { + "quant": "fp32", + "roc_auc": 0.9741325381944445, + "tpr_at_1pct_fpr": 0.7027916666666667, + "auc_delta": 0.0, + "challenge_dr": 0.29296875, + "model_size_mb": 13.457, + "latency_warm_ms": 0.846, + "throughput_batch1": 1182.0 + }, + "ELF": { + "quant": "fp32", + "roc_auc": 0.9793294444444443, + "tpr_at_1pct_fpr": 0.546, + "auc_delta": -1e-06, + "challenge_dr": 0.04404145077720207, + "model_size_mb": 13.457, + "latency_warm_ms": 0.505, + "throughput_batch1": 1979.5 + }, + "PDF": { + "quant": "fp32", + "roc_auc": 0.9809688611111111, + "tpr_at_1pct_fpr": 0.8596666666666667, + "auc_delta": 0.0, + "challenge_dr": 0.4012422360248447, + "model_size_mb": 13.457, + "latency_warm_ms": 2.23, + "throughput_batch1": 448.4 + }, + "PE": { + "quant": "fp32", + "roc_auc": 0.9948456613505301, + "tpr_at_1pct_fpr": 0.9194977105036892, + "auc_delta": -0.00027, + "challenge_dr": 0.36915077989601386, + "model_size_mb": 140.154, + "latency_warm_ms": 4.402, + "throughput_batch1": 227.2 + }, + "Win32": { + "quant": "fp32", + "roc_auc": 0.9948896635312952, + "tpr_at_1pct_fpr": 0.9317385068918811, + "auc_delta": 2.9e-05, + "challenge_dr": 0.45271317829457364, + "model_size_mb": 140.154, + "latency_warm_ms": 4.693, + "throughput_batch1": 213.1 + }, + "Win64": { + "quant": "fp32", + "roc_auc": 0.9944386641629801, + "tpr_at_1pct_fpr": 0.9317508012820512, + "auc_delta": -0.000202, + "challenge_dr": 0.44103194103194104, + "model_size_mb": 140.154, + "latency_warm_ms": 5.621, + "throughput_batch1": 177.9 + }, + "all": { + "quant": "fp32", + "roc_auc": 0.9922211951695316, + "tpr_at_1pct_fpr": 0.8911692412221978, + "auc_delta": 6.7e-05, + "challenge_dr": 0.42264449722882025, + "model_size_mb": 140.154, + "latency_warm_ms": 4.788, + "throughput_batch1": 208.9 + } + }, + "hybrid": { + ".NET": { + "quant": "fp32", + "roc_auc": 0.9960508297936879, + "tpr_at_1pct_fpr": 0.9466423737636648, + "auc_delta": -3.8e-05, + "challenge_dr": 0.8057901085645356, + "model_size_mb": 5.326, + "latency_warm_ms": 0.151, + "throughput_batch1": 6622.5 + }, + "APK": { + "quant": "fp32", + "roc_auc": 0.9827782760416667, + "tpr_at_1pct_fpr": 0.8002916666666666, + "auc_delta": 0.000631, + "challenge_dr": 0.34375, + "model_size_mb": 5.326, + "latency_warm_ms": 0.145, + "throughput_batch1": 6896.6 + }, + "ELF": { + "quant": "fp32", + "roc_auc": 0.9899468888888888, + "tpr_at_1pct_fpr": 0.8826666666666667, + "auc_delta": 0.0, + "challenge_dr": 0.23834196891191708, + "model_size_mb": 5.326, + "latency_warm_ms": 0.16, + "throughput_batch1": 6250.0 + }, + "PDF": { + "quant": "fp32", + "roc_auc": 0.9879419722222221, + "tpr_at_1pct_fpr": 0.9283333333333333, + "auc_delta": -0.0, + "challenge_dr": 0.568944099378882, + "model_size_mb": 5.326, + "latency_warm_ms": 0.172, + "throughput_batch1": 5814.0 + }, + "PE": { + "quant": "fp32", + "roc_auc": 0.9981821473120159, + "tpr_at_1pct_fpr": 0.9736058067225211, + "auc_delta": -2.7e-05, + "challenge_dr": 0.5820747709829166, + "model_size_mb": 5.326, + "latency_warm_ms": 0.138, + "throughput_batch1": 7246.4 + }, + "Win32": { + "quant": "fp32", + "roc_auc": 0.9981537496665784, + "tpr_at_1pct_fpr": 0.973448330810294, + "auc_delta": 4.3e-05, + "challenge_dr": 0.5835658914728682, + "model_size_mb": 5.326, + "latency_warm_ms": 0.141, + "throughput_batch1": 7092.2 + }, + "Win64": { + "quant": "fp32", + "roc_auc": 0.9982428399016703, + "tpr_at_1pct_fpr": 0.9811498397435897, + "auc_delta": -4.6e-05, + "challenge_dr": 0.5945945945945946, + "model_size_mb": 5.326, + "latency_warm_ms": 0.422, + "throughput_batch1": 2369.7 + }, + "all": { + "quant": "fp32", + "roc_auc": 0.9971621806918503, + "tpr_at_1pct_fpr": 0.9512599668272017, + "auc_delta": 0.000152, + "challenge_dr": 0.5412509897070467, + "model_size_mb": 5.326, + "latency_warm_ms": 0.147, + "throughput_batch1": 6802.7 + } + }, + "lightgbm": { + ".NET": { + "quant": "fp32", + "roc_auc": 0.9979601608207316, + "tpr_at_1pct_fpr": 0.9561326232330918, + "auc_delta": 0.0, + "challenge_dr": 0.7961399276236429, + "model_size_mb": 5.302, + "latency_warm_ms": 0.05, + "throughput_batch1": 19869.9 + }, + "APK": { + "quant": "fp32", + "roc_auc": 0.9861369791666666, + "tpr_at_1pct_fpr": 0.8157083333333334, + "auc_delta": 0.0, + "challenge_dr": 0.3359375, + "model_size_mb": 5.302, + "latency_warm_ms": 0.031, + "throughput_batch1": 32150.9 + }, + "ELF": { + "quant": "fp32", + "roc_auc": 0.9929004444444444, + "tpr_at_1pct_fpr": 0.914, + "auc_delta": 0.0, + "challenge_dr": 0.30310880829015546, + "model_size_mb": 5.302, + "latency_warm_ms": 0.036, + "throughput_batch1": 28039.7 + }, + "PDF": { + "quant": "fp32", + "roc_auc": 0.9912552777777778, + "tpr_at_1pct_fpr": 0.9275, + "auc_delta": 0.0, + "challenge_dr": 0.5714285714285714, + "model_size_mb": 5.292, + "latency_warm_ms": 0.048, + "throughput_batch1": 20825.4 + }, + "PE": { + "quant": "fp32", + "roc_auc": 0.998270741516304, + "tpr_at_1pct_fpr": 0.9686069064805742, + "auc_delta": 0.0, + "challenge_dr": 0.5877692498143104, + "model_size_mb": 5.306, + "latency_warm_ms": 0.028, + "throughput_batch1": 35660.7 + }, + "Win32": { + "quant": "fp32", + "roc_auc": 0.9984861748002601, + "tpr_at_1pct_fpr": 0.9722133694526333, + "auc_delta": 0.0, + "challenge_dr": 0.6989147286821705, + "model_size_mb": 5.304, + "latency_warm_ms": 0.044, + "throughput_batch1": 22884.3 + }, + "Win64": { + "quant": "fp32", + "roc_auc": 0.9988180677742535, + "tpr_at_1pct_fpr": 0.9829527243589744, + "auc_delta": 0.0, + "challenge_dr": 0.597051597051597, + "model_size_mb": 5.303, + "latency_warm_ms": 0.039, + "throughput_batch1": 25566.9 + }, + "all": { + "quant": "fp32", + "roc_auc": 0.9969703959628942, + "tpr_at_1pct_fpr": 0.9450250794348634, + "auc_delta": 0.0, + "challenge_dr": 0.48392715756136184, + "model_size_mb": 5.308, + "latency_warm_ms": 0.068, + "throughput_batch1": 14713.2 + } + } + } +} \ No newline at end of file diff --git a/tabnet/tabnet_APK.onnx b/tabnet/tabnet_APK.onnx new file mode 100644 index 0000000000000000000000000000000000000000..5ecf848191637a39ab9c558edcd1641980fb8af5 --- /dev/null +++ b/tabnet/tabnet_APK.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02b709754a128df83cca4ef3cd5612a082a55c6a3cf6e325a948cf3dc5bca8a0 +size 13457431 diff --git a/tabnet/tabnet_APK.zip b/tabnet/tabnet_APK.zip new file mode 100644 index 0000000000000000000000000000000000000000..007bc3e03c0cf8dd3bb632a4434824a45c1b8814 --- /dev/null +++ b/tabnet/tabnet_APK.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ad0847d89be29b6dd626a153912dec53645276b770f6534158bc88b498454b4 +size 3250261 diff --git a/tabnet/tabnet_APK_benchmark.json b/tabnet/tabnet_APK_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..e3a45da0c7de9ef37a080de746a1d2097a68152c --- /dev/null +++ b/tabnet/tabnet_APK_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "tabnet", + "subset": "APK", + "quant": "fp32", + "model_size_mb": 13.457, + "original_size_mb": 3.25, + "size_reduction_pct": -314.1, + "latency_ms": { + "cold_batch1": 0.765, + "warm_batch1": 0.846, + "warm_batch16": 2.56, + "warm_batch256": 39.071, + "throughput_batch1": 1182.0, + "ms_per_sample_batch1": 0.846, + "throughput_batch16": 6250.0, + "ms_per_sample_batch16": 0.16, + "throughput_batch256": 6552.2, + "ms_per_sample_batch256": 0.1526 + }, + "original_latency_ms": { + "cold_batch1": 2.01, + "warm_batch1": 1.776, + "warm_batch16": 6.445, + "warm_batch256": 24.723, + "throughput_batch1": 563.1, + "ms_per_sample_batch1": 1.776, + "throughput_batch16": 2482.5, + "ms_per_sample_batch16": 0.4028, + "throughput_batch256": 10354.7, + "ms_per_sample_batch256": 0.0966 + }, + "test_metrics": { + "roc_auc": 0.9741325381944445, + "pr_auc": 0.9752553810358323, + "tpr_at_1pct_fpr": 0.7027916666666667, + "n_samples": 96000, + "n_positive": 48000, + "auc_delta_vs_original": 0.0 + }, + "peak_rss_delta_mb": 0.0, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/tabnet/tabnet_APK_metrics.json b/tabnet/tabnet_APK_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..40f3961005807fab7dbaf7f84ccb254468a12401 --- /dev/null +++ b/tabnet/tabnet_APK_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "tabnet", + "subset": "APK", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9741325381944445, + "pr_auc": 0.9752553810358323, + "tpr_at_1pct_fpr": 0.7027916666666667, + "n_samples": 96000, + "n_positive": 48000, + "auc_delta_vs_original": 0.0 + }, + "challenge_metrics": { + "detection_rate": 0.29296875, + "threshold": 0.978909432888031 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/tabnet/tabnet_ELF.onnx b/tabnet/tabnet_ELF.onnx new file mode 100644 index 0000000000000000000000000000000000000000..5f14ea7dd3242e7b89c141ddaf92c97ed4442c48 --- /dev/null +++ b/tabnet/tabnet_ELF.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a3c904503610ad01a550b2467b279ca78c71554dd9668e382c06ace1cc0a041 +size 13457431 diff --git a/tabnet/tabnet_ELF.zip b/tabnet/tabnet_ELF.zip new file mode 100644 index 0000000000000000000000000000000000000000..86ae02159f39793e7fc7bee750e414722fba4fd2 --- /dev/null +++ b/tabnet/tabnet_ELF.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70a9d7feb6a4f31abbbd8285337bb16b245a9870ac801001336873762894428d +size 3245118 diff --git a/tabnet/tabnet_ELF_benchmark.json b/tabnet/tabnet_ELF_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..e4f0c27b4721c92733373f6f66a80a3ee91ce2e4 --- /dev/null +++ b/tabnet/tabnet_ELF_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "tabnet", + "subset": "ELF", + "quant": "fp32", + "model_size_mb": 13.457, + "original_size_mb": 3.245, + "size_reduction_pct": -314.7, + "latency_ms": { + "cold_batch1": 0.682, + "warm_batch1": 0.505, + "ms_per_sample_batch1": 0.5052, + "throughput_batch1": 1979.5, + "warm_batch16": 2.198, + "ms_per_sample_batch16": 0.1374, + "throughput_batch16": 7278.2, + "warm_batch256": 31.163, + "ms_per_sample_batch256": 0.1217, + "throughput_batch256": 8214.8 + }, + "original_latency_ms": { + "cold_batch1": 1.952, + "warm_batch1": 1.805, + "ms_per_sample_batch1": 1.8048, + "throughput_batch1": 554.1, + "warm_batch16": 4.991, + "ms_per_sample_batch16": 0.3119, + "throughput_batch16": 3206.0, + "warm_batch256": 22.953, + "ms_per_sample_batch256": 0.0897, + "throughput_batch256": 11153.0 + }, + "test_metrics": { + "roc_auc": 0.9793294444444443, + "pr_auc": 0.9665705945334059, + "tpr_at_1pct_fpr": 0.546, + "n_samples": 12000, + "n_positive": 6000, + "auc_delta_vs_original": -1e-06 + }, + "peak_rss_delta_mb": 0.0, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/tabnet/tabnet_ELF_metrics.json b/tabnet/tabnet_ELF_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..fdf01d9d0332f2a9ac0ad0480ea6ee49195b16bb --- /dev/null +++ b/tabnet/tabnet_ELF_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "tabnet", + "subset": "ELF", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9793294444444443, + "pr_auc": 0.9665705945334059, + "tpr_at_1pct_fpr": 0.546, + "n_samples": 12000, + "n_positive": 6000, + "auc_delta_vs_original": -1e-06 + }, + "challenge_metrics": { + "detection_rate": 0.04404145077720207, + "threshold": 0.9998235106468201 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/tabnet/tabnet_PDF.onnx b/tabnet/tabnet_PDF.onnx new file mode 100644 index 0000000000000000000000000000000000000000..569677d576b116c935cc3b5985304a4f2ebc0cc7 --- /dev/null +++ b/tabnet/tabnet_PDF.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e868262178ca7236bd02432cd8c3d36eb4194d7a5156cab88d544204cb5d70c0 +size 13457431 diff --git a/tabnet/tabnet_PDF.zip b/tabnet/tabnet_PDF.zip new file mode 100644 index 0000000000000000000000000000000000000000..91afdfa12b7ee90e63fdd57e074ff4a7b9ab4e72 --- /dev/null +++ b/tabnet/tabnet_PDF.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7e0276a57454c5aeccdf963e2a5982e53c253835989782a8f4f66bdb96286d7 +size 3244282 diff --git a/tabnet/tabnet_PDF_benchmark.json b/tabnet/tabnet_PDF_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..8c92a9e1df596a99340b8a31c4e0cc8705ffb431 --- /dev/null +++ b/tabnet/tabnet_PDF_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "tabnet", + "subset": "PDF", + "quant": "fp32", + "model_size_mb": 13.457, + "original_size_mb": 3.244, + "size_reduction_pct": -314.8, + "latency_ms": { + "cold_batch1": 0.639, + "warm_batch1": 2.23, + "ms_per_sample_batch1": 2.2303, + "throughput_batch1": 448.4, + "warm_batch16": 5.116, + "ms_per_sample_batch16": 0.3197, + "throughput_batch16": 3127.5, + "warm_batch256": 33.16, + "ms_per_sample_batch256": 0.1295, + "throughput_batch256": 7720.2 + }, + "original_latency_ms": { + "cold_batch1": 2.018, + "warm_batch1": 1.677, + "ms_per_sample_batch1": 1.6769, + "throughput_batch1": 596.3, + "warm_batch16": 6.752, + "ms_per_sample_batch16": 0.422, + "throughput_batch16": 2369.6, + "warm_batch256": 28.73, + "ms_per_sample_batch256": 0.1122, + "throughput_batch256": 8910.6 + }, + "test_metrics": { + "roc_auc": 0.9809688611111111, + "pr_auc": 0.985679691731748, + "tpr_at_1pct_fpr": 0.8596666666666667, + "n_samples": 24000, + "n_positive": 12000, + "auc_delta_vs_original": 0.0 + }, + "peak_rss_delta_mb": 0.2, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/tabnet/tabnet_PDF_metrics.json b/tabnet/tabnet_PDF_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..3286d0d77cc410b41b2dd05d1fe9e1dab24dc409 --- /dev/null +++ b/tabnet/tabnet_PDF_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "tabnet", + "subset": "PDF", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9809688611111111, + "pr_auc": 0.985679691731748, + "tpr_at_1pct_fpr": 0.8596666666666667, + "n_samples": 24000, + "n_positive": 12000, + "auc_delta_vs_original": 0.0 + }, + "challenge_metrics": { + "detection_rate": 0.4012422360248447, + "threshold": 0.7277986407279968 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/tabnet/tabnet_PE.onnx b/tabnet/tabnet_PE.onnx new file mode 100644 index 0000000000000000000000000000000000000000..4295732b3639e8a9f10b22136edfe8f228037dea --- /dev/null +++ b/tabnet/tabnet_PE.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95401beee88232d57bbf000cb2b9446ba8702274d39feacb080713010c04e0d1 +size 140154413 diff --git a/tabnet/tabnet_PE.zip b/tabnet/tabnet_PE.zip new file mode 100644 index 0000000000000000000000000000000000000000..657c891803a60f5b5c2b79bb2bc19b6d5770eb3c --- /dev/null +++ b/tabnet/tabnet_PE.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55e128776443d988494ac28d8a75b7003e89169f1f91010f83eb0147f0cc9de8 +size 7429880 diff --git a/tabnet/tabnet_PE_benchmark.json b/tabnet/tabnet_PE_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..5c4ca69f5e592095a097257868659053b93562de --- /dev/null +++ b/tabnet/tabnet_PE_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "tabnet", + "subset": "PE", + "quant": "fp32", + "model_size_mb": 140.154, + "original_size_mb": 7.43, + "size_reduction_pct": -1786.3, + "latency_ms": { + "cold_batch1": 5.664, + "warm_batch1": 4.402, + "warm_batch16": 10.607, + "warm_batch256": 156.092, + "throughput_batch1": 227.2, + "ms_per_sample_batch1": 4.402, + "throughput_batch16": 1508.4, + "ms_per_sample_batch16": 0.6629, + "throughput_batch256": 1640.1, + "ms_per_sample_batch256": 0.6097 + }, + "original_latency_ms": { + "cold_batch1": 10.918, + "warm_batch1": 10.183, + "warm_batch16": 21.887, + "warm_batch256": 111.527, + "throughput_batch1": 98.2, + "ms_per_sample_batch1": 10.183, + "throughput_batch16": 731.0, + "ms_per_sample_batch16": 1.3679, + "throughput_batch256": 2295.4, + "ms_per_sample_batch256": 0.4357 + }, + "test_metrics": { + "roc_auc": 0.9948456613505301, + "pr_auc": 0.9951360774507333, + "tpr_at_1pct_fpr": 0.9194977105036892, + "n_samples": 100000, + "n_positive": 50011, + "auc_delta_vs_original": -0.00027 + }, + "peak_rss_delta_mb": 0.3, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/tabnet/tabnet_PE_metrics.json b/tabnet/tabnet_PE_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..b679321997886e64824d5d57c2ec2b37fa4922be --- /dev/null +++ b/tabnet/tabnet_PE_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "tabnet", + "subset": "PE", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9948456613505301, + "pr_auc": 0.9951360774507333, + "tpr_at_1pct_fpr": 0.9194977105036892, + "n_samples": 100000, + "n_positive": 50011, + "auc_delta_vs_original": -0.00027 + }, + "challenge_metrics": { + "detection_rate": 0.36915077989601386, + "threshold": 0.7964108440279961 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/tabnet/tabnet_Win32.onnx b/tabnet/tabnet_Win32.onnx new file mode 100644 index 0000000000000000000000000000000000000000..bd93544e3d358067e92b068c2fd0300ae64887ff --- /dev/null +++ b/tabnet/tabnet_Win32.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc7b39cde7ccfb8cb2f06cafed933684c6ebc61a7302b9a05a3eb2cf3af94531 +size 140154413 diff --git a/tabnet/tabnet_Win32.zip b/tabnet/tabnet_Win32.zip new file mode 100644 index 0000000000000000000000000000000000000000..bef3ff433f60602f68a163bf13b9d93ece59b82e --- /dev/null +++ b/tabnet/tabnet_Win32.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed51c6834747a3497e8d2f72e78e1e84bbf2dae03a5d337ea6cf590acb6b168f +size 7434137 diff --git a/tabnet/tabnet_Win32_benchmark.json b/tabnet/tabnet_Win32_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..4608d095703bf42938185763c3a1030f98225046 --- /dev/null +++ b/tabnet/tabnet_Win32_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "tabnet", + "subset": "Win32", + "quant": "fp32", + "model_size_mb": 140.154, + "original_size_mb": 7.434, + "size_reduction_pct": -1785.3, + "latency_ms": { + "cold_batch1": 5.309, + "warm_batch1": 4.693, + "warm_batch16": 11.64, + "warm_batch256": 164.592, + "throughput_batch1": 213.1, + "ms_per_sample_batch1": 4.693, + "throughput_batch16": 1374.6, + "ms_per_sample_batch16": 0.7275, + "throughput_batch256": 1555.4, + "ms_per_sample_batch256": 0.6429 + }, + "original_latency_ms": { + "cold_batch1": 9.701, + "warm_batch1": 9.188, + "warm_batch16": 23.93, + "warm_batch256": 106.658, + "throughput_batch1": 108.8, + "ms_per_sample_batch1": 9.188, + "throughput_batch16": 668.6, + "ms_per_sample_batch16": 1.4956, + "throughput_batch256": 2400.2, + "ms_per_sample_batch256": 0.4166 + }, + "test_metrics": { + "roc_auc": 0.9948896635312952, + "pr_auc": 0.9948405944927753, + "tpr_at_1pct_fpr": 0.9317385068918811, + "n_samples": 100000, + "n_positive": 50204, + "auc_delta_vs_original": 2.9e-05 + }, + "peak_rss_delta_mb": 5.3, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/tabnet/tabnet_Win32_metrics.json b/tabnet/tabnet_Win32_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..00760a02b5ea437eb7185e96091131b66f96d3ce --- /dev/null +++ b/tabnet/tabnet_Win32_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "tabnet", + "subset": "Win32", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9948896635312952, + "pr_auc": 0.9948405944927753, + "tpr_at_1pct_fpr": 0.9317385068918811, + "n_samples": 100000, + "n_positive": 50204, + "auc_delta_vs_original": 2.9e-05 + }, + "challenge_metrics": { + "detection_rate": 0.45271317829457364, + "threshold": 0.9470166277885437 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/tabnet/tabnet_Win64.onnx b/tabnet/tabnet_Win64.onnx new file mode 100644 index 0000000000000000000000000000000000000000..381f2192e14309e6f45eb3739274dd9619ce81b7 --- /dev/null +++ b/tabnet/tabnet_Win64.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:161a17a5daf2418b8583a08e01007d13c5619d5feeefee31596ba9bcdaa1f47f +size 140154413 diff --git a/tabnet/tabnet_Win64.zip b/tabnet/tabnet_Win64.zip new file mode 100644 index 0000000000000000000000000000000000000000..d45c08593b1c5f9b341a5b982cee48c093b826e6 --- /dev/null +++ b/tabnet/tabnet_Win64.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:260bf4136861d1b3a04ff90eb7903b41e6a01193978c320c13d1c62df1bba103 +size 7429140 diff --git a/tabnet/tabnet_Win64_benchmark.json b/tabnet/tabnet_Win64_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..9378778e0ad786bc53bda750e0f7136af51c69af --- /dev/null +++ b/tabnet/tabnet_Win64_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "tabnet", + "subset": "Win64", + "quant": "fp32", + "model_size_mb": 140.154, + "original_size_mb": 7.429, + "size_reduction_pct": -1786.6, + "latency_ms": { + "cold_batch1": 5.52, + "warm_batch1": 5.621, + "warm_batch16": 12.261, + "warm_batch256": 176.112, + "throughput_batch1": 177.9, + "ms_per_sample_batch1": 5.621, + "throughput_batch16": 1305.0, + "ms_per_sample_batch16": 0.7663, + "throughput_batch256": 1453.6, + "ms_per_sample_batch256": 0.6879 + }, + "original_latency_ms": { + "cold_batch1": 9.918, + "warm_batch1": 9.95, + "warm_batch16": 27.51, + "warm_batch256": 111.345, + "throughput_batch1": 100.5, + "ms_per_sample_batch1": 9.95, + "throughput_batch16": 581.6, + "ms_per_sample_batch16": 1.7194, + "throughput_batch256": 2299.2, + "ms_per_sample_batch256": 0.4349 + }, + "test_metrics": { + "roc_auc": 0.9944386641629801, + "pr_auc": 0.99484068139891, + "tpr_at_1pct_fpr": 0.9317508012820512, + "n_samples": 100000, + "n_positive": 49920, + "auc_delta_vs_original": -0.000202 + }, + "peak_rss_delta_mb": 7.9, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/tabnet/tabnet_Win64_metrics.json b/tabnet/tabnet_Win64_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..ef14ddf45f7ae28a68084f316edbb90873420998 --- /dev/null +++ b/tabnet/tabnet_Win64_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "tabnet", + "subset": "Win64", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9944386641629801, + "pr_auc": 0.99484068139891, + "tpr_at_1pct_fpr": 0.9317508012820512, + "n_samples": 100000, + "n_positive": 49920, + "auc_delta_vs_original": -0.000202 + }, + "challenge_metrics": { + "detection_rate": 0.44103194103194104, + "threshold": 0.9233863472938537 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/tabnet/tabnet_all.onnx b/tabnet/tabnet_all.onnx new file mode 100644 index 0000000000000000000000000000000000000000..858416b0537f222d130eaf0294c1d0277cb69c14 --- /dev/null +++ b/tabnet/tabnet_all.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39ac01a51c2dd388514f16f53ae7b36ff0e2af10ac91de359f572955c2893e89 +size 140154413 diff --git a/tabnet/tabnet_all.zip b/tabnet/tabnet_all.zip new file mode 100644 index 0000000000000000000000000000000000000000..144abf57ee9269d64157239408f335282eb75b16 --- /dev/null +++ b/tabnet/tabnet_all.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8be87a5bc2a01b9041d73bc92cb19727efee7eb17135711a90ec3d55afe8a16 +size 7432616 diff --git a/tabnet/tabnet_all_benchmark.json b/tabnet/tabnet_all_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..95c9ab8f4a5ea44b30355869ef6a8d75cc72fcbd --- /dev/null +++ b/tabnet/tabnet_all_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "tabnet", + "subset": "all", + "quant": "fp32", + "model_size_mb": 140.154, + "original_size_mb": 7.433, + "size_reduction_pct": -1785.6, + "latency_ms": { + "cold_batch1": 5.73, + "warm_batch1": 4.788, + "warm_batch16": 13.262, + "warm_batch256": 181.755, + "throughput_batch1": 208.9, + "ms_per_sample_batch1": 4.788, + "throughput_batch16": 1206.5, + "ms_per_sample_batch16": 0.8289, + "throughput_batch256": 1408.5, + "ms_per_sample_batch256": 0.71 + }, + "original_latency_ms": { + "cold_batch1": 8.63, + "warm_batch1": 10.467, + "warm_batch16": 22.009, + "warm_batch256": 106.04, + "throughput_batch1": 95.5, + "ms_per_sample_batch1": 10.467, + "throughput_batch16": 727.0, + "ms_per_sample_batch16": 1.3756, + "throughput_batch256": 2414.2, + "ms_per_sample_batch256": 0.4142 + }, + "test_metrics": { + "roc_auc": 0.9922211951695316, + "pr_auc": 0.9915223853617919, + "tpr_at_1pct_fpr": 0.8911692412221978, + "n_samples": 100000, + "n_positive": 50041, + "auc_delta_vs_original": 6.7e-05 + }, + "peak_rss_delta_mb": 7.9, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/tabnet/tabnet_all_metrics.json b/tabnet/tabnet_all_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..8890c7cd2f346796e76b80e7b7823693de52a81f --- /dev/null +++ b/tabnet/tabnet_all_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "tabnet", + "subset": "all", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9922211951695316, + "pr_auc": 0.9915223853617919, + "tpr_at_1pct_fpr": 0.8911692412221978, + "n_samples": 100000, + "n_positive": 50041, + "auc_delta_vs_original": 6.7e-05 + }, + "challenge_metrics": { + "detection_rate": 0.42264449722882025, + "threshold": 0.9068153047561646 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file diff --git a/tabnet/tabnet_dotnet.onnx b/tabnet/tabnet_dotnet.onnx new file mode 100644 index 0000000000000000000000000000000000000000..e706df808e6484e663e4f70897263a302cc350d9 --- /dev/null +++ b/tabnet/tabnet_dotnet.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e30615811f1463e09ac55a4e08d27eda932e91009342ab940552d3f6e8733ae +size 140154413 diff --git a/tabnet/tabnet_dotnet.zip b/tabnet/tabnet_dotnet.zip new file mode 100644 index 0000000000000000000000000000000000000000..6752e6be097bfcd1ca8d02e2cfa6aaf3a09e2aac --- /dev/null +++ b/tabnet/tabnet_dotnet.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0791981a206559c891ee76bbe6d7537df9beae842bca2c3a4f68034c9b6c57dd +size 7422109 diff --git a/tabnet/tabnet_dotnet_benchmark.json b/tabnet/tabnet_dotnet_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..b46e055c625e063a081425de094457e9cefb7d41 --- /dev/null +++ b/tabnet/tabnet_dotnet_benchmark.json @@ -0,0 +1,43 @@ +{ + "model": "tabnet", + "subset": ".NET", + "quant": "fp32", + "model_size_mb": 140.154, + "original_size_mb": 7.422, + "size_reduction_pct": -1788.4, + "latency_ms": { + "cold_batch1": 5.084, + "warm_batch1": 5.465, + "warm_batch16": 13.02, + "warm_batch256": 176.076, + "throughput_batch1": 183.0, + "ms_per_sample_batch1": 5.465, + "throughput_batch16": 1228.9, + "ms_per_sample_batch16": 0.8137, + "throughput_batch256": 1453.9, + "ms_per_sample_batch256": 0.6878 + }, + "original_latency_ms": { + "cold_batch1": 9.044, + "warm_batch1": 9.268, + "warm_batch16": 25.687, + "warm_batch256": 111.125, + "throughput_batch1": 107.9, + "ms_per_sample_batch1": 9.268, + "throughput_batch16": 622.9, + "ms_per_sample_batch16": 1.6054, + "throughput_batch256": 2303.7, + "ms_per_sample_batch256": 0.4341 + }, + "test_metrics": { + "roc_auc": 0.9923499432769739, + "pr_auc": 0.9916205586874096, + "tpr_at_1pct_fpr": 0.8700196211908862, + "n_samples": 100000, + "n_positive": 49946, + "auc_delta_vs_original": -0.000126 + }, + "peak_rss_delta_mb": 5.3, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "platform_short": "darwin-arm64" +} \ No newline at end of file diff --git a/tabnet/tabnet_dotnet_metrics.json b/tabnet/tabnet_dotnet_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..4fd7a3c95fde488fad6779c1ddc764b15c1e56fd --- /dev/null +++ b/tabnet/tabnet_dotnet_metrics.json @@ -0,0 +1,19 @@ +{ + "model": "tabnet", + "subset": ".NET", + "quant": "fp32", + "test_metrics": { + "roc_auc": 0.9923499432769739, + "pr_auc": 0.9916205586874096, + "tpr_at_1pct_fpr": 0.8700196211908862, + "n_samples": 100000, + "n_positive": 49946, + "auc_delta_vs_original": -0.000126 + }, + "challenge_metrics": { + "detection_rate": 0.6996381182147166, + "threshold": 0.9473323392868042 + }, + "platform": "macOS-15.7.7-arm64-arm-64bit", + "auc_gate_failed": false +} \ No newline at end of file