Add ensemble meta-learners, scalers and calibrators

README.md

@@ -0,0 +1,59 @@
+---
+license: other
+license_name: polyform-noncommercial-1.0.0
+license_link: https://polyformproject.org/licenses/noncommercial/1.0.0
+tags:
+  - deepfake-detection
+  - ensemble
+---
+
+# DeepSafe Ensemble Artifacts
+
+The meta-learners that turn 19 individual detector scores into one calibrated
+verdict, for [DeepSafe](https://github.com/deepsafehq/deepsafe-bench).
+
+Unlike the model code and weights in the other DeepSafe repositories, **these
+are first-party**: trained by us, licensed PolyForm Noncommercial 1.0.0, same
+as the project.
+
+## Contents
+
+| Modality | Meta-learner | Held-out AUC |
+|---|---|---|
+| Image | LightGBM over 7 models | 0.9466 |
+| Audio | Random Forest over 3 models | 0.8290 |
+| Video | XGBoost over 9 models | 0.6694 |
+
+Each modality ships four files:
+
+- `<modality>_meta_learner.pkl` — the trained model
+- `<modality>_scaler.pkl` — feature scaling
+- `<modality>_calibrator.pkl` — Platt calibration, so scores read as probabilities
+- `<modality>_config.json` — feature order and fallback weights
+
+Trained on the 15,499-sample medium evaluation tier. Without these, the
+inference server produces per-model scores but no ensemble verdict.
+
+## Usage
+
+`setup.sh` fetches these automatically. Manually:
+
+```python
+from huggingface_hub import snapshot_download
+snapshot_download("deepsafe/ensemble", local_dir="models/ensemble/artifacts")
+```
+
+## The numbers are the point
+
+Held-out video AUC is **0.6694**. That is barely above chance on generators the
+models were not trained for, and it is lower than the cross-validated figure
+produced during training. We publish the held-out number because the gap
+between the two is the finding. See
+[BENCHMARK.md](https://github.com/deepsafehq/deepsafe-bench/blob/main/BENCHMARK.md).
+
+## Security note
+
+These are Python pickles, which execute code on load. Only load them from a
+source you trust. DeepSafe's loader refuses any pickle outside its configured
+artifacts directory, but that is a guardrail, not a guarantee. If you are
+security-sensitive, retrain your own with `deepsafe fit --tier 1`.

audio_calibrator.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07c382f323a548522bb3f153c10211ff98af56ef571b84ccd8083fb61a27a9af
+size 713

audio_config.json

@@ -0,0 +1,34 @@
+{
+  "model_order": [
+    "shiftyspeech",
+    "safeear",
+    "nes2net"
+  ],
+  "algorithm": "RandomForestClassifier",
+  "hyperparams": {
+    "max_depth": 8,
+    "n_estimators": 500,
+    "random_state": 42
+  },
+  "cv_auc": 0.8748,
+  "cv_f1_at_05": 0.8749,
+  "optimal_threshold": 0.47,
+  "optimal_f1": 0.8787,
+  "ece_before_calibration": 0.0132,
+  "ece_after_calibration": 0.0338,
+  "calibrated_auc": 0.8748,
+  "calibrated_optimal_threshold": 0.39,
+  "calibrated_optimal_f1": 0.8789,
+  "calibrator_saved": true,
+  "n_samples": 3500,
+  "n_real": 1000,
+  "n_fake": 2500,
+  "measured_model_aucs": {
+    "shiftyspeech": 0.7347,
+    "safeear": 0.6035,
+    "nes2net": 0.8407
+  },
+  "training_date": "2026-04-12",
+  "training_data": "monolith_eval_medium_15499_patched",
+  "note": "3-model RandomForestClassifier trained on 15,499-sample medium eval. Replaces small-eval (50 sample) artifacts."
+}

audio_meta_learner.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea2389cd393b42880d6f1672a9281b1ff7789193fd36bbd6512027c97afe6d88
+size 7928136

audio_scaler.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11b6d94e752a1247301e6790f8b50ed3d1a9a7c355132172d925ab4e028fd296
+size 488

image_calibrator.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3083c3e12fd399aa4d3ec67f9cf9b78a2c29ff72895a63b04680b07e8951d56
+size 713

image_config.json

@@ -0,0 +1,48 @@
+{
+  "model_order": [
+    "aide",
+    "cospy",
+    "effort",
+    "fsd",
+    "npr",
+    "universal",
+    "yermandy"
+  ],
+  "algorithm": "LGBMClassifier",
+  "hyperparams": {
+    "colsample_bytree": 1.0,
+    "learning_rate": 0.05,
+    "max_depth": 4,
+    "min_child_weight": 0.001,
+    "n_estimators": 200,
+    "random_state": 42,
+    "reg_alpha": 0.0,
+    "reg_lambda": 1.0,
+    "subsample": 1.0
+  },
+  "cv_auc": 0.9705,
+  "cv_f1_at_05": 0.9033,
+  "optimal_threshold": 0.41,
+  "optimal_f1": 0.9065,
+  "ece_before_calibration": 0.0096,
+  "ece_after_calibration": 0.0281,
+  "calibrated_auc": 0.9705,
+  "calibrated_optimal_threshold": 0.35,
+  "calibrated_optimal_f1": 0.9065,
+  "calibrator_saved": true,
+  "n_samples": 10000,
+  "n_real": 5000,
+  "n_fake": 5000,
+  "measured_model_aucs": {
+    "aide": 0.7478,
+    "cospy": 0.8679,
+    "effort": 0.8353,
+    "fsd": 0.8961,
+    "npr": 0.6373,
+    "universal": 0.6711,
+    "yermandy": 0.5912
+  },
+  "training_date": "2026-04-12",
+  "training_data": "monolith_eval_medium_15499_patched",
+  "note": "7-model LGBMClassifier trained on 15,499-sample medium eval. Replaces small-eval (50 sample) artifacts."
+}

image_meta_learner.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1536f01547326da6436d4462411205cf333a35abbf128c6cbf3b13e913c44cd3
+size 318970

image_scaler.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71e2c7ba983fdd80a3b9abb41276af90003d92b396a446bc5bc226d120dfdade
+size 584

video_calibrator.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bfd34dd98bf48a11337d88dca299577d03f9d1d2e8232a717808714e323084f
+size 713

video_config.json

@@ -0,0 +1,48 @@
+{
+  "model_order": [
+    "fakestormer",
+    "sbi",
+    "dfd_fcg",
+    "pwtf_dvd",
+    "lipfd",
+    "recce",
+    "mintime",
+    "npr_video",
+    "univfd_video"
+  ],
+  "algorithm": "XGBClassifier",
+  "hyperparams": {
+    "learning_rate": 0.05,
+    "max_depth": 3,
+    "n_estimators": 300,
+    "random_state": 42,
+    "reg_lambda": 10.0
+  },
+  "cv_auc": 0.8898,
+  "cv_f1_at_05": 0.8133,
+  "optimal_threshold": 0.48,
+  "optimal_f1": 0.8147,
+  "ece_before_calibration": 0.0219,
+  "ece_after_calibration": 0.0221,
+  "calibrated_auc": 0.8898,
+  "calibrated_optimal_threshold": 0.48,
+  "calibrated_optimal_f1": 0.8141,
+  "calibrator_saved": true,
+  "n_samples": 1999,
+  "n_real": 999,
+  "n_fake": 1000,
+  "measured_model_aucs": {
+    "fakestormer": 0.6618,
+    "sbi": 0.6781,
+    "dfd_fcg": 0.5531,
+    "pwtf_dvd": 0.585,
+    "lipfd": 0.6312,
+    "recce": 0.6625,
+    "mintime": 0.5813,
+    "npr_video": 0.8022,
+    "univfd_video": 0.6798
+  },
+  "training_date": "2026-04-12",
+  "training_data": "monolith_eval_medium_15499_patched",
+  "note": "9-model XGBClassifier trained on 15,499-sample medium eval. Replaces small-eval (50 sample) artifacts."
+}

video_meta_learner.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b68f468744cfee759ce18641c5622b15d5877e9046db366312a0db9e5ab92750
+size 346122

video_scaler.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c489f49490a0fafa115c7ec3cc1bd49fb196c5881e4cae04516e219847f110db
+size 632