| { |
| "seed": 42, |
| "comparisons": [ |
| { |
| "model": "discovery/CryptoCodeDetector", |
| "metric": "F1 (held-out real code)", |
| "n": 2415, |
| "qtrust": { |
| "baseline": "qtrust (rules+AST+ML ensemble)", |
| "precision": 0.952, |
| "recall": 0.953, |
| "f1": 0.9525, |
| "accuracy": 0.9271 |
| }, |
| "baselines": [ |
| { |
| "baseline": "rules-only (static layer)", |
| "precision": 0.9794, |
| "recall": 0.5127, |
| "f1": 0.673, |
| "accuracy": 0.6178 |
| }, |
| { |
| "baseline": "majority (always crypto)", |
| "precision": 0.7673, |
| "recall": 1.0, |
| "f1": 0.8683, |
| "accuracy": 0.7673 |
| }, |
| { |
| "baseline": "random coin-flip", |
| "precision": 0.7686, |
| "recall": 0.4895, |
| "f1": 0.5981, |
| "accuracy": 0.4952 |
| } |
| ], |
| "best_baseline": "majority (always crypto)", |
| "relative_gain": 0.097 |
| }, |
| { |
| "model": "discovery/CryptoCodeDetector", |
| "metric": "F1 on adversarial holdout (132 \u00a729 cases)", |
| "n": 132, |
| "qtrust": { |
| "baseline": "qtrust (rules+AST+ML ensemble)", |
| "precision": 0.8, |
| "recall": 0.4, |
| "f1": 0.5333 |
| }, |
| "baselines": [ |
| { |
| "baseline": "rules-only (static layer)", |
| "precision": 0.7778, |
| "recall": 0.35, |
| "f1": 0.4828 |
| } |
| ], |
| "best_baseline": "rules-only (static layer)", |
| "relative_gain": 0.1046 |
| }, |
| { |
| "model": "discovery/AlgorithmPurposeClassifier", |
| "metric": "accuracy (held-out real triples)", |
| "n": 294, |
| "qtrust": { |
| "baseline": "qtrust (context-aware)", |
| "accuracy": 1.0 |
| }, |
| "baselines": [ |
| { |
| "baseline": "family-prior-only (no context)", |
| "accuracy": 1.0 |
| }, |
| { |
| "baseline": "majority purpose", |
| "accuracy": 0.483 |
| } |
| ], |
| "best_baseline": "family-prior-only (no context)", |
| "relative_gain": 0.0 |
| }, |
| { |
| "model": "migration/PQCRecommender", |
| "metric": "family-correct rate (real triples)", |
| "n": 294, |
| "qtrust": { |
| "baseline": "qtrust (purpose-aware catalog)", |
| "accuracy": 0.7007 |
| }, |
| "baselines": [ |
| { |
| "baseline": "always-ML-KEM-768", |
| "accuracy": 0.0782 |
| } |
| ], |
| "best_baseline": "always-ML-KEM-768", |
| "relative_gain": 7.9565 |
| }, |
| { |
| "model": "risk/QuantumExposureModel", |
| "metric": "MAE (held-out real TLS hosts)", |
| "n_train_hosts": 221, |
| "n_eval": 56, |
| "qtrust": { |
| "baseline": "qtrust (calibrated, host-held-out)", |
| "mae": 0.0 |
| }, |
| "baselines": [ |
| { |
| "baseline": "constant-mean predictor", |
| "mae": 7.4412 |
| } |
| ], |
| "best_baseline": "constant-mean predictor", |
| "relative_gain": 1.0 |
| }, |
| { |
| "model": "monitoring/CryptoAnomalyDetector", |
| "metric": "false-positive rate (held-out real hosts)", |
| "n": 56, |
| "qtrust": { |
| "baseline": "qtrust (baseline+zscore)", |
| "fp_rate": 0.0 |
| }, |
| "baselines": [ |
| { |
| "baseline": "always-alert", |
| "fp_rate": 1.0 |
| } |
| ], |
| "best_baseline": "always-alert", |
| "relative_gain": 1.0 |
| }, |
| { |
| "model": "vendor/SupplyChainRiskModel", |
| "metric": "ranking agreement (real NVD vendors)", |
| "n": 16, |
| "qtrust": { |
| "baseline": "qtrust (5-layer propagation)", |
| "kendall_tau": 0.7176, |
| "ndcg@5": 0.9722 |
| }, |
| "baselines": [ |
| { |
| "baseline": "random ordering", |
| "kendall_tau": -0.0695 |
| }, |
| { |
| "baseline": "uniform scores", |
| "kendall_tau": 0.0 |
| } |
| ], |
| "best_baseline": "uniform scores", |
| "relative_gain": 0.7176 |
| } |
| ], |
| "summary": { |
| "comparisons_run": 7, |
| "models_beat_best_baseline": 6, |
| "mean_relative_gain": 1.5537 |
| } |
| } |
