Spaces:
Sleeping
Sleeping
Download evaluation/deployment_gate.py from Sahilalaknur/smartdocs-api: direct link, hf CLI and curl.
- Browser
- Download file 8.93 kB
-
https://huggingface.co/spaces/Sahilalaknur/smartdocs-api/resolve/main/evaluation/deployment_gate.py
- Command line
-
hf download hf://spaces/Sahilalaknur/smartdocs-api/evaluation/deployment_gate.py
-
curl -L -o deployment_gate.py https://huggingface.co/spaces/Sahilalaknur/smartdocs-api/resolve/main/evaluation/deployment_gate.py
8.93 kB
| """ | |
| evaluation/deployment_gate.py | |
| WHY THIS EXISTS: Prevents deploying a SmartDocs version that fails Hindi users. | |
| Reads the evaluation_report.json produced by ragas_evaluator.py and enforces | |
| hard thresholds. Exit code 1 blocks the Railway / CI deploy pipeline. | |
| Usage in CI: | |
| python evaluation/deployment_gate.py | |
| # Exit 0 = deploy approved | |
| # Exit 1 = deploy blocked (reason printed) | |
| LAW 16: faithfulness_hindi < 0.85 OR language_accuracy < 0.92 → BLOCK DEPLOYMENT. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import logging | |
| import sys | |
| from dataclasses import dataclass | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| from typing import Any | |
| logger = logging.getLogger(__name__) | |
| # --------------------------------------------------------------------------- | |
| # Deployment thresholds — single source of truth | |
| # --------------------------------------------------------------------------- | |
| THRESHOLDS: dict[str, tuple[str, float, str]] = { | |
| # (metric_path, minimum_value, human_readable_name) | |
| "faithfulness_en": ("english.faithfulness", 0.90, "English faithfulness"), | |
| "faithfulness_hi": ("hindi.faithfulness", 0.85, "Hindi faithfulness"), | |
| "relevancy_en": ("english.answer_relevancy", 0.85, "English answer relevancy"), | |
| "relevancy_hi": ("hindi.answer_relevancy", 0.83, "Hindi answer relevancy"), | |
| "precision_en": ("english.context_precision", 0.80, "English context precision"), | |
| "precision_hi": ("hindi.context_precision", 0.80, "Hindi context precision"), | |
| "recall_en": ("english.context_recall", 0.75, "English context recall"), | |
| "recall_hi": ("hindi.context_recall", 0.75, "Hindi context recall"), | |
| "hallucination_rate": ("hallucination_rate", 0.10, "Hallucination rate", True), # lower is better | |
| "faithfulness_ratio": ("faithfulness_ratio", 0.97, "Hindi/English faithfulness ratio"), | |
| } | |
| # Metrics where LOWER is better (inverted threshold check) | |
| LOWER_IS_BETTER = {"hallucination_rate"} | |
| # --------------------------------------------------------------------------- | |
| # Gate result dataclass | |
| # --------------------------------------------------------------------------- | |
| class GateResult: | |
| approved: bool | |
| blocking_failures: list[dict] # [{"metric": str, "value": float, "threshold": float}] | |
| warnings: list[str] | |
| report_timestamp: str | |
| evaluated_at: str | |
| def print_report(self) -> None: | |
| sep = "=" * 64 | |
| print(f"\n{sep}") | |
| print("SMARTDOCS DEPLOYMENT GATE") | |
| print(f"Report timestamp: {self.report_timestamp}") | |
| print(f"Evaluated at: {self.evaluated_at}") | |
| print(sep) | |
| if self.approved: | |
| print("\n✅ DEPLOYMENT APPROVED\n") | |
| print(" All metrics above threshold.") | |
| else: | |
| print("\n🚫 DEPLOYMENT BLOCKED\n") | |
| for failure in self.blocking_failures: | |
| direction = ">" if failure["metric"] not in LOWER_IS_BETTER else "<" | |
| print( | |
| f" âœ-- {failure['name']:<35} " | |
| f"actual={failure['value']:.4f} " | |
| f"threshold{direction}{failure['threshold']:.4f}" | |
| ) | |
| if self.warnings: | |
| print("\n Warnings:") | |
| for w in self.warnings: | |
| print(f" âš {w}") | |
| print(sep) | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "approved": self.approved, | |
| "blocking_failures": self.blocking_failures, | |
| "warnings": self.warnings, | |
| "report_timestamp": self.report_timestamp, | |
| "evaluated_at": self.evaluated_at, | |
| } | |
| # --------------------------------------------------------------------------- | |
| # Helper: safe nested key access | |
| # --------------------------------------------------------------------------- | |
| def _get_nested(data: dict, dotted_path: str) -> float | None: | |
| """ | |
| Resolves a dot-separated key path against a nested dict. | |
| Returns None if any key is missing. | |
| Example: _get_nested(report, "english.faithfulness") → 0.91 | |
| """ | |
| parts = dotted_path.split(".") | |
| obj: Any = data | |
| for part in parts: | |
| if not isinstance(obj, dict) or part not in obj: | |
| return None | |
| obj = obj[part] | |
| try: | |
| return float(obj) | |
| except (TypeError, ValueError): | |
| return None | |
| # --------------------------------------------------------------------------- | |
| # Gate logic | |
| # --------------------------------------------------------------------------- | |
| def check_gate(report: dict) -> GateResult: | |
| """ | |
| Applies all deployment thresholds to an evaluation report dict. | |
| Args: | |
| report: Dict produced by EvaluationReport.to_dict() in ragas_evaluator.py. | |
| Returns: | |
| GateResult with approved=True if all thresholds pass. | |
| """ | |
| blocking_failures: list[dict] = [] | |
| warnings: list[str] = [] | |
| for metric_key, threshold_spec in THRESHOLDS.items(): | |
| # Handle both 4-tuple (with lower_is_better flag) and 3-tuple | |
| if len(threshold_spec) == 4: | |
| path, threshold, name, _ = threshold_spec | |
| else: | |
| path, threshold, name = threshold_spec[:3] | |
| value = _get_nested(report, path) | |
| if value is None: | |
| warnings.append(f"Metric '{metric_key}' not found in report — skipped") | |
| continue | |
| if metric_key in LOWER_IS_BETTER: | |
| failed = value >= threshold | |
| else: | |
| failed = value < threshold | |
| if failed: | |
| blocking_failures.append({ | |
| "metric": metric_key, | |
| "name": name, | |
| "path": path, | |
| "value": value, | |
| "threshold": threshold, | |
| }) | |
| # Special check: if both en and hi faithfulness are present, assert ratio | |
| en_faith = _get_nested(report, "english.faithfulness") | |
| hi_faith = _get_nested(report, "hindi.faithfulness") | |
| if en_faith and hi_faith and en_faith > 0: | |
| ratio = hi_faith / en_faith | |
| if ratio < 0.97: | |
| blocking_failures.append({ | |
| "metric": "faithfulness_ratio_computed", | |
| "name": "Computed Hindi/English faithfulness ratio", | |
| "path": "computed", | |
| "value": ratio, | |
| "threshold": 0.97, | |
| }) | |
| return GateResult( | |
| approved=len(blocking_failures) == 0, | |
| blocking_failures=blocking_failures, | |
| warnings=warnings, | |
| report_timestamp=report.get("timestamp", "unknown"), | |
| evaluated_at=datetime.now(timezone.utc).isoformat(), | |
| ) | |
| def load_and_check(report_path: Path) -> GateResult: | |
| """ | |
| Loads evaluation report from disk and runs the gate check. | |
| Args: | |
| report_path: Path to evaluation_report.json. | |
| Returns: | |
| GateResult. | |
| Raises: | |
| FileNotFoundError: If report does not exist. | |
| json.JSONDecodeError: If report is malformed. | |
| """ | |
| if not report_path.exists(): | |
| raise FileNotFoundError( | |
| f"Evaluation report not found at {report_path}. " | |
| "Run: python evaluation/ragas_evaluator.py first." | |
| ) | |
| with open(report_path, "r", encoding="utf-8") as f: | |
| report = json.load(f) | |
| return check_gate(report) | |
| def write_gate_result(result: GateResult, output_path: Path) -> None: | |
| """Writes GateResult to JSON for CI artifact storage.""" | |
| output_path.parent.mkdir(parents=True, exist_ok=True) | |
| with open(output_path, "w", encoding="utf-8") as f: | |
| json.dump(result.to_dict(), f, indent=2) | |
| logger.info("Gate result written to %s", output_path) | |
| # --------------------------------------------------------------------------- | |
| # CLI entry point | |
| # --------------------------------------------------------------------------- | |
| def main() -> int: | |
| """ | |
| CLI entry point. | |
| Exit code 0 = deployment approved. | |
| Exit code 1 = deployment blocked. | |
| Exit code 2 = report not found (treat as blocked). | |
| """ | |
| logging.basicConfig( | |
| level=logging.INFO, | |
| format="%(asctime)s %(levelname)s — %(message)s", | |
| stream=sys.stdout, | |
| ) | |
| report_path = Path(__file__).parent / "evaluation_report.json" | |
| gate_output_path = Path(__file__).parent / "gate_result.json" | |
| try: | |
| result = load_and_check(report_path) | |
| except FileNotFoundError as exc: | |
| print(f"\n🚫 DEPLOYMENT BLOCKED — {exc}\n") | |
| return 2 | |
| except json.JSONDecodeError as exc: | |
| print(f"\n🚫 DEPLOYMENT BLOCKED — Malformed evaluation report: {exc}\n") | |
| return 2 | |
| result.print_report() | |
| write_gate_result(result, gate_output_path) | |
| return 0 if result.approved else 1 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |