Spaces:
Paused
Paused
File size: 5,663 Bytes
5e0b58b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 | """Non-executing shadow mode for scientific AI proposals."""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Sequence
from .contracts import Interpretation, Observation, Proposal, ScientificContract
from .firewall import AIAuthority, ScientificAIFirewall
@dataclass(frozen=True, slots=True)
class ShadowResult:
"""A marked shadow output that has not been applied to system state."""
contract: ScientificContract
executed: bool = False
def to_dict(self) -> dict[str, Any]:
return {"executed": self.executed, "contract": self.contract.to_dict()}
@dataclass(frozen=True, slots=True)
class ShadowProposalMetrics:
"""Deterministic metrics for comparing shadow proposals with outcomes."""
precision: float
recall: float
false_positive_rate: float
prediction_accuracy: float
brier_score: float
expected_calibration_error: float
utility: float
sample_count: int
def to_dict(self) -> dict[str, Any]:
return {
"precision": self.precision,
"recall": self.recall,
"false_positive_rate": self.false_positive_rate,
"prediction_accuracy": self.prediction_accuracy,
"brier_score": self.brier_score,
"expected_calibration_error": self.expected_calibration_error,
"utility": self.utility,
"sample_count": self.sample_count,
}
def evaluate_shadow_proposals(
predicted: Sequence[bool],
actual: Sequence[bool],
*,
confidence: Sequence[float] | None = None,
utility: Sequence[float] | None = None,
) -> ShadowProposalMetrics:
"""Evaluate proposals without interpreting or executing their contents."""
if not predicted or len(predicted) != len(actual):
raise ValueError(
"Predicted and actual proposal labels must have equal non-zero length."
)
if confidence is not None and len(confidence) != len(predicted):
raise ValueError("Confidence values must match proposal count.")
if utility is not None and len(utility) != len(predicted):
raise ValueError("Utility values must match proposal count.")
if confidence is not None and any(value < 0 or value > 1 for value in confidence):
raise ValueError("Confidence values must be between 0 and 1.")
true_positive = sum(
predicted_value and actual_value
for predicted_value, actual_value in zip(predicted, actual)
)
false_positive = sum(
predicted_value and not actual_value
for predicted_value, actual_value in zip(predicted, actual)
)
false_negative = sum(
not predicted_value and actual_value
for predicted_value, actual_value in zip(predicted, actual)
)
true_negative = sum(
not predicted_value and not actual_value
for predicted_value, actual_value in zip(predicted, actual)
)
count = len(predicted)
precision = (
true_positive / (true_positive + false_positive)
if true_positive + false_positive
else 0.0
)
recall = (
true_positive / (true_positive + false_negative)
if true_positive + false_negative
else 0.0
)
false_positive_rate = (
false_positive / (false_positive + true_negative)
if false_positive + true_negative
else 0.0
)
accuracy = (true_positive + true_negative) / count
probabilities = list(confidence or [float(value) for value in predicted])
brier_score = (
sum(
(probability - float(outcome)) ** 2
for probability, outcome in zip(probabilities, actual)
)
/ count
)
calibration_bins = [0.0] * 10
calibration_counts = [0] * 10
for probability, outcome in zip(probabilities, actual):
index = min(int(probability * 10), 9)
calibration_bins[index] += float(outcome)
calibration_counts[index] += 1
expected_calibration_error = (
sum(
abs(
calibration_bins[index] / calibration_counts[index] - (index + 0.5) / 10
)
* calibration_counts[index]
for index in range(10)
if calibration_counts[index]
)
/ count
)
measured_utility = sum(utility or [float(value) for value in predicted]) / count
return ShadowProposalMetrics(
precision=precision,
recall=recall,
false_positive_rate=false_positive_rate,
prediction_accuracy=accuracy,
brier_score=brier_score,
expected_calibration_error=expected_calibration_error,
utility=measured_utility,
sample_count=count,
)
class ShadowMode:
"""Allow observation, interpretation and proposals without execution."""
def __init__(self) -> None:
self._firewall = ScientificAIFirewall(AIAuthority.PROPOSAL_ONLY)
def observe(self, payload: object, *, source: str = "shadow") -> ShadowResult:
self._firewall.authorize("observe")
return ShadowResult(
Observation.create(payload=payload, source=source, authority="read_only")
)
def interpret(self, payload: object, *, source: str = "shadow") -> ShadowResult:
self._firewall.authorize("interpret")
return ShadowResult(
Interpretation.create(payload=payload, source=source, authority="read_only")
)
def propose(self, payload: object, *, source: str = "shadow") -> ShadowResult:
self._firewall.authorize("propose")
return ShadowResult(
Proposal.create(payload=payload, source=source, authority="proposal_only")
)
|