File size: 5,962 Bytes
287ca83 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 | """Decisions, questions and the ``/v1/systemone`` wire format.
A *decision* is a state (text, JSON or a list) plus optional images and one or more typed
questions. Each question has an ordered list of labels:
* ``choice``: labels are the option names; ``definitions`` holds each option's criterion.
* ``score``: labels are the level descriptions, index = level (2 to 10 levels).
* ``noul``: labels are always ``["no", "yes"]``; ``definitions`` holds the
``[false, true]`` criteria when given.
Requests carry ``state``, ``questions`` and optionally ``images`` (``data:`` URLs); responses
carry one answer per question with ``probabilities`` (choice, score) or ``noul`` (the yes
probability).
"""
from __future__ import annotations
import math
from dataclasses import dataclass, field
from typing import Any
QUESTION_TYPES = ("choice", "score", "noul")
NOUL_LABELS = ["no", "yes"]
MAX_CHOICE_OPTIONS = 255
MIN_SCORE_LEVELS, MAX_SCORE_LEVELS = 2, 10
MAX_IMAGES = 8
class SchemaError(ValueError):
"""A request that does not follow the schema."""
def normalise(values: list[float]) -> list[float]:
total = float(sum(values))
return [float(v) / total for v in values]
@dataclass
class Question:
type: str
instructions: str
labels: list[str]
definitions: list[str | None] | None = None
def validate(self) -> None:
if self.type not in QUESTION_TYPES:
raise SchemaError(f"unknown question type {self.type!r}")
if not isinstance(self.instructions, str):
raise SchemaError("instructions must be a string")
n = len(self.labels)
if self.type == "noul":
if self.labels != NOUL_LABELS:
raise SchemaError(f"noul labels must be {NOUL_LABELS}, got {self.labels}")
elif self.type == "choice":
if not 2 <= n <= MAX_CHOICE_OPTIONS:
raise SchemaError(f"choice needs 2-{MAX_CHOICE_OPTIONS} options, got {n}")
if len(set(self.labels)) != n:
raise SchemaError("choice labels must be unique")
elif not MIN_SCORE_LEVELS <= n <= MAX_SCORE_LEVELS:
raise SchemaError(f"score needs {MIN_SCORE_LEVELS}-{MAX_SCORE_LEVELS} levels, got {n}")
if any(not isinstance(label, str) or not label for label in self.labels):
raise SchemaError("labels must be non-empty strings")
if self.definitions is not None and len(self.definitions) != n:
raise SchemaError("definitions must align with labels")
@classmethod
def from_wire(cls, q: dict[str, Any]) -> "Question":
qtype = q.get("type")
instructions = q.get("instructions", "") or ""
criteria = q.get("criteria")
if qtype == "choice":
if isinstance(criteria, dict):
labels = [str(k) for k in criteria]
definitions = [None if v is None else str(v) for v in criteria.values()]
elif isinstance(criteria, list):
labels, definitions = [str(c) for c in criteria], None
else:
raise SchemaError("choice question needs criteria")
question = cls("choice", instructions, labels, definitions)
elif qtype == "score":
if not isinstance(criteria, list):
raise SchemaError("score question needs a list of levels")
question = cls("score", instructions, [str(c) for c in criteria], None)
elif qtype == "noul":
definitions = None
if isinstance(criteria, dict):
definitions = [criteria.get("false"), criteria.get("true")]
question = cls("noul", instructions, list(NOUL_LABELS), definitions)
else:
raise SchemaError(f"unknown question type {qtype!r}")
question.validate()
return question
@dataclass
class Decision:
"""One state with its typed questions."""
id: str
state: Any
questions: dict[str, Question]
images: list[dict[str, Any]] = field(default_factory=list)
def validate(self) -> None:
if not isinstance(self.state, (str, dict, list)):
raise SchemaError("state must be a string, object or list")
if not self.questions:
raise SchemaError("a decision needs at least one question")
for name, q in self.questions.items():
if not name:
raise SchemaError("question names must be non-empty")
q.validate()
if len(self.images) > MAX_IMAGES:
raise SchemaError(f"at most {MAX_IMAGES} images per request")
def choice_confidence(probs: list[float]) -> float:
"""(p_max - 1/n) / (1 - 1/n): 0 for a uniform distribution, 1 for a certain one."""
n = len(probs)
return (max(probs) - 1.0 / n) / (1.0 - 1.0 / n)
def score_confidence(probs: list[float]) -> float:
"""1 - (probability-weighted distance from the peak level) / (same for a uniform distribution)."""
n = len(probs)
peak = max(range(n), key=lambda i: probs[i])
spread = sum(p * abs(i - peak) for i, p in enumerate(probs))
uniform = sum(abs(i - peak) for i in range(n)) / n
return 1.0 - spread / uniform if uniform else 1.0
def make_answer(question: Question, probs: list[float]) -> dict[str, Any]:
"""Format a distribution over ``question.labels`` as a ``/v1/systemone`` answer."""
probs = normalise(probs)
if question.type == "noul":
return {"type": "noul", "noul": probs[1]}
if question.type == "choice":
best = max(range(len(probs)), key=lambda i: probs[i])
return {"type": "choice", "choice": question.labels[best],
"probabilities": dict(zip(question.labels, probs)),
"confidence": choice_confidence(probs)}
expected = sum(i * p for i, p in enumerate(probs))
return {"type": "score", "score": expected, "confidence": score_confidence(probs),
"probabilities": {str(i): p for i, p in enumerate(probs)}}
|