src/solomon/reliability.py

"""What the readout says about its own answer, with nothing fitted behind it.

A question is decoded correctly when every unit of it is, so the readout's statement about a question is
the product of its statement about each unit. No parameter is fitted here; the only parameter on this path
is the per-task temperature in `solomon/calibration.py`.

The product assumes the units of a question are independent. That has never been validated as a joint
probability: it orders multi-candidate questions well and it is not a calibrated joint. Read the
per-candidate numbers if you need a magnitude. `solomon.service.ORDERING_DISCLOSURE` says the same thing
in the response itself.
"""
import numpy as np
from scipy.special import logsumexp

TASKS = ('boolean', 'single', 'ordered', 'multilabel', 'entity')


def unit_correct_probability(unit, t=1.0):
    """The readout's stated P(this unit is decoded correctly).

    Noul   -> max(p, 1-p) with p = P(yes); the prediction is argmax, so this is P(prediction right).
    Choice -> the listed top-1 probability.
    t rescales the logits; t = 1 is the raw readout.
    """
    if unit['kind'] == 'noul':
        x = np.asarray([unit['z'], 0.0], float) / t
    else:
        x = np.asarray(unit['logits'], float) / t
    return float(np.exp(x.max() - logsumexp(x)))


def question_reliability(units, t=1.0):
    """P(every unit correct) under the readout, assuming unit independence within a question."""
    return float(np.prod([unit_correct_probability(u, t) for u in units]))