File size: 3,548 Bytes
c19b933 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 | """
Minimal usage example for hv-tail.
python example.py
Scores five contrasting queries, shows the requery-history effect,
and demonstrates calibration on both narrow-gap and wide-gap labeled
data.
"""
from hv_tail import HVTail
QUERIES = [
("mean", "what is a black hole"),
("borderline",
"explain this in the style of a Feynman diagram but not the "
"standard textbook version"),
("tail",
"in the language of category theory, is there a functor from the "
"black hole information paradox to a broader framework of "
"unitarity that doesn't assume the standard measurement postulate?"),
("meta",
"your reasoning on that last answer seems inconsistent with what "
"you said earlier. can you explain your calibration? are you sure "
"the assumption you made is defensible?"),
("undecidable",
"is it even possible to prove whether the halting problem is "
"well-defined, or does it depend on the model of computation?"),
]
HISTORY = [
"what is a black hole",
"but why does the horizon have no local features",
"and does that mean information is destroyed",
]
def main() -> None:
m = HVTail()
print("=" * 72)
print("Scores for contrasting queries")
print("=" * 72)
print(f" {'name':<12} {'score':>7} {'above':>6} {'below':>6} "
f"{'conf':>6} recommendation")
print(" " + "-" * 68)
for name, q in QUERIES:
r = m.score(q)
print(
f" {name:<12} {r['raw_score']:>7.3f} "
f"{str(r['above_ceiling']):>6} "
f"{str(r['below_ceiling']):>6} "
f"{r['confidence']:>6.3f} {r['recommendation']}"
)
print()
print("=" * 72)
print("Requery history effect")
print("=" * 72)
q = "then how does Hawking radiation preserve unitarity"
r_no = m.score(q)
r_with = m.score(q, history=HISTORY)
print(f" query: {q}")
print(f" without history : raw={r_no['raw_score']:.3f} "
f"above={r_no['above_ceiling']} rec={r_no['recommendation']}")
print(f" with history : raw={r_with['raw_score']:.3f} "
f"above={r_with['above_ceiling']} rec={r_with['recommendation']}")
print()
print("=" * 72)
print("Calibration: narrow gap")
print("=" * 72)
m2 = HVTail()
narrow_above = [
QUERIES[2][1], # tail
QUERIES[3][1], # meta
"is there a way to prove this without the standard theorem?",
]
narrow_below = [
QUERIES[0][1], # mean
"how do I make coffee",
"what time is it",
]
a, b = m2.calibrate(narrow_above, narrow_below)
print(f" above={a:.4f} below={b:.4f} gap={a-b:.4f}")
print()
print("=" * 72)
print("Calibration: wide gap")
print("=" * 72)
m3 = HVTail()
wide_above = [
QUERIES[2][1], # tail
QUERIES[3][1], # meta
"is there a way to prove the category of endofunctors is "
"monoidal without appealing to the standard coherence theorem?",
]
wide_below = [
QUERIES[0][1], # mean
"how do I make coffee",
"what time is it",
"explain photosynthesis",
"what is the capital of France",
]
a, b = m3.calibrate(wide_above, wide_below)
print(f" above={a:.4f} below={b:.4f} gap={a-b:.4f}")
print()
print("=" * 72)
print("Verbose example (the tail query)")
print("=" * 72)
print(m.explain(QUERIES[2][1]))
if __name__ == "__main__":
main() |