Upload ocm26400/information.py with huggingface_hub
Browse files- ocm26400/information.py +59 -0
ocm26400/information.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Théorie de l'information — Shannon — vague 3.
|
| 2 |
+
|
| 3 |
+
* Entropie de Shannon H(X) = −Σ p·log₂(p) (mesure d'incertitude, bits).
|
| 4 |
+
* Entropie conjointe H(X,Y), conditionnelle H(X|Y).
|
| 5 |
+
* Information mutuelle I(X;Y) = H(X) − H(X|Y).
|
| 6 |
+
* Divergence KL (Kullback-Leibler) entre 2 distributions.
|
| 7 |
+
Vérifiable : formules exactes, H ∈ [0, log₂(n)].
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
import math
|
| 11 |
+
from typing import Dict, List
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def entropy(probs: List[float]) -> float:
|
| 15 |
+
"""H(X) = −Σ p·log₂(p). probs = distribution de probabilité (somme = 1)."""
|
| 16 |
+
return -sum(p * math.log2(p) for p in probs if p > 0)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def entropy_counts(counts: List[int]) -> float:
|
| 20 |
+
"""Entropie depuis des comptages (normalise en distribution)."""
|
| 21 |
+
total = sum(counts)
|
| 22 |
+
if total == 0:
|
| 23 |
+
return 0.0
|
| 24 |
+
return entropy([c / total for c in counts])
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def max_entropy(n: int) -> float:
|
| 28 |
+
"""Entropie maximale pour n événements = log₂(n) (distribution uniforme)."""
|
| 29 |
+
return math.log2(n) if n > 0 else 0.0
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def joint_entropy(joint: List[List[float]]) -> float:
|
| 33 |
+
"""H(X,Y) depuis une matrice de probabilités jointes."""
|
| 34 |
+
flat = [p for row in joint for p in row]
|
| 35 |
+
return -sum(p * math.log2(p) for p in flat if p > 0)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def kl_divergence(p: List[float], q: List[float]) -> float:
|
| 39 |
+
"""D_KL(P‖Q) = Σ p·log₂(p/q). 0 si P=Q."""
|
| 40 |
+
return sum(pi * math.log2(pi / qi) for pi, qi in zip(p, q) if pi > 0 and qi > 0)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def mutual_information(px: List[float], py: List[float], joint: List[List[float]]) -> float:
|
| 44 |
+
"""I(X;Y) = ΣΣ p(x,y)·log₂(p(x,y)/(p(x)p(y)))."""
|
| 45 |
+
i = 0.0
|
| 46 |
+
for r, pxy_row in enumerate(joint):
|
| 47 |
+
for c, pxy in enumerate(pxy_row):
|
| 48 |
+
if pxy > 0 and px[r] > 0 and py[c] > 0:
|
| 49 |
+
i += pxy * math.log2(pxy / (px[r] * py[c]))
|
| 50 |
+
return i
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
if __name__ == "__main__":
|
| 54 |
+
print("[info] H(pièce équilibrée) =", entropy([0.5, 0.5]), "(=1 bit)")
|
| 55 |
+
print("[info] H(pièce biaisée 0.9/0.1) =", round(entropy([0.9, 0.1]), 4))
|
| 56 |
+
print("[info] H max pour 8 états =", max_entropy(8), "(=3 bits)")
|
| 57 |
+
print("[info] KL(uniforme‖biaisée) =", round(kl_divergence([0.5, 0.5], [0.9, 0.1]), 4))
|
| 58 |
+
print("[info] I(X;Y) X=Y parfaite =", round(mutual_information([0.5, 0.5], [0.5, 0.5],
|
| 59 |
+
[[0.5, 0], [0, 0.5]]), 4), "(=1 bit, dépendance parfaite)")
|