Download kdflow/loss/reverse_kl_div.py from codemaivanngu/ctopd: direct link, hf CLI and curl.
- Browser
- Download file 761 Bytes
-
https://huggingface.co/codemaivanngu/ctopd/resolve/main/kdflow/loss/reverse_kl_div.py
- Command line
-
hf download hf://codemaivanngu/ctopd/kdflow/loss/reverse_kl_div.py
-
curl -L -o reverse_kl_div.py https://huggingface.co/codemaivanngu/ctopd/resolve/main/kdflow/loss/reverse_kl_div.py
761 Bytes
| import torch | |
| from kdflow.loss import register_loss | |
| def compute_reverse_kl_div( | |
| student_logits, | |
| teacher_logits, | |
| temperature=1.0, | |
| reduction="none", | |
| **kwargs | |
| ): | |
| student_logits = student_logits / temperature | |
| teacher_logits = teacher_logits / temperature | |
| student_log_probs = torch.log_softmax(student_logits, -1, dtype=torch.float32) | |
| teacher_log_probs = torch.log_softmax(teacher_logits, -1, dtype=torch.float32) | |
| student_probs = student_log_probs.exp() | |
| rkl_div = (student_probs * (student_log_probs - teacher_log_probs)).sum(-1) | |
| if reduction == "mean": | |
| rkl_div = rkl_div.mean() | |
| elif reduction == "sum": | |
| rkl_div = rkl_div.sum() | |
| return rkl_div |