Upload overlay/subsystems/sdr_semantic.py with huggingface_hub

overlay/subsystems/sdr_semantic.py

@@ -265,7 +265,42 @@ class SemanticFoldingSDR(nn.Module):
             self.delta_u = nn.Parameter(torch.randn(vocab_size, delta_rank) * 1e-4)
             self.delta_v = nn.Parameter(torch.randn(delta_rank, n_bits) * 1e-4)
 
-        # ------------------------------------------------------------------\n        # Contrastive retina parameter (Retina-D).\n        #\n        # Audit 2026-05-09 issue #21: this parameter was previously named\n        # `retina_logits`, which UNCONDITIONALLY SHADOWED the learnable-mode\n        # binary retina logits assigned above (also named `retina_logits`,\n        # shape [V, n_bits]). The shadow silently broke binary_only(),\n        # binary_softplus_topk(), and the symmetric-difference debug paths\n        # in learnable mode — they all index `retina_logits[token_ids]` and\n        # expected n_bits columns but were getting contrastive_rank columns.\n        #\n        # Renamed to `retina_contrastive` to remove the shadow. The\n        # contrastive_loss() and contrastive-retina debug code paths in this\n        # module are updated to use the new name; the optimizer group in\n        # hydra/model.py is updated to bind to `retina_contrastive`.\n        #\n        # retina_contrastive: [V, contrastive_rank] — a compact learned\n        # embedding for each vocabulary token. Cosine similarity on these\n        # logits is the contrastive signal. Using a low-rank projection\n        # (contrastive_rank << n_bits) keeps the similarity computation\n        # cheap: O(V_unique * contrastive_rank) vs O(V * n_bits).\n        #\n        # Initialised with unit-norm rows so cosine similarities start near\n        # zero (random ~1/sqrt(contrastive_rank)) rather than being uniformly\n        # positive from a raw randn init. This avoids a cold-start\n        # degenerate state where all pairs look similar and the loss\n        # provides no gradient direction.\n        # ------------------------------------------------------------------\n        self.retina_contrastive = None\n        if contrastive_rank > 0:\n            _logit_init = torch.randn(vocab_size, contrastive_rank)\n            # Normalise rows to unit length so cosine starts meaningful\n            _logit_init = _logit_init / (_logit_init.norm(dim=-1, keepdim=True) + 1e-8)\n            self.retina_contrastive = nn.Parameter(_logit_init * 0.1)\n\n        # Step counter as plain Python int — avoids GPU buffer mutation that
+        # ------------------------------------------------------------------
+        # Contrastive retina parameter (Retina-D).
+        #
+        # Audit 2026-05-09 issue #21: this parameter was previously named
+        # `retina_logits`, which UNCONDITIONALLY SHADOWED the learnable-mode
+        # binary retina logits assigned above (also named `retina_logits`,
+        # shape [V, n_bits]). The shadow silently broke binary_only(),
+        # binary_softplus_topk(), and the symmetric-difference debug paths
+        # in learnable mode — they all index `retina_logits[token_ids]` and
+        # expected n_bits columns but were getting contrastive_rank columns.
+        #
+        # Renamed to `retina_contrastive` to remove the shadow. The
+        # contrastive_loss() and contrastive-retina debug code paths in this
+        # module are updated to use the new name; the optimizer group in
+        # hydra/model.py is updated to bind to `retina_contrastive`.
+        #
+        # retina_contrastive: [V, contrastive_rank] — a compact learned
+        # embedding for each vocabulary token. Cosine similarity on these
+        # logits is the contrastive signal. Using a low-rank projection
+        # (contrastive_rank << n_bits) keeps the similarity computation
+        # cheap: O(V_unique * contrastive_rank) vs O(V * n_bits).
+        #
+        # Initialised with unit-norm rows so cosine similarities start near
+        # zero (random ~1/sqrt(contrastive_rank)) rather than being uniformly
+        # positive from a raw randn init. This avoids a cold-start
+        # degenerate state where all pairs look similar and the loss
+        # provides no gradient direction.
+        # ------------------------------------------------------------------
+        self.retina_contrastive = None
+        if contrastive_rank > 0:
+            _logit_init = torch.randn(vocab_size, contrastive_rank)
+            # Normalise rows to unit length so cosine starts meaningful
+            _logit_init = _logit_init / (_logit_init.norm(dim=-1, keepdim=True) + 1e-8)
+            self.retina_contrastive = nn.Parameter(_logit_init * 0.1)
+
+        # Step counter as plain Python int — avoids GPU buffer mutation that
         # invalidates torch.compile guards on every maybe_som_update call.
         # Persisted manually via state_dict hooks for checkpointing.
         self._som_step: int = 0