multimodalart HF Staff commited on
Commit
cd8304e
·
verified ·
1 Parent(s): 03c68c4

Upload matcha/models/models.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. matcha/models/models.py +52 -1
matcha/models/models.py CHANGED
@@ -9,6 +9,7 @@ from transformers import EsmConfig
9
  from transformers.models.esm.modeling_esm import EsmEncoder
10
 
11
  from matcha.models.unimol_distance_embedding import DistanceBias
 
12
  from matcha.utils.preprocessing import lig_feature_dims
13
  from matcha.utils.rotation import expm_SO3
14
  from matcha.utils.transforms import (compute_batch_ligand_centers,
@@ -19,6 +20,52 @@ from matcha.utils.log import get_logger
19
  logger = get_logger(__name__)
20
 
21
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
22
  ### ATOM ENCODER ###
23
 
24
  class AtomEncoder(torch.nn.Module):
@@ -664,8 +711,12 @@ class MatchaModel(nn.Module):
664
  batch.ligand.num_rotatable_bonds, lig_seq.dtype)
665
 
666
  # Transformer forward pass
 
 
 
667
  tor_embeddings = self.tor_decoder(
668
- tor_inputs, attention_mask=extended_attention_mask)['last_hidden_state']
 
669
 
670
  v_tor = self.tor_head(tor_embeddings)
671
 
 
9
  from transformers.models.esm.modeling_esm import EsmEncoder
10
 
11
  from matcha.models.unimol_distance_embedding import DistanceBias
12
+
13
  from matcha.utils.preprocessing import lig_feature_dims
14
  from matcha.utils.rotation import expm_SO3
15
  from matcha.utils.transforms import (compute_batch_ligand_centers,
 
20
  logger = get_logger(__name__)
21
 
22
 
23
+
24
+ ### ROTARY POSITION EMBEDDINGS (transformers compatibility) ###
25
+ #
26
+ # transformers >= 5.0 removed the internal RotaryEmbedding module from
27
+ # EsmSelfAttention: callers must pass position_embeddings=(cos, sin) to
28
+ # EsmEncoder.forward. Matcha's torsion decoder was written against the
29
+ # transformers 4.57.2 API where EsmSelfAttention computed rotary internally
30
+ # from a registered `inv_freq` buffer (present in the released checkpoints).
31
+ # The helper below reproduces the exact 4.57.2 math
32
+ # (inv_freq = 1/(10000^(arange(0,dim,2)/dim)); cos/sin of outer(t, inv_freq)
33
+ # concatenated with itself), so checkpoint numerics are unchanged.
34
+ def esm_rotary_cos_sin(hidden_states: torch.Tensor, head_dim: int,
35
+ seq_len: int):
36
+ """cos/sin tables with the exact transformers 4.57.2 RotaryEmbedding math.
37
+
38
+ Returns shape (1, seq_len, head_dim), the `cos[position_ids]` layout that
39
+ transformers 5.x EsmSelfAttention expects (it unsqueezes the head dim
40
+ internally to broadcast over (batch, heads, seq, head_dim)).
41
+ """
42
+ inv_freq = 1.0 / (
43
+ 10000 ** (torch.arange(0, head_dim, 2, dtype=torch.int64,
44
+ device=hidden_states.device).float() / head_dim))
45
+ t = torch.arange(seq_len, device=hidden_states.device).type_as(inv_freq)
46
+ freqs = torch.outer(t, inv_freq)
47
+ emb = torch.cat((freqs, freqs), dim=-1).to(hidden_states.device)
48
+ return emb.cos()[None, :, :], emb.sin()[None, :, :]
49
+
50
+
51
+ def _esm_position_embeddings(x: torch.Tensor, encoder: EsmEncoder):
52
+ """Return position_embeddings for EsmEncoder.forward in a
53
+ transformers-version-agnostic way: None on 4.x (self-computed rotary),
54
+ (cos, sin) on 5.x+ (must be passed explicitly)."""
55
+ if _esm_position_embeddings.rotary is None:
56
+ _esm_position_embeddings.rotary = getattr(
57
+ encoder.layer[0].attention.self, "rotary_embeddings", None)
58
+ if _esm_position_embeddings.rotary is not None:
59
+ # transformers 4.x: EsmSelfAttention owns a RotaryEmbedding module
60
+ return None
61
+ # transformers 5.x: attention head size = hidden_size // num_attention_heads
62
+ head_dim = encoder.config.hidden_size // encoder.config.num_attention_heads
63
+ return esm_rotary_cos_sin(x, head_dim, seq_len=x.shape[1])
64
+
65
+
66
+ _esm_position_embeddings.rotary = None
67
+
68
+
69
  ### ATOM ENCODER ###
70
 
71
  class AtomEncoder(torch.nn.Module):
 
711
  batch.ligand.num_rotatable_bonds, lig_seq.dtype)
712
 
713
  # Transformer forward pass
714
+ # (transformers 5.x needs explicit rotary cos/sin; 4.x computes them
715
+ # internally — _esm_position_embeddings handles both)
716
+ tor_position_embeddings = _esm_position_embeddings(tor_inputs, self.tor_decoder)
717
  tor_embeddings = self.tor_decoder(
718
+ tor_inputs, attention_mask=extended_attention_mask,
719
+ position_embeddings=tor_position_embeddings)['last_hidden_state']
720
 
721
  v_tor = self.tor_head(tor_embeddings)
722