FlorianVal commited on
Commit
8a7d9c3
·
verified ·
1 Parent(s): 2136da5

Handle tensor outputs from Qwen decoder layers

Browse files
Files changed (1) hide show
  1. src/model_adapters.py +6 -4
src/model_adapters.py CHANGED
@@ -107,14 +107,16 @@ class LlamaStyleAdapter(ModelAdapter):
107
  hidden_states,
108
  attention_mask=attention_mask,
109
  position_ids=position_ids,
110
- past_key_values=past_key_values,
111
  use_cache=use_cache,
112
  position_embeddings=position_embeddings,
113
  cache_position=cache_position,
114
  )
115
- hidden_states = layer_outputs[0]
116
- new_kv = layer_outputs[1] if len(layer_outputs) > 1 else None
117
- return hidden_states, new_kv
 
 
118
 
119
  def apply_final_norm(self, hidden_states: Tensor) -> Tensor:
120
  return self._norm(hidden_states)
 
107
  hidden_states,
108
  attention_mask=attention_mask,
109
  position_ids=position_ids,
110
+ past_key_value=past_key_values,
111
  use_cache=use_cache,
112
  position_embeddings=position_embeddings,
113
  cache_position=cache_position,
114
  )
115
+ # Recent Transformers decoder layers return a tensor directly. Indexing
116
+ # it like a tuple would drop the batch dimension after the first layer.
117
+ if isinstance(layer_outputs, torch.Tensor):
118
+ return layer_outputs, None
119
+ return layer_outputs[0], layer_outputs[1] if len(layer_outputs) > 1 else None
120
 
121
  def apply_final_norm(self, hidden_states: Tensor) -> Tensor:
122
  return self._norm(hidden_states)