cafierom commited on
Commit
dd66952
·
verified ·
1 Parent(s): c194ceb

Upload inflections_funcs.py

Browse files
Files changed (1) hide show
  1. inflections_funcs.py +19 -20
inflections_funcs.py CHANGED
@@ -103,6 +103,9 @@ def calculate_score_vectors(model: AutoModelForCausalLM, generated_dicts: Any) -
103
  '''
104
  Creates a score vector for each beam containing the probability of each
105
  token that was chosen.
 
 
 
106
  '''
107
  # Number of sequences generated
108
  num_sequences = generated_dicts.sequences.shape[0]
@@ -113,25 +116,21 @@ def calculate_score_vectors(model: AutoModelForCausalLM, generated_dicts: Any) -
113
  # Input length (prompt length)
114
  input_len = total_len - gen_len
115
 
116
- # To get the accurate probabilities for the final beams,
117
- # we pass the final sequences back through the model.
118
- with torch.no_grad():
119
- outputs = model(generated_dicts.sequences)
120
- logits = outputs.logits # shape: (num_sequences, total_len, vocab_size)
121
-
122
- # Convert logits to probabilities
123
- probs = F.softmax(logits, dim=-1) # shape: (num_sequences, total_len, vocab_size)
124
-
125
- score_vectors = []
126
- for i in range(num_sequences):
127
- beam_probs = []
128
- # The generated tokens are at positions [input_len, total_len)
129
- # The probability for the token at position t is found at logits index t-1
130
- for t in range(input_len, total_len):
131
- token_id = generated_dicts.sequences[i, t]
132
- # Probability for token at position t is at index t-1 in the logits
133
- prob = probs[i, t - 1, token_id].item()
134
- beam_probs.append(prob)
135
- score_vectors.append(beam_probs)
136
 
137
  return score_vectors
 
103
  '''
104
  Creates a score vector for each beam containing the probability of each
105
  token that was chosen.
106
+
107
+ Optimized to use generated_dicts.scores instead of a full model forward pass
108
+ to prevent GPU timeouts on ZeroGPU.
109
  '''
110
  # Number of sequences generated
111
  num_sequences = generated_dicts.sequences.shape[0]
 
116
  # Input length (prompt length)
117
  input_len = total_len - gen_len
118
 
119
+ # Stack the transition scores (logits) from the generation process
120
+ # generated_dicts.scores is a tuple of length gen_len, each element (num_beams, vocab_size)
121
+ all_logits = torch.stack(generated_dicts.scores, dim=0) # shape: (gen_len, num_sequences, vocab_size)
122
+
123
+ # Convert logits to probabilities across the vocab dimension
124
+ all_probs = F.softmax(all_logits, dim=-1) # shape: (gen_len, num_sequences, vocab_size)
125
+
126
+ score_vectors = []
127
+ for i in range(num_sequences):
128
+ beam_probs = []
129
+ # Extract the probability for the specific token that was chosen at each step
130
+ for t in range(gen_len):
131
+ token_id = generated_dicts.sequences[i, input_len + t]
132
+ prob = all_probs[t, i, token_id].item()
133
+ beam_probs.append(prob)
134
+ score_vectors.append(beam_probs)
 
 
 
 
135
 
136
  return score_vectors