ggomarobot commited on
Commit
5c831b0
·
verified ·
1 Parent(s): b270769

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +12 -2
README.md CHANGED
@@ -197,13 +197,23 @@ def decode_predictions(input_ids: List[int], predictions: List[int], tokenizer,
197
 
198
  segmented = segment_predictions(tokens, labels)
199
 
200
- for segment in segmented[:-1]: # Decode all segments except the last one
 
201
  sentence = tokenizer.decode(segment, skip_special_tokens=False, clean_up_tokenization_spaces=False).strip()
202
  if sentence:
203
  sentences.append(sentence)
204
 
205
  # Handle carry-over for the last segment
206
- carry_over = segmented[-1] if segmented[-1] and labels[len(tokens) - len(segmented[-1])] != 1 else []
 
 
 
 
 
 
 
 
 
207
 
208
  return sentences, carry_over
209
 
 
197
 
198
  segmented = segment_predictions(tokens, labels)
199
 
200
+ # Decode all segments except the last one
201
+ for segment in segmented[:-1]:
202
  sentence = tokenizer.decode(segment, skip_special_tokens=False, clean_up_tokenization_spaces=False).strip()
203
  if sentence:
204
  sentences.append(sentence)
205
 
206
  # Handle carry-over for the last segment
207
+ if segmented[-1]:
208
+ last_label = labels[len(tokens) - 1] # Index of the last token in tokens
209
+
210
+ if last_label == 1: # If the last segment ends with 1, decode it
211
+ sentence = tokenizer.decode(segmented[-1], skip_special_tokens=False, clean_up_tokenization_spaces=False).strip()
212
+ if sentence:
213
+ sentences.append(sentence)
214
+ carry_over = [] # No carry-over needed
215
+ else: # If the last segment ends with 0, set it as carry-over
216
+ carry_over = segmented[-1]
217
 
218
  return sentences, carry_over
219