Update README.md
Browse files
README.md
CHANGED
|
@@ -197,13 +197,23 @@ def decode_predictions(input_ids: List[int], predictions: List[int], tokenizer,
|
|
| 197 |
|
| 198 |
segmented = segment_predictions(tokens, labels)
|
| 199 |
|
| 200 |
-
|
|
|
|
| 201 |
sentence = tokenizer.decode(segment, skip_special_tokens=False, clean_up_tokenization_spaces=False).strip()
|
| 202 |
if sentence:
|
| 203 |
sentences.append(sentence)
|
| 204 |
|
| 205 |
# Handle carry-over for the last segment
|
| 206 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 207 |
|
| 208 |
return sentences, carry_over
|
| 209 |
|
|
|
|
| 197 |
|
| 198 |
segmented = segment_predictions(tokens, labels)
|
| 199 |
|
| 200 |
+
# Decode all segments except the last one
|
| 201 |
+
for segment in segmented[:-1]:
|
| 202 |
sentence = tokenizer.decode(segment, skip_special_tokens=False, clean_up_tokenization_spaces=False).strip()
|
| 203 |
if sentence:
|
| 204 |
sentences.append(sentence)
|
| 205 |
|
| 206 |
# Handle carry-over for the last segment
|
| 207 |
+
if segmented[-1]:
|
| 208 |
+
last_label = labels[len(tokens) - 1] # Index of the last token in tokens
|
| 209 |
+
|
| 210 |
+
if last_label == 1: # If the last segment ends with 1, decode it
|
| 211 |
+
sentence = tokenizer.decode(segmented[-1], skip_special_tokens=False, clean_up_tokenization_spaces=False).strip()
|
| 212 |
+
if sentence:
|
| 213 |
+
sentences.append(sentence)
|
| 214 |
+
carry_over = [] # No carry-over needed
|
| 215 |
+
else: # If the last segment ends with 0, set it as carry-over
|
| 216 |
+
carry_over = segmented[-1]
|
| 217 |
|
| 218 |
return sentences, carry_over
|
| 219 |
|