Update app.py
Browse files
app.py
CHANGED
|
@@ -12,7 +12,7 @@ def load_and_preprocess_data():
|
|
| 12 |
tokenizer = LongformerTokenizer.from_pretrained('allenai/longformer-base-4096')
|
| 13 |
|
| 14 |
# Preprocess the dataset: tokenize inputs
|
| 15 |
-
|
| 16 |
# Dynamic max length; each example will be processed to match tokenization length
|
| 17 |
inputs = {
|
| 18 |
'input_ids': [],
|
|
|
|
| 12 |
tokenizer = LongformerTokenizer.from_pretrained('allenai/longformer-base-4096')
|
| 13 |
|
| 14 |
# Preprocess the dataset: tokenize inputs
|
| 15 |
+
def preprocess_function(examples):
|
| 16 |
# Dynamic max length; each example will be processed to match tokenization length
|
| 17 |
inputs = {
|
| 18 |
'input_ids': [],
|