| |
|
|
| xcopa_connectors = {"cause": " Izan ere,", "effect": " Beraz,"} |
|
|
|
|
| def xcopa_doc_to_text(doc): |
| conn = xcopa_connectors[doc["question"]] |
| return doc["premise"].strip() + f"{conn}" |
|
|
|
|
| def xcopa_doc_to_choice(doc): |
| def convert_choice(choice): |
| return choice[0].lower() + choice[1:] |
|
|
| return [convert_choice(doc["choice1"]), convert_choice(doc["choice2"])] |
|
|
|
|
| |
|
|
|
|
| def paws_process_docs(dataset): |
| empty_docs = [] |
|
|
| def _process_doc(doc): |
| if doc["sentence1"] not in [None, ""] and doc["sentence2"] not in [None, ""]: |
| |
| if doc["sentence1"].endswith((".", ",", ";")): |
| doc["sentence1"] = doc["sentence1"][:-1] |
| |
| doc["sentence2"] = lowercase_first_letter(doc["sentence2"]) |
| return doc |
| else: |
| empty_docs.append(doc) |
| return doc |
|
|
| def lowercase_first_letter(text): |
| return text[0].lower() + text[1:] |
|
|
| return dataset.filter( |
| lambda doc: doc["sentence1"] not in [None, ""] |
| and doc["sentence2"] not in [None, ""] |
| ).map(_process_doc) |
|
|
