| from collections import defaultdict |
|
|
| import fasttext |
| import pandas as pd |
| from sklearn.metrics import classification_report |
| from tqdm import tqdm; tqdm.pandas() |
| |
| import io |
| from pathlib import Path |
| import numpy as np |
| import pandas as pd |
| import requests |
| from sklearn.metrics import accuracy_score |
| from sklearn.metrics import classification_report |
| from sklearn.metrics import precision_recall_fscore_support |
|
|
|
|
| names = pd.read_csv( |
| io.StringIO(requests.get("https://iso639-3.sil.org/sites/iso639-3/files/downloads/iso-639-3.tab").text |
| ), sep="\t").set_index("Id").rename( |
| columns={"Ref_Name": "name"} |
| )[["name"]].to_dict()["name"] |
| tato_names = pd.read_html( |
| "https://tatoeba.org/en/stats/sentences_by_language" |
| )[0].rename( |
| columns={"Unnamed: 2": "code", "Language": "name"} |
| ).set_index("code")[["name"]].to_dict()["name"] |
| names.update(tato_names) |
|
|
| |
| |
| |
| |
|
|
|
|
| def pandas_classification_report(y_true, y_pred, labels=None): |
| metrics_summary = precision_recall_fscore_support( |
| y_true=y_true, |
| y_pred=y_pred, |
| labels=labels) |
| weighted_avg = list(precision_recall_fscore_support( |
| y_true=y_true, |
| y_pred=y_pred, |
| labels=labels, |
| average='weighted')) |
| macro_avg = list(precision_recall_fscore_support( |
| y_true=y_true, |
| y_pred=y_pred, |
| labels=labels, |
| average='macro')) |
| accuracy = [np.nan, np.nan, accuracy_score(y_true=y_true, y_pred=y_pred), np.nan] |
| metrics_sum_index = ['precision', 'recall', 'f1-score', 'support'] |
| class_report_df = pd.DataFrame( |
| list(metrics_summary), |
| index=metrics_sum_index, |
| columns=labels) |
|
|
| support = class_report_df.loc['support'] |
| total = support.sum() |
| weighted_avg[-1] = total |
| macro_avg[-1] = total |
| accuracy[-1] = total |
|
|
| class_report_df['accuracy'] = accuracy |
| class_report_df['weighted avg'] = weighted_avg |
| class_report_df['macro avg'] = macro_avg |
| report = class_report_df.T |
| report["support"] = report["support"].astype(int) |
| return report |
|
|
|
|
| scores_text = "" |
| for model_name in ("nordic-lid.bin", "nordic-lid_all.bin"): |
| print( |
| f""" |
| ------------ |
| {model_name} |
| ------------ |
| """) |
| model = fasttext.load_model(model_name) |
|
|
| train = pd.read_csv("train.csv") |
| ddict = defaultdict(lambda: "---") |
| for k in train.lang.unique().tolist(): |
| ddict[k] = k |
|
|
| train["nordic-lid"] = train.progress_apply(lambda row: ddict[model.predict(row["text"].replace("\n", " "))[0][0][-3:]], axis=1) |
| print("TRAIN") |
| print(model.test("train.txt")) |
| print(classification_report(train["lang"], train["nordic-lid"], digits=4)) |
|
|
| val = pd.read_csv("validation.csv") |
| val["nordic-lid"] = val.progress_apply(lambda row: ddict[model.predict(row["text"].replace("\n", " "))[0][0][-3:]], axis=1) |
| print("VALIDATION") |
| print(model.test("validation.txt")) |
| print(classification_report(val["lang"], val["nordic-lid"], digits=4)) |
|
|
| test = pd.read_csv("test.csv") |
| test["nordic-lid"] = test.progress_apply(lambda row: ddict[model.predict(row["text"].replace("\n", " "))[0][0][-3:]], axis=1) |
| print("TEST") |
| print(model.test("test.txt")) |
| print(classification_report(test["lang"], test["nordic-lid"], digits=4)) |
|
|
| if "_all" in model_name: |
| train = pd.read_csv("train_all.csv") |
| ddict = defaultdict(lambda: "---") |
| for k in train.lang.unique().tolist(): |
| ddict[k] = k |
|
|
| train["nordic-lid"] = train.progress_apply(lambda row: ddict[model.predict(row["text"].replace("\n", " "))[0][0][-3:]], axis=1) |
| print("TRAIN ALL") |
| print(model.test("train_all.txt")) |
| print(classification_report(train["lang"], train["nordic-lid"], digits=4)) |
|
|
| val = pd.read_csv("validation_all.csv") |
| val["nordic-lid"] = val.progress_apply(lambda row: ddict[model.predict(row["text"].replace("\n", " "))[0][0][-3:]], axis=1) |
| print("VALIDATION ALL") |
| print(model.test("validation_all.txt")) |
| print(classification_report(val["lang"], val["nordic-lid"], digits=4)) |
|
|
| test = pd.read_csv("test_all.csv") |
| test["nordic-lid"] = test.progress_apply(lambda row: ddict[model.predict(row["text"].replace("\n", " "))[0][0][-3:]], axis=1) |
| print("TEST ALL") |
| print(model.test("test_all.txt")) |
| print(classification_report(test["lang"], test["nordic-lid"], digits=4)) |
|
|
| langs = pd.read_csv("train_all.csv").lang.unique().tolist() |
| else: |
| langs = pd.read_csv("train.csv").lang.unique().tolist() |
|
|
| langs_df = pd.DataFrame({"ISO-639-3": langs}).sort_values("ISO-639-3") |
| langs_df["Language"] = langs_df["ISO-639-3"].apply(names.__getitem__) |
| langs_df = langs_df.set_index("ISO-639-3") |
|
|
| report_df = pandas_classification_report(test["nordic-lid"], test["lang"], sorted(langs)) |
| scores = report_df.join(langs_df) |
| scores.columns = map(str.title, scores.columns) |
| scores.index.name = "ISO-639-3" |
| scores = scores[["Language"] + [col.title() for col in scores.columns if col != "Language"]] |
| scores_text += f"## {model_name}\n\n{scores.reset_index().to_markdown(index=False, floatfmt='.4f')}\n\n" |
|
|
| print() |
|
|
| print(scores_text) |
| Path("./scores.md").write_text(scores_text) |
|
|
