Spaces:
Sleeping
Sleeping
File size: 2,939 Bytes
ee461be | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 | import json
from pathlib import Path
import pandas as pd
from sklearn.metrics import classification_report, confusion_matrix
def save_evaluation_artifacts(
output_dir,
tasks,
label_maps,
y_true,
raw_predictions,
corrected_predictions,
probabilities,
global_indices,
):
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
prediction_variants = {
"raw": raw_predictions,
"corrected": corrected_predictions,
}
for variant, predictions in prediction_variants.items():
for task in tasks:
id2label = label_maps[task]["id2label"]
label_ids = list(range(len(id2label)))
label_names = [
id2label[str(label_id)]
for label_id in label_ids
]
report = classification_report(
y_true[task],
predictions[task],
labels=label_ids,
target_names=label_names,
zero_division=0,
output_dict=True,
)
report_path = (output_dir / f"{variant}_{task}_classification_report.json")
with open(report_path, "w", encoding="utf-8") as file:
json.dump(
report,
file,
indent=2,
ensure_ascii=False,
)
matrix = confusion_matrix(
y_true[task],
predictions[task],
labels=label_ids,
)
matrix_path = (output_dir / f"{variant}_{task}_confusion_matrix.csv")
pd.DataFrame(
matrix,
index=label_names,
columns=label_names,
).to_csv(
matrix_path,
encoding="utf-8",
)
rows = []
for row_index, global_index in enumerate(global_indices):
row = {"global_index": int(global_index)}
raw_exact = True
corrected_exact = True
for task in tasks:
id2label = label_maps[task]["id2label"]
true_id = int(y_true[task][row_index])
raw_id = int(raw_predictions[task][row_index])
corrected_id = int(corrected_predictions[task][row_index])
row[f"{task}_true"] = id2label[str(true_id)]
row[f"{task}_raw"] = id2label[str(raw_id)]
row[f"{task}_corrected"] = id2label[str(corrected_id)]
row[f"{task}_confidence"] = float(probabilities[task][row_index, raw_id])
raw_exact &= true_id == raw_id
corrected_exact &= true_id == corrected_id
row["raw_exact_match"] = bool(raw_exact)
row["corrected_exact_match"] = bool(corrected_exact)
rows.append(row)
pd.DataFrame(rows).to_csv(
output_dir / "test_predictions.csv",
index=False,
encoding="utf-8",
) |