import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import os
import numpy as np
sns.set_theme(style="whitegrid", font="Noto Sans",
font_scale=1.5)
this_graph_palette = sns.color_palette("husl", 4)
sns.set_palette(this_graph_palette)
dataset_names = ["1-munchkin",
"3-vicheanmas", "2-chinchilla", "4-scottishfold"]
# cademic_dataset_names = ["Dataset 1", "Dataset 3", "Dataset 2", "Dataset 4"]
academic_dataset_names = [
"Simple Dataset", "Complicated Dataset", "Actual Dataset", "Extended Random Dataset"]
learning_rate = [0.005, 0.01, 0.05, 0.1, 0.5]
overall_accuracy_df = pd.DataFrame()
model_learning_rates = ["1e-07", "1e-08"]
iteration = 100
modifiers = [
"",
"_remove_outlier",
"_augmented",
"_remove_outlier_augmented",
"_early_stopping",
"_remove_outlier_early_stopping",
"_augmented_early_stopping",
"_remove_outlier_augmented_early_stopping",
]
displayed_modifiers = [
"Original",
"Remove Outlier",
"Augmentation",
"Remove Outlier + Augmentation",
"Early Stopping",
"Remove Outlier + Early Stopping",
"Augmentation + Early Stopping",
"Remove Outlier + Augmentation + Early Stopping",
]
def to_kebab_case(value):
return "-".join(value.lower().split())
def plotting_learning_curve(use_academic_name=False, round_no=0,
model_version=1):
dataset_index = 0
for dataset_name in dataset_names:
modifier_index = 0
for modifer in modifiers:
accuracy_list = []
# Find Best Accuracy
# For Each Model Learning Rate
fig, ax = plt.subplots(2, 5, figsize=(20, 8))
model_learning_rate_index = 0
learning_rate_index = 0
if use_academic_name:
fig.suptitle(
f"Learning Curve of {academic_dataset_names[dataset_index]} - {displayed_modifiers[modifier_index]}",
fontdict={"fontsize": 20, "fontweight": "bold"},
)
else:
fig.suptitle(
f"Learning Curve of {academic_dataset_names[dataset_index]} - {displayed_modifiers[modifier_index]}",
fontdict={"fontsize": 20, "fontweight": "bold"},
)
for model_learning_rate in model_learning_rates:
directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{modifer}"
# For Each Learning Rate
for lr in learning_rate:
epoch_error_file = (
f"{directory_name}/round{round_no + 1}/{iteration}-{lr}.csv"
)
epoch_error = pd.read_csv(epoch_error_file)
ax[model_learning_rate_index][learning_rate_index].plot(
epoch_error["epoch"],
epoch_error["error_percent"],
label="Training",
linewidth=4
)
ax[model_learning_rate_index][learning_rate_index].plot(
epoch_error["epoch"],
epoch_error["validate_error_percent"],
label="Validation", linewidth=4
)
ax[model_learning_rate_index][learning_rate_index].set_title(
f"{lr}/{model_learning_rate}",
fontdict={"fontsize": 16},
)
# ax[model_learning_rate_index][learning_rate_index].set_xlabel(
# "Epoch"
# )
# ax[model_learning_rate_index][learning_rate_index].set_ylabel(
# "Error RMSPE"
# )
ax[model_learning_rate_index][learning_rate_index].set_ylim(
0, 100)
ax[model_learning_rate_index][learning_rate_index].set_xlim(
0, 100)
# ax[model_learning_rate_index][learning_rate_index].legend(
# loc="lower right"
# )
learning_rate_index += 1
model_learning_rate_index += 1
learning_rate_index = 0
handles, labels = ax[0][0].get_legend_handles_labels()
fig.legend(handles, labels, loc="lower center",
ncol=2, fontsize=12)
fig.tight_layout(pad=2.0)
plt.tight_layout()
os.makedirs(
f"result/learning_curve/by-modifer/{dataset_name}", exist_ok=True
)
if modifer == "":
modifer = "original"
plt.savefig(
f"result/learning_curve/by-modifer/{dataset_name}/{modifer}.png"
)
plt.close(fig)
accuracy_list = pd.DataFrame(accuracy_list)
modifier_index += 1
dataset_index += 1
def plotting_learning_curve_by_rate(use_academic_name=False, round_no=0,
model_version=1):
dataset_index = 0
for dataset_name in dataset_names:
# Find Best Accuracy
# For Each Model Learning Rate
model_learning_rate_index = 0
learning_rate_index = 0
for model_learning_rate in model_learning_rates:
# For Each Learning Rate
for lr in learning_rate:
fig, ax = plt.subplots(2, 4, figsize=(20, 8))
for modifier_index, modifer in enumerate(modifiers):
directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{modifer}"
epoch_error_file = (
f"{directory_name}/round{round_no + 1}/{iteration}-{lr}.csv"
)
epoch_error = pd.read_csv(epoch_error_file)
row_index = 0 if modifier_index < 4 else 1
col_index = modifier_index % 4
ax[row_index][col_index].plot(
epoch_error["epoch"],
epoch_error["error_percent"],
label="Training",
)
ax[row_index][col_index].plot(
epoch_error["epoch"],
epoch_error["validate_error_percent"],
label="Validation",
)
if use_academic_name:
ax[row_index][col_index].set_title(
f"{academic_dataset_names[dataset_index]}\n{displayed_modifiers[modifier_index]}\n{lr}/{model_learning_rate}",
fontdict={"fontsize": 12},
)
else:
ax[row_index][col_index].set_title(
f"{dataset_name}\n{displayed_modifiers[modifier_index]}\n{lr}/{model_learning_rate}",
fontdict={"fontsize": 12},
)
ax[row_index][col_index].set_xlabel(
f'Epoch\n Lasted Training Error {epoch_error.iloc[-1]["error_percent"]:.2f} \n Lasted Validation Error {epoch_error.iloc[-1]["validate_error_percent"]:.2f}'
)
ax[row_index][col_index].set_ylabel("Error RMSPE")
ax[row_index][col_index].set_ylim(0, 100)
ax[row_index][col_index].set_xlim(0, 100)
ax[row_index][col_index].legend(loc="lower right")
learning_rate_index += 1
fig.tight_layout(pad=2.0)
os.makedirs(
f"result/learning_curve/by-rate/{dataset_name}", exist_ok=True
)
plt.savefig(
f"result/learning_curve/by-rate/{dataset_name}/{model_learning_rate}-{lr}.png"
)
plt.close(fig)
model_learning_rate_index += 1
learning_rate_index = 0
dataset_index += 1
def plot_graph_grid(use_academic_name=False, use_early_stopping=False, round_no=0, model_version=1):
dataset_index = 0
early_stopping_title = "With Early Stopping" if use_early_stopping else ""
for dataset_name in dataset_names:
# Find Best Accuracy
# For Each Model Learning Rate
if use_early_stopping:
filter_modifier = modifiers[4:]
filter_display_modifier = displayed_modifiers[4:]
else:
filter_modifier = modifiers[:4]
filter_display_modifier = displayed_modifiers[:4]
model_learning_rate_index = 0
learning_rate_index = 0
for model_learning_rate in model_learning_rates:
# For Each Learning Rate
fig, ax = plt.subplots(5, 4, figsize=(20, 14))
if use_academic_name:
fig.suptitle(
f"Learning Curve of {academic_dataset_names[dataset_index]} - β ={model_learning_rate} {early_stopping_title}",
fontdict={"fontsize": 14, "fontweight": "bold"},
)
else:
fig.suptitle(
f"Learning Curve of {academic_dataset_names[dataset_index]} - β ={model_learning_rate} {early_stopping_title}",
fontdict={"fontsize": 14, "fontweight": "bold"},
)
learning_rate_index = 0
for lr in learning_rate:
for modifier_index, modifer in enumerate(filter_modifier):
directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{modifer}"
epoch_error_file = (
f"{directory_name}/round{round_no + 1}/{iteration}-{lr}.csv"
)
epoch_error = pd.read_csv(epoch_error_file)
col_index = modifier_index
ax[learning_rate_index][col_index].plot(
epoch_error["epoch"],
epoch_error["error_percent"],
label="Training",
linewidth=4,
)
ax[learning_rate_index][col_index].plot(
epoch_error["epoch"],
epoch_error["validate_error_percent"],
label="Validation",
linewidth=4,
)
if use_academic_name:
ax[learning_rate_index][col_index].set_title(
f"{filter_display_modifier[modifier_index]}\n α = {lr}",
fontdict={"fontsize": 20},
)
# best_train_epoch = epoch_error[
# epoch_error["error_percent"]
# == epoch_error["error_percent"].min()
# ]
# best_validate_epoch = epoch_error[
# epoch_error["validate_error_percent"]
# == epoch_error["validate_error_percent"].min()
# ]
# try:
# best_train_epoch = best_train_epoch.iloc[0]
# best_train_epoch = best_train_epoch["epoch"]
# best_validate_epoch = best_validate_epoch.iloc[0]
# best_validate_epoch = best_validate_epoch["epoch"]
# except:
# best_train_epoch = np.nan
# best_validate_epoch = np.nan
# ax[learning_rate_index][col_index].set_xlabel(
# f'Train Err {epoch_error.iloc[-1]["error_percent"]:.2f} , Validate Err {epoch_error.iloc[-1]["validate_error_percent"]:.2f} \n Best Epoch Train {best_train_epoch:.0f} , Validate {best_validate_epoch:.0f}'
# )
# ax[learning_rate_index][col_index].set_ylabel(
# 'Error RMSPE')
ax[learning_rate_index][col_index].set_ylim(0, 100)
ax[learning_rate_index][col_index].set_xlim(0, 100)
# ax[learning_rate_index][col_index].legend(
# loc='lower right')
learning_rate_index += 1
handles, labels = ax[0][0].get_legend_handles_labels()
fig.legend(handles, labels, loc="lower center",
ncol=2, fontsize=12)
fig.tight_layout(pad=2.0)
early_stopping_modifier = "/early-stopping" if use_early_stopping else ""
if round_no > 0:
early_stopping_modifier = f"/round-{round_no}/{early_stopping_modifier}"
os.makedirs(
f"result/learning_curve/grid-rate-modifier/{dataset_name}{early_stopping_modifier}",
exist_ok=True,
)
plt.savefig(
f"result/learning_curve/grid-rate-modifier/{dataset_name}{early_stopping_modifier}/{model_learning_rate}.png"
)
plt.close(fig)
model_learning_rate_index += 1
learning_rate_index = 0
dataset_index += 1
# TODO: Not Implement
def plot_model_weight(use_academic_name=False, use_early_stopping=False, round_no=0, model_version=1):
dataset_index = 0
early_stopping_title = "With Early Stopping" if use_early_stopping else ""
for dataset_name in dataset_names:
# Find Best Accuracy
# For Each Model Learning Rate
if use_early_stopping:
filter_modifier = modifiers[4:]
filter_display_modifier = displayed_modifiers[4:]
else:
filter_modifier = modifiers[:4]
filter_display_modifier = displayed_modifiers[:4]
model_learning_rate_index = 0
learning_rate_index = 0
for model_learning_rate in model_learning_rates:
# For Each Learning Rate
fig, ax = plt.subplots(5, 4, figsize=(15, 14))
if use_academic_name:
fig.suptitle(
f"Weight Adjustment of {academic_dataset_names[dataset_index]} - β ={model_learning_rate} {early_stopping_title}",
fontdict={"fontsize": 14, "fontweight": "bold"},
)
else:
fig.suptitle(
f"Weight Adjustment of {academic_dataset_names[dataset_index]} - β ={model_learning_rate} {early_stopping_title}",
fontdict={"fontsize": 14, "fontweight": "bold"},
)
learning_rate_index = 0
for lr in learning_rate:
for modifier_index, modifer in enumerate(filter_modifier):
directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{modifer}"
epoch_error_file = (
f"{directory_name}/round{round_no + 1}/{iteration}-{lr}.csv"
)
epoch_error = pd.read_csv(epoch_error_file)
adjustment_file = (
f"{directory_name}/round{round_no + 1}/sample-payload-list-{iteration}-{lr}.csv"
)
adjustment_data = pd.read_csv(adjustment_file)
col_index = modifier_index
ax[learning_rate_index][col_index].plot(
adjustment_data["model_weight_1"],
label="Material Element Weight",
)
ax[learning_rate_index][col_index].plot(
adjustment_data["model_weight_2"],
label="Labor Element Weight",
)
ax[learning_rate_index][col_index].plot(
adjustment_data["model_weight_3"],
label="Utiltiy Cost Element Weight",
)
if use_academic_name:
ax[learning_rate_index][col_index].set_title(
f"{filter_display_modifier[modifier_index]}\n α = {lr}",
fontdict={"fontsize": 12},
)
best_train_epoch = epoch_error[
epoch_error["error_percent"]
== epoch_error["error_percent"].min()
]
best_validate_epoch = epoch_error[
epoch_error["validate_error_percent"]
== epoch_error["validate_error_percent"].min()
]
try:
best_train_epoch = best_train_epoch.iloc[0]
best_train_epoch = best_train_epoch["epoch"]
best_validate_epoch = best_validate_epoch.iloc[0]
best_validate_epoch = best_validate_epoch["epoch"]
except Exception as e:
best_train_epoch = np.nan
best_validate_epoch = np.nan
ax[learning_rate_index][col_index].set_xlabel(
f'Train Err {epoch_error.iloc[-1]["error_percent"]:.2f} , Validate Err {epoch_error.iloc[-1]["validate_error_percent"]:.2f} \n Best Epoch Train {best_train_epoch:.0f} , Validate {best_validate_epoch:.0f}'
)
ax[learning_rate_index][col_index].set_xticks([])
learning_rate_index += 1
handles, labels = ax[0][0].get_legend_handles_labels()
fig.legend(handles, labels, loc="lower center",
ncol=2, fontsize=12)
fig.tight_layout(pad=2.0)
early_stopping_modifier = "/early-stopping" if use_early_stopping else ""
if round_no > 0:
early_stopping_modifier = f"/round-{round_no}/{early_stopping_modifier}"
os.makedirs(
f"result/learning_curve/hyperparameter/model-weight-{dataset_name}{early_stopping_modifier}",
exist_ok=True,
)
plt.savefig(
f"result/learning_curve/hyperparameter/model-weight-{dataset_name}{early_stopping_modifier}/{model_learning_rate}.png"
)
plt.close(fig)
model_learning_rate_index += 1
learning_rate_index = 0
dataset_index += 1
def find_amount_of_employee(sample_df):
column_list = sample_df.columns.tolist()
employee_column_list = [
col for col in column_list if "employee_weight_" in col]
number_of_employee = len(employee_column_list)
return number_of_employee
def plot_element_weights(use_academic_name=False, use_early_stopping=False, round_no=0, model_version=1):
dataset_index = 0
early_stopping_title = "With Early Stopping" if use_early_stopping else ""
for dataset_name in dataset_names:
# Find Best Accuracy
# For Each Model Learning Rate
if use_early_stopping:
filter_modifier = modifiers[4:]
filter_display_modifier = displayed_modifiers[4:]
else:
filter_modifier = modifiers[:4]
filter_display_modifier = displayed_modifiers[:4]
model_learning_rate_index = 0
learning_rate_index = 0
for model_learning_rate in model_learning_rates:
# For Each Learning Rate
fig, ax = plt.subplots(5, 4, figsize=(18, 17))
if use_academic_name:
fig.suptitle(
f"Bias Adjustment of {academic_dataset_names[dataset_index]} - β ={model_learning_rate} {early_stopping_title}",
fontdict={"fontsize": 14, "fontweight": "bold"},
)
else:
fig.suptitle(
f"Model Element Weight Adjustment of {academic_dataset_names[dataset_index]} - β ={model_learning_rate} {early_stopping_title}",
fontdict={"fontsize": 14, "fontweight": "bold"},
)
learning_rate_index = 0
for lr in learning_rate:
for modifier_index, modifer in enumerate(filter_modifier):
directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{modifer}"
epoch_error_file = (
f"{directory_name}/round{round_no + 1}/{iteration}-{lr}.csv"
)
epoch_error = pd.read_csv(epoch_error_file)
adjustment_file = (
f"{directory_name}/round{round_no + 1}/sample-payload-list-{iteration}-{lr}.csv"
)
adjustment_data = pd.read_csv(adjustment_file)
col_index = modifier_index
# Material Weights
ax[learning_rate_index][col_index].plot(
adjustment_data["material_weight_1"],
label="A Crab", color="LightPink"
)
ax[learning_rate_index][col_index].plot(
adjustment_data["material_weight_2"],
label="C Crab", color="SkyBlue"
)
# In Case of Not Exist Material Weight 4 in Actual Case
try:
ax[learning_rate_index][col_index].plot(
adjustment_data["material_weight_4"],
label="Loss Crab", color="MediumPurple"
)
ax[learning_rate_index][col_index].plot(
adjustment_data["material_weight_3"],
label="Small Crab", color="LightGreen"
)
except:
ax[learning_rate_index][col_index].plot(
adjustment_data["material_weight_3"],
label="Small and Loss Crab", color="MediumPurple"
)
# Employee Weights
number_of_employee = find_amount_of_employee(
adjustment_data)
employee_colors = ["PaleTurquoise", "Cyan",
"LightCyan", "Turquoise", "DarkTurquoise"]
emp_index = 0
for emp in range(number_of_employee):
ax[learning_rate_index][col_index].plot(
adjustment_data[f"employee_weight_{emp + 1}"],
label=f"Employee {emp}",
color=employee_colors[emp % len(employee_colors)]
)
emp_index += 1
# Utility Cost Weights
ax[learning_rate_index][col_index].plot(
adjustment_data["capital_cost_weight_1"],
label="Electricity Cost",
color="gold"
)
ax[learning_rate_index][col_index].plot(
adjustment_data["capital_cost_weight_2"],
label="Water Supply Cost",
color="yellow"
)
if use_academic_name:
ax[learning_rate_index][col_index].set_title(
f"{filter_display_modifier[modifier_index]}\n α = {lr}",
fontdict={"fontsize": 12},
)
best_train_epoch = epoch_error[
epoch_error["error_percent"]
== epoch_error["error_percent"].min()
]
best_validate_epoch = epoch_error[
epoch_error["validate_error_percent"]
== epoch_error["validate_error_percent"].min()
]
try:
best_train_epoch = best_train_epoch.iloc[0]
best_train_epoch = best_train_epoch["epoch"]
best_validate_epoch = best_validate_epoch.iloc[0]
best_validate_epoch = best_validate_epoch["epoch"]
except Exception as e:
best_train_epoch = np.nan
best_validate_epoch = np.nan
# ax[learning_rate_index][col_index].set_xlabel(
# f'Train Err {epoch_error.iloc[-1]["error_percent"]:.2f} , Validate Err {epoch_error.iloc[-1]["validate_error_percent"]:.2f} \n Best Epoch Train {best_train_epoch:.0f} , Validate {best_validate_epoch:.0f}'
# )
ax[learning_rate_index][col_index].set_xticks([])
ax[learning_rate_index][col_index].set_ylim(-2000, 10000)
# Ensure all values are positive before setting the y-axis to logarithmic scale
# if (adjustment_data.select_dtypes(include=[np.number]) > 0).all().all():
# ax[learning_rate_index][col_index].set_yscale('log')
# else:
# print(
# f"Warning: Non-positive values detected in dataset {dataset_name}, modifier {modifer}, learning rate {lr}. Skipping logarithmic scale.")
learning_rate_index += 1
handles, labels = ax[0][0].get_legend_handles_labels()
fig.legend(handles, labels, loc="lower center",
ncol=8, fontsize=12)
fig.tight_layout(pad=3.0)
early_stopping_modifier = "/early-stopping" if use_early_stopping else ""
if round_no > 0:
early_stopping_modifier = f"/round-{round_no}/{early_stopping_modifier}"
os.makedirs(
f"result/learning_curve/hyperparameter/element-weight-{dataset_name}{early_stopping_modifier}",
exist_ok=True,
)
plt.savefig(
f"result/learning_curve/hyperparameter/element-weight-{dataset_name}{early_stopping_modifier}/{model_learning_rate}.png"
)
plt.close(fig)
model_learning_rate_index += 1
learning_rate_index = 0
dataset_index += 1
def plot_element_bias(use_academic_name=False, use_early_stopping=False, round_no=0, model_version=1):
dataset_index = 0
early_stopping_title = "With Early Stopping" if use_early_stopping else ""
for dataset_name in dataset_names:
# Find Best Accuracy
# For Each Model Learning Rate
if use_early_stopping:
filter_modifier = modifiers[4:]
filter_display_modifier = displayed_modifiers[4:]
else:
filter_modifier = modifiers[:4]
filter_display_modifier = displayed_modifiers[:4]
model_learning_rate_index = 0
learning_rate_index = 0
for model_learning_rate in model_learning_rates:
# For Each Learning Rate
fig, ax = plt.subplots(5, 4, figsize=(15, 14))
if use_academic_name:
fig.suptitle(
f"Biases Adjustment of {academic_dataset_names[dataset_index]} - β ={model_learning_rate} {early_stopping_title}",
fontdict={"fontsize": 14, "fontweight": "bold"},
)
else:
fig.suptitle(
f"Biases Adjustment of {academic_dataset_names[dataset_index]} - β ={model_learning_rate} {early_stopping_title}",
fontdict={"fontsize": 14, "fontweight": "bold"},
)
learning_rate_index = 0
for lr in learning_rate:
for modifier_index, modifer in enumerate(filter_modifier):
directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{modifer}"
epoch_error_file = (
f"{directory_name}/round{round_no + 1}/{iteration}-{lr}.csv"
)
epoch_error = pd.read_csv(epoch_error_file)
adjustment_file = (
f"{directory_name}/round{round_no + 1}/sample-payload-list-{iteration}-{lr}.csv"
)
adjustment_data = pd.read_csv(adjustment_file)
col_index = modifier_index
ax[learning_rate_index][col_index].plot(
adjustment_data["material_bias_1"],
label="Material Element Bias",
)
ax[learning_rate_index][col_index].plot(
adjustment_data["employee_bias_1"],
label="Labor Element Bias",
)
ax[learning_rate_index][col_index].plot(
adjustment_data["capital_cost_bias_1"],
label="Utility Cost Element Bias",
)
ax[learning_rate_index][col_index].plot(
adjustment_data["model_bias"],
label="Model Bias",
)
if use_academic_name:
ax[learning_rate_index][col_index].set_title(
f"{filter_display_modifier[modifier_index]}\n α = {lr}",
fontdict={"fontsize": 12},
)
best_train_epoch = epoch_error[
epoch_error["error_percent"]
== epoch_error["error_percent"].min()
]
best_validate_epoch = epoch_error[
epoch_error["validate_error_percent"]
== epoch_error["validate_error_percent"].min()
]
try:
best_train_epoch = best_train_epoch.iloc[0]
best_train_epoch = best_train_epoch["epoch"]
best_validate_epoch = best_validate_epoch.iloc[0]
best_validate_epoch = best_validate_epoch["epoch"]
except Exception as e:
best_train_epoch = np.nan
best_validate_epoch = np.nan
ax[learning_rate_index][col_index].set_xlabel(
f'Train Err {epoch_error.iloc[-1]["error_percent"]:.2f} , Validate Err {epoch_error.iloc[-1]["validate_error_percent"]:.2f} \n Best Epoch Train {best_train_epoch:.0f} , Validate {best_validate_epoch:.0f}'
)
ax[learning_rate_index][col_index].set_xticks([])
learning_rate_index += 1
handles, labels = ax[0][0].get_legend_handles_labels()
fig.legend(handles, labels, loc="lower center",
ncol=2, fontsize=12)
fig.tight_layout(pad=2.0)
early_stopping_modifier = "/early-stopping" if use_early_stopping else ""
if round_no > 0:
early_stopping_modifier = f"/round-{round_no}/{early_stopping_modifier}"
os.makedirs(
f"result/learning_curve/hyperparameter/bias-{dataset_name}{early_stopping_modifier}",
exist_ok=True,
)
plt.savefig(
f"result/learning_curve/hyperparameter/bias-{dataset_name}{early_stopping_modifier}/{model_learning_rate}.png"
)
plt.close(fig)
model_learning_rate_index += 1
learning_rate_index = 0
dataset_index += 1
def plot_each_case(use_academic_name=False, use_early_stopping=False, round_no=0, model_version=1, selected_case=""):
modifier_index = modifiers.index(selected_case)
model_learning_rate_index = 0
for model_learning_rate in model_learning_rates:
dataset_index = 0
fig, ax = plt.subplots(4, 5, figsize=(24, 15))
fig.suptitle(
f"Learning Curve of {displayed_modifiers[modifier_index]} Dataset with Model Learning Rate (β) of {model_learning_rate}",
fontdict={"fontsize": 18, "fontweight": "bold"},
)
for dataset_name in dataset_names:
learning_rate_index = 0
# For Each Learning Rate
learning_rate_index = 0
for lr in (learning_rate):
directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{selected_case}"
epoch_error_file = (
f"{directory_name}/round{round_no + 1}/{iteration}-{lr}.csv"
)
epoch_error = pd.read_csv(epoch_error_file)
ax[dataset_index][learning_rate_index].plot(
epoch_error["epoch"],
epoch_error["error_percent"],
label="Training",
linewidth=4
)
ax[dataset_index][learning_rate_index].plot(
epoch_error["epoch"],
epoch_error["validate_error_percent"],
label="Validation",
linewidth=4
)
if learning_rate_index == 0:
ax[dataset_index][learning_rate_index].set_title(
f"{academic_dataset_names[dataset_index]} \nα={lr}",
fontsize=20,
)
else:
ax[dataset_index][learning_rate_index].set_title(
f"\nα={lr}",
fontsize=20,
)
# best_train_epoch = epoch_error[
# epoch_error["error_percent"]
# == epoch_error["error_percent"].min()
# ]
# best_validate_epoch = epoch_error[
# epoch_error["validate_error_percent"]
# == epoch_error["validate_error_percent"].min()
# ]
# try:
# best_train_epoch = best_train_epoch.iloc[0]
# best_train_epoch = best_train_epoch["epoch"]
# best_validate_epoch = best_validate_epoch.iloc[0]
# best_validate_epoch = best_validate_epoch["epoch"]
# except:
# best_train_epoch = np.nan
# best_validate_epoch = np.nan
# ax[dataset_index][learning_rate_index].set_xlabel(
# f'Train Err {epoch_error.iloc[-1]["error_percent"]:.2f} , Val Err {epoch_error.iloc[-1]["validate_error_percent"]:.2f} \n Lowest Point Iteration on Train {best_train_epoch:.0f} , Validate {best_validate_epoch:.0f}'
# )
# ax[learning_rate_index][col_index].set_ylabel(
# 'Error RMSPE')
ax[dataset_index][learning_rate_index].set_ylim(0, 115)
ax[dataset_index][learning_rate_index].set_xlim(0, 100)
# ax[learning_rate_index][col_index].legend(
# loc='lower right')
learning_rate_index += 1
dataset_index += 1
handles, labels = ax[0][0].get_legend_handles_labels()
fig.legend(handles, labels, loc="lower center",
ncol=2, fontsize=18)
fig.tight_layout(pad=2.0)
early_stopping_modifier = "/early-stopping" if use_early_stopping else ""
if round_no > 0:
early_stopping_modifier = f"/round-{round_no}/{early_stopping_modifier}"
casename = to_kebab_case(displayed_modifiers[modifier_index])
os.makedirs(
f"result/learning_curve/by-case/{casename}{early_stopping_modifier}",
exist_ok=True,
)
plt.savefig(
f"result/learning_curve/by-case/{casename}{early_stopping_modifier}/{model_learning_rate}.png"
)
plt.close(fig)
model_learning_rate_index += 1
def plot_each_case_reversed(use_academic_name=False, use_early_stopping=False, round_no=0, model_version=1, selected_case=""):
modifier_index = modifiers.index(selected_case)
model_learning_rate_index = 0
for model_learning_rate in model_learning_rates:
dataset_index = 0
fig, ax = plt.subplots(5, 4, figsize=(24, 20))
fig.suptitle(
f"Learning Curve of {displayed_modifiers[modifier_index]} Dataset with Model Learning Rate (β) of {model_learning_rate}",
fontdict={"fontsize": 20, "fontweight": "bold"},
)
learning_rate_index = 0
for lr in (learning_rate):
dataset_index = 0
# For Each Learning Rate
for dataset_name in dataset_names:
directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{selected_case}"
epoch_error_file = (
f"{directory_name}/round{round_no + 1}/{iteration}-{lr}.csv"
)
epoch_error = pd.read_csv(epoch_error_file)
ax[learning_rate_index][dataset_index].plot(
epoch_error["epoch"],
epoch_error["error_percent"],
label="Training",
linewidth=4
)
ax[learning_rate_index][dataset_index].plot(
epoch_error["epoch"],
epoch_error["validate_error_percent"],
label="Validation",
linewidth=4
)
if dataset_index == 0:
ax[learning_rate_index][dataset_index].set_title(
f"α={lr}\n{academic_dataset_names[dataset_index]}",
fontsize=24,
)
else:
ax[learning_rate_index][dataset_index].set_title(
f"\n{academic_dataset_names[dataset_index]}",
fontsize=24,
)
# best_train_epoch = epoch_error[
# epoch_error["error_percent"]
# == epoch_error["error_percent"].min()
# ]
# best_validate_epoch = epoch_error[
# epoch_error["validate_error_percent"]
# == epoch_error["validate_error_percent"].min()
# ]
# try:
# best_train_epoch = best_train_epoch.iloc[0]
# best_train_epoch = best_train_epoch["epoch"]
# best_validate_epoch = best_validate_epoch.iloc[0]
# best_validate_epoch = best_validate_epoch["epoch"]
# except:
# best_train_epoch = np.nan
# best_validate_epoch = np.nan
# ax[learning_rate_index][dataset_index].set_xlabel(
# f'Train Err {epoch_error.iloc[-1]["error_percent"]:.2f} , Val Err {epoch_error.iloc[-1]["validate_error_percent"]:.2f} \n Lowest Point Iteration on Train {best_train_epoch:.0f} , Validate {best_validate_epoch:.0f}'
# )
# ax[learning_rate_index][col_index].set_ylabel(
# 'Error RMSPE')
ax[learning_rate_index][dataset_index].set_ylim(0, 115)
ax[learning_rate_index][dataset_index].set_xlim(0, 100)
# ax[learning_rate_index][col_index].legend(
# loc='lower right')
dataset_index += 1
learning_rate_index += 1
handles, labels = ax[0][0].get_legend_handles_labels()
fig.legend(handles, labels, loc="lower center",
ncol=2, fontsize=18)
fig.tight_layout(pad=2.0)
early_stopping_modifier = "/early-stopping" if use_early_stopping else ""
if round_no > 0:
early_stopping_modifier = f"/round-{round_no}/{early_stopping_modifier}"
casename = to_kebab_case(displayed_modifiers[modifier_index])
os.makedirs(
f"result/learning_curve/by-case/{casename}{early_stopping_modifier}",
exist_ok=True,
)
plt.savefig(
f"result/learning_curve/by-case/{casename}{early_stopping_modifier}/{model_learning_rate}-reversed.png"
)
plt.close(fig)
model_learning_rate_index += 1
def plot_each_element_weights(use_academic_name=False, use_early_stopping=False, round_no=0, model_version=1, selected_case="", learning_rate=0.01):
early_stopping_title = "With Early Stopping" if use_early_stopping else ""
model_learning_rate_index = 0
modifier_index = modifiers.index(selected_case)
this_graph_palette = sns.color_palette("husl", 9)
sns.set_palette(this_graph_palette)
for model_learning_rate in model_learning_rates:
# For Each Learning Rate
dataset_index = 0
fig, ax = plt.subplots(1, 4, figsize=(18, 5))
if use_academic_name:
fig.suptitle(
f"Model Element Weight Adjustment of {displayed_modifiers[modifier_index]} {academic_dataset_names[dataset_index]} \n β ={model_learning_rate} α = {learning_rate} {early_stopping_title}",
fontdict={"fontsize": 16, "fontweight": "bold"},
)
else:
fig.suptitle(
f"Model Element Weight Adjustment of {displayed_modifiers[modifier_index]} {academic_dataset_names[dataset_index]} \n β ={model_learning_rate}α = {learning_rate} {early_stopping_title}",
fontdict={"fontsize": 16, "fontweight": "bold"},
)
for dataset_name in dataset_names:
directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{selected_case}"
epoch_error_file = (
f"{directory_name}/round{round_no + 1}/{iteration}-{learning_rate}.csv"
)
epoch_error = pd.read_csv(epoch_error_file)
adjustment_file = (
f"{directory_name}/round{round_no + 1}/sample-payload-list-{iteration}-{learning_rate}.csv"
)
adjustment_data = pd.read_csv(adjustment_file)
# Material Weights
ax[dataset_index].plot(
adjustment_data["material_weight_1"],
label="A Crab"
)
ax[dataset_index].plot(
adjustment_data["material_weight_2"],
label="C Crab"
)
# In Case of Not Exist Material Weight 4 in Actual Case
try:
ax[dataset_index].plot(
adjustment_data["material_weight_4"],
label="Loss Crab"
)
ax[dataset_index].plot(
adjustment_data["material_weight_3"],
label="Small Crab"
)
except Exception as e:
print(e)
ax[dataset_index].plot(
adjustment_data["material_weight_3"],
label="Small and Loss Crab"
)
# Utility Cost Weights
ax[dataset_index].plot(
adjustment_data["capital_cost_weight_1"],
label="Electricity Cost",
)
ax[dataset_index].plot(
adjustment_data["capital_cost_weight_2"],
label="Water Supply Cost",
)
# Employee Weights
number_of_employee = find_amount_of_employee(
adjustment_data)
employee_colors = ["PaleTurquoise", "Cyan",
"LightCyan", "Turquoise", "DarkTurquoise"]
emp_index = 0
for emp in range(number_of_employee):
ax[dataset_index].plot(
adjustment_data[f"employee_weight_{emp + 1}"],
label=f"Employee {emp}",
linestyle="--",
# color=employee_colors[emp % len(employee_colors)]
)
emp_index += 1
if use_academic_name:
ax[dataset_index].set_title(
f"{academic_dataset_names[dataset_index]}",
fontsize=18,
)
best_train_epoch = epoch_error[
epoch_error["error_percent"]
== epoch_error["error_percent"].min()
]
best_validate_epoch = epoch_error[
epoch_error["validate_error_percent"]
== epoch_error["validate_error_percent"].min()
]
try:
best_train_epoch = best_train_epoch.iloc[0]
best_train_epoch = best_train_epoch["epoch"]
best_validate_epoch = best_validate_epoch.iloc[0]
best_validate_epoch = best_validate_epoch["epoch"]
except Exception as e:
best_train_epoch = np.nan
best_validate_epoch = np.nan
# ax[dataset_index].set_xlabel(
# f'Train Err {epoch_error.iloc[-1]["error_percent"]:.2f} , Validate Err {epoch_error.iloc[-1]["validate_error_percent"]:.2f} \n Best Epoch Train {best_train_epoch:.0f} , Validate {best_validate_epoch:.0f}'
# )
ax[dataset_index].set_xticks([])
ax[dataset_index].set_ylim(0, 10000)
# Ensure all values are positive before setting the y-axis to logarithmic scale
# if (adjustment_data.select_dtypes(include=[np.number]) > 0).all().all():
# ax[dataset_index].set_yscale('log')
# else:
# print(
# f"Warning: Non-positive values detected in dataset {dataset_name}. Skipping logarithmic scale.")
dataset_index += 1
handles, labels = ax[3].get_legend_handles_labels()
print('labels', labels)
# Group employee labels together
employee_labels = [
label for label in labels if label.startswith("Employee")]
other_labels = [
label for label in labels if not label.startswith("Employee")]
# Combine employee labels into a single entry
if employee_labels:
other_labels.append("Employees")
labels = other_labels
fig.legend(handles, labels, loc="lower center",
ncol=8, fontsize=16)
fig.tight_layout(pad=2.0)
early_stopping_modifier = "/early-stopping" if use_early_stopping else ""
if round_no > 0:
early_stopping_modifier = f"/round-{round_no}/{early_stopping_modifier}"
casename = to_kebab_case(displayed_modifiers[modifier_index])
os.makedirs(
f"result/learning_curve/hyperparameter/element-weight-{learning_rate}{early_stopping_modifier}",
exist_ok=True,
)
plt.savefig(
f"result/learning_curve/hyperparameter/element-weight-{learning_rate}{early_stopping_modifier}/{casename}-{model_learning_rate}.png"
)
plt.close(fig)
model_learning_rate_index += 1
dataset_index += 1
sns.set_palette(sns.color_palette())
def plot_each_element_weights_select(use_academic_name=False, use_early_stopping=False, round_no=0, model_version=1, selected_case="", learning_rate=0.01, is_tail=False, round=1000):
early_stopping_title = "With Early Stopping" if use_early_stopping else ""
model_learning_rate_index = 0
modifier_index = modifiers.index(selected_case)
this_graph_palette = sns.color_palette("husl", 9)
sns.set_palette(this_graph_palette)
prefix_modifier = is_tail and f"Last {round} Round" or f"First {round} Round"
for model_learning_rate in model_learning_rates:
# For Each Learning Rate
dataset_index = 0
fig, ax = plt.subplots(1, 4, figsize=(18, 5))
if use_academic_name:
fig.suptitle(
f"Model Element Weight Adjustment of {displayed_modifiers[modifier_index]} {academic_dataset_names[dataset_index]} \n β ={model_learning_rate} α = {learning_rate} {early_stopping_title}",
fontdict={"fontsize": 16, "fontweight": "bold"},
)
else:
fig.suptitle(
f"Model Element Weight Adjustment of {displayed_modifiers[modifier_index]} {academic_dataset_names[dataset_index]} \n β ={model_learning_rate}α = {learning_rate} {early_stopping_title}",
fontdict={"fontsize": 16, "fontweight": "bold"},
)
for dataset_name in dataset_names:
directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{selected_case}"
epoch_error_file = (
f"{directory_name}/round{round_no + 1}/{iteration}-{learning_rate}.csv"
)
epoch_error = pd.read_csv(epoch_error_file)
adjustment_file = (
f"{directory_name}/round{round_no + 1}/sample-payload-list-{iteration}-{learning_rate}.csv"
)
adjustment_data = pd.read_csv(adjustment_file)
if is_tail:
# Select the last 1000 rows of the DataFrame
adjustment_data = adjustment_data.tail(round)
else:
adjustment_data = adjustment_data.head(round)
# Material Weights
ax[dataset_index].plot(
adjustment_data["material_weight_1"],
label="A Crab"
)
ax[dataset_index].plot(
adjustment_data["material_weight_2"],
label="C Crab"
)
# In Case of Not Exist Material Weight 4 in Actual Case
try:
ax[dataset_index].plot(
adjustment_data["material_weight_4"],
label="Loss Crab"
)
ax[dataset_index].plot(
adjustment_data["material_weight_3"],
label="Small Crab"
)
except Exception as e:
print(e)
ax[dataset_index].plot(
adjustment_data["material_weight_3"],
label="Small and Loss Crab"
)
# Employee Weights
number_of_employee = find_amount_of_employee(
adjustment_data)
employee_colors = ["PaleTurquoise", "Cyan",
"LightCyan", "Turquoise", "DarkTurquoise"]
emp_index = 0
for emp in range(number_of_employee):
ax[dataset_index].plot(
adjustment_data[f"employee_weight_{emp + 1}"],
label=f"Employee {emp}",
# color=employee_colors[emp % len(employee_colors)]
)
emp_index += 1
# Utility Cost Weights
ax[dataset_index].plot(
adjustment_data["capital_cost_weight_1"],
label="Electricity Cost",
)
ax[dataset_index].plot(
adjustment_data["capital_cost_weight_2"],
label="Water Supply Cost",
)
if use_academic_name:
ax[dataset_index].set_title(
f"{academic_dataset_names[dataset_index]}",
fontsize=16,
)
best_train_epoch = epoch_error[
epoch_error["error_percent"]
== epoch_error["error_percent"].min()
]
best_validate_epoch = epoch_error[
epoch_error["validate_error_percent"]
== epoch_error["validate_error_percent"].min()
]
try:
best_train_epoch = best_train_epoch.iloc[0]
best_train_epoch = best_train_epoch["epoch"]
best_validate_epoch = best_validate_epoch.iloc[0]
best_validate_epoch = best_validate_epoch["epoch"]
except Exception as e:
best_train_epoch = np.nan
best_validate_epoch = np.nan
# ax[dataset_index].set_xlabel(
# f'Train Err {epoch_error.iloc[-1]["error_percent"]:.2f} , Validate Err {epoch_error.iloc[-1]["validate_error_percent"]:.2f} \n Best Epoch Train {best_train_epoch:.0f} , Validate {best_validate_epoch:.0f}'
# )
ax[dataset_index].set_xticks([])
ax[dataset_index].set_ylim(0, 10000)
# Ensure all values are positive before setting the y-axis to logarithmic scale
# if (adjustment_data.select_dtypes(include=[np.number]) > 0).all().all():
# ax[dataset_index].set_yscale('log')
# else:
# print(
# f"Warning: Non-positive values detected in dataset {dataset_name}. Skipping logarithmic scale.")
dataset_index += 1
handles, labels = ax[3].get_legend_handles_labels()
print('labels', labels)
# Group employee labels together
employee_labels = [
label for label in labels if label.startswith("Employee")]
other_labels = [
label for label in labels if not label.startswith("Employee")]
# Combine employee labels into a single entry
if employee_labels:
other_labels.append("Employees")
labels = other_labels
fig.legend(handles, labels, loc="lower center",
ncol=8, fontsize=12)
fig.tight_layout(pad=2.0)
early_stopping_modifier = "/early-stopping" if use_early_stopping else ""
if round_no > 0:
early_stopping_modifier = f"/round-{round_no}/{early_stopping_modifier}"
prefix_filename = is_tail and f"tail-{round}" or f"head-{round}"
os.makedirs(
f"result/learning_curve/hyperparameter/element-weight-{learning_rate}{early_stopping_modifier}",
exist_ok=True,
)
plt.savefig(
f"result/learning_curve/hyperparameter/element-weight-{learning_rate}{early_stopping_modifier}/{prefix_filename}-{model_learning_rate}.png"
)
plt.close(fig)
model_learning_rate_index += 1
dataset_index += 1
sns.set_palette(sns.color_palette())
def plot_each_weights(use_academic_name=False, use_early_stopping=False, round_no=0, model_version=1, selected_case="", learning_rate=0.01):
early_stopping_title = "With Early Stopping" if use_early_stopping else ""
model_learning_rate_index = 0
modifier_index = modifiers.index(selected_case)
this_graph_palette = sns.color_palette("husl", 4)
sns.set_palette(this_graph_palette)
for model_learning_rate in model_learning_rates:
# For Each Learning Rate
dataset_index = 0
fig, ax = plt.subplots(1, 4, figsize=(18, 5))
if use_academic_name:
fig.suptitle(
f"Model Level Weight Adjustment of {displayed_modifiers[modifier_index]} {academic_dataset_names[dataset_index]} \n β ={model_learning_rate} α = {learning_rate} {early_stopping_title}",
fontdict={"fontsize": 16, "fontweight": "bold"},
)
else:
fig.suptitle(
f"Model Level Weight Adjustment of {displayed_modifiers[modifier_index]} {academic_dataset_names[dataset_index]} \n β ={model_learning_rate}α = {learning_rate} {early_stopping_title}",
fontdict={"fontsize": 16, "fontweight": "bold"},
)
for dataset_name in dataset_names:
directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{selected_case}"
epoch_error_file = (
f"{directory_name}/round{round_no + 1}/{iteration}-{learning_rate}.csv"
)
epoch_error = pd.read_csv(epoch_error_file)
adjustment_file = (
f"{directory_name}/round{round_no + 1}/sample-payload-list-{iteration}-{learning_rate}.csv"
)
adjustment_data = pd.read_csv(adjustment_file)
ax[dataset_index].plot(
adjustment_data["model_weight_1"],
label="Material Element Weight",
)
ax[dataset_index].plot(
adjustment_data["model_weight_2"],
label="Labor Element Weight", linewidth=3,
# linestyle='-',
)
ax[dataset_index].plot(
adjustment_data["model_weight_3"],
label="Utiltiy Cost Element Weight",
linewidth=3,
linestyle=':', color=sns.palettes.hls_palette(6)[4]
)
if use_academic_name:
ax[dataset_index].set_title(
f"{academic_dataset_names[dataset_index]}",
fontsize=20,
)
best_train_epoch = epoch_error[
epoch_error["error_percent"]
== epoch_error["error_percent"].min()
]
best_validate_epoch = epoch_error[
epoch_error["validate_error_percent"]
== epoch_error["validate_error_percent"].min()
]
try:
best_train_epoch = best_train_epoch.iloc[0]
best_train_epoch = best_train_epoch["epoch"]
best_validate_epoch = best_validate_epoch.iloc[0]
best_validate_epoch = best_validate_epoch["epoch"]
except Exception as e:
best_train_epoch = np.nan
best_validate_epoch = np.nan
# ax[dataset_index].set_xlabel(
# f'Train Err {epoch_error.iloc[-1]["error_percent"]:.2f} , Validate Err {epoch_error.iloc[-1]["validate_error_percent"]:.2f} \n Best Epoch Train {best_train_epoch:.0f} , Validate {best_validate_epoch:.0f}'
# )
ax[dataset_index].set_xticks([])
ax[dataset_index].set_ylim(0, 30)
# Ensure all values are positive before setting the y-axis to logarithmic scale
# if (adjustment_data.select_dtypes(include=[np.number]) > 0).all().all():
# ax[dataset_index].set_yscale('log')
# else:
# print(
# f"Warning: Non-positive values detected in dataset {dataset_name}. Skipping logarithmic scale.")
dataset_index += 1
handles, labels = ax[3].get_legend_handles_labels()
print('labels', labels)
# Group employee labels together
employee_labels = [
label for label in labels if label.startswith("Employee")]
other_labels = [
label for label in labels if not label.startswith("Employee")]
# Combine employee labels into a single entry
if employee_labels:
other_labels.append("Employees")
labels = other_labels
fig.legend(handles, labels, loc="lower center",
ncol=8, fontsize=16)
fig.tight_layout(pad=2.0)
early_stopping_modifier = "/early-stopping" if use_early_stopping else ""
if round_no > 0:
early_stopping_modifier = f"/round-{round_no}/{early_stopping_modifier}"
os.makedirs(
f"result/learning_curve/hyperparameter/model-weight-{learning_rate}{early_stopping_modifier}",
exist_ok=True,
)
plt.savefig(
f"result/learning_curve/hyperparameter/model-weight-{learning_rate}{early_stopping_modifier}/{model_learning_rate}-{selected_case}.png"
)
plt.close(fig)
model_learning_rate_index += 1
dataset_index += 1
sns.set_palette(sns.color_palette())
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import os
import numpy as np
sns.set_theme(style="whitegrid", font="Noto Sans",
font_scale=1.5)
this_graph_palette = sns.color_palette("husl", 4)
sns.set_palette(this_graph_palette)
dataset_names = ["1-munchkin",
"3-vicheanmas", "2-chinchilla", "4-scottishfold"]
# cademic_dataset_names = ["Dataset 1", "Dataset 3", "Dataset 2", "Dataset 4"]
academic_dataset_names = [
"Simple Dataset", "Complicated Dataset", "Actual Dataset", "Extended Random Dataset"]
learning_rate = [0.005, 0.01, 0.05, 0.1, 0.5]
overall_accuracy_df = pd.DataFrame()
model_learning_rates = ["1e-07", "1e-08"]
iteration = 100
modifiers = [
"",
"_remove_outlier",
"_augmented",
"_remove_outlier_augmented",
"_early_stopping",
"_remove_outlier_early_stopping",
"_augmented_early_stopping",
"_remove_outlier_augmented_early_stopping",
]
displayed_modifiers = [
"Original",
"Remove Outlier",
"Augmentation",
"Remove Outlier + Augmentation",
"Early Stopping",
"Remove Outlier + Early Stopping",
"Augmentation + Early Stopping",
"Remove Outlier + Augmentation + Early Stopping",
]
def to_kebab_case(value):
return "-".join(value.lower().split())
def plotting_learning_curve(use_academic_name=False, round_no=0,
model_version=1):
dataset_index = 0
for dataset_name in dataset_names:
modifier_index = 0
for modifer in modifiers:
accuracy_list = []
# Find Best Accuracy
# For Each Model Learning Rate
fig, ax = plt.subplots(2, 5, figsize=(20, 8))
model_learning_rate_index = 0
learning_rate_index = 0
if use_academic_name:
fig.suptitle(
f"Learning Curve of {academic_dataset_names[dataset_index]} - {displayed_modifiers[modifier_index]}",
fontdict={"fontsize": 20, "fontweight": "bold"},
)
else:
fig.suptitle(
f"Learning Curve of {academic_dataset_names[dataset_index]} - {displayed_modifiers[modifier_index]}",
fontdict={"fontsize": 20, "fontweight": "bold"},
)
for model_learning_rate in model_learning_rates:
directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{modifer}"
# For Each Learning Rate
for lr in learning_rate:
epoch_error_file = (
f"{directory_name}/round{round_no + 1}/{iteration}-{lr}.csv"
)
epoch_error = pd.read_csv(epoch_error_file)
ax[model_learning_rate_index][learning_rate_index].plot(
epoch_error["epoch"],
epoch_error["error_percent"],
label="Training",
linewidth=4
)
ax[model_learning_rate_index][learning_rate_index].plot(
epoch_error["epoch"],
epoch_error["validate_error_percent"],
label="Validation", linewidth=4
)
ax[model_learning_rate_index][learning_rate_index].set_title(
f"{lr}/{model_learning_rate}",
fontdict={"fontsize": 16},
)
# ax[model_learning_rate_index][learning_rate_index].set_xlabel(
# "Epoch"
# )
# ax[model_learning_rate_index][learning_rate_index].set_ylabel(
# "Error RMSPE"
# )