| import json | |
| import pandas as pd | |
| file_names = [#'./llm_for_code/datasets/codecontests/CN_py800_catmix_5rand_4fold_full_oripairs_matched.json' | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_oripairs_temp.json', | |
| # './dataset_lfs/CN_py800_catmix_5rand_4fold_200_oripairs_temp.json', | |
| # './dataset_lfs/CN_py800_catmix_5rand_4fold_200_7m2_oripairs_temp.json', | |
| # './dataset_lfs/CN_py800_catmix_5rand_4fold_200_7m3_oripairs_temp.json', | |
| # './dataset_lfs/CN_py800_catmix_5rand_4fold_200_7m4_oripairs_temp.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_0_2_oripairs.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_101_3_oripairs_temp.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_101_3_rem_oripairs_temp.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_101_3_rem2_oripairs_temp.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_200_0_oripairs_temp.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_200_101_2_oripairs_temp.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_200_101_oripairs_temp.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_200_110_oripairs.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_1s_1k_oripairs_temp.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_1s_1k_2_oripairs_temp.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_1s_1k_3_oripairs_temp.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_100_4_oripairs_temp.json', | |
| #"./dataset_lfs/allpairs_data_val_900_unperturbed.json", | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_2_oripairs_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_3_oripairs_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_4_oripairs_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_5_oripairs_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_oripairs_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_0_conc_1_oripairs_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_0_conc_oripairs_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_101_oripairs_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_2_oripairs.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_2_oripairs.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_3_oripairs.json', | |
| #'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_2_oripairs_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_3_oripairs.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_500_1_oripairs_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_500_2_oripairs_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_0s_conc_1_oripairs.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_0_conc_1_oripairs_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_0_conc_oripairs_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_0s_500_0_oripairs.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_conc_pos_2_oripairs_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_conc_pos_oripairs_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_5_oripairs_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_1_oripairs_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_10s_500_0_oripairs_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_10s_500_2_oripairs_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_2_oripairs.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_3_oripairs.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_oripairs_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_conc_5_oripairs_temp.json', | |
| ] | |
| file_names_per = [#'./dataset_lfs/CC_900_py_catmix_5rand_4fold_pairs_0_temp.json', | |
| # './dataset_lfs/CN_py800_catmix_5rand_4fold_200_pairs_0_temp.json', | |
| # './dataset_lfs/CN_py800_catmix_5rand_4fold_200_7m2_pairs_0_temp.json', | |
| # './dataset_lfs/CN_py800_catmix_5rand_4fold_200_7m3_pairs_0_temp.json', | |
| # './dataset_lfs/CN_py800_catmix_5rand_4fold_200_7m4_pairs_0_temp.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_0_2_pairs_0.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_101_3_pairs_0_temp.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_101_3_rem_pairs_0_temp.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_101_3_rem2_pairs_0_temp.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_200_0_pairs_0_temp.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_200_101_2_pairs_0_temp.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_200_101_pairs_0_temp.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_200_110_pairs_0.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_1s_1k_pairs_0_temp.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_1s_1k_2_pairs_0_temp.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_1s_1k_3_pairs_0_temp.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_100_4_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_2_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_3_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_4_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_5_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_0_conc_1_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_0_conc_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_101_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_2_pairs_0.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_2_pairs_0.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_3_pairs_0.json', | |
| #'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_2_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_3_pairs_0.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_500_1_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_500_2_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_0s_conc_1_pairs_0.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_0_conc_1_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_0_conc_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_0s_500_0_pairs_0.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_conc_pos_2_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_conc_pos_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_5_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_1_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_10s_500_0_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_10s_500_2_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_2_pairs_0.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_3_pairs_0.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_pairs_0_temp.json', | |
| './dataset_lfs/CC_900_py_catmix_5rand_4fold_train_conc_5_pairs_0_temp.json', | |
| ] | |
| #file_names = [ './dataset_lfs/CC_900_py_catmix_5rand_4fold_test_varc2_oripairs_temp.json', | |
| # #'./dataset_lfs/CC_900_py_catmix_5rand_4fold_val_oripairs.json' | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_test_varc_oripairs_temp.json', | |
| # './llm_for_code/datasets/codecontests/CC_900_py_catmix_5rand_4fold_test_varc0s1_oripairs.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_test_varc10s_oripairs_temp.json', | |
| #'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_2_pairs_0_temp_fulltrydup_train.json' | |
| #] | |
| # file_names = [ 'CC_1200_py_catmix_5rand_4fold_test_1s_v_oripairs.json', | |
| # 'CC_1200_py_catmix_5rand_4fold_test_all_v_oripairs.json', | |
| # 'CC_1200_py_catmix_5rand_4fold_test_all_v2_oripairs.json', | |
| # 'CC_1200_py_catmix_5rand_4fold_test_pos1s_v_oripairs.json', | |
| # 'CC_1200_py_catmix_5rand_4fold_test_v2_1s_oripairs_temp.json' | |
| # ] | |
| # file_names_per = [ 'CC_1200_py_catmix_5rand_4fold_test_1s_v_pairs_0.json', | |
| # 'CC_1200_py_catmix_5rand_4fold_test_all_v_pairs_0.json', | |
| # 'CC_1200_py_catmix_5rand_4fold_test_all_v2_pairs_0.json', | |
| # 'CC_1200_py_catmix_5rand_4fold_test_pos1s_v_pairs_0.json', | |
| # 'CC_1200_py_catmix_5rand_4fold_test_v2_1s_pairs_0_temp.json' | |
| # ] | |
| # file_names = [ 'CC_2200_py_catmix_5rand_1fold_test_1s_5_oripairs.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_all_2_oripairs.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_all_3_oripairs.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_all_4_oripairs.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_all_oripairs.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_ls_2_oripairs.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_ls_oripairs.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_0s_2_oripairs.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_0s_neg_oripairs.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_0s_neg2_oripairs.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_0s_neg22_oripairs.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_0s_oripairs.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_0s_pos_oripairs.json', | |
| # ] | |
| # file_names_per = [ 'CC_2200_py_catmix_5rand_1fold_test_1s_5_pairs_0.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_all_2_pairs_0.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_all_3_pairs_0.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_all_4_pairs_0.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_all_pairs_0.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_ls_2_pairs_0.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_ls_pairs_0.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_0s_2_pairs_0.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_0s_neg_pairs_0.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_0s_neg2_pairs_0.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_0s_neg22_pairs_0.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_0s_pairs_0.json', | |
| # 'CC_2200_py_catmix_5rand_1fold_test_0s_pos_pairs_0.json', | |
| # ] | |
| #file_names_per = [ './dataset_lfs/CC_900_py_catmix_5rand_4fold_test_varc2_pairs_0_temp.json', | |
| # # './dataset_lfs/CC_900_py_catmix_5rand_4fold_val_pairs_0.json' | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_test_varc_pairs_0_temp.json', | |
| # './llm_for_code/datasets/codecontests/CC_900_py_catmix_5rand_4fold_test_varc0s1_pairs_0.json', | |
| # './dataset_lfs/CC_900_py_catmix_5rand_4fold_test_varc10s_pairs_0_temp.json', | |
| #'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_2_pairs_0_temp_fulltrydup_train.json' | |
| #] | |
| # file_names = [ | |
| # 'CC_1600_py_catmix_5rand_4fold_test_all_v2_oripairs.json', | |
| # 'CC_1600_py_catmix_5rand_4fold_test_all_oripairs_temp.json', | |
| # 'CC_1600_py_catmix_5rand_4fold_test_1s_oripairs.json', | |
| # 'CC_1600_py_catmix_5rand_4fold_test_pos1s_v_oripairs.json', | |
| # 'CC_1600_py_catmix_5rand_4fold_test_pos1s_oripairs.json', | |
| # 'CC_1600_py_catmix_5rand_4fold_test_neg1s_v_oripairs.json', | |
| # 'CC_1600_py_catmix_5rand_4fold_test_neg1s_oripairs_temp.json', | |
| # 'CC_1600_py_catmix_5rand_4fold_test_neg1s_2_oripairs_temp.json', | |
| #] | |
| # file_names_per = [ | |
| # 'CC_1600_py_catmix_5rand_4fold_test_all_v2_pairs_0.json', | |
| # 'CC_1600_py_catmix_5rand_4fold_test_all_pairs_0_temp.json', | |
| # 'CC_1600_py_catmix_5rand_4fold_test_1s_pairs_0.json', | |
| # 'CC_1600_py_catmix_5rand_4fold_test_pos1s_v_pairs_0.json', | |
| # 'CC_1600_py_catmix_5rand_4fold_test_pos1s_pairs_0.json', | |
| # 'CC_1600_py_catmix_5rand_4fold_test_neg1s_v_pairs_0.json', | |
| # 'CC_1600_py_catmix_5rand_4fold_test_neg1s_pairs_0_temp.json', | |
| # 'CC_1600_py_catmix_5rand_4fold_test_neg1s_2_pairs_0_temp.json', | |
| # ] | |
| #file_names = ['CC_900_py_catmix_5rand_1fold_conctest_all_ind_oripairs.json', 'CC_900_py_catmix_5rand_1fold_conctest_1s_ind_oripairs.json',] | |
| #file_names_per = ['CC_900_py_catmix_5rand_1fold_conctest_all_ind_pairs_0.json' ,'CC_900_py_catmix_5rand_1fold_conctest_1s_ind_pairs_0.json'] | |
| # file_names = ['CC_900_py_catmix_5rand_1fold_conctest_1s_loop_oripairs_temp.json', | |
| # 'CC_900_py_catmix_5rand_1fold_conctest_all_loop_oripairs_temp.json'] | |
| # file_names_per = ['CC_900_py_catmix_5rand_1fold_conctest_1s_loop_pairs_0_temp.json', | |
| # 'CC_900_py_catmix_5rand_1fold_conctest_all_loop_pairs_0_temp.json',] | |
| # file_names = ['CC_900_py_catmix_5rand_1fold_conctest_boolinv_1s_oripairs.json', | |
| # 'CC_900_py_catmix_5rand_1fold_conctest_boolinv_1s2_oripairs.json', | |
| # 'CC_900_py_catmix_5rand_1fold_conctest_boolinv_1s3_oripairs.json', | |
| # 'CC_900_py_catmix_5rand_1fold_conctest_boolinv_all_oripairs.json', | |
| # 'CC_900_py_catmix_5rand_1fold_conctest_boolinv_all2_oripairs.json'] | |
| # file_names_per = ['CC_900_py_catmix_5rand_1fold_conctest_boolinv_1s_pairs_0.json', | |
| # 'CC_900_py_catmix_5rand_1fold_conctest_boolinv_1s2_pairs_0.json', | |
| # 'CC_900_py_catmix_5rand_1fold_conctest_boolinv_1s3_pairs_0.json', | |
| # 'CC_900_py_catmix_5rand_1fold_conctest_boolinv_all_pairs_0.json', | |
| # 'CC_900_py_catmix_5rand_1fold_conctest_boolinv_all2_pairs_0.json'] | |
| # file_names = ['CC_900_py_catmix_exact9rand_1fold_conctest_all_oripairs.json', # & 7 | |
| # 'CC_900_py_catmix_exact9rand_1fold_conctest_0s_oripairs.json', | |
| # 'CC_900_py_catmix_exact9rand_1fold_conctest_1s_oripairs.json',] | |
| # file_names_per = ['CC_900_py_catmix_exact9rand_1fold_conctest_all_pairs_0.json', | |
| # 'CC_900_py_catmix_exact9rand_1fold_conctest_0s_pairs_0.json', | |
| # 'CC_900_py_catmix_exact9rand_1fold_conctest_1s_pairs_0.json',] | |
| # file_names = ['CC_900_py_catmix_exact1rand_1fold_conctest_all_oripairs.json', # & 3 | |
| # #'CC_900_py_catmix_exact3rand_1fold_conctest_0s_oripairs.json', | |
| # 'CC_900_py_catmix_exact1rand_1fold_conctest_1s_oripairs.json',] | |
| # file_names_per = ['CC_900_py_catmix_exact1rand_1fold_conctest_all_pairs_0.json', | |
| # #'CC_900_py_catmix_exact3rand_1fold_conctest_0s_pairs_0.json', | |
| # 'CC_900_py_catmix_exact1rand_1fold_conctest_1s_pairs_0.json',] | |
| # file_names = ['CC_900_py_catmix_exact5rand_1fold_conctest_all_oripairs_temp.json', # & 7 | |
| # 'CC_900_py_catmix_exact5rand_1fold_conctest_0s_oripairs.json', | |
| # 'CC_900_py_catmix_exact5rand_1fold_conctest_1s_oripairs_temp.json', | |
| # 'CC_900_py_catmix_exact1rand_5fold_conctest_1s2_oripairs_temp.json', | |
| # 'CC_900_py_catmix_exact1rand_5fold_conctest_all2_oripairs_temp.json',] | |
| # file_names_per = ['CC_900_py_catmix_exact5rand_1fold_conctest_all_pairs_0_temp.json', | |
| # 'CC_900_py_catmix_exact5rand_1fold_conctest_0s_pairs_0.json', | |
| # 'CC_900_py_catmix_exact5rand_1fold_conctest_1s_pairs_0_temp.json', | |
| # 'CC_900_py_catmix_exact1rand_5fold_conctest_1s2_pairs_0_temp.json', | |
| # 'CC_900_py_catmix_exact1rand_5fold_conctest_all2_pairs_0_temp.json',] | |
| # file_names = [ 'CC_900_py_catmix_5rand_1fold_conctest_expr_0s_4_oripairs.json', | |
| # 'CC_900_py_catmix_5rand_1fold_conctest_expr_1s_4_oripairs.json', | |
| # 'CC_900_py_catmix_5rand_1fold_conctest_expr_all_4_oripairs.json', | |
| # ] | |
| # file_names_per = ['CC_900_py_catmix_5rand_1fold_conctest_expr_0s_4_pairs_0.json', | |
| # 'CC_900_py_catmix_5rand_1fold_conctest_expr_1s_4_pairs_0.json', | |
| # 'CC_900_py_catmix_5rand_1fold_conctest_expr_all_4_pairs_0.json', | |
| # ] | |
| # file_names = ['CC_900_py_catmix_5rand_1fold_train_conctest_loopvarexpr_all_oripairs_temp.json', | |
| # 'CC_900_py_catmix_5rand_1fold_train_conctest_loopvarexpr_0s_oripairs.json', | |
| # 'CC_900_py_catmix_5rand_1fold_train_conctest_loopvarexpr_1s_oripairs_temp.json', | |
| # 'CC_900_py_catmix_5rand_1fold_train_conctest_loopvarexpr_all_2_oripairs_temp.json', | |
| # 'CC_900_py_catmix_5rand_1fold_train_conctest_loopvarexpr_all_3_oripairs_temp.json', | |
| # 'CC_900_py_catmix_5rand_1fold_train_conctest_loopvarexpr_all_7_oripairs_temp.json'] | |
| # file_names_per = ['CC_900_py_catmix_5rand_1fold_train_conctest_loopvarexpr_all_pairs_0_temp.json', | |
| # 'CC_900_py_catmix_5rand_1fold_train_conctest_loopvarexpr_0s_pairs_0.json', | |
| # 'CC_900_py_catmix_5rand_1fold_train_conctest_loopvarexpr_1s_pairs_0_temp.json', | |
| # 'CC_900_py_catmix_5rand_1fold_train_conctest_loopvarexpr_all_2_pairs_0_temp.json', | |
| # 'CC_900_py_catmix_5rand_1fold_train_conctest_loopvarexpr_all_3_pairs_0_temp.json', | |
| # 'CC_900_py_catmix_5rand_1fold_train_conctest_loopvarexpr_all_7_pairs_0_temp.json'] | |
| # file_names = ['CC_900_py_catmix_5rand_1fold_test_conctest_all_sptask_1s_13_oripairs_temp.json', | |
| # 'CC_900_py_catmix_5rand_1fold_test_conctest_all_sptask_1s_14_oripairs_temp.json'] | |
| # file_names_per = ['CC_900_py_catmix_5rand_1fold_test_conctest_all_sptask_1s_13_pairs_0_temp.json', | |
| # 'CC_900_py_catmix_5rand_1fold_test_conctest_all_sptask_1s_14_pairs_0_temp.json'] | |
| file_names = ['CC_900_py_catmix_1rand_1fold_train_conctest_all_11_oripairs_temp.json', | |
| 'CC_900_py_catmix_1rand_1fold_train_conctest_all_12_oripairs_temp.json', | |
| 'CC_900_py_catmix_1rand_1fold_train_conctest_0s_15_oripairs.json', | |
| 'CC_900_py_catmix_1rand_1fold_train_conctest_0s_16_oripairs.json',] | |
| file_names_per = ['CC_900_py_catmix_1rand_1fold_train_conctest_all_11_pairs_0_temp.json', | |
| 'CC_900_py_catmix_1rand_1fold_train_conctest_all_12_pairs_0_temp.json', | |
| 'CC_900_py_catmix_1rand_1fold_train_conctest_0s_15_pairs_0.json', | |
| 'CC_900_py_catmix_1rand_1fold_train_conctest_0s_16_pairs_0.json',] | |
| file_names = ['CC_900_py_catmix_5rand_1fold_train_conctest_iffp_all_2_oripairs.json'] | |
| file_names_per = ['CC_900_py_catmix_5rand_1fold_train_conctest_iffp_all_2_pairs_0.json'] | |
| file_names = ['AtCoder_D_catmix_5rand_oripairs.json', | |
| 'AtCoder_D_catmix_5rand_1s_oripairs.json' ] | |
| file_names_per = ['AtCoder_D_catmix_5rand_pairs_0.json', | |
| 'AtCoder_D_catmix_5rand_1s_pairs_0.json'] | |
| file_names = ['CC_900_py_catmix_5rand_4fold_pyjava_bothper_oripairs.json', | |
| 'CC_900_py_catmix_5rand_4fold_pyjava_bothper_1s_3_oripairs.json', | |
| 'CC_900_py_catmix_5rand_4fold_pyjava_bothper_1s_oripairs.json', | |
| 'CC_900_py_catmix_5rand_4fold_pyjava_bothper_1s_2_oripairs.json', | |
| #'CC_900_py_catmix_5rand_4fold_pyjava_bothper_2_oripairs.json', | |
| ] | |
| file_names_per = ['CC_900_py_catmix_5rand_4fold_pyjava_bothper_pairs_0.json', | |
| 'CC_900_py_catmix_5rand_4fold_pyjava_bothper_1s_3_pairs_0.json', | |
| 'CC_900_py_catmix_5rand_4fold_pyjava_bothper_1s_pairs_0.json', | |
| 'CC_900_py_catmix_5rand_4fold_pyjava_bothper_1s_2_pairs_0.json', | |
| #'CC_900_py_catmix_5rand_4fold_pyjava_bothper_2_pairs_0.json', | |
| ] | |
| file_names = ['CC_900_py_catmix_5rand_4fold_pyjava_bothper_wneg_10s_3_oripairs.json', | |
| 'CC_900_py_catmix_5rand_4fold_pyjava_bothper_wneg_10s_2_oripairs_temp.json', | |
| 'CC_900_py_catmix_5rand_4fold_pyjava_bothper_wneg_10s_4_oripairs_temp.json', | |
| 'CC_900_py_catmix_5rand_4fold_pyjava_bothper_wneg_10s_oripairs_temp.json'] | |
| file_names_per = ['CC_900_py_catmix_5rand_4fold_pyjava_bothper_wneg_10s_3_pairs_0.json', | |
| 'CC_900_py_catmix_5rand_4fold_pyjava_bothper_wneg_10s_2_pairs_0_temp.json', | |
| 'CC_900_py_catmix_5rand_4fold_pyjava_bothper_wneg_10s_4_pairs_0_temp.json', | |
| 'CC_900_py_catmix_5rand_4fold_pyjava_bothper_wneg_10s_pairs_0_temp.json'] | |
| file_names = ["CC_900_py_catmix_5rand_1fold_pyjava_bothper_all_oripairs_temp.json", | |
| "CC_900_py_catmix_5rand_1fold_pyjava_bothper_all_2_oripairs_temp.json", | |
| "CC_900_py_catmix_5rand_1fold_pyjava_bothper_1s_oripairs_temp.json", | |
| "CC_900_py_catmix_5rand_1fold_pyjava_bothper_1s_2_oripairs_temp.json", | |
| "CC_900_py_catmix_5rand_1fold_pyjava_only_1s_50_oripairs_temp.json", | |
| "CC_900_py_catmix_5rand_1fold_pyjava_only_11s_oripairs_temp.json"] | |
| file_names_per = ["CC_900_py_catmix_5rand_1fold_pyjava_bothper_all_pairs_0_temp.json", | |
| "CC_900_py_catmix_5rand_1fold_pyjava_bothper_all_2_pairs_0_temp.json", | |
| "CC_900_py_catmix_5rand_1fold_pyjava_bothper_1s_pairs_0_temp.json", | |
| "CC_900_py_catmix_5rand_1fold_pyjava_bothper_1s_2_pairs_0_temp.json", | |
| "CC_900_py_catmix_5rand_1fold_pyjava_only_1s_50_pairs_0_temp.json", | |
| "CC_900_py_catmix_5rand_1fold_pyjava_only_11s_pairs_0_temp.json"] | |
| file_names = ["CC_900_py_catmix_5rand_1fold_pycpp_11s_oripairs_temp.json", | |
| "CC_900_py_catmix_5rand_1fold_pycpp_1s_2_oripairs_temp.json", | |
| "CC_900_py_catmix_5rand_1fold_pycpp_1s_oripairs_temp.json", | |
| "CC_900_py_catmix_5rand_1fold_pycpp_all_oripairs_temp.json", | |
| "CC_900_py_catmix_5rand_1fold_pycpp_all_2_oripairs_temp.json", | |
| "CC_900_py_catmix_5rand_1fold_pycpp_all_3s_oripairs_temp.json", | |
| ] | |
| file_names_per = ["CC_900_py_catmix_5rand_1fold_pycpp_11s_pairs_0_temp.json", | |
| "CC_900_py_catmix_5rand_1fold_pycpp_1s_2_pairs_0_temp.json", | |
| "CC_900_py_catmix_5rand_1fold_pycpp_1s_pairs_0_temp.json", | |
| "CC_900_py_catmix_5rand_1fold_pycpp_all_pairs_0_temp.json", | |
| "CC_900_py_catmix_5rand_1fold_pycpp_all_2_pairs_0_temp.json", | |
| "CC_900_py_catmix_5rand_1fold_pycpp_all_3s_pairs_0_temp.json", | |
| ] | |
| # file_names = ["CC_900_py_catmix_5rand_1fold_java_1s_2_oripairs_temp.json", | |
| # "CC_900_py_catmix_5rand_1fold_java_all_oripairs.json", | |
| # "CC_900_py_catmix_5rand_1fold_java_only_1s_oripairs_temp.json", | |
| # "CC_900_py_catmix_5rand_1fold_java_only_all_oripairs_temp.json"] | |
| # file_names_per = ["CC_900_py_catmix_5rand_1fold_java_1s_2_pairs_0_temp.json", | |
| # "CC_900_py_catmix_5rand_1fold_java_all_pairs_0.json", | |
| # "CC_900_py_catmix_5rand_1fold_java_only_1s_pairs_0_temp.json", | |
| # "CC_900_py_catmix_5rand_1fold_java_only_all_pairs_0_temp.json"] | |
| # file_names = ['CC_900_py_catmix_5rand_1fold_train_conctest_iffp_all_oripairs_temp.json', | |
| # 'CC_900_py_catmix_5rand_1fold_train_conctest_iffp_all_3_oripairs_temp.json', | |
| # 'CC_900_py_catmix_5rand_1fold_train_conctest_iffp_all_2_oripairs_temp.json', | |
| # 'CC_900_py_catmix_1rand_1fold_train_conctest_0s_15_oripairs_temp.json', | |
| # 'CC_900_py_catmix_1rand_1fold_train_conctest_0s_16_oripairs_temp.json'] | |
| # file_names_per = ['CC_900_py_catmix_5rand_1fold_train_conctest_iffp_all_pairs_0_temp.json', | |
| # 'CC_900_py_catmix_5rand_1fold_train_conctest_iffp_all_3_pairs_0_temp.json', | |
| # 'CC_900_py_catmix_5rand_1fold_train_conctest_iffp_all_2_pairs_0_temp.json', | |
| # 'CC_900_py_catmix_1rand_1fold_train_conctest_0s_15_pairs_0_temp.json', | |
| # 'CC_900_py_catmix_1rand_1fold_train_conctest_0s_16_pairs_0_temp.json'] | |
| file1 = './llm_for_code/datasets/codecontests/CC_900_py_catmix_5rand_1fold_pycpp_11s_oripairs_temp_fulltry.json' #CC_900_py_catmix_5rand_4fold_pyjava_bothper_oripairs_fulltry.json' #CC_900_py_catmix_5rand_4fold_oripairs_temp_fulltry.json' | |
| file2 = './llm_for_code/datasets/codecontests/CC_900_py_catmix_5rand_1fold_pycpp_11s_pairs_0_temp_fulltry.json' #CC_900_py_catmix_5rand_4fold_pyjava_bothper_pairs_0_fulltry.json' #CC_900_py_catmix_5rand_4fold_pairs_0_temp_fulltry.json' | |
| #file1 = './CC_2200_py_catmix_5rand_1fold_test_1s_5_oripairs_temp_fulltry.json' #'./llm_for_code/datasets/codecontests/CC_1200_py_catmix_5rand_4fold_test_1s_v_oripairs_fulltry.json' | |
| #file2 = './CC_2200_py_catmix_5rand_1fold_test_1s_5_pairs_0_temp_fulltry.json' #'./llm_for_code/datasets/codecontests/CC_1200_py_catmix_5rand_4fold_test_1s_v_pairs_0_fulltry.json' | |
| filt1 = [] | |
| with open(file1, 'r') as file: | |
| for line in file: | |
| filt1.append(json.loads(line)) | |
| filt2 = [] | |
| with open(file2, 'r') as file: | |
| for line in file: | |
| filt2.append(json.loads(line)) | |
| filtdf = pd.DataFrame(filt1[:]) | |
| filtdf_per = pd.DataFrame(filt2[:]) | |
| original_columns_df2 = filtdf_per.columns.tolist() | |
| filtdf_per.columns = [f"{col}_df2" for col in filtdf_per.columns] | |
| filtdf_combined = pd.concat([filtdf, filtdf_per], axis=1) | |
| data = [] | |
| data_per = [] | |
| for fn in file_names: | |
| with open("./dataset_lfs/"+fn, 'r') as file: | |
| for line in file: | |
| data.append(json.loads(line)) | |
| for fn in file_names_per: | |
| with open("./dataset_lfs/"+fn, 'r') as file: | |
| for line in file: | |
| data_per.append(json.loads(line)) | |
| print(len(data), len(data_per)) | |
| def replace_third(val): | |
| parts = val.split('_') | |
| if len(parts) >= 3: | |
| parts[2] = 'K' | |
| return '_'.join(parts) | |
| df = pd.DataFrame(data) | |
| #df['category'] = df['category'].apply(replace_third) | |
| df_per = pd.DataFrame(data_per) | |
| original_columns_df2 = df_per.columns.tolist() | |
| df_per.columns = [f"{col}_df2" for col in df_per.columns] | |
| df_combined = pd.concat([df, df_per], axis=1) | |
| print(df_combined.head(), len(df_combined)) | |
| df_combined = df_combined[~df_combined.apply(tuple, axis=1).isin(filtdf_combined.apply(tuple, axis=1))] | |
| prohib_probid= [ | |
| x.get('probid') for x in filtdf['prog1'] | |
| if isinstance(x, dict) and 'probid' in x | |
| ] | |
| prohib_probid += [ | |
| x.get('probid') for x in filtdf['prog2'] | |
| if isinstance(x, dict) and 'probid' in x | |
| ] | |
| # # # Extract 'A1' from df and filter | |
| # df_combined = df_combined[ ~df['prog1'].apply(lambda x: x.get('probid') if isinstance(x, dict) else None).isin(prohib_probid)] | |
| # df_combined = df_combined[ ~df['prog2'].apply(lambda x: x.get('probid') if isinstance(x, dict) else None).isin(prohib_probid)] | |
| # df_combined = df_combined[ ~df['prog1'].apply(lambda x: x.get('probid') if isinstance(x, dict) else None).str.startswith('p0').fillna(False)] | |
| print("filtered, ", len(df_combined)) | |
| # Step 2: Update only the 'cat' column | |
| #df_combined['category'] = df_combined['category'].str[:-1] + df_combined['category_df2'].str[-1] | |
| # Define a custom function to compare dictionaries in prog1 and prog2 | |
| def compare_dicts(row): | |
| #print(row) | |
| return (str(row['prog1_df2']['scode']), str(row['prog2_df2']['scode'])) # Convert dictionaries to tuple of strings for comparison | |
| # Apply the custom function and mark duplicates | |
| df_combined = df_combined.dropna(subset=['prog1_df2']) | |
| df_combined['unique_check'] = df_combined.apply(compare_dicts, axis=1) | |
| # Drop duplicates based on the unique_check column | |
| df_unique = df_combined.drop_duplicates(subset=['unique_check']) | |
| # Drop the helper column used for comparison | |
| df_unique = df_unique.drop(columns=['unique_check']) | |
| print(len(data),len(df_unique) ," are unique") | |
| #df = pd.DataFrame(data) | |
| print(df_unique['category'].value_counts()) | |
| def get_sample_size(category): | |
| catlen = {'2_0_0_1': (200-135-31), '2_0_0_0': (200-145-38),'1_1_0_1': (1000-845-155) , '1_1_0_0': (1000-818-182), '0_1_0_1': (200-200), '0_1_0_0': (200-200), '1_0_0_0' : (1000-742-144), '1_0_0_1':(1000-725-162) } | |
| # if category[0] != '0': | |
| # return 0 | |
| # catlen = {'0_1_0_0': (200-45), '0_1_0_1' : (200-36)} | |
| print(catlen[category]) | |
| return catlen[category] | |
| # For example: sample 10% of the group size, with a max of 50 | |
| # if category == '1_0_0_1' or category == '1_0_0_0': | |
| # return 1000 | |
| """ pycpp | |
| 1_1_0_1 845 | |
| 1_1_0_0 818 | |
| 1_0_0_0 742 | |
| 1_0_0_1 725 | |
| 2_0_0_0 145 | |
| 2_0_0_1 135 | |
| """ | |
| if category[0] == '1': | |
| return 300 | |
| # return 0 | |
| return 65 | |
| # Apply the custom sampling | |
| sampled_df = ( | |
| df_unique.groupby('category', group_keys=False) | |
| .apply(lambda x: x.sample(n=min(get_sample_size(x['category'].iloc[0]),len(x)), replace=False)) | |
| ) | |
| #sampled_df1 = df[~(df['category'].str[0] == '1')] | |
| #sampled_df = sampled_df[sampled_df['category'].str.startswith('2')] | |
| # Step 1: Identify and separate unique and duplicate rows | |
| #df_unique = df_combined.drop_duplicates(subset=['unique_check']) | |
| # df_duplicates = df_combined[~df_combined.index.isin(df_unique.index)] | |
| # df_duplicates = df_duplicates.drop(columns=['unique_check']) | |
| # # Step 2: Sample from df_unique first | |
| # # Step 3: Identify categories that still need more samples | |
| # remaining_counts = ( | |
| # sampled_df['category'].value_counts().reindex(df_combined['category'].unique(), fill_value=0) | |
| # ) | |
| # print("remaining_counts: ", remaining_counts) | |
| # # Step 4: Fill missing samples from df_duplicates | |
| # additional_samples = None | |
| # additional_samples = [] | |
| # for category, count in remaining_counts.items(): | |
| # required = get_sample_size(category) - count | |
| # if required > 0: | |
| # extra_samples = df_duplicates[df_duplicates['category'] == category].sample( | |
| # n=min(required, len(df_duplicates[df_duplicates['category'] == category])), | |
| # replace=False | |
| # ) | |
| # additional_samples.append(extra_samples) | |
| # # Concatenate the extra samples with the originally sampled DataFrame | |
| # if additional_samples: | |
| # additional_samples_df = pd.concat(additional_samples) | |
| # sampled_df = pd.concat([sampled_df, additional_samples_df]) | |
| # # #Final sampled dataframe | |
| sampled_df = sampled_df.reset_index(drop=True) | |
| print(sampled_df['category'].value_counts(), len(sampled_df)) | |
| df1_final = sampled_df.iloc[:, :df.shape[1]] | |
| df2_final = sampled_df.iloc[:, df.shape[1]:] | |
| print(df1_final.head(), df2_final.head()) | |
| df2_final.columns = original_columns_df2 | |
| with open(file_names[0][:-5]+"_fulltry_rem.json" , 'w+') as outfile: | |
| for _, sample in df1_final.iterrows(): | |
| outfile.write(f"{json.dumps(sample.to_dict())}\n") | |
| with open(file_names_per[0][:-5]+"_fulltry_rem.json" , 'w+') as outfile: | |
| for _, sample in df2_final.iterrows(): | |
| outfile.write(f"{json.dumps(sample.to_dict())}\n") | |
| print("changed file using", file_names_per[0]) |