Upload combine_perturbed4.py with huggingface_hub

combine_perturbed4.py

@@ -0,0 +1,264 @@
+import json
+import pandas as pd
+
+
+file_names = [#'./llm_for_code/datasets/codecontests/CN_py800_catmix_5rand_4fold_full_oripairs_matched.json'
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_oripairs_temp.json',
+# './dataset_lfs/CN_py800_catmix_5rand_4fold_200_oripairs_temp.json',
+# './dataset_lfs/CN_py800_catmix_5rand_4fold_200_7m2_oripairs_temp.json',
+# './dataset_lfs/CN_py800_catmix_5rand_4fold_200_7m3_oripairs_temp.json',
+# './dataset_lfs/CN_py800_catmix_5rand_4fold_200_7m4_oripairs_temp.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_0_2_oripairs.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_101_3_oripairs_temp.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_101_3_rem_oripairs_temp.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_101_3_rem2_oripairs_temp.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_200_0_oripairs_temp.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_200_101_2_oripairs_temp.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_200_101_oripairs_temp.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_200_110_oripairs.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_1s_1k_oripairs_temp.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_1s_1k_2_oripairs_temp.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_1s_1k_3_oripairs_temp.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_100_4_oripairs_temp.json',
+#"./dataset_lfs/allpairs_data_val_900_unperturbed.json",
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_2_oripairs_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_3_oripairs_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_4_oripairs_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_5_oripairs_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_oripairs_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_0_conc_1_oripairs_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_0_conc_oripairs_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_101_oripairs_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_2_oripairs.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_2_oripairs.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_3_oripairs.json',
+#'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_2_oripairs_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_3_oripairs.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_500_1_oripairs_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_500_2_oripairs_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_0s_conc_1_oripairs.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_0_conc_1_oripairs_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_0_conc_oripairs_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_0s_500_0_oripairs.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_conc_pos_2_oripairs_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_conc_pos_oripairs_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_5_oripairs_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_1_oripairs_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_10s_500_0_oripairs_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_10s_500_2_oripairs_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_2_oripairs.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_3_oripairs.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_oripairs_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_conc_5_oripairs_temp.json',
+
+
+
+]
+
+file_names_per = [#'./dataset_lfs/CC_900_py_catmix_5rand_4fold_pairs_0_temp.json',
+# './dataset_lfs/CN_py800_catmix_5rand_4fold_200_pairs_0_temp.json',
+# './dataset_lfs/CN_py800_catmix_5rand_4fold_200_7m2_pairs_0_temp.json',
+# './dataset_lfs/CN_py800_catmix_5rand_4fold_200_7m3_pairs_0_temp.json',
+# './dataset_lfs/CN_py800_catmix_5rand_4fold_200_7m4_pairs_0_temp.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_0_2_pairs_0.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_101_3_pairs_0_temp.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_101_3_rem_pairs_0_temp.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_101_3_rem2_pairs_0_temp.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_200_0_pairs_0_temp.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_200_101_2_pairs_0_temp.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_200_101_pairs_0_temp.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_200_110_pairs_0.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_1s_1k_pairs_0_temp.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_1s_1k_2_pairs_0_temp.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_1s_1k_3_pairs_0_temp.json',
+# './dataset_lfs/CC_900_py_catmix_5rand_4fold_100_4_pairs_0_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_2_pairs_0_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_3_pairs_0_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_4_pairs_0_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_5_pairs_0_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_pairs_0_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_0_conc_1_pairs_0_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_0_conc_pairs_0_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_101_pairs_0_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_2_pairs_0.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_2_pairs_0.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_3_pairs_0.json',
+#'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_2_pairs_0_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_3_pairs_0.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_500_1_pairs_0_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_500_2_pairs_0_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_0s_conc_1_pairs_0.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_0_conc_1_pairs_0_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_0_conc_pairs_0_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_0s_500_0_pairs_0.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_conc_pos_2_pairs_0_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_conc_pos_pairs_0_temp.json',
+
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_5_pairs_0_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_1_pairs_0_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_10s_500_0_pairs_0_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_10s_500_2_pairs_0_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_2_pairs_0.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_3_pairs_0.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_all_conc_pairs_0_temp.json',
+'./dataset_lfs/CC_900_py_catmix_5rand_4fold_train_conc_5_pairs_0_temp.json',
+]
+file1 = './llm_for_code/datasets/codecontests/CC_900_py_catmix_5rand_4fold_oripairs_temp_fulltry.json'
+file2 = './llm_for_code/datasets/codecontests/CC_900_py_catmix_5rand_4fold_pairs_0_temp_fulltry.json'
+
+filt1 = []
+with open(file1, 'r') as file:
+    for line in file:
+        filt1.append(json.loads(line))
+
+filt2 = []
+with open(file2, 'r') as file:
+    for line in file:
+        filt2.append(json.loads(line))
+
+filtdf = pd.DataFrame(filt1)
+filtdf_per = pd.DataFrame(filt2)
+original_columns_df2 = filtdf_per.columns.tolist()
+filtdf_per.columns = [f"{col}_df2" for col in filtdf_per.columns]
+filtdf_combined = pd.concat([filtdf, filtdf_per], axis=1)        
+data = []
+data_per = [[], [], [], []]
+df_per_list = []
+df_per = None
+for fn in file_names:
+    with open(fn, 'r') as file:
+        for line in file:
+            data.append(json.loads(line))
+
+print(len(data), len(data_per))            
+df = pd.DataFrame(data)
+
+df_combined = df.copy(deep=True)
+for per_id in range(4):            
+    for file in file_names_per:
+        fn = file[:file.rfind('0')] + str(per_id) + file[file.rfind('0')+1:]
+        #print(fn)
+        with open(fn, 'r') as file:
+            for line in file:
+                data_per[per_id].append(json.loads(line))
+    df_per = pd.DataFrame(data_per[per_id])           
+    df_per_list.append(df_per)
+    original_columns_df2 = df_per.columns.tolist()
+
+    df_per.columns = [f"{col}_df{per_id+2}" for col in df_per.columns]
+    df_combined = pd.concat([df_combined, df_per], axis=1)
+print(df_combined.head(), len(df_combined))
+df_combined = df_combined[~df_combined.apply(tuple, axis=1).isin(filtdf_combined.apply(tuple, axis=1))]
+prohib_probid= [
+    x.get('probid') for x in filtdf['prog1'] 
+    if isinstance(x, dict) and 'probid' in x
+] 
+prohib_probid += [
+    x.get('probid') for x in filtdf['prog2'] 
+    if isinstance(x, dict) and 'probid' in x
+]
+
+# Extract 'A1' from df and filter
+df_combined = df_combined[ ~df['prog1'].apply(lambda x: x.get('probid') if isinstance(x, dict) else None).isin(prohib_probid)]
+df_combined = df_combined[ ~df['prog2'].apply(lambda x: x.get('probid') if isinstance(x, dict) else None).isin(prohib_probid)]
+df_combined = df_combined[ ~df['prog1'].apply(lambda x: x.get('probid') if isinstance(x, dict) else None).str.startswith('p0').fillna(False)]
+
+print("filtered, ", len(df_combined))
+# Step 2: Update only the 'cat' column
+#df_combined['category'] = df_combined['category'].str[:-1] + df_combined['category_df2'].str[-1]
+# Define a custom function to compare dictionaries in prog1 and prog2
+def compare_dicts(row):
+    return (str(row['prog1']), str(row['prog2']))  # Convert dictionaries to tuple of strings for comparison
+
+# Apply the custom function and mark duplicates
+
+df_combined['unique_check'] = df_combined.apply(compare_dicts, axis=1)
+
+# Drop duplicates based on the unique_check column
+df_unique = df_combined.drop_duplicates(subset=['unique_check'])
+
+# Drop the helper column used for comparison
+df_unique = df_unique.drop(columns=['unique_check'])
+print(len(data),len(df_unique) ," are unique")     
+#df = pd.DataFrame(data)
+print(df['category'].value_counts())
+
+def get_sample_size(category):
+    #catlen = {'2_0_0_1': 0, '2_0_0_0': 73,'1_1_0_1': 151 , '1_1_0_0': 0, '0_1_0_0': 0, '0_1_0_1': 2, '1_0_0_0' : 112, '1_0_0_1':0 }
+    #return catlen[category]
+    # For example: sample 10% of the group size, with a max of 50
+    # if category == '0_1_0_1':
+        # return 3600
+    # if category == '1_1_0_1':
+        # return 3600
+    # # if category[0] == '1':
+        # # return 1000
+    # return 1200
+    return 400
+# Apply the custom sampling
+sampled_df = (
+    df_unique.groupby('category', group_keys=False)
+    .apply(lambda x: x.sample(n=min(get_sample_size(x['category'].iloc[0]),len(x)), replace=False))
+)
+#sampled_df1  = df[~(df['category'].str[0] == '1')]
+#sampled_df = sampled_df[sampled_df['category'].str.startswith('2')]
+
+
+# # Step 1: Identify and separate unique and duplicate rows
+# df_unique = df_combined.drop_duplicates(subset=['unique_check'])
+
+
+
+
+# df_duplicates = df_combined[~df_combined.index.isin(df_unique.index)]
+# df_duplicates = df_duplicates.drop(columns=['unique_check'])
+# # Step 2: Sample from df_unique first
+
+# # Step 3: Identify categories that still need more samples
+# remaining_counts = (
+    # sampled_df['category'].value_counts()
+    # .reindex(df_combined['category'].unique(), fill_value=0)
+# )
+
+# print(remaining_counts)
+# # Step 4: Fill missing samples from df_duplicates
+# additional_samples = []
+# for category, count in remaining_counts.items():
+    # required = get_sample_size(category) - count
+    # if required > 0:
+        # extra_samples = df_duplicates[df_duplicates['category'] == category].sample(
+            # n=min(required, len(df_duplicates[df_duplicates['category'] == category])),
+            # replace=False
+        # )
+        # additional_samples.append(extra_samples)
+
+# # Concatenate the extra samples with the originally sampled DataFrame
+# if additional_samples:
+    # additional_samples_df = pd.concat(additional_samples)
+    # sampled_df = pd.concat([sampled_df, additional_samples_df])
+
+# Final sampled dataframe
+sampled_df = sampled_df.reset_index(drop=True)
+
+
+
+
+
+print(sampled_df['category'].value_counts(), len(sampled_df), sampled_df.columns)    
+
+df1_final = sampled_df.iloc[:, :df.shape[1]]
+df2_final_list = []
+for per_id in range(4):
+    df2_final = sampled_df.iloc[:, df.shape[1]+df_per.shape[1]*(per_id):df.shape[1]+df_per.shape[1]*(per_id+1)]
+    print(df1_final.head(), df2_final.head())
+    df2_final.columns = original_columns_df2
+    df2_final_list.append(df2_final)
+df2_final = pd.concat(df2_final_list, ignore_index=True)
+print(len(df2_final))
+with open(file_names[0][:-5]+"_fulltrydup_train_2nem.json" , 'w+') as outfile:  
+    for _, sample in df1_final.iterrows():
+        outfile.write(f"{json.dumps(sample.to_dict())}\n")
+       
+with open(file_names_per[0][:-5]+"_fulltrydup_train_2nem.json" , 'w+') as outfile:  
+    for _, sample in df2_final.iterrows():
+        outfile.write(f"{json.dumps(sample.to_dict())}\n")