File size: 24,616 Bytes
fd74a4d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
import os
import re
import ast
import sys
import json
import pathlib
import argparse
import numpy as np
import editdistance
from typing import Union
from collections import defaultdict

module_path = str(pathlib.Path(os.path.abspath(__file__)).parent.parent.parent)
sys.path.append(module_path)

from src.data_generation.primitives import ProgramBFCCNode, ProgramVocabulary

def sort_dict_by_key(d: dict, reverse=False):
    return dict(sorted(d.items(), key=lambda x: x[0], reverse=reverse))

def safe_mean(x):
    if len(x) == 0: return 0
    return np.mean(x)

def parse_args():
    parser = argparse.ArgumentParser(description="Evaluate functional correctness of model generated program sequence.")
    parser.add_argument(
        "--model_predictions_path", type=str,
        help="Path to the model predictions file."
    )
    # parser.add_argument(
    #     "--search_mode", action="store_true",
    #     help="flag to indicate model preds are a folder of search steps instead of single pred file."
    # )
    parser.add_argument(
        "--vocab", type=str,
        default="abcdefghijkuvwxyz",
        help="vocabulary to be used for inputs/programs",
    )
    parser.add_argument(
        "--max_programs", type=int, default=5,
        help="Maximum programs the model is allowed to use.",
    )
    return parser.parse_args()

def classify_iod(iod: float):
    if iod < 0.2: return "IOD < 0.2"
    elif iod < 0.4: return "0.2 <= IOD < 0.4"
    elif iod < 0.6: return "0.4 <= IOD < 0.6"
    elif iod < 0.8: return "0.6 <= IOD < 0.8"
    else: return "0.8 <= IOD"

def reward(pred_final_output: list[str], inputs: list[str], targets: list[str]) -> float:
    pred_final_output_tok: list[str[str]] = [w.split() for w in pred_final_output] 
    inputs_tok: list[str[str]] = [w.split() for w in inputs] 
    targets_tok: list[str[str]] = [w.split() for w in targets] 

    pred_target_dist = sum(editdistance.eval(s, t) for s, t in zip(pred_final_output_tok, targets_tok))
    input_target_dist = sum(editdistance.eval(s, t) for s, t in zip(inputs_tok, targets_tok))

    return (1 - (pred_target_dist / input_target_dist))

def identity(x: list[str]) -> list[str]: return x

def find_degenerate_program_ids(input_dict: dict):
    prev_outputs = input_dict["inputs"]
    skipped_programs = [] # skip degenerate programs that don't alter any inputs.
    for program_ind,program in enumerate(input_dict['programs']):
        program = program.replace("\\","")
        program_node = ProgramBFCCNode.from_string(program, vocabulary=ProgramVocabulary(vocab_chars))
        next_inputs = program_node(prev_outputs)
        # print(prev_outputs, program, next_inputs)
        if next_inputs == prev_outputs:
            skipped_programs.append(program_ind)
        prev_outputs = next_inputs
        input_dict["programs"][program_ind] = program # just to fix the issue with exccess back slashes

    return skipped_programs

def extract_program(program: Union[str, ProgramBFCCNode], vocab_chars) -> ProgramBFCCNode:
    if isinstance(program, str):
        try: 
            program = ProgramBFCCNode.from_string(program, vocabulary=ProgramVocabulary(vocab_chars))
        except IndexError as e:
            # print(f"\x1b[31;1m{e}\x1b[0m")
            program = identity
        except AssertionError as e:
            # print(f"\x1b[31;1m{e}\x1b[0m")
            program = identity # replace invalid programs with identity programs.
    else: 
        # assert isinstance(program, ProgramBFCCNode), f"Invalid program type: {type(program)}"
        # print(f"\x1b[31;1mInvalid program type: {type(program)}. Replacing with identity. Check if this isn't the result of WRONG FORMATTING!!!\x1b[0m")
        program = identity

    return program

def eval_outputs(
        inputs: list[str], outputs: list[str], 
        pred_cascade: list[Union[str, ProgramBFCCNode]], 
        vocab_chars: str="abcdefghijkuvwxyz", max_programs: int=5):
    pred_outputs: list[str] = [i for i in inputs]
    valid_programs = 0
    invalid_programs = 0
    for program in pred_cascade[:max_programs]: # we should only pick the first max_programs (m in the paper) programs predicted by the model in the cascade (in case the model disobeys the instruction and uses more programs than max_programs)
        if isinstance(program, str):
            try: 
                program = ProgramBFCCNode.from_string(program, vocabulary=ProgramVocabulary(vocab_chars))
                valid_programs += 1
            except IndexError as e:
                # print(f"\x1b[31;1m{e}\x1b[0m")
                program = identity
                invalid_programs += 1
            except AssertionError as e:
                # print(f"\x1b[31;1m{e}\x1b[0m")
                program = identity # replace invalid programs with identity programs.
                invalid_programs += 1
        else: 
            # assert isinstance(program, ProgramBFCCNode), f"Invalid program type: {type(program)}"
            # print(f"\x1b[31;1mInvalid program type: {type(program)}. Replacing with identity. Check if this isn't the result of WRONG FORMATTING!!!\x1b[0m")
            program = identity
            invalid_programs += 1
        pred_outputs: list[str] = program(pred_outputs)
    
    return pred_outputs, valid_programs, invalid_programs

def eval_mean_pass_at_1(dataset, predictions: list[str], vocab_chars: str="abcdefghijkuvwxyz", max_programs: int=5):
    # passing_programs = 0
    passing_programs = []
    for rec, pred_cascade in zip(dataset, predictions):
        inputs = rec["inputs"]
        outputs = rec["outputs"]
        pred_outputs, valid_programs, invalid_programs = eval_outputs(inputs=inputs, outputs=outputs, pred_cascade=pred_cascade, vocab_chars=vocab_chars, max_programs=max_programs)
        test_pass_ratio = sum([int(o == po) for o,po in zip(outputs, pred_outputs)])/len(outputs)
        if test_pass_ratio == 1: passing_programs.append(1)
        else: passing_programs.append(0)

    return np.mean(passing_programs), passing_programs

def eval_rewards(dataset, predictions: list[str], vocab_chars: str="abcdefghijkuvwxyz", max_programs: int=5):
    rewards = []
    total_valid_programs = 0
    total_invalid_programs = 0
    for rec, pred_cascade in zip(dataset, predictions):
        inputs = rec["inputs"]
        outputs = rec["outputs"]
        pred_outputs, valid_programs, invalid_programs = eval_outputs(inputs=inputs, outputs=outputs, pred_cascade=pred_cascade, vocab_chars=vocab_chars, max_programs=max_programs)
        total_valid_programs += valid_programs
        total_invalid_programs += invalid_programs
        rewards.append(reward(pred_outputs, inputs, outputs))


    return np.mean(rewards), rewards, total_valid_programs/(total_invalid_programs+total_valid_programs)

def pass_at_k(n, c, k):
    """
    :param n: total number of samples
    :param c: number of correct samples
    :param k: k in pass@$k$
    """
    if n - c < k: return 1.0
    return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))

# def eval_mean_pass_at_k(dataset, predictions: list[list[str]], k: int=5, vocab_chars: str="abcdefghijkuvwxyz", max_programs: int=5):
#     n = len(predictions[0])
#     assert k <= n, f"k should be less than {n}"
#     instance_pass_at_ks = []
#     for rec, sampled_cascades in zip(dataset, predictions):
#         inputs = rec["inputs"]
#         outputs = rec["outputs"]
#         c = 0
#         for pred_cascade in sampled_cascades:
#             test_pass_ratio = eval_passing_test_cases(inputs=inputs, outputs=outputs, pred_cascade=pred_cascade, vocab_chars=vocab_chars, max_programs=max_programs)
#             if test_pass_ratio > 0.999: c += 1
#         instance_pass_at_k = pass_at_k(n, c, k)
#         instance_pass_at_ks.append(instance_pass_at_k)

#     return np.mean(instance_pass_at_ks)

def extract_first_python_block(markdown_text: str) -> str:
    """
    Extracts first Python code block from a markdown string.

    Args:
        markdown_text (str): The markdown text containing code blocks.

    Returns:
        List[str]: A list of Python code blocks.
    """
    code_block_pattern = r"```python\s(.*?)```"
    matches = re.findall(code_block_pattern, markdown_text, re.DOTALL)
    try: return [block.strip() for block in matches][0]
    except IndexError: return "[]"

def extract_last_python_block(markdown_text: str) -> str:
    """
    Extracts first Python code block from a markdown string.

    Args:
        markdown_text (str): The markdown text containing code blocks.

    Returns:
        List[str]: A list of Python code blocks.
    """
    code_block_pattern = r"```python\s(.*?)```"
    matches = re.findall(code_block_pattern, markdown_text, re.DOTALL)
    try: return [block.strip() for block in matches][-1]
    except IndexError: return "[]"

def extract_program_str_from_last_python_block(model_response: str) -> str:
    try: pred = extract_last_python_block(model_response) 
    except TypeError as e:
        if model_response is None: pass
        else: print(e)
        pred = "replace('x','x')"

    # wrap any replace(...) that isn’t already quoted in single quotes
    # e.g.  replace('kd','ka')  →  'replace('kd','ka')'
    pred = re.sub(
        r"(?<!['\"])(replace\([^)]*\))(?!['\"])",
        r'"\1"',
        pred
    )

    return pred

# def load_model_predictions(path: str):
#     raw_data = read_jsonl(path)
#     loaded_data = []
#     for rec in raw_data:
#         loaded_data.append(rec['input'])
#         loaded_data[-1]["prediction"] = extract_first_python_block(rec['output'])
#         print(loaded_data[-1]['prediction'])

#     return loaded_data

def load_model_predictions(path: str, extract_method) -> list[dict]: 
    raw_data: list[dict] = read_jsonl(path) 
    loaded_data = []
    for rec in raw_data: 
        # extract the code‐block text
        if isinstance(rec['output'], dict):
            rec['output'] = rec["output"]["text"]  # fix specific for QwQ file (should correct the prediction format for future)
        pred = extract_method(rec['output']) 

        # wrap any replace(...) that isn’t already quoted in single quotes
        # e.g.  replace('kd','ka')  →  'replace('kd','ka')'
        pred = re.sub(
            r"(?<!['\"])(replace\([^)]*\))(?!['\"])",
            r'"\1"',
            pred
        )

        # assign and print
        rec['input']["predictions"] = [pred]
        # print(pred)

        loaded_data.append(rec['input'])

    return loaded_data

def load_model_predictions_multi(path: str, extract_method) -> list[dict]: 
    raw_data: list[dict] = read_jsonl(path) 
    loaded_data = []
    null_ctr = 0
    for rec in raw_data: 
        # extract the code‐block text
        rec['input']["predictions"] = []
        for output in rec['outputs']: 
            try: pred = extract_method(output) 
            except TypeError as e:
                if output is None:
                    null_ctr += 1
                else: print(e)
                pred = "replace('x','x')"

            # wrap any replace(...) that isn’t already quoted in single quotes
            # e.g.  replace('kd','ka')  →  'replace('kd','ka')'
            pred = re.sub(
                r"(?<!['\"])(replace\([^)]*\))(?!['\"])",
                r'"\1"',
                pred
            )

            # assign and print
            rec['input']["predictions"].append(pred) 
            # print(pred)

        loaded_data.append(rec['input']) 
    print(f"found {null_ctr} nulls")

    return loaded_data 

def process_model_predictions_multi(raw_data: list[dict], extract_method) -> list[dict]: 
    loaded_data = []
    null_ctr = 0
    for rec in raw_data: 
        # extract the code‐block text
        rec['input']["predictions"] = []
        for output in rec['outputs']: 
            try: pred = extract_method(output) 
            except TypeError as e:
                if output is None:
                    null_ctr += 1
                else: print(e)
                pred = "replace('x','x')"

            # wrap any replace(...) that isn’t already quoted in single quotes
            # e.g.  replace('kd','ka')  →  'replace('kd','ka')'
            pred = re.sub(
                r"(?<!['\"])(replace\([^)]*\))(?!['\"])",
                r'"\1"',
                pred
            )

            # assign and print
            rec['input']["predictions"].append(pred) 
            # print(pred)

        loaded_data.append(rec['input']) 
    print(f"found {null_ctr} nulls")

    return loaded_data 

def normalized_levenshtein(s1: str, s2: str) -> float:
    distance = editdistance.eval(s1, s2)
    max_len = max(len(s1), len(s2))
    return distance / max_len if max_len > 0 else 0.0

def try_ast_literal_eval(string): # do some minor fixes.
    """Takes program sequence string and returns the extracted sequence of actual replace programs."""
    try:
        try: 
            program_seq = ast.literal_eval(string)
            if isinstance(program_seq, str):
                program_seq = [program_seq] 
            # print(program_seq, type(program_seq))
            try: 
                assert isinstance(program_seq, list), f"program sequence of list form couldn't be extracted or derived: {program_seq}"
                return program_seq
            except AssertionError: return []
        except ValueError: return []
    except SyntaxError: return [] # empty prediction if no code blocks found.

def get_instance_complexities(model_preds, max_programs: int, vocab_chars):
    instance_complexity = [[0 for _ in range(7)] for _ in range(len(model_preds))] # index 0-3: B,F,CB,CF counts, index 4: effective 
    pred_cascade_complexity_scores = [[] for _ in range(len(model_preds))] # score based on summing up the lengths of the alphas and betas.
    gt_cascade_complexity_scores = [0 for _ in range(len(model_preds))] # score based on summing up the lengths of the alphas and betas.

    for ind,rec in enumerate(model_preds):
        prev_outputs = rec['inputs']
        rec["skipped_programs"] = [] # skip degenerate programs that don't alter any inputs.

        for program_ind,program in enumerate(rec['programs']):
            program = program.replace("\\","")
            program_node = ProgramBFCCNode.from_string(program, vocabulary=ProgramVocabulary(vocab_chars))
            cascade_complexity_score = (len(program_node.predicate) + len(program_node.transform))
            gt_cascade_complexity_scores[ind] += cascade_complexity_score
            next_inputs = program_node(prev_outputs)
            # print(prev_outputs, program, next_inputs)
            if next_inputs == prev_outputs:
                rec["skipped_programs"].append(program_ind)
            prev_outputs = next_inputs
            rec['programs'][program_ind] = program

        for pred_ind,prediction in enumerate(rec['predictions']):
            pred_cascade_complexity_scores[ind] = [0 for _ in range(len(rec['predictions']))]
            pred_cascade = try_ast_literal_eval(prediction)
            for program_ind, program in enumerate(pred_cascade):
                try: program = program.replace("\\","")
                except AttributeError: program = "" 
                try: 
                    program_node = ProgramBFCCNode.from_string(program, vocabulary=ProgramVocabulary('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!@#$%^&*()_+-=1234567890 §{}[]|\?"/?><,.'), max_window_size=20) # deliberately add a very huge vocab so we can count the true complexity.
                    cascade_complexity_score = (len(program_node.predicate) + len(program_node.transform))
                    pred_cascade_complexity_scores[ind][pred_ind] += cascade_complexity_score
                except AssertionError:
                    pred_cascade_complexity_scores[ind][pred_ind] += 0 # invalid programs get a zero score.

        # edit distance complexity feature not affected by the skipped_programs.
        instance_complexity[ind][5] = np.mean([normalized_levenshtein(i,o) for i,o in zip(rec["inputs"], rec["outputs"])])
        # effective cascade length after skipping degenerate programs.
        instance_complexity[ind][4] = len(rec['programs'])-len(rec["skipped_programs"]) # effective cascade length
        # no. of words that have changed.
        instance_complexity[ind][6] = sum((i != o) for i,o in zip(rec['inputs'], rec['outputs']))/5

        # effective relation counts after removing degenerate programs.
        for link in rec["bfcc_dag"]:
            if link[0] in rec["skipped_programs"] or link[-1] in rec["skipped_programs"]: continue
            if link[1][0] == "N": pass
            elif link[1][0] == "B" and link[0] < link[-1]: # bleeding.
                instance_complexity[ind][0] += 1
            elif link[1][0] == "F" and link[0] < link[-1]: # feeding.
                instance_complexity[ind][1] += 1
            elif link[1][0] == "B" and link[0] > link[-1]: # counter-bleeding.
                instance_complexity[ind][2] += 1
            elif link[1][0] == "F" and link[0] > link[-1]: # counter-feeding.
                instance_complexity[ind][3] += 1

    return instance_complexity, gt_cascade_complexity_scores, pred_cascade_complexity_scores

def pick_highest_reward_prog_seq(programs: list[str], inputs: list[str], outputs: list[str], max_programs: int=5, vocab_chars=None):
    progs_and_rewards = []
    for prog_seq_string in programs:
        pred_cascade = try_ast_literal_eval(prog_seq_string)
        pred_outputs, valid_programs, invalid_programs = eval_outputs(inputs=inputs, outputs=outputs, pred_cascade=pred_cascade, vocab_chars=vocab_chars, max_programs=max_programs)
        progs_and_rewards.append((pred_cascade[:max_programs], reward(pred_outputs, inputs, outputs)))
    progs_and_rewards = sorted(progs_and_rewards, reverse=True, key=lambda x: x[1]) # sort by reward.
    # print(progs_and_rewards[0][0])

    return progs_and_rewards[0][0]    

def pick_highest_reward_outputs(programs: list[str], inputs: list[str], outputs: list[str], max_programs: int=5, vocab_chars=None):
    pred_outputs_and_rewards = []
    for prog_seq_string in programs:
        pred_cascade = try_ast_literal_eval(prog_seq_string)
        pred_outputs, valid_programs, invalid_programs = eval_outputs(inputs=inputs, outputs=outputs, pred_cascade=pred_cascade, vocab_chars=vocab_chars, max_programs=max_programs)
        pred_outputs_and_rewards.append((pred_outputs, reward(pred_outputs, inputs, outputs)))
    pred_outputs_and_rewards = sorted(pred_outputs_and_rewards, reverse=True, key=lambda x: x[1]) # sort by reward.
    # print(progs_and_rewards[0][0])

    return pred_outputs_and_rewards[0][0]    

# main
if __name__ == "__main__":
    from src.data_generation.utils import read_jsonl

    args = parse_args()
    
    vocab_chars = args.vocab
    model_predictions_path = args.model_predictions_path
    metric_values_path = os.path.join("metric_values", pathlib.Path(model_predictions_path).stem+".json")

    for extraction_type,extract_method in {
        "extract first code block": extract_first_python_block,
        "extract last code block": extract_last_python_block,
    }.items():
        # try:
        data = load_model_predictions_multi(model_predictions_path, extract_method=extract_method)
        best_predictions = [pick_highest_reward_prog_seq(programs=rec['predictions'], inputs=rec['inputs'], outputs=rec['outputs'], max_programs=args.max_programs, vocab_chars=vocab_chars) for rec in data]
        # print([len(p) for p in best_predictions])
        # except KeyError:
        #     data = load_model_predictions(model_predictions_path, extract_method=extract_method)
        #     best_predictions = [try_ast_literal_eval(rec['predictions'][0]) for rec in data]
        # [try_ast_literal_eval(rec['prediction']) for rec in data]
        # exit() 
        instance_complexities, gt_cascade_complexity_scores, pred_cascade_complexity_scores = get_instance_complexities(data, max_programs=args.max_programs, vocab_chars=vocab_chars)
        # print(instance_complexities[0])

        mean_pass_at_1, inst_level_passing_progs = eval_mean_pass_at_1(data, best_predictions, vocab_chars=vocab_chars, max_programs=args.max_programs)
        reward_at_1, inst_level_rewards, valid_program_rate = eval_rewards(data, best_predictions, vocab_chars=vocab_chars, max_programs=args.max_programs)

        metric_values = []
        for passing, reward_val in zip(inst_level_passing_progs, inst_level_rewards):
            if passing == 1: assert reward_val == 1, f"reward: {reward_val}, passing: {passing}"
            if reward_val < 1: assert passing == 0
            metric_values.append({"passing": passing, "reward": reward_val})
        with open(metric_values_path, "w") as f:
            json.dump(metric_values, f, indent=4)

        # sampled_predictions = [[ast.literal_eval(pred) for pred in rec['prediction']] for rec in data] 
        # pass_at_1 = eval_mean_pass_at_k(data, sampled_predictions, k=1) 
        # pass_at_3 = eval_mean_pass_at_k(data, sampled_predictions, k=3)
        # pass_at_5 = eval_mean_pass_at_k(data, sampled_predictions, k=5)
        # pass_at_10 = eval_mean_pass_at_k(data, sampled_predictions, k=10)
        
        print(extraction_type)
        print(f"pass@1: {mean_pass_at_1:.4f}")
        print(f"reward@1: {reward_at_1:.4f}")
        print(f"valid program rate: {valid_program_rate:.4f}")
        print(f"gt complexity: {np.mean(gt_cascade_complexity_scores):.2f}")
        print(f"pred complexity: {np.mean([safe_mean(p) for p in pred_cascade_complexity_scores]):.2f}")
        
        perf_by_cascade_length = defaultdict(lambda: [])
        perf_by_num_relns = defaultdict(lambda: [])
        perf_by_num_uniq_relns = defaultdict(lambda: [])
        perf_by_io_dist = defaultdict(lambda: [])

        cascade_length_dist = defaultdict(lambda: 0)
        num_relns_dist = defaultdict(lambda: 0)
        io_dist = defaultdict(lambda: 0)
        
        for i in range(len(data)):
            effective_cascade_length = instance_complexities[i][4]
            num_relns = sum(instance_complexities[i][:4])
            num_uniq_relns = sum([int(rel_count > 0) for rel_count in instance_complexities[i][:4]])
            io_distance = instance_complexities[i][5]
            
            perf_by_cascade_length[effective_cascade_length].append(inst_level_passing_progs[i])
            perf_by_num_relns[num_relns].append(inst_level_passing_progs[i])
            perf_by_num_uniq_relns[num_uniq_relns].append(inst_level_passing_progs[i])
            perf_by_io_dist[classify_iod(io_distance)].append(inst_level_passing_progs[i])
            # assert effective_cascade_length == instance_complexities[i][4]
            cascade_length_dist[effective_cascade_length] += 1
            num_relns_dist[num_relns] += 1
            io_dist[classify_iod(io_distance)] += 1

        perf_by_cascade_length = dict(perf_by_cascade_length)
        for k in perf_by_cascade_length:
            perf_by_cascade_length[k] = (round(sum(perf_by_cascade_length[k])/len(perf_by_cascade_length[k]),4),len(perf_by_cascade_length[k]))
        for k in perf_by_num_relns:
            perf_by_num_relns[k] = (round(sum(perf_by_num_relns[k])/len(perf_by_num_relns[k]),4),len(perf_by_num_relns[k]))
        for k in perf_by_io_dist:
            perf_by_io_dist[k] = (round(sum(perf_by_io_dist[k])/len(perf_by_io_dist[k]),4),len(perf_by_io_dist[k]))
        for k in perf_by_num_uniq_relns:
            perf_by_num_uniq_relns[k] = (round(sum(perf_by_num_uniq_relns[k])/len(perf_by_num_uniq_relns[k]),4),len(perf_by_num_uniq_relns[k]))

        cascade_length_dist = dict(cascade_length_dist)
        num_relns_dist = dict(num_relns_dist)
        io_dist = dict(io_dist)
    
        perf_by_cascade_length = dict(perf_by_cascade_length)
        perf_by_num_relns = dict(perf_by_num_relns)
        perf_by_io_dist = dict(perf_by_io_dist)
        
        print("Perf. by Cascade Length:", sort_dict_by_key(perf_by_cascade_length))
        print("Perf. by # Relations:", sort_dict_by_key(perf_by_num_relns))
        print("Perf. by # Unique Relations (e.g 1101 category = 3):", sort_dict_by_key(perf_by_num_uniq_relns))
        print("Perf. by IOD:", {k: perf_by_io_dist.get(k) for k in ['IOD < 0.2', '0.2 <= IOD < 0.4', '0.4 <= IOD < 0.6', '0.6 <= IOD < 0.8']})

        # print(cascade_length_dist)
        # print(num_relns_dist)
        # print(io_dist)

    # print("pass@3:", pass_at_3) 
    # print("pass@5:", pass_at_5)
    # print("pass@10:", pass_at_10)