| import os |
| import subprocess |
| from concurrent.futures import ThreadPoolExecutor |
|
|
|
|
| os.environ['split'] = 'genre' |
| MIN_APPS_PER_CAT = 5 |
|
|
| split = os.getenv('split', '') |
| suffix_split = f'-{split}' if split else '' |
|
|
| GT = f'../gts/det/interactable/{split}.json' |
| RESULT_DIR = f'./results{suffix_split}' |
| RAW_RESULT_DIR = f'{RESULT_DIR}/raw' |
| GUIDANCES = [ |
| |
| |
| f'../results/ours/realfse_union3_gemini15pro_ape_d_object_bbox_gpu0123_i_bf.json', |
| |
| f'../results/baselines/{split}/det/interactable/YOLO.json', |
| |
| f'../results/baselines/{split}/det/interactable/internVL-E2E.json', |
| f'../results/baselines/{split}/det/interactable/Claude4_5-sonnet-E2E.json' |
| ] |
|
|
| CAT_COMPARE_GUIDANCE = f'../results/ours/realfse_union3_gemini15pro_ape_d_object_bbox_gpu0123_i_bf.json' |
| BASELINE_GUIDANCE = f'../results/baselines/{split}/det/interactable/YOLO.json' |
|
|
| EVAL_UTILS_ROOT = './eval_utils' |
|
|
|
|
| GEN_INTERACT_SCRIPT = f'{EVAL_UTILS_ROOT}/gen_interact.py' |
| TIME_EVAL_SCRIPT = f'{EVAL_UTILS_ROOT}/time_evaluation.py' |
| CAT_EVAL_SCRIPT = f'{EVAL_UTILS_ROOT}/cat_evaluation.py' |
| AVG_SCRIPT = f'{EVAL_UTILS_ROOT}/time_avg_metric.py' |
| TIME_PLOT_SCRIPT = f'{EVAL_UTILS_ROOT}/time_plot.py' |
| CAT_PLOT_SCRIPT = f'{EVAL_UTILS_ROOT}/cat_plot.py' |
| TO_LATEX_SCRIPT = f'{EVAL_UTILS_ROOT}/cat_2latex.py' |
|
|
| ROUND = 30 |
|
|
| os.environ['ROUND'] = str(ROUND) |
|
|
| os.makedirs(RAW_RESULT_DIR, exist_ok=True) |
|
|
|
|
| def generate_interact(inter_path, guide=None): |
| if not os.path.exists(inter_path + '.json'): |
| cli = f'python {GEN_INTERACT_SCRIPT} -a {GT} -o {inter_path + ".json"}' |
| if guide: |
| cli += f' -g {guide}' |
| print(cli) |
| subprocess.run(cli.split()) |
| else: |
| print(f'{inter_path}.json already exists') |
| return inter_path |
|
|
|
|
| def time_eval_interact(inter_path, guide=None): |
| cli = f'python {TIME_EVAL_SCRIPT} -a {GT} -i {inter_path + ".json"} -o {inter_path + ".csv"}' |
| if guide: |
| cli += f' -p {guide}' |
| print(cli) |
| subprocess.run(cli.split()) |
| return inter_path |
|
|
|
|
| def avg_metric(series): |
| cli = f'python {AVG_SCRIPT} -s {series}' |
| print(cli) |
| subprocess.run(cli.split()) |
|
|
|
|
| def cat_eval(series, guide=None): |
| cli = f'python {CAT_EVAL_SCRIPT} -a {GT} -is {RAW_RESULT_DIR}/{series} -o {RESULT_DIR}/cat_eval/{series}' |
| if guide: |
| cli += f' -p {guide}' |
| print(cli) |
| subprocess.run(cli.split()) |
|
|
|
|
| def cat_plot(): |
| cli = (f'python {CAT_PLOT_SCRIPT} ' |
| f'-r {RESULT_DIR}/cat_eval/{os.path.basename(BASELINE_GUIDANCE).split(".")[0]} ' |
| f'-g {RESULT_DIR}/cat_eval/{os.path.basename(CAT_COMPARE_GUIDANCE).split(".")[0]} ' |
| f'-t coverage_rate ' |
| f'-m {MIN_APPS_PER_CAT}' |
| ) |
| print(cli) |
| subprocess.run(cli.split()) |
|
|
| |
| |
| |
| |
| |
| |
|
|
| |
| |
| |
| |
| cli = (f'python {CAT_PLOT_SCRIPT} ' |
| f'-r {RESULT_DIR}/cat_eval/{os.path.basename(BASELINE_GUIDANCE).split(".")[0]} ' |
| f'-g {RESULT_DIR}/cat_eval/{os.path.basename(CAT_COMPARE_GUIDANCE).split(".")[0]} ' |
| f'-t effective_interacts_cnt ' |
| f'-m {MIN_APPS_PER_CAT}' |
| ) |
| print(cli) |
| subprocess.run(cli.split()) |
|
|
|
|
| def main(): |
| with ThreadPoolExecutor(max_workers=16) as executor: |
| futures = [] |
| for i in range(ROUND): |
| random_inter_path = os.path.join(RAW_RESULT_DIR, 'random' + str(i)) |
| futures.append(executor.submit(generate_interact, random_inter_path)) |
| for future in futures: |
| inter_path = future.result() |
| executor.submit(time_eval_interact, inter_path) |
|
|
| avg_metric('random') |
| cat_eval('random') |
|
|
|
|
| for guide in GUIDANCES: |
| with ThreadPoolExecutor(max_workers=16) as executor: |
| futures = [] |
| for i in range(ROUND): |
| guide_inter_path = os.path.join(RAW_RESULT_DIR, os.path.basename(guide).split('.')[0] + str(i)) |
| futures.append(executor.submit(generate_interact, guide_inter_path, guide)) |
| for future in futures: |
| inter_path = future.result() |
| executor.submit(time_eval_interact, inter_path, guide) |
|
|
| avg_metric(os.path.basename(guide).split(".")[0]) |
| cat_eval(os.path.basename(guide).split(".")[0], guide) |
| pass |
|
|
| cli = f'python {TIME_PLOT_SCRIPT}' |
| print(cli) |
| subprocess.run(cli.split()) |
|
|
| cat_plot() |
|
|
| cli = f'python {TO_LATEX_SCRIPT} -gt {GT} -m {MIN_APPS_PER_CAT}' |
| print(cli) |
| subprocess.run(cli.split()) |
|
|
|
|
| if __name__ == '__main__': |
| main() |
|
|
