evaluation/context_eval_baselines.py

# Description: Evaluate baselines on the context dataset
#
# Usage: python context_eval_baselines.py

import argparse
import os
import subprocess
from os.path import join as pjoin

RESULTS_ROOT = './results/baselines/cat'
EVAL_ROOT = './eval_context/results'

GT_PATH = './gts/context/context/cat.json'
EVAL_DIMENSION = 's'
METHODS = list(dict.fromkeys([
    'CenterNet2', 'FasterRCNN', 'YOLO', 'UIED', 'Xianyu',
    # 'GPT4V-E2E', 'Gemini-E2E',
    'OmniParser', 'CogVLM', 'Seed-E2E', 'Claude4_5-sonnet-E2E',
    'Gemini-2_5-pro-E2E', 'GPT5-E2E', 'O4-E2E',
    'Qwen3-VL-plus-E2E', 'internVL-E2E']))
LLM_METHODS = list(dict.fromkeys([
    # 'GPT4V-E2E', 'Gemini-E2E',
    'OmniParser', 'CogVLM', 'Seed-E2E', 'Claude4_5-sonnet-E2E',
    'Gemini-2_5-pro-E2E', 'GPT5-E2E', 'O4-E2E',
    'Qwen3-VL-plus-E2E', 'internVL-E2E']))


evaluator_script = './context_eval.py'


def validate_unique_methods(methods):
    seen = set()
    duplicates = []
    for method in methods:
        if method in seen:
            duplicates.append(method)
        seen.add(method)
    if duplicates:
        raise ValueError(f"Duplicate methods in context baseline config: {duplicates}")


def main(args):
    validate_unique_methods(METHODS)
    validate_unique_methods(LLM_METHODS)

    format = 'det'
    task = 'semantics'
    for iou in [0.75, 0.8, 0.85, 0.9, 0.95]:
        for method in METHODS:
            result_path = pjoin(RESULTS_ROOT, format, task, method + '.json')
            if not os.path.exists(result_path):
                continue
            eval_result_path = pjoin(EVAL_ROOT, method + f'@{iou:.2f}.csv')
            if os.path.exists(eval_result_path):
                print(f'{eval_result_path} exists, skipping...')
                continue
            cli = [
                'python',
                evaluator_script,
                '-d',
                EVAL_DIMENSION,
                '-g',
                GT_PATH,
                '-p',
                result_path,
                '-o',
                eval_result_path,
                '-i',
                str(iou),
            ]
            if method not in LLM_METHODS:
                cli.append('-n')
            print(' '.join(cli))
            subprocess.run(cli, check=True)


def build_parser():
    return argparse.ArgumentParser()


if __name__ == '__main__':
    parser = build_parser()
    args = parser.parse_args()
    main(args)