diff --git a/jevbench/adapters/__init__.py b/jevbench/adapters/__init__.py index c71d476..6aacaa8 100644 --- a/jevbench/adapters/__init__.py +++ b/jevbench/adapters/__init__.py @@ -17,6 +17,7 @@ Raw provider responses are preserved by the runner, not here. from .base import DecisionResult # noqa: F401 from .typesafe import TypeSafeAdapter # noqa: F401 +from .openjev_image import OpenJevImageAdapter # noqa: F401 from .openai_compat import OpenAICompatAdapter # noqa: F401 from .systemone_list import SystemOneListAdapter # noqa: F401 from .gradio_space import GradioSpaceAdapter # noqa: F401 diff --git a/jevbench/adapters/openjev_image.py b/jevbench/adapters/openjev_image.py new file mode 100644 index 0000000..7aab898 --- /dev/null +++ b/jevbench/adapters/openjev_image.py @@ -0,0 +1,30 @@ +"""OpenJev image Decisions transport; normalization is disclosed, never fitted. + +Task state: {"text": "An image: <>", "image_data": ""}. +Keeping the image bytes in the task binds them into the existing dataset hash. +Uses the unchanged JevBench scoring/ledger/raw-response path. +""" +from .typesafe import TypeSafeAdapter + + +class OpenJevImageAdapter(TypeSafeAdapter): + name = "openjev_image" + + def build_request(self, task): + state = task.state + if not isinstance(state, dict) or not isinstance(state.get("image_data"), str): + raise ValueError("openjev_image requires state.image_data as base64 or a data URI") + if not isinstance(state.get("text", ""), str): + raise ValueError("state.text must be text") + body = super().build_request(task) + body["state"] = state.get("text", "An image: <>") + body["image_data"] = state["image_data"] + return body + + def run(self, task): + result = super().run(task) + result.probs_source = "normalized_entailment_v1" + if result.ok and result.raw.get("probability_method") != "normalized_entailment_v1": + result.ok = False + result.error = "Endpoint did not declare normalized_entailment_v1" + return result diff --git a/jevbench/cli.py b/jevbench/cli.py index da38bea..fb79845 100644 --- a/jevbench/cli.py +++ b/jevbench/cli.py @@ -23,7 +23,7 @@ import time from .adapters import (GradioSpaceAdapter, LocalOpenJevAdapter, NeedleLocalAdapter, OpenAICompatAdapter, SystemOneListAdapter, RemoteInprocAdapter, SemIfDirectAdapter, SgSystemOneAdapter, So1DeciderAdapter, - TypeSafeAdapter, DjevAdapter, + TypeSafeAdapter, OpenJevImageAdapter, DjevAdapter, LayaLocalAdapter, Gliner2LocalAdapter, VerdictLocalAdapter, PawLocalAdapter, ClassifierDevAdapter, CertoLocalAdapter, QwenFlashLinearAdapter) from .budget import Ledger @@ -55,7 +55,7 @@ def cmd_run(args) -> int: if args.limit: tasks = tasks[: args.limit] - kinds = {"typesafe": TypeSafeAdapter, "systemone_list": SystemOneListAdapter, + kinds = {"typesafe": TypeSafeAdapter, "openjev_image": OpenJevImageAdapter, "systemone_list": SystemOneListAdapter, "gradio_space": GradioSpaceAdapter, "local_openjev": LocalOpenJevAdapter, "openai_compat": OpenAICompatAdapter, "needle_local": NeedleLocalAdapter, @@ -161,7 +161,7 @@ def main(argv=None) -> int: p_run = sub.add_parser("run", help="run the benchmark over tasks") p_run.add_argument("--tasks", required=True) p_run.add_argument("--adapter", required=True, - choices=["typesafe", "systemone_list", "gradio_space", + choices=["typesafe", "openjev_image", "systemone_list", "gradio_space", "local_openjev", "openai_compat", "needle_local", "semif_direct", "so1_decider", "remote_inproc", "sg_system_one", "djev", "laya_local", "gliner2_local",