File size: 4,021 Bytes
26de23c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | diff --git a/jevbench/adapters/__init__.py b/jevbench/adapters/__init__.py
index c71d476..6aacaa8 100644
--- a/jevbench/adapters/__init__.py
+++ b/jevbench/adapters/__init__.py
@@ -17,6 +17,7 @@ Raw provider responses are preserved by the runner, not here.
from .base import DecisionResult # noqa: F401
from .typesafe import TypeSafeAdapter # noqa: F401
+from .openjev_image import OpenJevImageAdapter # noqa: F401
from .openai_compat import OpenAICompatAdapter # noqa: F401
from .systemone_list import SystemOneListAdapter # noqa: F401
from .gradio_space import GradioSpaceAdapter # noqa: F401
diff --git a/jevbench/adapters/openjev_image.py b/jevbench/adapters/openjev_image.py
new file mode 100644
index 0000000..7aab898
--- /dev/null
+++ b/jevbench/adapters/openjev_image.py
@@ -0,0 +1,30 @@
+"""OpenJev image Decisions transport; normalization is disclosed, never fitted.
+
+Task state: {"text": "An image: <<IMG>>", "image_data": "<base64>"}.
+Keeping the image bytes in the task binds them into the existing dataset hash.
+Uses the unchanged JevBench scoring/ledger/raw-response path.
+"""
+from .typesafe import TypeSafeAdapter
+
+
+class OpenJevImageAdapter(TypeSafeAdapter):
+ name = "openjev_image"
+
+ def build_request(self, task):
+ state = task.state
+ if not isinstance(state, dict) or not isinstance(state.get("image_data"), str):
+ raise ValueError("openjev_image requires state.image_data as base64 or a data URI")
+ if not isinstance(state.get("text", ""), str):
+ raise ValueError("state.text must be text")
+ body = super().build_request(task)
+ body["state"] = state.get("text", "An image: <<IMG>>")
+ body["image_data"] = state["image_data"]
+ return body
+
+ def run(self, task):
+ result = super().run(task)
+ result.probs_source = "normalized_entailment_v1"
+ if result.ok and result.raw.get("probability_method") != "normalized_entailment_v1":
+ result.ok = False
+ result.error = "Endpoint did not declare normalized_entailment_v1"
+ return result
diff --git a/jevbench/cli.py b/jevbench/cli.py
index da38bea..fb79845 100644
--- a/jevbench/cli.py
+++ b/jevbench/cli.py
@@ -23,7 +23,7 @@ import time
from .adapters import (GradioSpaceAdapter, LocalOpenJevAdapter, NeedleLocalAdapter,
OpenAICompatAdapter, SystemOneListAdapter,
RemoteInprocAdapter, SemIfDirectAdapter, SgSystemOneAdapter, So1DeciderAdapter,
- TypeSafeAdapter, DjevAdapter,
+ TypeSafeAdapter, OpenJevImageAdapter, DjevAdapter,
LayaLocalAdapter, Gliner2LocalAdapter, VerdictLocalAdapter, PawLocalAdapter,
ClassifierDevAdapter, CertoLocalAdapter, QwenFlashLinearAdapter)
from .budget import Ledger
@@ -55,7 +55,7 @@ def cmd_run(args) -> int:
if args.limit:
tasks = tasks[: args.limit]
- kinds = {"typesafe": TypeSafeAdapter, "systemone_list": SystemOneListAdapter,
+ kinds = {"typesafe": TypeSafeAdapter, "openjev_image": OpenJevImageAdapter, "systemone_list": SystemOneListAdapter,
"gradio_space": GradioSpaceAdapter, "local_openjev": LocalOpenJevAdapter,
"openai_compat": OpenAICompatAdapter,
"needle_local": NeedleLocalAdapter,
@@ -161,7 +161,7 @@ def main(argv=None) -> int:
p_run = sub.add_parser("run", help="run the benchmark over tasks")
p_run.add_argument("--tasks", required=True)
p_run.add_argument("--adapter", required=True,
- choices=["typesafe", "systemone_list", "gradio_space",
+ choices=["typesafe", "openjev_image", "systemone_list", "gradio_space",
"local_openjev", "openai_compat", "needle_local",
"semif_direct", "so1_decider", "remote_inproc",
"sg_system_one", "djev", "laya_local", "gliner2_local",
|