benchmarking: refresh run logs and shim

benchmarking/shim.py

@@ -12,13 +12,14 @@ reverse-proxies everything else untouched:
   content[{token, logprob, top_logprobs}]; harness expects legacy
   {token_logprobs[], top_logprobs[]}. Translated field-for-field.
 - /v1/chat/completions: system messages are folded into the following user
-  turn (Gemma supports only user/model roles; same info, valid roles).
+  turn, but only for families without a system role (Gemma). See families.py.
 - Sidecar: every completions request + raw response (incl. reasoning_content)
   is appended to thinking-log.jsonl for later inspection. Scoring untouched.
 
 Usage: python3 shim.py [upstream] [port]
 """
 import json
+import os
 import sys
 import time
 import urllib.request
@@ -26,7 +27,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
 UPSTREAM = sys.argv[1] if len(sys.argv) > 1 else "http://host.docker.internal:8080"
 PORT = int(sys.argv[2]) if len(sys.argv) > 2 else 8081
-THINK_LOG = "benchmarking/thinking-log.jsonl"
+THINK_LOG = os.path.join(os.path.dirname(os.path.abspath(__file__)), "thinking-log.jsonl")
 
 
 def _up(method, path, body=None):
@@ -75,6 +76,18 @@ def _info():
             "chat_template": tmpl}
 
 
+def _family():
+    sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+    import families
+    try:
+        _, raw = _up("GET", "/props")
+        tmpl = json.loads(raw).get("chat_template", "")
+    except Exception:
+        tmpl = ""
+    name = families.detect(tmpl, sys.argv[3] if len(sys.argv) > 3 else "")
+    sys.stderr.write(f"shim: family={name}\n")
+    return families.profile(name)
+
 def _fix_logprobs(raw):
     try:
         obj = json.loads(raw)
@@ -110,7 +123,7 @@ def _fold_system(body):
 
 
 INFO = _info()
-
+FAMILY = _family()
 
 class H(BaseHTTPRequestHandler):
     protocol_version = "HTTP/1.1"
@@ -153,7 +166,7 @@ class H(BaseHTTPRequestHandler):
                             {"tokens": body.get("tokens", [])})
             text = json.loads(raw).get("content", "")
             return self._send(code, {"prompt": text})
-        if self.path == "/v1/chat/completions":
+        if self.path == "/v1/chat/completions" and FAMILY["fold_system"]:
             body = _fold_system(body)
         try:
             code, raw = _up("POST", self.path, body)