File size: 8,426 Bytes
09d4173 b6b0992 09d4173 3c08bbf 09d4173 3c08bbf 09d4173 b6b0992 09d4173 b6b0992 09d4173 b6b0992 09d4173 b6b0992 09d4173 b6b0992 09d4173 3c08bbf 09d4173 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 | """CLI for the independent adapter process."""
import argparse
import logging
import math
import os
from .labels import DEFAULT_LABEL_SCHEME, LABEL_SCHEME_ENV, LABEL_SCHEMES
logger = logging.getLogger(__name__)
def positive_temperature(text: str) -> float:
"""argparse type for --default-temperature: a finite number above zero."""
try:
value = float(text)
except ValueError:
raise argparse.ArgumentTypeError(f"not a number: {text!r}") from None
if not math.isfinite(value) or value <= 0:
raise argparse.ArgumentTypeError("must be finite and greater than zero")
return value
def temperature_map(text: str) -> dict[str, float]:
"""argparse type for --temperature-by-type: 'choice=1.6,noul=1.2,score=1.8'."""
result = {}
for part in filter(None, (item.strip() for item in text.split(","))):
key, sep, value = part.partition("=")
key = key.strip()
if not sep or key not in ("choice", "noul", "score") or key in result:
raise argparse.ArgumentTypeError(
f"expected type=T pairs for choice, noul, score; got {part!r}"
)
result[key] = positive_temperature(value.strip())
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--engine-url", default="http://127.0.0.1:30000")
parser.add_argument("--model", required=True, help="Engine's served model name")
parser.add_argument("--alias", default="jev-latest")
parser.add_argument("--host", default="127.0.0.1")
parser.add_argument("--port", type=int, default=30120)
parser.add_argument("--timeout", type=float, default=120.0)
parser.add_argument("--max-concurrency", type=int, default=32)
parser.add_argument(
"--tokenizer-model", help="Optional official text tokenizer repo"
)
parser.add_argument("--tokenizer-revision", help="Immutable tokenizer revision")
parser.add_argument(
"--default-temperature",
type=positive_temperature,
default=os.environ.get("JEV_DEFAULT_TEMPERATURE", "1.0"),
help="Temperature applied when a request omits options.temperature "
"(label logits are divided by T before softmax; an explicit request "
"value wins). Env: JEV_DEFAULT_TEMPERATURE. Default 1.0.",
)
parser.add_argument(
"--temperature-by-type",
type=temperature_map,
default=os.environ.get("JEV_TEMPERATURE_BY_TYPE", ""),
help="Per-answer-type defaults used instead of --default-temperature when "
"a request omits options.temperature, e.g. 'choice=1.6,noul=1.2,score=1.8'. "
"Types not listed use --default-temperature. Env: JEV_TEMPERATURE_BY_TYPE.",
)
parser.add_argument(
"--prompt-wording",
choices=("served", "native"),
default=os.environ.get("JEV_PROMPT_WORDING", "served"),
help="'served' (default, unchanged behaviour): this adapter's own "
"prompt text (Context:/.../Options: A: name: description). 'native': "
"the jevbench-hard runner's own contract instead (run_suites_rotation.py: "
"State:/Question:/Options: A. name: description), which the runner's "
"own ablation measured as worth about +6 points on jevbench-hard by "
"wording alone. Requires the engine tokenizer's first labels to be "
"canonical single-token A-Z (true for essentially all "
"English-vocabulary tokenizers; jev_adapter.protocol.build_prompt "
"raises per-request otherwise). With --tokenizer-model/"
"--tokenizer-revision also set, the adapter additionally extracts the "
"model's default system-prompt text from a dedicated HF-template load "
"of that same pin and prepends it explicitly, since a mistral-common "
"native tokenizer (the default for Ministral) never auto-injects it. "
"See --native-system-prompt to disable this. Env: JEV_PROMPT_WORDING.",
)
parser.add_argument(
"--native-system-prompt",
choices=("auto", "none"),
default=os.environ.get("JEV_NATIVE_SYSTEM_PROMPT", "auto"),
help="Only takes effect with --prompt-wording native; ignored "
"otherwise. 'auto' (default, unchanged behaviour): with "
"--tokenizer-model/--tokenizer-revision also set, extract the "
"model's default system-prompt text from the pinned tokenizer's chat "
"template (see --prompt-wording) and prepend it as an explicit "
"system message; warns at startup if no tokenizer pin is given or "
"the template defines no default system prompt. 'none': render the "
"native wording with no system message at all -- skips the "
"extraction entirely and never warns about a missing prompt (useful "
"when the extra system-prompt tokens are not wanted, e.g. the "
"Ministral default is 531 tokens, about 80%% of a typical request). "
"Env: JEV_NATIVE_SYSTEM_PROMPT.",
)
parser.add_argument(
"--label-scheme",
choices=LABEL_SCHEMES,
default=os.environ.get(LABEL_SCHEME_ENV) or DEFAULT_LABEL_SCHEME,
help="Answer labels for questions with more than 26 options. 'upper' "
"(default): A-Z, then AA, AB, ... (uppercase only; labels that are not "
"single tokens are skipped). 'served': the original A-Z, a-z, 0-9, "
"then AA, ... order (pre-v2.2 serving). Both give A-Z to "
"questions with at most 26 options. Reported in /v1/models and in "
"response metadata. Env: JEV_LABEL_SCHEME.",
)
# argparse runs the type converter on the string default too, so a bad
# JEV_DEFAULT_TEMPERATURE fails at startup with the same message.
args = parser.parse_args()
if args.label_scheme not in LABEL_SCHEMES:
parser.error(
f"JEV_LABEL_SCHEME must be one of {', '.join(LABEL_SCHEMES)}; "
f"got {args.label_scheme!r}"
)
if args.timeout <= 0 or args.max_concurrency < 1:
parser.error("timeout and max-concurrency must be positive")
if bool(args.tokenizer_model) != bool(args.tokenizer_revision):
parser.error("tokenizer-model and tokenizer-revision must be supplied together")
import uvicorn
from .server import create_app
from .sglang import SGLangBackend
native_tokenizer = None
native_system_prompt = None
if args.tokenizer_model:
from .native_tokenizer import NativeTokenizer
native_tokenizer = NativeTokenizer.from_pretrained(
args.tokenizer_model, args.tokenizer_revision
)
if args.prompt_wording == "native" and args.native_system_prompt == "auto":
native_system_prompt = NativeTokenizer.native_default_system_prompt(
args.tokenizer_model, args.tokenizer_revision
)
if (
args.prompt_wording == "native"
and args.native_system_prompt == "auto"
and native_system_prompt is None
):
logger.warning(
"prompt-wording=native without a default system prompt (pass "
"--tokenizer-model/--tokenizer-revision to extract and prepend the "
"model's own default system message, or the model's template "
"defines none); serving the native wording text without it."
)
logger.info(
"prompt_wording=%s native_system_prompt=%s (%s) label_scheme=%s",
args.prompt_wording,
args.native_system_prompt,
"system message attached"
if native_system_prompt
else "no system message attached",
args.label_scheme,
)
backend = SGLangBackend(
args.engine_url,
args.model,
api_key=os.environ.get("SGLANG_API_KEY"),
timeout=args.timeout,
max_connections=args.max_concurrency,
native_tokenizer=native_tokenizer,
prompt_wording=args.prompt_wording,
native_system_prompt=native_system_prompt,
label_scheme=args.label_scheme,
)
app = create_app(
backend,
api_key=os.environ.get("JEV_API_KEY"),
max_concurrency=args.max_concurrency,
alias=args.alias,
default_temperature=args.default_temperature,
temperature_by_type=args.temperature_by_type,
prompt_wording=args.prompt_wording,
)
uvicorn.run(app, host=args.host, port=args.port)
if __name__ == "__main__":
main()
|