#!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # Copyright 2026 DevelopingDad """Self-contained native API diagnostics; no external datasets or dependencies.""" import argparse import json import time import urllib.request MODEL = 'DevelopingDad/Gemma4-12B-it-Fast' def post(base, payload): request = urllib.request.Request(base.rstrip('/') + '/chat/completions', data=json.dumps(payload).encode(), headers={'Content-Type':'application/json'}) return urllib.request.urlopen(request, timeout=180) def request(messages, **kwargs): result = {'model': MODEL, 'messages': messages, 'temperature':0, 'seed':91027, 'max_tokens':256, 'chat_template_kwargs':{'enable_thinking':False}} result.update(kwargs) return result def evaluate(base): results = {'schema':'gemma4-fast-generic-api-screen/v1', 'hardware':'one NVIDIA DGX Spark GB10', 'runtime':'vLLM 0.31.0, weight-only target, MTP8, FP8 KV', 'scope':'bounded native API diagnostic; not a standardized quality benchmark', 'sampling':{'temperature':0,'seed':91027}, 'checks':[], 'generation':[]} prompts = [ ('explanation', 'Explain how condensation forms on a cold glass. Include a concrete example and explain the role of temperature.'), ('comparison', 'Compare a binary search tree with a hash table. Explain lookup, insertion, ordering and a suitable use case for each.'), ('code', 'Implement a Python function that merges two sorted integer lists. Include type hints, a docstring, and explain the time and space complexity.'), ] for name, prompt in prompts: body = request([{'role':'user','content':prompt}], max_tokens=512, stream=True, stream_options={'include_usage':True}, cache_salt='generic-public-screen-'+str(time.time_ns())) start = time.monotonic(); first = None; usage = {}; finish = None; chunks = []; reasoning = [] with post(base, body) as response: for line in response: if not line.startswith(b'data: '): continue raw = line[6:].strip() if raw == b'[DONE]': break item = json.loads(raw) if item.get('usage'): usage = item['usage'] for choice in item.get('choices', []): delta = choice.get('delta', {}) if delta.get('content'): if first is None: first = time.monotonic() chunks.append(delta['content']) if delta.get('reasoning_content'): reasoning.append(delta['reasoning_content']) if choice.get('finish_reason'): finish = choice['finish_reason'] end = time.monotonic(); tokens = usage.get('completion_tokens',0) # Completion usage can include final control tokens; this is an explicitly # approximate decode rate, not the old conservative delivery metric. results['generation'].append({'case':name,'max_tokens':512,'finish_reason':finish, 'prompt_tokens':usage.get('prompt_tokens'), 'completion_tokens':tokens, 'first_content_s':None if first is None else round(first-start,4), 'elapsed_s':round(end-start,4), 'approximate_completion_tokens_per_s_after_first_content':None if first is None else round(tokens/max(end-first,1e-9),3), 'nonempty_content':bool(''.join(chunks).strip()),'unexpected_reasoning':bool(reasoning)}) # Explicit reasoning and a completion allowance exceeding its reasoning cap. body = request([{'role':'user','content':'Calculate 137 * 24 - 89. Give the final integer.'}], max_tokens=4096, thinking_token_budget=2048, chat_template_kwargs={'enable_thinking':True}) start = time.monotonic() with post(base,body) as response: item=json.load(response) answer=item['choices'][0]['message'].get('content') or '' results['checks'].append({'case':'reasoning_arithmetic','passed':'3199' in answer.replace(',',''), 'finish_reason':item['choices'][0]['finish_reason'],'elapsed_s':round(time.monotonic()-start,4), 'completion_tokens':item.get('usage',{}).get('completion_tokens'), 'reasoning_tokens':(item.get('usage',{}).get('completion_tokens_details') or {}).get('reasoning_tokens')}) schema={'type':'object','properties':{'color':{'type':'string','enum':['blue']},'count':{'type':'integer','enum':[3]}},'required':['color','count'],'additionalProperties':False} body=request([{'role':'user','content':'Return color blue and count 3 using the supplied JSON schema.'}], response_format={'type':'json_schema','json_schema':{'name':'sample','strict':True,'schema':schema}}) with post(base,body) as response: item=json.load(response) try: passed=json.loads(item['choices'][0]['message']['content'])=={'color':'blue','count':3} except (ValueError,TypeError,KeyError): passed=False results['checks'].append({'case':'strict_json_schema','passed':passed,'finish_reason':item['choices'][0]['finish_reason']}) tool={'type':'function','function':{'name':'get_temperature','description':'Read a temperature for a named city.', 'parameters':{'type':'object','properties':{'city':{'type':'string'}},'required':['city'],'additionalProperties':False}}} messages=[{'role':'user','content':'Use the temperature tool for Paris.'}] body=request(messages,tools=[tool],tool_choice={'type':'function','function':{'name':'get_temperature'}}) with post(base,body) as response: item=json.load(response) message=item['choices'][0]['message']; calls=message.get('tool_calls') or [] passed=bool(calls) and calls[0]['function']['name']=='get_temperature' and json.loads(calls[0]['function']['arguments']).get('city','').lower()=='paris' results['checks'].append({'case':'tool_call','passed':passed,'finish_reason':item['choices'][0]['finish_reason']}) if passed: messages.extend([{'role':'assistant','content':message.get('content'),'tool_calls':calls}, {'role':'tool','tool_call_id':calls[0]['id'],'content':'{"city":"Paris","temperature_celsius":17}'}]) with post(base,request(messages,tools=[tool],tool_choice='none')) as response: item=json.load(response) answer=item['choices'][0]['message'].get('content') or '' results['checks'].append({'case':'tool_result_continuation','passed':'17' in answer,'finish_reason':item['choices'][0]['finish_reason']}) results['all_checks_passed']=all(x['passed'] for x in results['checks']) and len(results['checks'])==4 results['generation_nonempty']=all(x['nonempty_content'] for x in results['generation']) return results def main(): parser=argparse.ArgumentParser(description=__doc__) parser.add_argument('--base-url',default='http://127.0.0.1:8000/v1') parser.add_argument('--output',default='generic-results.json') args=parser.parse_args() results=evaluate(args.base_url) with open(args.output,'w') as handle: json.dump(results,handle,indent=2); handle.write('\n') print(json.dumps(results,indent=2)) return 0 if results['all_checks_passed'] and results['generation_nonempty'] else 1 if __name__=='__main__': raise SystemExit(main())