File size: 1,424 Bytes
3680485 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | #!/usr/bin/env python3
"""Send one inference request with a fixed repetition penalty of 1.05."""
import argparse
import json
import urllib.request
from pathlib import Path
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--prompt', required=True)
p.add_argument('--base-url', default='http://127.0.0.1:28080/v1')
p.add_argument('--model', default='Safin-1.5')
p.add_argument('--max-tokens', type=int, default=65536)
p.add_argument('--output')
a = p.parse_args()
body = {
'model': a.model, 'messages': [{'role': 'user', 'content': a.prompt}],
'temperature': 1.0, 'top_p': 0.95, 'seed': 42,
'max_tokens': a.max_tokens, 'presence_penalty': 0.0,
'repetition_penalty': 1.05,
'chat_template_kwargs': {'enable_thinking': True}, 'stream': False,
}
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
req = urllib.request.Request(
a.base_url.rstrip('/') + '/chat/completions',
data=json.dumps(body).encode(), headers={'Content-Type': 'application/json'},
)
with opener.open(req, timeout=3600) as response:
raw = json.load(response)
if a.output:
Path(a.output).write_text(json.dumps({'request': body, 'response': raw}, ensure_ascii=False, indent=2) + '\n')
print(raw['choices'][0]['message'].get('content') or '')
if __name__ == '__main__':
main()
|