hecz-showcase's picture TechWithRay's picture
[Ling] Add a working Foundry Local CPU adapter for the BF16 model (#1)
d00cac4
Raw History Blame Contribute Delete
3.56 kB
{
"request": {
"model": "ling-pr-bf16-cpu:1",
"messages": [
{
"role": "user",
"content": "What is 2 + 2? Reply with just the answer."
}
],
"chat_template_kwargs": {
"enable_thinking": false
},
"temperature": 0,
"max_tokens": 8,
"stream": true,
"stream_options": {
"include_usage": true
}
},
"endpoint": "/catalogs/local (discovered dynamically)",
"runs": [
{
"warmup": false,
"text": "4",
"ttft_seconds": 61.050051799975336,
"generation_seconds": 63.483419400057755,
"usage": {
"completion_tokens": 1,
"completion_tokens_details": {
"reasoning_tokens": 0
},
"prompt_tokens": 35,
"prompt_tokens_details": {
"cached_tokens": 0
},
"total_tokens": 36
},
"last_content_seconds": 61.050051799975336,
"decode_tokens_per_second": null,
"events": [
{
"elapsed_seconds": 61.04775440006051,
"payload": {
"choices": [
{
"delta": {
"content": "",
"role": "assistant"
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1793132091,
"id": "chatcmpl-d21bd5cdf954ede0",
"model": "ling-pr-bf16-cpu:1",
"object": "chat.completion.chunk",
"system_fingerprint": null
}
},
{
"elapsed_seconds": 61.050051799975336,
"payload": {
"choices": [
{
"delta": {
"content": "4"
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1793132091,
"id": "chatcmpl-d21bd5cdf954ede0",
"model": "ling-pr-bf16-cpu:1",
"object": "chat.completion.chunk",
"system_fingerprint": null
}
},
{
"elapsed_seconds": 63.483419400057755,
"payload": {
"choices": [
{
"delta": {},
"finish_reason": "stop",
"index": 0,
"logprobs": null
}
],
"created": 1793132091,
"id": "chatcmpl-d21bd5cdf954ede0",
"model": "ling-pr-bf16-cpu:1",
"object": "chat.completion.chunk",
"system_fingerprint": null
}
},
{
"elapsed_seconds": 63.48882740002591,
"payload": {
"choices": [],
"created": 1793132091,
"id": "chatcmpl-d21bd5cdf954ede0",
"model": "ling-pr-bf16-cpu:1",
"object": "chat.completion.chunk",
"system_fingerprint": null,
"usage": {
"completion_tokens": 1,
"completion_tokens_details": {
"reasoning_tokens": 0
},
"prompt_tokens": 35,
"prompt_tokens_details": {
"cached_tokens": 0
},
"total_tokens": 36
}
}
}
]
}
],
"median_ttft_seconds": 61.050051799975336,
"median_decode_tokens_per_second": null
}