File size: 1,173 Bytes
60faa09 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 | import { InferenceClient } from "@huggingface/inference";
export async function callHuggingFace({
model, messages, temperature = 1.0, maxTokens = 2048,
stream = false, apiKey, onToken,
}) {
const client = new InferenceClient(apiKey);
const modelName = model.replace(/^hf:/, "");
const chatMessages = messages.map((m) => ({ role: m.role, content: m.content }));
if (stream) {
let fullText = "";
const out = await client.chatCompletionStream({
model: modelName,
messages: chatMessages,
temperature: Number(temperature),
max_tokens: Number(maxTokens),
});
for await (const chunk of out) {
const delta = chunk.choices?.[0]?.delta?.content;
if (delta) { fullText += delta; onToken?.(delta); }
}
return { text: fullText, usage: null };
}
const out = await client.chatCompletion({
model: modelName,
messages: chatMessages,
temperature: Number(temperature),
max_tokens: Number(maxTokens),
});
return {
text: out.choices?.[0]?.message?.content || "",
usage: {
promptTokens: out.usage?.prompt_tokens,
completionTokens: out.usage?.completion_tokens,
},
};
}
|