Kev-0.8B-RKNN3-RK1828 / host /kev_daemon.cpp
jj90's picture
Add model card, deployment guide and host-side code
a846b3d verified
Raw History Blame Contribute Delete
15.4 kB
// Copyright (c) 2026 by Rockchip Electronics Co., Ltd. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "rknn3_session_test_common.h"
// ---- 我们加的:output_callback 里直接拿到 logits 张量并落盘 ----
// sampling_callback 不会被调用(运行时内部采样),但 output_tensors[0] 就是 vocabulary logits
static int g_kev_dumped = 0;
static int g_kev_seq = 0; // 当前第几条序列,用来给 logits 文件编号
static char g_out_path[512] = "D:\\rk1828-console\\kevrunner\\out\\cur.bin";
static int kev_output_callback(void* userdata, rknn3_tensor* outs, uint32_t n,
LLMOutputCallbackState state) {
(void)userdata;
if (!g_kev_dumped && n > 0 && state == RKLLM_OUTPUT_CALLBACK_PREFILL_FINISHED) {
const char* p = g_out_path;
FILE* f = fopen(p, "wb");
if (f) {
size_t sz = outs[0].mem->size;
fwrite(outs[0].mem->virt_addr, 1, sz, f);
fclose(f);
printf("[kev] dumped %zu bytes, state=%d, name=%s\n", sz, (int)state,
outs[0].attr->name);
g_kev_dumped = 1;
} else {
printf("[kev] fopen failed\n");
}
fflush(stdout);
}
return 0;
}
// ---- 极简 base64 解码,给 TOK 命令用(文本里有空格换行,用 base64 最省事)----
static int b64val(char c) {
if (c >= 'A' && c <= 'Z') return c - 'A';
if (c >= 'a' && c <= 'z') return c - 'a' + 26;
if (c >= '0' && c <= '9') return c - '0' + 52;
if (c == '+') return 62;
if (c == '/') return 63;
return -1;
}
static int b64decode(const char* in, int inlen, char* out, int outmax) {
int v = 0, bits = 0, n = 0;
for (int i = 0; i < inlen; i++) {
if (in[i] == '=' || in[i] == '\n' || in[i] == '\r') continue;
int d = b64val(in[i]);
if (d < 0) continue;
v = (v << 6) | d;
bits += 6;
if (bits >= 8) {
bits -= 8;
if (n < outmax) out[n++] = (char)((v >> bits) & 0xFF);
}
}
return n;
}
int main(int argc, char** argv)
{
// keep_history / ignore_eos_token / key_path are optional and parsed by
// argc position: passing fewer falls back to the defaults in [] below.
// Usage: %s <rknn> <weight> <tokenizer> <embedding> <max_context_len> <max_new_tokens> <core_mask> [keep_history] [ignore_eos_token]
// [key_path]
if (argc < 8 || argc > 12) { // 12 = 多一个 ids 文件
LOGW(
"Usage: %s <rknn_path> <weight_path> <tokenizer.gguf> <embedding.bin> <max_context_len> <max_new_tokens> <core_mask> [keep_history] [ignore_eos_token] [key_path]\n ",
argv[0]);
LOGW(
"Such as: %s ./model/Qwen2.5-0.5B.rknn ./model/Qwen2.5-0.5B.weight ./model/Qwen2.5-0.5B.tokenizer.gguf ./model/Qwen2.5-0.5B-embedding.bin 1024 256 0xff 0 0 ./model/key.env\n",
argv[0]);
LOGW(
"For encrypted model: %s ./model/Qwen2.5-0.5B.rknn ./model/Qwen2.5-0.5B.weight ./model/Qwen2.5-0.5B.tokenizer.gguf ./model/Qwen2.5-0.5B-embedding.bin 1024 256 0xff 0 0 ./model/key.env\n",
argv[0]);
return -1;
}
int ret = 0;
int i = 0; // 已处理的序列数(放在这里避免 goto 跨初始化)
char* model_path = argv[1];
char* weight_path = argv[2];
const char* tokenizer_path = argv[3];
const char* embedding_path = argv[4];
int32_t max_context_len = atoi(argv[5]);
int32_t max_new_tokens = atoi(argv[6]);
uint32_t core_mask = strtoul(argv[7], nullptr, 16);
// Optional positional args; defaults keep the old behaviour.
int keep_history = (argc > 8) ? atoi(argv[8]) : 0;
bool ignore_eos_token = (argc > 9) ? (atoi(argv[9]) != 0) : false;
const char* key_path = (argc > 10) ? argv[10] : nullptr;
const char* ids_path = (argc > 11) ? argv[11] : nullptr; // 每行一串 token id
// key 文件是占位参数;不存在就当没给,否则会被当成加密模型去解密
if (key_path) {
FILE* kf = fopen(key_path, "r");
if (kf) fclose(kf);
else {
printf("[kev] key file '%s' not found, ignore it\n", key_path);
key_path = nullptr;
}
}
rknn3_devices devs;
rknn3_context ctx = 0;
rknn3_input_output_num io_num;
VocabInfo vocab_info;
Tokenizer* tokenizer = nullptr;
struct stat emb_st;
struct embedding_info embedding_info;
RKLLMCallback callback;
rknn3_test_session_userdata_t session_userdata;
rknn3_llm_param params;
int n_params = 1;
RKLLMRunState state;
rknn3_lora loras_enabled[RKNN3_MAX_LORA_NUM];
rknn3_llm_config llm_config;
rknn3_kvcache_policy kvcache_policy = RKNN3_KVCACHE_POLICY_NORMAL;
rknn3_kvcache_policy_param kvcache_policy_param;
int llm_n_inputs = 1;
rknn3_llm_input* llm_inputs[llm_n_inputs];
rknn3_llm_infer_param llm_infer_param;
memset(&devs, 0, sizeof(devs));
memset(&io_num, 0, sizeof(io_num));
memset(&vocab_info, 0, sizeof(vocab_info));
memset(&emb_st, 0, sizeof(emb_st));
memset(&embedding_info, 0, sizeof(embedding_info));
memset(&callback, 0, sizeof(callback));
memset(&params, 0, sizeof(params));
memset(&state, 0, sizeof(state));
memset(&llm_config, 0, sizeof(llm_config));
memset(loras_enabled, 0, sizeof(loras_enabled));
memset(&kvcache_policy_param, 0, sizeof(kvcache_policy_param));
state.loras_enabled = loras_enabled;
memset(llm_inputs, 0, sizeof(llm_inputs));
memset(&llm_infer_param, 0, sizeof(llm_infer_param));
memset(&session_userdata, 0, sizeof(session_userdata));
// set output tensors index for output callback
rknn3_tensor output_tensors[1];
int n_output_tensors = 1;
int output_tensors_index[1] = {0};
memset(output_tensors, 0, sizeof(output_tensors));
std::vector<std::string> random_prompts = {
"请解释一下相对论的基本概念。",
"Please explain the basic concept of relativity",
};
int test_num = random_prompts.size();
// 若给了 ids 文件,就走批量模式:每行一串 token id,逐条 prefill 并 dump logits
std::vector<std::vector<int32_t>> seqs;
if (ids_path) {
FILE* fp = fopen(ids_path, "r");
if (!fp) { printf("cannot open ids file: %s\n", ids_path); return -1; }
char line[65536];
while (fgets(line, sizeof(line), fp)) {
std::vector<int32_t> one;
char* p = line;
while (*p) {
char* end = nullptr;
long v = strtol(p, &end, 10);
if (end == p) break;
one.push_back((int32_t)v);
p = end;
}
if (!one.empty()) seqs.push_back(one);
}
fclose(fp);
test_num = (int)seqs.size();
printf("[kev] loaded %d sequences from %s\n", test_num, ids_path);
}
printf("*******************************NEW TEST**********************************\n");
// find devices (devs already zero-initialised above)
const char* device_id = nullptr;
ret = rknn3_find_devices(&devs);
if (ret != RKNN3_SUCCESS || devs.n_devices == 0) {
printf("rknn3_find_devices failed, ret = %d\n", ret);
goto exit;
}
// init context and model
device_id = devs.devices[0].id;
ret = init_context_and_model(&ctx, model_path, weight_path, core_mask, key_path, device_id);
if (ret != RKNN3_SUCCESS) {
printf("rknn3_session_init_model fail! ret = %d\n", ret);
goto exit;
}
// Get Model Output Info for output callback (if needed)
if (n_output_tensors > 0) {
ret = setup_output_tensors(ctx, output_tensors, n_output_tensors, output_tensors_index, &io_num);
if (ret < 0) {
goto exit;
}
}
// get tokenizer and embedding
ret = get_tokenizer_and_embedding(tokenizer_path, &vocab_info, &tokenizer, &embedding_info, embedding_path, &emb_st);
if (ret < 0) {
printf("get_tokenizer_and_embedding failed, ret = %d\n", ret);
goto exit;
}
ret = rknn3_query(ctx, RKNN3_QUERY_LLM_CONFIG, &llm_config, sizeof(rknn3_llm_config));
if (ret != RKNN3_SUCCESS) {
printf("rknn3_query llm config failed! ret=%d", ret);
goto exit;
}
// Set basic session parameters
build_default_llm_params(&params, &vocab_info, max_context_len, ignore_eos_token);
// init session
session_userdata.session = rknn3_session_init(ctx, &params, n_params);
if (!session_userdata.session) {
printf("rknn3_session_init failed, ret = %d\n", ret);
goto exit;
}
// set session callback
build_default_callback(&callback, tokenizer, &embedding_info, &session_userdata);
// sampling_callback 不会被运行时调用(内部采样),logits 从 output_callback 拿
callback.sampling_callback = NULL;
callback.sampling_userdata = NULL;
// 同时打开 output_callback:若它能触发而 sampling 不能,说明 sampling 被门控
callback.output_callback = kev_output_callback;
callback.output_userdata = &embedding_info;
callback.output_tensors = output_tensors;
callback.n_output_tensors = n_output_tensors;
ret = rknn3_session_set_callback(session_userdata.session, &callback);
if (ret != RKNN3_SUCCESS) {
printf("rknn3_session_set_callback failed, ret = %d\n", ret);
goto exit;
}
// set/update chat template
if (0) {
std::string system_prompt = "<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n";
std::string prompt_prefix = "<|im_start|>user\n";
std::string prompt_postfix = "<|im_end|>\n<|im_start|>assistant\n";
ret = rknn3_session_set_chat_template(session_userdata.session, system_prompt.c_str(), prompt_prefix.c_str(), prompt_postfix.c_str());
if (ret != RKNN3_SUCCESS) {
printf("rknn3_session_set_chat_template failed, ret = %d\n", ret);
goto exit;
}
}
// set kvcache policy
ret = rknn3_session_set_kvcache_policy(session_userdata.session, kvcache_policy, nullptr);
if (ret != RKNN3_SUCCESS) {
printf("rknn3_session_set_kvcache_policy failed, ret=%d\n", ret);
goto exit;
}
print_llm_config(&llm_config);
// Test the generic session inference: each iteration builds a typical prompt request,
// runs one round of generation to show the standard prompt input workflow.
char in_line[65536];
// 关键:stdout 接管道时 C 运行库默认全缓冲,READY_DAEMON 会卡在缓冲区里,
// 父进程永远等不到。必须显式改成行缓冲(只 fflush 一次不够,后面每行都要)。
setvbuf(stdout, NULL, _IOLBF, 0);
printf("READY_DAEMON\n");
fflush(stdout);
while (fgets(in_line, sizeof(in_line), stdin)) {
if (strncmp(in_line, "quit", 4) == 0) break;
// OUT <path> -> 指定下一次 dump 的落盘路径
if (strncmp(in_line, "OUT ", 4) == 0) {
char* q = in_line + 4;
while (*q == ' ') q++;
int n = (int)strlen(q);
while (n > 0 && (q[n-1] == '\n' || q[n-1] == '\r')) n--;
if (n > 0 && n < (int)sizeof(g_out_path)) {
memcpy(g_out_path, q, n);
g_out_path[n] = 0;
printf("OUTSET %s\n", g_out_path);
fflush(stdout);
}
continue;
}
// TOK <base64 text> -> 打印 "IDS n id1 id2 ..."
if (strncmp(in_line, "TOK ", 4) == 0) {
static char tbuf[65536];
static int32_t tids[8192];
int raw = b64decode(in_line + 4, (int)strlen(in_line + 4), tbuf, sizeof(tbuf) - 1);
tbuf[raw] = 0;
printf("[kev] TOK raw=%d head='%.40s' tok=%p\n", raw, tbuf, (void*)tokenizer);
fflush(stdout);
int n = tokenizer->Tokenize(tbuf, raw, tids, 8192);
printf("IDS %d", n);
for (int k = 0; k < n; k++) printf(" %d", tids[k]);
printf("\n");
fflush(stdout);
continue;
}
// R <ids> = 清空 kvcache 后从头 prefill(keep_history=0)
// A <ids> = 接在上一次之后追加 prefill(keep_history=1)
// 不带前缀 = 老行为(等价 R)
bool append = false;
char* pp = in_line;
if (in_line[0] == 'A' && in_line[1] == ' ') { append = true; pp = in_line + 2; }
else if (in_line[0] == 'R' && in_line[1] == ' ') { pp = in_line + 2; }
{
std::vector<int32_t> one;
char* p = pp;
while (*p) {
char* e = nullptr;
long v = strtol(p, &e, 10);
if (e == p) break;
one.push_back((int32_t)v);
p = e;
}
if (one.empty()) continue;
seqs.push_back(one);
}
printf("\n--------------------Input[%d]-------------------- \n", i);
llm_infer_param = build_default_infer_param(max_new_tokens, append ? 1 : 0);
g_kev_seq = i;
g_kev_dumped = 0;
// A 模式接着上次的 kvcache;R 模式(默认)从头开始
if (!append) {
rknn3_session_clear_kvcache(session_userdata.session, RKNN3_KVCACHE_CLEAR_ALL);
}
llm_inputs[0] = build_llm_input_tokens(seqs.back().data(), seqs.back().size(), false, false);
printf("\n--------------------Output---------------------- \n");
rknn3_test_timing_mark_start(&session_userdata.timing);
ret = rknn3_session_run(session_userdata.session, llm_inputs[0], llm_n_inputs, &llm_infer_param);
if (ret != RKNN3_SUCCESS) {
printf("rknn3_session_run failed, ret = %d\n", ret);
goto exit;
}
rknn3_test_timing_mark_end(&session_userdata.timing);
printf("DONE %d\n", i); // 每处理完一条就给客户端一个信号
fflush(stdout);
i++;
ret = rknn3_session_query_state(session_userdata.session, &state);
if (ret != RKNN3_SUCCESS) {
printf("rknn3_session_query_state failed, ret=%d\n", ret);
goto exit;
}
print_run_state(&state, true);
// clear kvcache
if (state.n_total_tokens >= (state.n_max_tokens - max_new_tokens)) {
ret = rknn3_session_clear_kvcache(session_userdata.session, RKNN3_KVCACHE_CLEAR_ALL);
if (ret != RKNN3_SUCCESS) {
printf("rknn3_session_clear_kvcache failed, ret=%d\n", ret);
goto exit;
}
}
print_perf_stats(&state, &session_userdata.timing);
// Free any heap memory owned by this iteration's input(s).
for (int j = 0; j < llm_n_inputs; j++) {
release_llm_input(ctx, llm_inputs[j]);
}
}
exit:
for (int j = 0; j < llm_n_inputs; j++) {
release_llm_input(ctx, llm_inputs[j]);
}
if (n_output_tensors > 0) {
cleanup_output_tensors(ctx, output_tensors, n_output_tensors);
}
if (session_userdata.session) {
rknn3_session_destroy(session_userdata.session);
}
if (ctx) {
rknn3_destroy(ctx);
}
embedding_info_release(&embedding_info, &emb_st);
if (tokenizer) {
delete tokenizer;
}
printf("*******************************END TEST**********************************\n");
fflush(stdout);
return 0;
}