// Copyright (c) 2026 by Rockchip Electronics Co., Ltd. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "rknn3_session_test_common.h" // ---- 我们加的:output_callback 里直接拿到 logits 张量并落盘 ---- // sampling_callback 不会被调用(运行时内部采样),但 output_tensors[0] 就是 vocabulary logits static int g_kev_dumped = 0; static int g_kev_seq = 0; // 当前第几条序列,用来给 logits 文件编号 static char g_out_path[512] = "D:\\rk1828-console\\kevrunner\\out\\cur.bin"; static int kev_output_callback(void* userdata, rknn3_tensor* outs, uint32_t n, LLMOutputCallbackState state) { (void)userdata; if (!g_kev_dumped && n > 0 && state == RKLLM_OUTPUT_CALLBACK_PREFILL_FINISHED) { const char* p = g_out_path; FILE* f = fopen(p, "wb"); if (f) { size_t sz = outs[0].mem->size; fwrite(outs[0].mem->virt_addr, 1, sz, f); fclose(f); printf("[kev] dumped %zu bytes, state=%d, name=%s\n", sz, (int)state, outs[0].attr->name); g_kev_dumped = 1; } else { printf("[kev] fopen failed\n"); } fflush(stdout); } return 0; } // ---- 极简 base64 解码,给 TOK 命令用(文本里有空格换行,用 base64 最省事)---- static int b64val(char c) { if (c >= 'A' && c <= 'Z') return c - 'A'; if (c >= 'a' && c <= 'z') return c - 'a' + 26; if (c >= '0' && c <= '9') return c - '0' + 52; if (c == '+') return 62; if (c == '/') return 63; return -1; } static int b64decode(const char* in, int inlen, char* out, int outmax) { int v = 0, bits = 0, n = 0; for (int i = 0; i < inlen; i++) { if (in[i] == '=' || in[i] == '\n' || in[i] == '\r') continue; int d = b64val(in[i]); if (d < 0) continue; v = (v << 6) | d; bits += 6; if (bits >= 8) { bits -= 8; if (n < outmax) out[n++] = (char)((v >> bits) & 0xFF); } } return n; } int main(int argc, char** argv) { // keep_history / ignore_eos_token / key_path are optional and parsed by // argc position: passing fewer falls back to the defaults in [] below. // Usage: %s [keep_history] [ignore_eos_token] // [key_path] if (argc < 8 || argc > 12) { // 12 = 多一个 ids 文件 LOGW( "Usage: %s [keep_history] [ignore_eos_token] [key_path]\n ", argv[0]); LOGW( "Such as: %s ./model/Qwen2.5-0.5B.rknn ./model/Qwen2.5-0.5B.weight ./model/Qwen2.5-0.5B.tokenizer.gguf ./model/Qwen2.5-0.5B-embedding.bin 1024 256 0xff 0 0 ./model/key.env\n", argv[0]); LOGW( "For encrypted model: %s ./model/Qwen2.5-0.5B.rknn ./model/Qwen2.5-0.5B.weight ./model/Qwen2.5-0.5B.tokenizer.gguf ./model/Qwen2.5-0.5B-embedding.bin 1024 256 0xff 0 0 ./model/key.env\n", argv[0]); return -1; } int ret = 0; int i = 0; // 已处理的序列数(放在这里避免 goto 跨初始化) char* model_path = argv[1]; char* weight_path = argv[2]; const char* tokenizer_path = argv[3]; const char* embedding_path = argv[4]; int32_t max_context_len = atoi(argv[5]); int32_t max_new_tokens = atoi(argv[6]); uint32_t core_mask = strtoul(argv[7], nullptr, 16); // Optional positional args; defaults keep the old behaviour. int keep_history = (argc > 8) ? atoi(argv[8]) : 0; bool ignore_eos_token = (argc > 9) ? (atoi(argv[9]) != 0) : false; const char* key_path = (argc > 10) ? argv[10] : nullptr; const char* ids_path = (argc > 11) ? argv[11] : nullptr; // 每行一串 token id // key 文件是占位参数;不存在就当没给,否则会被当成加密模型去解密 if (key_path) { FILE* kf = fopen(key_path, "r"); if (kf) fclose(kf); else { printf("[kev] key file '%s' not found, ignore it\n", key_path); key_path = nullptr; } } rknn3_devices devs; rknn3_context ctx = 0; rknn3_input_output_num io_num; VocabInfo vocab_info; Tokenizer* tokenizer = nullptr; struct stat emb_st; struct embedding_info embedding_info; RKLLMCallback callback; rknn3_test_session_userdata_t session_userdata; rknn3_llm_param params; int n_params = 1; RKLLMRunState state; rknn3_lora loras_enabled[RKNN3_MAX_LORA_NUM]; rknn3_llm_config llm_config; rknn3_kvcache_policy kvcache_policy = RKNN3_KVCACHE_POLICY_NORMAL; rknn3_kvcache_policy_param kvcache_policy_param; int llm_n_inputs = 1; rknn3_llm_input* llm_inputs[llm_n_inputs]; rknn3_llm_infer_param llm_infer_param; memset(&devs, 0, sizeof(devs)); memset(&io_num, 0, sizeof(io_num)); memset(&vocab_info, 0, sizeof(vocab_info)); memset(&emb_st, 0, sizeof(emb_st)); memset(&embedding_info, 0, sizeof(embedding_info)); memset(&callback, 0, sizeof(callback)); memset(¶ms, 0, sizeof(params)); memset(&state, 0, sizeof(state)); memset(&llm_config, 0, sizeof(llm_config)); memset(loras_enabled, 0, sizeof(loras_enabled)); memset(&kvcache_policy_param, 0, sizeof(kvcache_policy_param)); state.loras_enabled = loras_enabled; memset(llm_inputs, 0, sizeof(llm_inputs)); memset(&llm_infer_param, 0, sizeof(llm_infer_param)); memset(&session_userdata, 0, sizeof(session_userdata)); // set output tensors index for output callback rknn3_tensor output_tensors[1]; int n_output_tensors = 1; int output_tensors_index[1] = {0}; memset(output_tensors, 0, sizeof(output_tensors)); std::vector random_prompts = { "请解释一下相对论的基本概念。", "Please explain the basic concept of relativity", }; int test_num = random_prompts.size(); // 若给了 ids 文件,就走批量模式:每行一串 token id,逐条 prefill 并 dump logits std::vector> seqs; if (ids_path) { FILE* fp = fopen(ids_path, "r"); if (!fp) { printf("cannot open ids file: %s\n", ids_path); return -1; } char line[65536]; while (fgets(line, sizeof(line), fp)) { std::vector one; char* p = line; while (*p) { char* end = nullptr; long v = strtol(p, &end, 10); if (end == p) break; one.push_back((int32_t)v); p = end; } if (!one.empty()) seqs.push_back(one); } fclose(fp); test_num = (int)seqs.size(); printf("[kev] loaded %d sequences from %s\n", test_num, ids_path); } printf("*******************************NEW TEST**********************************\n"); // find devices (devs already zero-initialised above) const char* device_id = nullptr; ret = rknn3_find_devices(&devs); if (ret != RKNN3_SUCCESS || devs.n_devices == 0) { printf("rknn3_find_devices failed, ret = %d\n", ret); goto exit; } // init context and model device_id = devs.devices[0].id; ret = init_context_and_model(&ctx, model_path, weight_path, core_mask, key_path, device_id); if (ret != RKNN3_SUCCESS) { printf("rknn3_session_init_model fail! ret = %d\n", ret); goto exit; } // Get Model Output Info for output callback (if needed) if (n_output_tensors > 0) { ret = setup_output_tensors(ctx, output_tensors, n_output_tensors, output_tensors_index, &io_num); if (ret < 0) { goto exit; } } // get tokenizer and embedding ret = get_tokenizer_and_embedding(tokenizer_path, &vocab_info, &tokenizer, &embedding_info, embedding_path, &emb_st); if (ret < 0) { printf("get_tokenizer_and_embedding failed, ret = %d\n", ret); goto exit; } ret = rknn3_query(ctx, RKNN3_QUERY_LLM_CONFIG, &llm_config, sizeof(rknn3_llm_config)); if (ret != RKNN3_SUCCESS) { printf("rknn3_query llm config failed! ret=%d", ret); goto exit; } // Set basic session parameters build_default_llm_params(¶ms, &vocab_info, max_context_len, ignore_eos_token); // init session session_userdata.session = rknn3_session_init(ctx, ¶ms, n_params); if (!session_userdata.session) { printf("rknn3_session_init failed, ret = %d\n", ret); goto exit; } // set session callback build_default_callback(&callback, tokenizer, &embedding_info, &session_userdata); // sampling_callback 不会被运行时调用(内部采样),logits 从 output_callback 拿 callback.sampling_callback = NULL; callback.sampling_userdata = NULL; // 同时打开 output_callback:若它能触发而 sampling 不能,说明 sampling 被门控 callback.output_callback = kev_output_callback; callback.output_userdata = &embedding_info; callback.output_tensors = output_tensors; callback.n_output_tensors = n_output_tensors; ret = rknn3_session_set_callback(session_userdata.session, &callback); if (ret != RKNN3_SUCCESS) { printf("rknn3_session_set_callback failed, ret = %d\n", ret); goto exit; } // set/update chat template if (0) { std::string system_prompt = "<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n"; std::string prompt_prefix = "<|im_start|>user\n"; std::string prompt_postfix = "<|im_end|>\n<|im_start|>assistant\n"; ret = rknn3_session_set_chat_template(session_userdata.session, system_prompt.c_str(), prompt_prefix.c_str(), prompt_postfix.c_str()); if (ret != RKNN3_SUCCESS) { printf("rknn3_session_set_chat_template failed, ret = %d\n", ret); goto exit; } } // set kvcache policy ret = rknn3_session_set_kvcache_policy(session_userdata.session, kvcache_policy, nullptr); if (ret != RKNN3_SUCCESS) { printf("rknn3_session_set_kvcache_policy failed, ret=%d\n", ret); goto exit; } print_llm_config(&llm_config); // Test the generic session inference: each iteration builds a typical prompt request, // runs one round of generation to show the standard prompt input workflow. char in_line[65536]; // 关键:stdout 接管道时 C 运行库默认全缓冲,READY_DAEMON 会卡在缓冲区里, // 父进程永远等不到。必须显式改成行缓冲(只 fflush 一次不够,后面每行都要)。 setvbuf(stdout, NULL, _IOLBF, 0); printf("READY_DAEMON\n"); fflush(stdout); while (fgets(in_line, sizeof(in_line), stdin)) { if (strncmp(in_line, "quit", 4) == 0) break; // OUT -> 指定下一次 dump 的落盘路径 if (strncmp(in_line, "OUT ", 4) == 0) { char* q = in_line + 4; while (*q == ' ') q++; int n = (int)strlen(q); while (n > 0 && (q[n-1] == '\n' || q[n-1] == '\r')) n--; if (n > 0 && n < (int)sizeof(g_out_path)) { memcpy(g_out_path, q, n); g_out_path[n] = 0; printf("OUTSET %s\n", g_out_path); fflush(stdout); } continue; } // TOK -> 打印 "IDS n id1 id2 ..." if (strncmp(in_line, "TOK ", 4) == 0) { static char tbuf[65536]; static int32_t tids[8192]; int raw = b64decode(in_line + 4, (int)strlen(in_line + 4), tbuf, sizeof(tbuf) - 1); tbuf[raw] = 0; printf("[kev] TOK raw=%d head='%.40s' tok=%p\n", raw, tbuf, (void*)tokenizer); fflush(stdout); int n = tokenizer->Tokenize(tbuf, raw, tids, 8192); printf("IDS %d", n); for (int k = 0; k < n; k++) printf(" %d", tids[k]); printf("\n"); fflush(stdout); continue; } // R = 清空 kvcache 后从头 prefill(keep_history=0) // A = 接在上一次之后追加 prefill(keep_history=1) // 不带前缀 = 老行为(等价 R) bool append = false; char* pp = in_line; if (in_line[0] == 'A' && in_line[1] == ' ') { append = true; pp = in_line + 2; } else if (in_line[0] == 'R' && in_line[1] == ' ') { pp = in_line + 2; } { std::vector one; char* p = pp; while (*p) { char* e = nullptr; long v = strtol(p, &e, 10); if (e == p) break; one.push_back((int32_t)v); p = e; } if (one.empty()) continue; seqs.push_back(one); } printf("\n--------------------Input[%d]-------------------- \n", i); llm_infer_param = build_default_infer_param(max_new_tokens, append ? 1 : 0); g_kev_seq = i; g_kev_dumped = 0; // A 模式接着上次的 kvcache;R 模式(默认)从头开始 if (!append) { rknn3_session_clear_kvcache(session_userdata.session, RKNN3_KVCACHE_CLEAR_ALL); } llm_inputs[0] = build_llm_input_tokens(seqs.back().data(), seqs.back().size(), false, false); printf("\n--------------------Output---------------------- \n"); rknn3_test_timing_mark_start(&session_userdata.timing); ret = rknn3_session_run(session_userdata.session, llm_inputs[0], llm_n_inputs, &llm_infer_param); if (ret != RKNN3_SUCCESS) { printf("rknn3_session_run failed, ret = %d\n", ret); goto exit; } rknn3_test_timing_mark_end(&session_userdata.timing); printf("DONE %d\n", i); // 每处理完一条就给客户端一个信号 fflush(stdout); i++; ret = rknn3_session_query_state(session_userdata.session, &state); if (ret != RKNN3_SUCCESS) { printf("rknn3_session_query_state failed, ret=%d\n", ret); goto exit; } print_run_state(&state, true); // clear kvcache if (state.n_total_tokens >= (state.n_max_tokens - max_new_tokens)) { ret = rknn3_session_clear_kvcache(session_userdata.session, RKNN3_KVCACHE_CLEAR_ALL); if (ret != RKNN3_SUCCESS) { printf("rknn3_session_clear_kvcache failed, ret=%d\n", ret); goto exit; } } print_perf_stats(&state, &session_userdata.timing); // Free any heap memory owned by this iteration's input(s). for (int j = 0; j < llm_n_inputs; j++) { release_llm_input(ctx, llm_inputs[j]); } } exit: for (int j = 0; j < llm_n_inputs; j++) { release_llm_input(ctx, llm_inputs[j]); } if (n_output_tensors > 0) { cleanup_output_tensors(ctx, output_tensors, n_output_tensors); } if (session_userdata.session) { rknn3_session_destroy(session_userdata.session); } if (ctx) { rknn3_destroy(ctx); } embedding_info_release(&embedding_info, &emb_st); if (tokenizer) { delete tokenizer; } printf("*******************************END TEST**********************************\n"); fflush(stdout); return 0; }