Text Classification
GGUF
English
rknn
rknn3
rk1828
rk1820
rockchip
npu
edge-ai
kev
decision
w4a16
Eval Results (legacy)
conversational
Instructions to use jj90/Kev-0.8B-RKNN3-RK1828 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- llama.cpp
How to use jj90/Kev-0.8B-RKNN3-RK1828 with llama.cpp:
Install (macOS, Linux)
curl -LsSf https://llama.app/install.sh | sh # Start a local OpenAI-compatible server with a web UI: llama serve -hf jj90/Kev-0.8B-RKNN3-RK1828 # Run inference directly in the terminal: llama cli -hf jj90/Kev-0.8B-RKNN3-RK1828
Install from WinGet (Windows)
winget install llama.cpp # Start a local OpenAI-compatible server with a web UI: llama serve -hf jj90/Kev-0.8B-RKNN3-RK1828 # Run inference directly in the terminal: llama cli -hf jj90/Kev-0.8B-RKNN3-RK1828
Build from source code
git clone https://github.com/ggml-org/llama.cpp.git cd llama.cpp cmake -B build cmake --build build -j --target llama-app # Start a local OpenAI-compatible server with a web UI: ./build/bin/llama serve -hf jj90/Kev-0.8B-RKNN3-RK1828 # Run inference directly in the terminal: ./build/bin/llama cli -hf jj90/Kev-0.8B-RKNN3-RK1828
Use Docker
docker model run hf.co/jj90/Kev-0.8B-RKNN3-RK1828
- Llama new
- LM Studio
- Jan
- Ollama
How to use jj90/Kev-0.8B-RKNN3-RK1828 with Ollama:
ollama run hf.co/jj90/Kev-0.8B-RKNN3-RK1828
- Unsloth Desktop
- Pi
How to use jj90/Kev-0.8B-RKNN3-RK1828 with Pi:
Start the llama.cpp server
# Install llama.cpp: brew install llama.cpp # Start a local OpenAI-compatible server: llama serve -hf jj90/Kev-0.8B-RKNN3-RK1828
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "llama-cpp": { "baseUrl": "http://localhost:9931/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "jj90/Kev-0.8B-RKNN3-RK1828" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Docker Model Runner
How to use jj90/Kev-0.8B-RKNN3-RK1828 with Docker Model Runner:
docker model run hf.co/jj90/Kev-0.8B-RKNN3-RK1828
- Lemonade
How to use jj90/Kev-0.8B-RKNN3-RK1828 with Lemonade:
Pull the model
# Download Lemonade from https://lemonade-server.ai/ lemonade pull jj90/Kev-0.8B-RKNN3-RK1828
Run and chat with the model
lemonade run user.Kev-0.8B-RKNN3-RK1828-{{QUANT_TAG}}List all available models
lemonade list
- Hermes Agent
How to use jj90/Kev-0.8B-RKNN3-RK1828 with Hermes Agent:
Start the llama.cpp server
# Install llama.cpp: brew install llama.cpp # Start a local OpenAI-compatible server: llama serve -hf jj90/Kev-0.8B-RKNN3-RK1828
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:9931/v1 hermes config set model.default jj90/Kev-0.8B-RKNN3-RK1828
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use jj90/Kev-0.8B-RKNN3-RK1828 with OpenClaw:
Start the llama.cpp server
# Install llama.cpp: brew install llama.cpp # Start a local OpenAI-compatible server: llama serve -hf jj90/Kev-0.8B-RKNN3-RK1828
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:9931/v1 \ --custom-model-id "jj90/Kev-0.8B-RKNN3-RK1828" \ --custom-provider-id llama-cpp \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
Download host/kev_daemon.cpp from jj90/Kev-0.8B-RKNN3-RK1828: direct link, hf CLI and curl.
- Browser
- Download file 15.4 kB
-
https://huggingface.co/jj90/Kev-0.8B-RKNN3-RK1828/resolve/main/host/kev_daemon.cpp
- Command line
-
hf download hf://jj90/Kev-0.8B-RKNN3-RK1828/host/kev_daemon.cpp
-
curl -L -o kev_daemon.cpp https://huggingface.co/jj90/Kev-0.8B-RKNN3-RK1828/resolve/main/host/kev_daemon.cpp
15.4 kB
| // Copyright (c) 2026 by Rockchip Electronics Co., Ltd. All Rights Reserved. | |
| // | |
| // Licensed under the Apache License, Version 2.0 (the "License"); | |
| // you may not use this file except in compliance with the License. | |
| // You may obtain a copy of the License at | |
| // | |
| // http://www.apache.org/licenses/LICENSE-2.0 | |
| // | |
| // Unless required by applicable law or agreed to in writing, software | |
| // distributed under the License is distributed on an "AS IS" BASIS, | |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| // See the License for the specific language governing permissions and | |
| // limitations under the License. | |
| // ---- 我们加的:output_callback 里直接拿到 logits 张量并落盘 ---- | |
| // sampling_callback 不会被调用(运行时内部采样),但 output_tensors[0] 就是 vocabulary logits | |
| static int g_kev_dumped = 0; | |
| static int g_kev_seq = 0; // 当前第几条序列,用来给 logits 文件编号 | |
| static char g_out_path[512] = "D:\\rk1828-console\\kevrunner\\out\\cur.bin"; | |
| static int kev_output_callback(void* userdata, rknn3_tensor* outs, uint32_t n, | |
| LLMOutputCallbackState state) { | |
| (void)userdata; | |
| if (!g_kev_dumped && n > 0 && state == RKLLM_OUTPUT_CALLBACK_PREFILL_FINISHED) { | |
| const char* p = g_out_path; | |
| FILE* f = fopen(p, "wb"); | |
| if (f) { | |
| size_t sz = outs[0].mem->size; | |
| fwrite(outs[0].mem->virt_addr, 1, sz, f); | |
| fclose(f); | |
| printf("[kev] dumped %zu bytes, state=%d, name=%s\n", sz, (int)state, | |
| outs[0].attr->name); | |
| g_kev_dumped = 1; | |
| } else { | |
| printf("[kev] fopen failed\n"); | |
| } | |
| fflush(stdout); | |
| } | |
| return 0; | |
| } | |
| // ---- 极简 base64 解码,给 TOK 命令用(文本里有空格换行,用 base64 最省事)---- | |
| static int b64val(char c) { | |
| if (c >= 'A' && c <= 'Z') return c - 'A'; | |
| if (c >= 'a' && c <= 'z') return c - 'a' + 26; | |
| if (c >= '0' && c <= '9') return c - '0' + 52; | |
| if (c == '+') return 62; | |
| if (c == '/') return 63; | |
| return -1; | |
| } | |
| static int b64decode(const char* in, int inlen, char* out, int outmax) { | |
| int v = 0, bits = 0, n = 0; | |
| for (int i = 0; i < inlen; i++) { | |
| if (in[i] == '=' || in[i] == '\n' || in[i] == '\r') continue; | |
| int d = b64val(in[i]); | |
| if (d < 0) continue; | |
| v = (v << 6) | d; | |
| bits += 6; | |
| if (bits >= 8) { | |
| bits -= 8; | |
| if (n < outmax) out[n++] = (char)((v >> bits) & 0xFF); | |
| } | |
| } | |
| return n; | |
| } | |
| int main(int argc, char** argv) | |
| { | |
| // keep_history / ignore_eos_token / key_path are optional and parsed by | |
| // argc position: passing fewer falls back to the defaults in [] below. | |
| // Usage: %s <rknn> <weight> <tokenizer> <embedding> <max_context_len> <max_new_tokens> <core_mask> [keep_history] [ignore_eos_token] | |
| // [key_path] | |
| if (argc < 8 || argc > 12) { // 12 = 多一个 ids 文件 | |
| LOGW( | |
| "Usage: %s <rknn_path> <weight_path> <tokenizer.gguf> <embedding.bin> <max_context_len> <max_new_tokens> <core_mask> [keep_history] [ignore_eos_token] [key_path]\n ", | |
| argv[0]); | |
| LOGW( | |
| "Such as: %s ./model/Qwen2.5-0.5B.rknn ./model/Qwen2.5-0.5B.weight ./model/Qwen2.5-0.5B.tokenizer.gguf ./model/Qwen2.5-0.5B-embedding.bin 1024 256 0xff 0 0 ./model/key.env\n", | |
| argv[0]); | |
| LOGW( | |
| "For encrypted model: %s ./model/Qwen2.5-0.5B.rknn ./model/Qwen2.5-0.5B.weight ./model/Qwen2.5-0.5B.tokenizer.gguf ./model/Qwen2.5-0.5B-embedding.bin 1024 256 0xff 0 0 ./model/key.env\n", | |
| argv[0]); | |
| return -1; | |
| } | |
| int ret = 0; | |
| int i = 0; // 已处理的序列数(放在这里避免 goto 跨初始化) | |
| char* model_path = argv[1]; | |
| char* weight_path = argv[2]; | |
| const char* tokenizer_path = argv[3]; | |
| const char* embedding_path = argv[4]; | |
| int32_t max_context_len = atoi(argv[5]); | |
| int32_t max_new_tokens = atoi(argv[6]); | |
| uint32_t core_mask = strtoul(argv[7], nullptr, 16); | |
| // Optional positional args; defaults keep the old behaviour. | |
| int keep_history = (argc > 8) ? atoi(argv[8]) : 0; | |
| bool ignore_eos_token = (argc > 9) ? (atoi(argv[9]) != 0) : false; | |
| const char* key_path = (argc > 10) ? argv[10] : nullptr; | |
| const char* ids_path = (argc > 11) ? argv[11] : nullptr; // 每行一串 token id | |
| // key 文件是占位参数;不存在就当没给,否则会被当成加密模型去解密 | |
| if (key_path) { | |
| FILE* kf = fopen(key_path, "r"); | |
| if (kf) fclose(kf); | |
| else { | |
| printf("[kev] key file '%s' not found, ignore it\n", key_path); | |
| key_path = nullptr; | |
| } | |
| } | |
| rknn3_devices devs; | |
| rknn3_context ctx = 0; | |
| rknn3_input_output_num io_num; | |
| VocabInfo vocab_info; | |
| Tokenizer* tokenizer = nullptr; | |
| struct stat emb_st; | |
| struct embedding_info embedding_info; | |
| RKLLMCallback callback; | |
| rknn3_test_session_userdata_t session_userdata; | |
| rknn3_llm_param params; | |
| int n_params = 1; | |
| RKLLMRunState state; | |
| rknn3_lora loras_enabled[RKNN3_MAX_LORA_NUM]; | |
| rknn3_llm_config llm_config; | |
| rknn3_kvcache_policy kvcache_policy = RKNN3_KVCACHE_POLICY_NORMAL; | |
| rknn3_kvcache_policy_param kvcache_policy_param; | |
| int llm_n_inputs = 1; | |
| rknn3_llm_input* llm_inputs[llm_n_inputs]; | |
| rknn3_llm_infer_param llm_infer_param; | |
| memset(&devs, 0, sizeof(devs)); | |
| memset(&io_num, 0, sizeof(io_num)); | |
| memset(&vocab_info, 0, sizeof(vocab_info)); | |
| memset(&emb_st, 0, sizeof(emb_st)); | |
| memset(&embedding_info, 0, sizeof(embedding_info)); | |
| memset(&callback, 0, sizeof(callback)); | |
| memset(¶ms, 0, sizeof(params)); | |
| memset(&state, 0, sizeof(state)); | |
| memset(&llm_config, 0, sizeof(llm_config)); | |
| memset(loras_enabled, 0, sizeof(loras_enabled)); | |
| memset(&kvcache_policy_param, 0, sizeof(kvcache_policy_param)); | |
| state.loras_enabled = loras_enabled; | |
| memset(llm_inputs, 0, sizeof(llm_inputs)); | |
| memset(&llm_infer_param, 0, sizeof(llm_infer_param)); | |
| memset(&session_userdata, 0, sizeof(session_userdata)); | |
| // set output tensors index for output callback | |
| rknn3_tensor output_tensors[1]; | |
| int n_output_tensors = 1; | |
| int output_tensors_index[1] = {0}; | |
| memset(output_tensors, 0, sizeof(output_tensors)); | |
| std::vector<std::string> random_prompts = { | |
| "请解释一下相对论的基本概念。", | |
| "Please explain the basic concept of relativity", | |
| }; | |
| int test_num = random_prompts.size(); | |
| // 若给了 ids 文件,就走批量模式:每行一串 token id,逐条 prefill 并 dump logits | |
| std::vector<std::vector<int32_t>> seqs; | |
| if (ids_path) { | |
| FILE* fp = fopen(ids_path, "r"); | |
| if (!fp) { printf("cannot open ids file: %s\n", ids_path); return -1; } | |
| char line[65536]; | |
| while (fgets(line, sizeof(line), fp)) { | |
| std::vector<int32_t> one; | |
| char* p = line; | |
| while (*p) { | |
| char* end = nullptr; | |
| long v = strtol(p, &end, 10); | |
| if (end == p) break; | |
| one.push_back((int32_t)v); | |
| p = end; | |
| } | |
| if (!one.empty()) seqs.push_back(one); | |
| } | |
| fclose(fp); | |
| test_num = (int)seqs.size(); | |
| printf("[kev] loaded %d sequences from %s\n", test_num, ids_path); | |
| } | |
| printf("*******************************NEW TEST**********************************\n"); | |
| // find devices (devs already zero-initialised above) | |
| const char* device_id = nullptr; | |
| ret = rknn3_find_devices(&devs); | |
| if (ret != RKNN3_SUCCESS || devs.n_devices == 0) { | |
| printf("rknn3_find_devices failed, ret = %d\n", ret); | |
| goto exit; | |
| } | |
| // init context and model | |
| device_id = devs.devices[0].id; | |
| ret = init_context_and_model(&ctx, model_path, weight_path, core_mask, key_path, device_id); | |
| if (ret != RKNN3_SUCCESS) { | |
| printf("rknn3_session_init_model fail! ret = %d\n", ret); | |
| goto exit; | |
| } | |
| // Get Model Output Info for output callback (if needed) | |
| if (n_output_tensors > 0) { | |
| ret = setup_output_tensors(ctx, output_tensors, n_output_tensors, output_tensors_index, &io_num); | |
| if (ret < 0) { | |
| goto exit; | |
| } | |
| } | |
| // get tokenizer and embedding | |
| ret = get_tokenizer_and_embedding(tokenizer_path, &vocab_info, &tokenizer, &embedding_info, embedding_path, &emb_st); | |
| if (ret < 0) { | |
| printf("get_tokenizer_and_embedding failed, ret = %d\n", ret); | |
| goto exit; | |
| } | |
| ret = rknn3_query(ctx, RKNN3_QUERY_LLM_CONFIG, &llm_config, sizeof(rknn3_llm_config)); | |
| if (ret != RKNN3_SUCCESS) { | |
| printf("rknn3_query llm config failed! ret=%d", ret); | |
| goto exit; | |
| } | |
| // Set basic session parameters | |
| build_default_llm_params(¶ms, &vocab_info, max_context_len, ignore_eos_token); | |
| // init session | |
| session_userdata.session = rknn3_session_init(ctx, ¶ms, n_params); | |
| if (!session_userdata.session) { | |
| printf("rknn3_session_init failed, ret = %d\n", ret); | |
| goto exit; | |
| } | |
| // set session callback | |
| build_default_callback(&callback, tokenizer, &embedding_info, &session_userdata); | |
| // sampling_callback 不会被运行时调用(内部采样),logits 从 output_callback 拿 | |
| callback.sampling_callback = NULL; | |
| callback.sampling_userdata = NULL; | |
| // 同时打开 output_callback:若它能触发而 sampling 不能,说明 sampling 被门控 | |
| callback.output_callback = kev_output_callback; | |
| callback.output_userdata = &embedding_info; | |
| callback.output_tensors = output_tensors; | |
| callback.n_output_tensors = n_output_tensors; | |
| ret = rknn3_session_set_callback(session_userdata.session, &callback); | |
| if (ret != RKNN3_SUCCESS) { | |
| printf("rknn3_session_set_callback failed, ret = %d\n", ret); | |
| goto exit; | |
| } | |
| // set/update chat template | |
| if (0) { | |
| std::string system_prompt = "<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n"; | |
| std::string prompt_prefix = "<|im_start|>user\n"; | |
| std::string prompt_postfix = "<|im_end|>\n<|im_start|>assistant\n"; | |
| ret = rknn3_session_set_chat_template(session_userdata.session, system_prompt.c_str(), prompt_prefix.c_str(), prompt_postfix.c_str()); | |
| if (ret != RKNN3_SUCCESS) { | |
| printf("rknn3_session_set_chat_template failed, ret = %d\n", ret); | |
| goto exit; | |
| } | |
| } | |
| // set kvcache policy | |
| ret = rknn3_session_set_kvcache_policy(session_userdata.session, kvcache_policy, nullptr); | |
| if (ret != RKNN3_SUCCESS) { | |
| printf("rknn3_session_set_kvcache_policy failed, ret=%d\n", ret); | |
| goto exit; | |
| } | |
| print_llm_config(&llm_config); | |
| // Test the generic session inference: each iteration builds a typical prompt request, | |
| // runs one round of generation to show the standard prompt input workflow. | |
| char in_line[65536]; | |
| // 关键:stdout 接管道时 C 运行库默认全缓冲,READY_DAEMON 会卡在缓冲区里, | |
| // 父进程永远等不到。必须显式改成行缓冲(只 fflush 一次不够,后面每行都要)。 | |
| setvbuf(stdout, NULL, _IOLBF, 0); | |
| printf("READY_DAEMON\n"); | |
| fflush(stdout); | |
| while (fgets(in_line, sizeof(in_line), stdin)) { | |
| if (strncmp(in_line, "quit", 4) == 0) break; | |
| // OUT <path> -> 指定下一次 dump 的落盘路径 | |
| if (strncmp(in_line, "OUT ", 4) == 0) { | |
| char* q = in_line + 4; | |
| while (*q == ' ') q++; | |
| int n = (int)strlen(q); | |
| while (n > 0 && (q[n-1] == '\n' || q[n-1] == '\r')) n--; | |
| if (n > 0 && n < (int)sizeof(g_out_path)) { | |
| memcpy(g_out_path, q, n); | |
| g_out_path[n] = 0; | |
| printf("OUTSET %s\n", g_out_path); | |
| fflush(stdout); | |
| } | |
| continue; | |
| } | |
| // TOK <base64 text> -> 打印 "IDS n id1 id2 ..." | |
| if (strncmp(in_line, "TOK ", 4) == 0) { | |
| static char tbuf[65536]; | |
| static int32_t tids[8192]; | |
| int raw = b64decode(in_line + 4, (int)strlen(in_line + 4), tbuf, sizeof(tbuf) - 1); | |
| tbuf[raw] = 0; | |
| printf("[kev] TOK raw=%d head='%.40s' tok=%p\n", raw, tbuf, (void*)tokenizer); | |
| fflush(stdout); | |
| int n = tokenizer->Tokenize(tbuf, raw, tids, 8192); | |
| printf("IDS %d", n); | |
| for (int k = 0; k < n; k++) printf(" %d", tids[k]); | |
| printf("\n"); | |
| fflush(stdout); | |
| continue; | |
| } | |
| // R <ids> = 清空 kvcache 后从头 prefill(keep_history=0) | |
| // A <ids> = 接在上一次之后追加 prefill(keep_history=1) | |
| // 不带前缀 = 老行为(等价 R) | |
| bool append = false; | |
| char* pp = in_line; | |
| if (in_line[0] == 'A' && in_line[1] == ' ') { append = true; pp = in_line + 2; } | |
| else if (in_line[0] == 'R' && in_line[1] == ' ') { pp = in_line + 2; } | |
| { | |
| std::vector<int32_t> one; | |
| char* p = pp; | |
| while (*p) { | |
| char* e = nullptr; | |
| long v = strtol(p, &e, 10); | |
| if (e == p) break; | |
| one.push_back((int32_t)v); | |
| p = e; | |
| } | |
| if (one.empty()) continue; | |
| seqs.push_back(one); | |
| } | |
| printf("\n--------------------Input[%d]-------------------- \n", i); | |
| llm_infer_param = build_default_infer_param(max_new_tokens, append ? 1 : 0); | |
| g_kev_seq = i; | |
| g_kev_dumped = 0; | |
| // A 模式接着上次的 kvcache;R 模式(默认)从头开始 | |
| if (!append) { | |
| rknn3_session_clear_kvcache(session_userdata.session, RKNN3_KVCACHE_CLEAR_ALL); | |
| } | |
| llm_inputs[0] = build_llm_input_tokens(seqs.back().data(), seqs.back().size(), false, false); | |
| printf("\n--------------------Output---------------------- \n"); | |
| rknn3_test_timing_mark_start(&session_userdata.timing); | |
| ret = rknn3_session_run(session_userdata.session, llm_inputs[0], llm_n_inputs, &llm_infer_param); | |
| if (ret != RKNN3_SUCCESS) { | |
| printf("rknn3_session_run failed, ret = %d\n", ret); | |
| goto exit; | |
| } | |
| rknn3_test_timing_mark_end(&session_userdata.timing); | |
| printf("DONE %d\n", i); // 每处理完一条就给客户端一个信号 | |
| fflush(stdout); | |
| i++; | |
| ret = rknn3_session_query_state(session_userdata.session, &state); | |
| if (ret != RKNN3_SUCCESS) { | |
| printf("rknn3_session_query_state failed, ret=%d\n", ret); | |
| goto exit; | |
| } | |
| print_run_state(&state, true); | |
| // clear kvcache | |
| if (state.n_total_tokens >= (state.n_max_tokens - max_new_tokens)) { | |
| ret = rknn3_session_clear_kvcache(session_userdata.session, RKNN3_KVCACHE_CLEAR_ALL); | |
| if (ret != RKNN3_SUCCESS) { | |
| printf("rknn3_session_clear_kvcache failed, ret=%d\n", ret); | |
| goto exit; | |
| } | |
| } | |
| print_perf_stats(&state, &session_userdata.timing); | |
| // Free any heap memory owned by this iteration's input(s). | |
| for (int j = 0; j < llm_n_inputs; j++) { | |
| release_llm_input(ctx, llm_inputs[j]); | |
| } | |
| } | |
| exit: | |
| for (int j = 0; j < llm_n_inputs; j++) { | |
| release_llm_input(ctx, llm_inputs[j]); | |
| } | |
| if (n_output_tensors > 0) { | |
| cleanup_output_tensors(ctx, output_tensors, n_output_tensors); | |
| } | |
| if (session_userdata.session) { | |
| rknn3_session_destroy(session_userdata.session); | |
| } | |
| if (ctx) { | |
| rknn3_destroy(ctx); | |
| } | |
| embedding_info_release(&embedding_info, &emb_st); | |
| if (tokenizer) { | |
| delete tokenizer; | |
| } | |
| printf("*******************************END TEST**********************************\n"); | |
| fflush(stdout); | |
| return 0; | |
| } | |