File size: 15,367 Bytes
a846b3d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
// Copyright (c) 2026 by Rockchip Electronics Co., Ltd. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "rknn3_session_test_common.h"

// ---- 我们加的:output_callback 里直接拿到 logits 张量并落盘 ----
// sampling_callback 不会被调用(运行时内部采样),但 output_tensors[0] 就是 vocabulary logits
static int g_kev_dumped = 0;
static int g_kev_seq    = 0;   // 当前第几条序列,用来给 logits 文件编号
static char g_out_path[512] = "D:\\rk1828-console\\kevrunner\\out\\cur.bin";

static int kev_output_callback(void* userdata, rknn3_tensor* outs, uint32_t n,
                               LLMOutputCallbackState state) {
  (void)userdata;
  if (!g_kev_dumped && n > 0 && state == RKLLM_OUTPUT_CALLBACK_PREFILL_FINISHED) {
    const char* p = g_out_path;
    FILE* f = fopen(p, "wb");
    if (f) {
      size_t sz = outs[0].mem->size;
      fwrite(outs[0].mem->virt_addr, 1, sz, f);
      fclose(f);
      printf("[kev] dumped %zu bytes, state=%d, name=%s\n", sz, (int)state,
             outs[0].attr->name);
      g_kev_dumped = 1;
    } else {
      printf("[kev] fopen failed\n");
    }
    fflush(stdout);
  }
  return 0;
}

// ---- 极简 base64 解码,给 TOK 命令用(文本里有空格换行,用 base64 最省事)----
static int b64val(char c) {
  if (c >= 'A' && c <= 'Z') return c - 'A';
  if (c >= 'a' && c <= 'z') return c - 'a' + 26;
  if (c >= '0' && c <= '9') return c - '0' + 52;
  if (c == '+') return 62;
  if (c == '/') return 63;
  return -1;
}
static int b64decode(const char* in, int inlen, char* out, int outmax) {
  int v = 0, bits = 0, n = 0;
  for (int i = 0; i < inlen; i++) {
    if (in[i] == '=' || in[i] == '\n' || in[i] == '\r') continue;
    int d = b64val(in[i]);
    if (d < 0) continue;
    v = (v << 6) | d;
    bits += 6;
    if (bits >= 8) {
      bits -= 8;
      if (n < outmax) out[n++] = (char)((v >> bits) & 0xFF);
    }
  }
  return n;
}

int main(int argc, char** argv)
{
  // keep_history / ignore_eos_token / key_path are optional and parsed by
  // argc position: passing fewer falls back to the defaults in [] below.
  // Usage: %s <rknn> <weight> <tokenizer> <embedding> <max_context_len> <max_new_tokens> <core_mask> [keep_history] [ignore_eos_token]
  // [key_path]
  if (argc < 8 || argc > 12) {   // 12 = 多一个 ids 文件
    LOGW(
      "Usage: %s <rknn_path> <weight_path> <tokenizer.gguf> <embedding.bin> <max_context_len> <max_new_tokens> <core_mask> [keep_history] [ignore_eos_token] [key_path]\n ",
      argv[0]);
    LOGW(
      "Such as: %s ./model/Qwen2.5-0.5B.rknn ./model/Qwen2.5-0.5B.weight ./model/Qwen2.5-0.5B.tokenizer.gguf ./model/Qwen2.5-0.5B-embedding.bin 1024 256 0xff 0 0 ./model/key.env\n",
      argv[0]);
    LOGW(
      "For encrypted model: %s ./model/Qwen2.5-0.5B.rknn ./model/Qwen2.5-0.5B.weight ./model/Qwen2.5-0.5B.tokenizer.gguf ./model/Qwen2.5-0.5B-embedding.bin 1024 256 0xff 0 0 ./model/key.env\n",
      argv[0]);

    return -1;
  }

  int         ret             = 0;
  int         i               = 0;   // 已处理的序列数(放在这里避免 goto 跨初始化)
  char*       model_path      = argv[1];
  char*       weight_path     = argv[2];
  const char* tokenizer_path  = argv[3];
  const char* embedding_path  = argv[4];
  int32_t     max_context_len = atoi(argv[5]);
  int32_t     max_new_tokens  = atoi(argv[6]);
  uint32_t    core_mask       = strtoul(argv[7], nullptr, 16);
  // Optional positional args; defaults keep the old behaviour.
  int         keep_history     = (argc > 8) ? atoi(argv[8]) : 0;
  bool        ignore_eos_token = (argc > 9) ? (atoi(argv[9]) != 0) : false;
  const char* key_path         = (argc > 10) ? argv[10] : nullptr;
  const char* ids_path         = (argc > 11) ? argv[11] : nullptr;  // 每行一串 token id

  // key 文件是占位参数;不存在就当没给,否则会被当成加密模型去解密
  if (key_path) {
    FILE* kf = fopen(key_path, "r");
    if (kf) fclose(kf);
    else {
      printf("[kev] key file '%s' not found, ignore it\n", key_path);
      key_path = nullptr;
    }
  }

  rknn3_devices                 devs;
  rknn3_context                 ctx = 0;
  rknn3_input_output_num        io_num;
  VocabInfo                     vocab_info;
  Tokenizer*                    tokenizer = nullptr;
  struct stat                   emb_st;
  struct embedding_info         embedding_info;
  RKLLMCallback                 callback;
  rknn3_test_session_userdata_t session_userdata;
  rknn3_llm_param               params;
  int                           n_params = 1;
  RKLLMRunState                 state;
  rknn3_lora                    loras_enabled[RKNN3_MAX_LORA_NUM];
  rknn3_llm_config              llm_config;
  rknn3_kvcache_policy          kvcache_policy = RKNN3_KVCACHE_POLICY_NORMAL;
  rknn3_kvcache_policy_param    kvcache_policy_param;
  int                           llm_n_inputs = 1;
  rknn3_llm_input*              llm_inputs[llm_n_inputs];
  rknn3_llm_infer_param         llm_infer_param;

  memset(&devs, 0, sizeof(devs));
  memset(&io_num, 0, sizeof(io_num));
  memset(&vocab_info, 0, sizeof(vocab_info));
  memset(&emb_st, 0, sizeof(emb_st));
  memset(&embedding_info, 0, sizeof(embedding_info));
  memset(&callback, 0, sizeof(callback));
  memset(&params, 0, sizeof(params));
  memset(&state, 0, sizeof(state));
  memset(&llm_config, 0, sizeof(llm_config));
  memset(loras_enabled, 0, sizeof(loras_enabled));
  memset(&kvcache_policy_param, 0, sizeof(kvcache_policy_param));
  state.loras_enabled = loras_enabled;
  memset(llm_inputs, 0, sizeof(llm_inputs));
  memset(&llm_infer_param, 0, sizeof(llm_infer_param));
  memset(&session_userdata, 0, sizeof(session_userdata));

  // set output tensors index for output callback
  rknn3_tensor output_tensors[1];
  int          n_output_tensors        = 1;
  int          output_tensors_index[1] = {0};
  memset(output_tensors, 0, sizeof(output_tensors));

  std::vector<std::string> random_prompts = {
    "请解释一下相对论的基本概念。",
    "Please explain the basic concept of relativity",
  };
  int test_num = random_prompts.size();

  // 若给了 ids 文件,就走批量模式:每行一串 token id,逐条 prefill 并 dump logits
  std::vector<std::vector<int32_t>> seqs;
  if (ids_path) {
    FILE* fp = fopen(ids_path, "r");
    if (!fp) { printf("cannot open ids file: %s\n", ids_path); return -1; }
    char line[65536];
    while (fgets(line, sizeof(line), fp)) {
      std::vector<int32_t> one;
      char* p = line;
      while (*p) {
        char* end = nullptr;
        long v = strtol(p, &end, 10);
        if (end == p) break;
        one.push_back((int32_t)v);
        p = end;
      }
      if (!one.empty()) seqs.push_back(one);
    }
    fclose(fp);
    test_num = (int)seqs.size();
    printf("[kev] loaded %d sequences from %s\n", test_num, ids_path);
  }

  printf("*******************************NEW TEST**********************************\n");

  // find devices (devs already zero-initialised above)
  const char* device_id = nullptr;
  ret                   = rknn3_find_devices(&devs);
  if (ret != RKNN3_SUCCESS || devs.n_devices == 0) {
    printf("rknn3_find_devices failed, ret = %d\n", ret);
    goto exit;
  }

  // init context and model
  device_id = devs.devices[0].id;
  ret       = init_context_and_model(&ctx, model_path, weight_path, core_mask, key_path, device_id);
  if (ret != RKNN3_SUCCESS) {
    printf("rknn3_session_init_model fail! ret = %d\n", ret);
    goto exit;
  }

  // Get Model Output Info for output callback (if needed)
  if (n_output_tensors > 0) {
    ret = setup_output_tensors(ctx, output_tensors, n_output_tensors, output_tensors_index, &io_num);
    if (ret < 0) {
      goto exit;
    }
  }

  // get tokenizer and embedding
  ret = get_tokenizer_and_embedding(tokenizer_path, &vocab_info, &tokenizer, &embedding_info, embedding_path, &emb_st);
  if (ret < 0) {
    printf("get_tokenizer_and_embedding failed, ret = %d\n", ret);
    goto exit;
  }

  ret = rknn3_query(ctx, RKNN3_QUERY_LLM_CONFIG, &llm_config, sizeof(rknn3_llm_config));
  if (ret != RKNN3_SUCCESS) {
    printf("rknn3_query llm config failed! ret=%d", ret);
    goto exit;
  }

  // Set basic session parameters
  build_default_llm_params(&params, &vocab_info, max_context_len, ignore_eos_token);

  // init session
  session_userdata.session = rknn3_session_init(ctx, &params, n_params);
  if (!session_userdata.session) {
    printf("rknn3_session_init failed, ret = %d\n", ret);
    goto exit;
  }

  // set session callback
  build_default_callback(&callback, tokenizer, &embedding_info, &session_userdata);
  // sampling_callback 不会被运行时调用(内部采样),logits 从 output_callback 拿
  callback.sampling_callback = NULL;
  callback.sampling_userdata = NULL;
  // 同时打开 output_callback:若它能触发而 sampling 不能,说明 sampling 被门控
  callback.output_callback   = kev_output_callback;
  callback.output_userdata   = &embedding_info;
  callback.output_tensors    = output_tensors;
  callback.n_output_tensors  = n_output_tensors;

  ret = rknn3_session_set_callback(session_userdata.session, &callback);
  if (ret != RKNN3_SUCCESS) {
    printf("rknn3_session_set_callback failed, ret = %d\n", ret);
    goto exit;
  }

  // set/update chat template
  if (0) {
    std::string system_prompt  = "<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n";
    std::string prompt_prefix  = "<|im_start|>user\n";
    std::string prompt_postfix = "<|im_end|>\n<|im_start|>assistant\n";
    ret = rknn3_session_set_chat_template(session_userdata.session, system_prompt.c_str(), prompt_prefix.c_str(), prompt_postfix.c_str());
    if (ret != RKNN3_SUCCESS) {
      printf("rknn3_session_set_chat_template failed, ret = %d\n", ret);
      goto exit;
    }
  }

  // set kvcache policy
  ret = rknn3_session_set_kvcache_policy(session_userdata.session, kvcache_policy, nullptr);
  if (ret != RKNN3_SUCCESS) {
    printf("rknn3_session_set_kvcache_policy failed, ret=%d\n", ret);
    goto exit;
  }

  print_llm_config(&llm_config);

  // Test the generic session inference: each iteration builds a typical prompt request, 
  // runs one round of generation to show the standard prompt input workflow.
  char          in_line[65536];
  // 关键:stdout 接管道时 C 运行库默认全缓冲,READY_DAEMON 会卡在缓冲区里,
  // 父进程永远等不到。必须显式改成行缓冲(只 fflush 一次不够,后面每行都要)。
  setvbuf(stdout, NULL, _IOLBF, 0);
  printf("READY_DAEMON\n");
  fflush(stdout);
  while (fgets(in_line, sizeof(in_line), stdin)) {
    if (strncmp(in_line, "quit", 4) == 0) break;

    // OUT <path>  ->  指定下一次 dump 的落盘路径
    if (strncmp(in_line, "OUT ", 4) == 0) {
      char* q = in_line + 4;
      while (*q == ' ') q++;
      int n = (int)strlen(q);
      while (n > 0 && (q[n-1] == '\n' || q[n-1] == '\r')) n--;
      if (n > 0 && n < (int)sizeof(g_out_path)) {
        memcpy(g_out_path, q, n);
        g_out_path[n] = 0;
        printf("OUTSET %s\n", g_out_path);
        fflush(stdout);
      }
      continue;
    }

    // TOK <base64 text>  ->  打印 "IDS n id1 id2 ..."
    if (strncmp(in_line, "TOK ", 4) == 0) {
      static char tbuf[65536];
      static int32_t tids[8192];
      int raw = b64decode(in_line + 4, (int)strlen(in_line + 4), tbuf, sizeof(tbuf) - 1);
      tbuf[raw] = 0;
      printf("[kev] TOK raw=%d head='%.40s' tok=%p\n", raw, tbuf, (void*)tokenizer);
      fflush(stdout);
      int n = tokenizer->Tokenize(tbuf, raw, tids, 8192);
      printf("IDS %d", n);
      for (int k = 0; k < n; k++) printf(" %d", tids[k]);
      printf("\n");
      fflush(stdout);
      continue;
    }
    // R <ids> = 清空 kvcache 后从头 prefill(keep_history=0)
    // A <ids> = 接在上一次之后追加 prefill(keep_history=1)
    // 不带前缀 = 老行为(等价 R)
    bool append = false;
    char* pp    = in_line;
    if (in_line[0] == 'A' && in_line[1] == ' ') { append = true; pp = in_line + 2; }
    else if (in_line[0] == 'R' && in_line[1] == ' ') { pp = in_line + 2; }
    {
      std::vector<int32_t> one;
      char* p = pp;
      while (*p) {
        char* e = nullptr;
        long v = strtol(p, &e, 10);
        if (e == p) break;
        one.push_back((int32_t)v);
        p = e;
      }
      if (one.empty()) continue;
      seqs.push_back(one);
    }
    printf("\n--------------------Input[%d]-------------------- \n", i);

    llm_infer_param = build_default_infer_param(max_new_tokens, append ? 1 : 0);

    g_kev_seq    = i;
    g_kev_dumped = 0;
    // A 模式接着上次的 kvcache;R 模式(默认)从头开始
    if (!append) {
      rknn3_session_clear_kvcache(session_userdata.session, RKNN3_KVCACHE_CLEAR_ALL);
    }
    llm_inputs[0] = build_llm_input_tokens(seqs.back().data(), seqs.back().size(), false, false);

    printf("\n--------------------Output---------------------- \n");

    rknn3_test_timing_mark_start(&session_userdata.timing);
    ret = rknn3_session_run(session_userdata.session, llm_inputs[0], llm_n_inputs, &llm_infer_param);
    if (ret != RKNN3_SUCCESS) {
      printf("rknn3_session_run failed, ret = %d\n", ret);
      goto exit;
    }
    rknn3_test_timing_mark_end(&session_userdata.timing);
    printf("DONE %d\n", i);   // 每处理完一条就给客户端一个信号
    fflush(stdout);
    i++;

    ret = rknn3_session_query_state(session_userdata.session, &state);
    if (ret != RKNN3_SUCCESS) {
      printf("rknn3_session_query_state failed, ret=%d\n", ret);
      goto exit;
    }

    print_run_state(&state, true);

    // clear kvcache
    if (state.n_total_tokens >= (state.n_max_tokens - max_new_tokens)) {
      ret = rknn3_session_clear_kvcache(session_userdata.session, RKNN3_KVCACHE_CLEAR_ALL);
      if (ret != RKNN3_SUCCESS) {
        printf("rknn3_session_clear_kvcache failed, ret=%d\n", ret);
        goto exit;
      }
    }

    print_perf_stats(&state, &session_userdata.timing);

    // Free any heap memory owned by this iteration's input(s).
    for (int j = 0; j < llm_n_inputs; j++) {
      release_llm_input(ctx, llm_inputs[j]);
    }
  }

exit:
  for (int j = 0; j < llm_n_inputs; j++) {
    release_llm_input(ctx, llm_inputs[j]);
  }

  if (n_output_tensors > 0) {
    cleanup_output_tensors(ctx, output_tensors, n_output_tensors);
  }

  if (session_userdata.session) {
    rknn3_session_destroy(session_userdata.session);
  }

  if (ctx) {
    rknn3_destroy(ctx);
  }

  embedding_info_release(&embedding_info, &emb_st);

  if (tokenizer) {
    delete tokenizer;
  }

  printf("*******************************END TEST**********************************\n");

  fflush(stdout);

  return 0;
}