| |
| |
| |
|
|
| |
|
|
| import json |
| import re |
| import sys |
|
|
| result_jsonl = sys.argv[1] |
| test_jsonl = sys.argv[2] |
| codec_jsonl = sys.argv[3] |
|
|
| |
| llm_token_list = [] |
| with open(result_jsonl) as f: |
| for line in f: |
| arr = line.strip() |
| numbers = re.findall(r"speech_(\d+)", arr) |
| result = " ".join(numbers) |
| llm_token_list.append(result) |
|
|
| |
| json_lines = open(test_jsonl).readlines() |
| assert len(llm_token_list) == len( |
| json_lines |
| ), "llm_token_list and json_lines length mismatch" |
|
|
| data = [] |
| for i, line in enumerate(json_lines): |
| item = json.loads(line) |
| |
| item["llm_token"] = llm_token_list[i] |
| data.append(item) |
|
|
| |
| with open(codec_jsonl, "w") as f: |
| for x in data: |
| f.write(json.dumps(x, ensure_ascii=False) + "\n") |
|
|
