| import os
|
| import sys
|
| import hashlib
|
|
|
| COMFY_ROOT = r"D:\ComfyUI_Python312\ComfyUI"
|
|
|
| sys.path.insert(0, COMFY_ROOT)
|
| os.chdir(COMFY_ROOT)
|
|
|
| from transformers import AutoTokenizer
|
|
|
| print("=" * 80)
|
| print("Tokenizer compatibility test")
|
| print("=" * 80)
|
|
|
|
|
|
|
| print("\nLoading SenseNova tokenizer...")
|
|
|
| sense_tok = AutoTokenizer.from_pretrained(
|
| "sensenova/SenseNova-U1.5-8B-MoT",
|
| trust_remote_code=True,
|
| )
|
|
|
|
|
| print("Loading Qwen3-VL tokenizer used by H3...")
|
|
|
| h3_tok = AutoTokenizer.from_pretrained(
|
| "Qwen/Qwen3-VL-32B-Instruct",
|
| trust_remote_code=True,
|
| )
|
|
|
| print("\nSenseNova vocab size:", len(sense_tok))
|
| print("H3/Qwen vocab size: ", len(h3_tok))
|
|
|
| sense_vocab = sense_tok.get_vocab()
|
| h3_vocab = h3_tok.get_vocab()
|
|
|
| common_tokens = set(sense_vocab) & set(h3_vocab)
|
|
|
| same_id = 0
|
| different_id = []
|
|
|
| for token in common_tokens:
|
| a = sense_vocab[token]
|
| b = h3_vocab[token]
|
|
|
| if a == b:
|
| same_id += 1
|
| else:
|
| different_id.append((token, a, b))
|
|
|
| print()
|
| print("Common token strings:", len(common_tokens))
|
| print("Same token ID: ", same_id)
|
| print("Different token ID: ", len(different_id))
|
|
|
| if common_tokens:
|
| print(
|
| "ID agreement:",
|
| f"{100.0 * same_id / len(common_tokens):.6f}%"
|
| )
|
|
|
| print("\nFirst ID mismatches:")
|
|
|
| for item in different_id[:30]:
|
| print(item)
|
|
|
|
|
|
|
|
|
|
|
| max_compare = min(len(sense_tok), len(h3_tok))
|
|
|
| id_mismatches = []
|
|
|
| for i in range(max_compare):
|
| a = sense_tok.convert_ids_to_tokens(i)
|
| b = h3_tok.convert_ids_to_tokens(i)
|
|
|
| if a != b:
|
| id_mismatches.append((i, a, b))
|
|
|
| print()
|
| print("ID->token mismatches:", len(id_mismatches))
|
|
|
| for x in id_mismatches[:30]:
|
| print(x)
|
|
|
|
|
|
|
|
|
|
|
| prompts = [
|
| "a red cube on a blue sphere",
|
| "a woman holding a transparent glass bottle",
|
| "three people standing behind a wooden table",
|
| "the word HELLO printed on a white sign",
|
| "a metallic robot illuminated from the left",
|
| "a hand holding a glass of water",
|
| "two cars parked beside a brick building",
|
| "a woman reflected in a mirror",
|
| ]
|
|
|
| print("\n" + "=" * 80)
|
| print("PROMPT TOKENIZATION")
|
| print("=" * 80)
|
|
|
| all_prompt_equal = True
|
|
|
| for text in prompts:
|
|
|
| s = sense_tok.encode(
|
| text,
|
| add_special_tokens=False,
|
| )
|
|
|
| h = h3_tok.encode(
|
| text,
|
| add_special_tokens=False,
|
| )
|
|
|
| equal = s == h
|
|
|
| if not equal:
|
| all_prompt_equal = False
|
|
|
| print()
|
| print(text)
|
| print("equal:", equal)
|
| print("SenseNova:", s)
|
| print("H3: ", h)
|
|
|
| print("\n" + "=" * 80)
|
|
|
| if (
|
| len(different_id) == 0
|
| and len(id_mismatches) == 0
|
| and all_prompt_equal
|
| ):
|
| print("RESULT: TOKENIZERS ARE IDENTICAL FOR TESTED VOCAB")
|
| else:
|
| print("RESULT: TOKENIZERS DIFFER")
|
|
|
| print("=" * 80) |
