import os
import sys
import hashlib

COMFY_ROOT = r"D:\ComfyUI_Python312\ComfyUI"

sys.path.insert(0, COMFY_ROOT)
os.chdir(COMFY_ROOT)

from transformers import AutoTokenizer

print("=" * 80)
print("Tokenizer compatibility test")
print("=" * 80)

# SenseNova tokenizer — скачиваются только tokenizer/config файлы,
# а НЕ 45 GB weights.
print("\nLoading SenseNova tokenizer...")

sense_tok = AutoTokenizer.from_pretrained(
    "sensenova/SenseNova-U1.5-8B-MoT",
    trust_remote_code=True,
)

# MiniMax H3 uses Qwen3-VL-32B tokenizer.
print("Loading Qwen3-VL tokenizer used by H3...")

h3_tok = AutoTokenizer.from_pretrained(
    "Qwen/Qwen3-VL-32B-Instruct",
    trust_remote_code=True,
)

print("\nSenseNova vocab size:", len(sense_tok))
print("H3/Qwen vocab size:  ", len(h3_tok))

sense_vocab = sense_tok.get_vocab()
h3_vocab = h3_tok.get_vocab()

common_tokens = set(sense_vocab) & set(h3_vocab)

same_id = 0
different_id = []

for token in common_tokens:
    a = sense_vocab[token]
    b = h3_vocab[token]

    if a == b:
        same_id += 1
    else:
        different_id.append((token, a, b))

print()
print("Common token strings:", len(common_tokens))
print("Same token ID:       ", same_id)
print("Different token ID:  ", len(different_id))

if common_tokens:
    print(
        "ID agreement:",
        f"{100.0 * same_id / len(common_tokens):.6f}%"
    )

print("\nFirst ID mismatches:")

for item in different_id[:30]:
    print(item)

# -----------------------------------------------------------
# Direct ID -> token comparison
# -----------------------------------------------------------

max_compare = min(len(sense_tok), len(h3_tok))

id_mismatches = []

for i in range(max_compare):
    a = sense_tok.convert_ids_to_tokens(i)
    b = h3_tok.convert_ids_to_tokens(i)

    if a != b:
        id_mismatches.append((i, a, b))

print()
print("ID->token mismatches:", len(id_mismatches))

for x in id_mismatches[:30]:
    print(x)

# -----------------------------------------------------------
# Real prompt test
# -----------------------------------------------------------

prompts = [
    "a red cube on a blue sphere",
    "a woman holding a transparent glass bottle",
    "three people standing behind a wooden table",
    "the word HELLO printed on a white sign",
    "a metallic robot illuminated from the left",
    "a hand holding a glass of water",
    "two cars parked beside a brick building",
    "a woman reflected in a mirror",
]

print("\n" + "=" * 80)
print("PROMPT TOKENIZATION")
print("=" * 80)

all_prompt_equal = True

for text in prompts:

    s = sense_tok.encode(
        text,
        add_special_tokens=False,
    )

    h = h3_tok.encode(
        text,
        add_special_tokens=False,
    )

    equal = s == h

    if not equal:
        all_prompt_equal = False

    print()
    print(text)
    print("equal:", equal)
    print("SenseNova:", s)
    print("H3:       ", h)

print("\n" + "=" * 80)

if (
    len(different_id) == 0
    and len(id_mismatches) == 0
    and all_prompt_equal
):
    print("RESULT: TOKENIZERS ARE IDENTICAL FOR TESTED VOCAB")
else:
    print("RESULT: TOKENIZERS DIFFER")

print("=" * 80)