File size: 2,344 Bytes
15034cc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 | """Runs GLM-OCR in PyTorch (fp32) on a few lines and keeps what goes into and out of its parts, as reference for the ONNX export.
python3 ref_capture.py OUT_DIR PICTURE_DIR N -> ~/hw-bench/glm_onnx/ref/<line>.npz"""
import json, os, sys, numpy as np, torch
from PIL import Image
from transformers import AutoProcessor, AutoModelForImageTextToText
out, pics, n = sys.argv[1], sys.argv[2], int(sys.argv[3])
torch.set_num_threads(4)
path = os.path.expanduser('~/hw-bench/models/glm-ocr')
proc = AutoProcessor.from_pretrained(path)
model = AutoModelForImageTextToText.from_pretrained(path, dtype=torch.float32).eval()
os.makedirs(os.path.expanduser('~/hw-bench/glm_onnx/ref'), exist_ok=True)
cap = {}
vis = model.model.visual
vis.register_forward_hook(lambda m, a, k, o: cap.update(vis_in=a[0].detach().clone(), vis_grid=k['grid_thw'].detach().clone(), vis_out=o.pooler_output.detach().clone()), with_kwargs=True)
lm = model.model.language_model
def pre(m, a, k):
if 'prefill_pos' not in cap:
cap['prefill_pos'] = k['position_ids'].detach().clone(); cap['prefill_embeds'] = k['inputs_embeds'].detach().clone()
lm.register_forward_pre_hook(pre, with_kwargs=True)
for i in list(json.load(open(f'{out}/truth.json')))[:n]:
cap.clear()
msgs = [{'role': 'user', 'content': [{'type': 'image', 'image': Image.open(f'{out}/{pics}/{i}.png').convert('RGB')}, {'type': 'text', 'text': 'Text Recognition:'}]}]
inp = proc.apply_chat_template(msgs, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors='pt')
print(i, {k: tuple(v.shape) for k, v in inp.items()})
inp.pop('token_type_ids', None)
with torch.no_grad(): gen = model.generate(**inp, max_new_tokens=60, do_sample=False)
ids = gen[0][inp['input_ids'].shape[1]:].tolist()
print(' ->', proc.decode(ids, skip_special_tokens=True), '| tokens', ids[:12], len(ids))
np.savez(os.path.expanduser(f'~/hw-bench/glm_onnx/ref/{i}.npz'), input_ids=inp['input_ids'].numpy(), pixel_values=inp['pixel_values'].numpy(),
grid=inp['image_grid_thw'].numpy(), mm=inp.get('mm_token_type_ids', torch.zeros(1)).numpy(), pos=cap['prefill_pos'].numpy(),
vis_out=cap['vis_out'].numpy(), vis_in=cap['vis_in'].numpy(), generated=np.array(ids))
if 'mm_token_type_ids' in inp: print(' mm types', inp['mm_token_type_ids'].shape)
|