File size: 2,344 Bytes
15034cc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
"""Runs GLM-OCR in PyTorch (fp32) on a few lines and keeps what goes into and out of its parts, as reference for the ONNX export.
  python3 ref_capture.py OUT_DIR PICTURE_DIR N  ->  ~/hw-bench/glm_onnx/ref/<line>.npz"""
import json, os, sys, numpy as np, torch
from PIL import Image
from transformers import AutoProcessor, AutoModelForImageTextToText
out, pics, n = sys.argv[1], sys.argv[2], int(sys.argv[3])
torch.set_num_threads(4)
path = os.path.expanduser('~/hw-bench/models/glm-ocr')
proc = AutoProcessor.from_pretrained(path)
model = AutoModelForImageTextToText.from_pretrained(path, dtype=torch.float32).eval()
os.makedirs(os.path.expanduser('~/hw-bench/glm_onnx/ref'), exist_ok=True)
cap = {}
vis = model.model.visual
vis.register_forward_hook(lambda m, a, k, o: cap.update(vis_in=a[0].detach().clone(), vis_grid=k['grid_thw'].detach().clone(), vis_out=o.pooler_output.detach().clone()), with_kwargs=True)
lm = model.model.language_model
def pre(m, a, k):
    if 'prefill_pos' not in cap:
        cap['prefill_pos'] = k['position_ids'].detach().clone(); cap['prefill_embeds'] = k['inputs_embeds'].detach().clone()
lm.register_forward_pre_hook(pre, with_kwargs=True)
for i in list(json.load(open(f'{out}/truth.json')))[:n]:
    cap.clear()
    msgs = [{'role': 'user', 'content': [{'type': 'image', 'image': Image.open(f'{out}/{pics}/{i}.png').convert('RGB')}, {'type': 'text', 'text': 'Text Recognition:'}]}]
    inp = proc.apply_chat_template(msgs, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors='pt')
    print(i, {k: tuple(v.shape) for k, v in inp.items()})
    inp.pop('token_type_ids', None)
    with torch.no_grad(): gen = model.generate(**inp, max_new_tokens=60, do_sample=False)
    ids = gen[0][inp['input_ids'].shape[1]:].tolist()
    print('  ->', proc.decode(ids, skip_special_tokens=True), '| tokens', ids[:12], len(ids))
    np.savez(os.path.expanduser(f'~/hw-bench/glm_onnx/ref/{i}.npz'), input_ids=inp['input_ids'].numpy(), pixel_values=inp['pixel_values'].numpy(),
             grid=inp['image_grid_thw'].numpy(), mm=inp.get('mm_token_type_ids', torch.zeros(1)).numpy(), pos=cap['prefill_pos'].numpy(),
             vis_out=cap['vis_out'].numpy(), vis_in=cap['vis_in'].numpy(), generated=np.array(ids))
    if 'mm_token_type_ids' in inp: print('  mm types', inp['mm_token_type_ids'].shape)