File size: 2,594 Bytes
cf814c5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
from __future__ import annotations
from pathlib import Path
from typing import Any
import torch
from PIL import Image

BASE_PROMPT = 'Judge whether the financial document page is relevant to the query. Pay special attention to company/entity, metric, year/period, unit/currency, statement type, and accounting scope. Answer True or False. Query: {query}{constraints}'

def constraint_text(item: dict[str, Any], enabled: bool) -> str:
    if not enabled:
        return ''
    c = item.get('constraints', {})
    values = {'entity': c.get('entity'), 'metric': c.get('canonical_metrics', []), 'year': c.get('year_terms', []), 'period': c.get('period_terms', []), 'unit': c.get('unit_terms', []), 'statement': c.get('statement_type'), 'scope': c.get('accounting_scope', [])}
    rendered = '; '.join((f'{key}={value}' for key, value in values.items() if value and value != 'unknown'))
    return f'\nParsed financial constraints: {rendered}.' if rendered else ''

def message(query: str, image: Image.Image, constraints: str) -> list[dict[str, Any]]:
    return [{'role': 'user', 'content': [{'type': 'image', 'image': image}, {'type': 'text', 'text': BASE_PROMPT.format(query=query, constraints=constraints)}]}]

@torch.inference_mode()
def score_batch(processor, model, token_ids: list[int], query: str, constraints: dict[str, Any], paths: list[Path]) -> list[float]:
    images = [Image.open(path).convert('RGB') for path in paths]
    rendered = constraint_text({'constraints': constraints}, True)
    texts = [processor.apply_chat_template(message(query, image, rendered), tokenize=False, add_generation_prompt=True) for image in images]
    inputs = processor(text=texts, images=images, return_tensors='pt', padding=True)
    inputs = {key: value.to('cuda') for key, value in inputs.items()}
    base = model.get_base_model()
    input_ids = inputs['input_ids']
    inputs_embeds = base.model.embed_tokens(input_ids)
    pixel_values = inputs['pixel_values'].type(base.visual.get_dtype())
    image_embeds = base.visual(pixel_values, grid_thw=inputs.get('image_grid_thw'))
    image_mask = (input_ids == base.config.image_token_id).unsqueeze(-1).expand_as(inputs_embeds)
    inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds.to(inputs_embeds.device, inputs_embeds.dtype))
    outputs = base.model(input_ids=None, attention_mask=inputs.get('attention_mask'), inputs_embeds=inputs_embeds, use_cache=False, return_dict=True)
    logits = base.lm_head(outputs.last_hidden_state[:, -1, :]).float()
    return (logits[:, token_ids[0]] - logits[:, token_ids[1]]).cpu().tolist()