from __future__ import annotations from pathlib import Path from typing import Any import torch from PIL import Image BASE_PROMPT = 'Judge whether the financial document page is relevant to the query. Pay special attention to company/entity, metric, year/period, unit/currency, statement type, and accounting scope. Answer True or False. Query: {query}{constraints}' def constraint_text(item: dict[str, Any], enabled: bool) -> str: if not enabled: return '' c = item.get('constraints', {}) values = {'entity': c.get('entity'), 'metric': c.get('canonical_metrics', []), 'year': c.get('year_terms', []), 'period': c.get('period_terms', []), 'unit': c.get('unit_terms', []), 'statement': c.get('statement_type'), 'scope': c.get('accounting_scope', [])} rendered = '; '.join((f'{key}={value}' for key, value in values.items() if value and value != 'unknown')) return f'\nParsed financial constraints: {rendered}.' if rendered else '' def message(query: str, image: Image.Image, constraints: str) -> list[dict[str, Any]]: return [{'role': 'user', 'content': [{'type': 'image', 'image': image}, {'type': 'text', 'text': BASE_PROMPT.format(query=query, constraints=constraints)}]}] @torch.inference_mode() def score_batch(processor, model, token_ids: list[int], query: str, constraints: dict[str, Any], paths: list[Path]) -> list[float]: images = [Image.open(path).convert('RGB') for path in paths] rendered = constraint_text({'constraints': constraints}, True) texts = [processor.apply_chat_template(message(query, image, rendered), tokenize=False, add_generation_prompt=True) for image in images] inputs = processor(text=texts, images=images, return_tensors='pt', padding=True) inputs = {key: value.to('cuda') for key, value in inputs.items()} base = model.get_base_model() input_ids = inputs['input_ids'] inputs_embeds = base.model.embed_tokens(input_ids) pixel_values = inputs['pixel_values'].type(base.visual.get_dtype()) image_embeds = base.visual(pixel_values, grid_thw=inputs.get('image_grid_thw')) image_mask = (input_ids == base.config.image_token_id).unsqueeze(-1).expand_as(inputs_embeds) inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds.to(inputs_embeds.device, inputs_embeds.dtype)) outputs = base.model(input_ids=None, attention_mask=inputs.get('attention_mask'), inputs_embeds=inputs_embeds, use_cache=False, return_dict=True) logits = base.lm_head(outputs.last_hidden_state[:, -1, :]).float() return (logits[:, token_ids[0]] - logits[:, token_ids[1]]).cpu().tolist()