Instructions to use yoohoo233/FinConRank with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use yoohoo233/FinConRank with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
File size: 2,594 Bytes
cf814c5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 | from __future__ import annotations
from pathlib import Path
from typing import Any
import torch
from PIL import Image
BASE_PROMPT = 'Judge whether the financial document page is relevant to the query. Pay special attention to company/entity, metric, year/period, unit/currency, statement type, and accounting scope. Answer True or False. Query: {query}{constraints}'
def constraint_text(item: dict[str, Any], enabled: bool) -> str:
if not enabled:
return ''
c = item.get('constraints', {})
values = {'entity': c.get('entity'), 'metric': c.get('canonical_metrics', []), 'year': c.get('year_terms', []), 'period': c.get('period_terms', []), 'unit': c.get('unit_terms', []), 'statement': c.get('statement_type'), 'scope': c.get('accounting_scope', [])}
rendered = '; '.join((f'{key}={value}' for key, value in values.items() if value and value != 'unknown'))
return f'\nParsed financial constraints: {rendered}.' if rendered else ''
def message(query: str, image: Image.Image, constraints: str) -> list[dict[str, Any]]:
return [{'role': 'user', 'content': [{'type': 'image', 'image': image}, {'type': 'text', 'text': BASE_PROMPT.format(query=query, constraints=constraints)}]}]
@torch.inference_mode()
def score_batch(processor, model, token_ids: list[int], query: str, constraints: dict[str, Any], paths: list[Path]) -> list[float]:
images = [Image.open(path).convert('RGB') for path in paths]
rendered = constraint_text({'constraints': constraints}, True)
texts = [processor.apply_chat_template(message(query, image, rendered), tokenize=False, add_generation_prompt=True) for image in images]
inputs = processor(text=texts, images=images, return_tensors='pt', padding=True)
inputs = {key: value.to('cuda') for key, value in inputs.items()}
base = model.get_base_model()
input_ids = inputs['input_ids']
inputs_embeds = base.model.embed_tokens(input_ids)
pixel_values = inputs['pixel_values'].type(base.visual.get_dtype())
image_embeds = base.visual(pixel_values, grid_thw=inputs.get('image_grid_thw'))
image_mask = (input_ids == base.config.image_token_id).unsqueeze(-1).expand_as(inputs_embeds)
inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds.to(inputs_embeds.device, inputs_embeds.dtype))
outputs = base.model(input_ids=None, attention_mask=inputs.get('attention_mask'), inputs_embeds=inputs_embeds, use_cache=False, return_dict=True)
logits = base.lm_head(outputs.last_hidden_state[:, -1, :]).float()
return (logits[:, token_ids[0]] - logits[:, token_ids[1]]).cpu().tolist()
|