File size: 3,306 Bytes
f65336d
 
 
 
 
 
 
 
 
 
 
 
 
 
524fd9d
f65336d
 
 
 
 
524fd9d
 
 
 
 
 
 
 
 
 
 
f65336d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
"""Read typed fields from backbone hidden states."""

from __future__ import annotations

import inspect
import json
from pathlib import Path
from typing import Any

import torch
from safetensors.torch import load_file
from . import model as cm


def load_head(path: Path, *, match_bfloat16: bool = False) -> cm.ClairicHead:
    configuration = json.loads((path / "clair_head_config.json").read_text())
    accepted = inspect.signature(cm.ClairicHead).parameters
    head = cm.ClairicHead(
        **{name: value for name, value in configuration.items() if name in accepted}
    )
    state = load_file(path / "clair_head.safetensors")
    if match_bfloat16:
        # Serving constructs the head in BF16 before widening it to FP32.
        # The length bias has an explicit FP32 initialization.
        state = {
            name: value.bfloat16().float()
            if value.is_floating_point() and name != "length_bias"
            else value
            for name, value in state.items()
        }
    head.load_state_dict(state, strict=True)
    return head.float().eval()


def read_slots(
    states: torch.Tensor, output_weight: torch.Tensor, chunk: int = 256
) -> tuple[torch.Tensor, torch.Tensor]:
    ids, probabilities = [], []
    for rows in states.split(chunk):
        logits = (rows @ output_weight.t()).float()
        best = logits.max(dim=-1)
        ids.append(best.indices)
        probabilities.append(best.values - torch.logsumexp(logits, dim=-1))
    return torch.cat(ids), torch.cat(probabilities)


@torch.inference_mode()
def read_record(
    head: cm.ClairicHead,
    states: torch.Tensor,
    item: cm.EncodedRecord,
    output_weight: torch.Tensor,
) -> dict[str, Any]:
    start, length, kinds, queries, items, slot_fields, slots = (
        cm.parse_prediction_layout(
            cm.prediction_layout(item),
            len(item.input_ids),
            with_slots=True,
        )
    )
    views = cm.prediction_views(kinds, queries, items, states.device)
    output = head.forward_states(states, start, start + length, views)
    if slots:
        width = max(map(len, slots))
        positions = torch.tensor(
            [row + [row[-1]] * (width - len(row)) for row in slots],
            dtype=torch.long,
            device=states.device,
        )
        if head.answer_slot_routing:
            field_queries = views["query_positions"][slot_fields]
            slot_kinds = torch.tensor(
                [
                    cm.ROUTE_KINDS.index(
                        "tags" if kinds[row] == "text_list" else kinds[row]
                    )
                    for row in slot_fields
                ],
                dtype=torch.long,
                device=states.device,
            )
            lengths = torch.tensor(
                [len(row) for row in slots], dtype=torch.long, device=states.device
            )
            slot_states = head.answer_slot_states(
                states, field_queries, slot_kinds, positions, lengths, start + length
            ).flatten(0, 1)
        else:
            slot_states = states[positions.flatten()]
        ids, logprobs = read_slots(slot_states, output_weight)
        output["slot_ids"] = ids.view(positions.shape)
        output["slot_log_probs"] = logprobs.view(positions.shape)
    return output