File size: 5,150 Bytes
0ca529a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
from typing import Optional

import torch
import torch.distributed as dist
import torch.nn as nn
from peft import LoraConfig, TaskType, get_peft_model
from peft.tuners.lora import LoraLayer
from transformers import AutoModelForCausalLM, AutoModelForImageTextToText, AutoConfig

from kdflow.utils import get_tokenizer
from kdflow.models.ring_attn_utils import gather_and_pad_tensor, unpad_and_slice_tensor


class DistillModel(nn.Module):
    """
    Base class for student models in knowledge distillation (modified from OpenRLHF/openrlhf/models/actor.py).

    Args:
        args (Arguments): Arguments.
        strategy (Strategy): Strategy for student model loading and training.
        device_map (dict, optional): Device mapping for loading the model onto specific devices. Defaults to None.
    """

    def __init__(
        self,
        strategy,
        device_map=None,
        **kwargs,
    ) -> None:
        super().__init__()
        self.strategy = strategy
        self.args = strategy.args
        self.temperature = self.args.rollout.temperature
        model_name_or_path = self.args.model.student_name_or_path

        # Support multiple attention mechanism implementations
        attn_impl = self.args.model.attn_implementation

        self.model_config = AutoConfig.from_pretrained(model_name_or_path, trust_remote_code=True)
        
        # Determine if this is a Vision-Language model
        self.is_vl_model = hasattr(self.model_config, "vision_config")
        
        if self.is_vl_model:
            model_class = AutoModelForImageTextToText
        elif self.args.model.use_liger_kernel:
            from liger_kernel.transformers import AutoLigerKernelForCausalLM
            model_class = AutoLigerKernelForCausalLM
        else:
            model_class = AutoModelForCausalLM

        if hasattr(self.model_config, "text_config"):
            self.hidden_size = self.model_config.text_config.hidden_size
        else:
            self.hidden_size = self.model_config.hidden_size
        
        self.model = strategy.load_hf_model(
            model_class, 
            model_name_or_path, 
            attn_impl, 
            self.model_config, 
        )
        
        # LoRA
        if self.args.model.lora_rank > 0:
            # https://github.com/huggingface/peft/issues/137
            self.model.enable_input_require_grads()
            lora_config = LoraConfig(
                task_type=TaskType.CAUSAL_LM,
                r=self.args.model.lora_rank,
                lora_alpha=self.args.model.lora_alpha,
                target_modules=self.args.model.target_modules,
                lora_dropout=self.args.model.lora_dropout,
                bias="none",
            )
            self.model = get_peft_model(self.model, lora_config)

        self.tokenizer = get_tokenizer(model_name_or_path, self.model)

        # https://github.com/huggingface/transformers/issues/26877
        # Use `model.generate(use_cache=True)` instead.`
        self.model.config.use_cache = False

        # packing samples using Flash Attention 2
        self.packing_samples = self.args.data.packing_samples
        
        self._print_model()

    def forward(
        self,
        sequences: torch.LongTensor,
        attention_mask: Optional[torch.Tensor] = None,
        allgather_logits=False,
        ring_attn_group: Optional[dist.ProcessGroup] = None,
        **kwargs,
    ) -> torch.Tensor:
        """Returns action log probs"""
        batch, seqlen = sequences.size()
        foward_attention_mask = attention_mask
        if self.packing_samples:
            sequences, position_ids, rolled_sequences, ring_attn_pad_len, indices = unpad_and_slice_tensor(
                sequences, attention_mask, ring_attn_group
            )
            foward_attention_mask = None
        else:
            position_ids = attention_mask.long().cumsum(-1) - 1
            position_ids.masked_fill_(attention_mask == 0, 1)

        output = self.model(
            sequences, 
            attention_mask=foward_attention_mask, 
            position_ids=position_ids, 
            **kwargs
        )
        # lm_head is patched to identity (skip=True), so output["logits"]
        # are actually final hidden states.
        output = {"hidden_states": [output["logits"]]}
            
        if allgather_logits and self.packing_samples:
            output["hidden_states"][-1] = gather_and_pad_tensor(
                output["hidden_states"][-1], ring_attn_group, 
                ring_attn_pad_len, indices, batch, seqlen
            ).squeeze(-2)
        return output

    def _print_model(self):
        self.strategy.print(f"Student Model: \n  {self.model}")
    
    def gradient_checkpointing_enable(self):
        self.model.gradient_checkpointing_enable(
            gradient_checkpointing_kwargs={
                "use_reentrant": self.args.train.gradient_checkpointing_use_reentrant
            }
        )

    def gradient_checkpointing_disable(self):
        self.model.gradient_checkpointing_disable()

    def print_trainable_parameters(self):
        self.model.print_trainable_parameters()