Hugging Face's logo

WaitXian
/
liqi

Model card Files
xet
 Community
liqi
File size: 2,811 Bytes
0ab2566
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
## TODO: 实现一个映射层，将输出得到的特征维度映射到embed空间当中
from models.proj_models import LLM2EmbedProjector, LLM2EmbedAddProjector
from typing import Optional
from logger import get_logger
from transformers.modeling_utils import load_state_dict
from transformers.trainer_utils import get_last_checkpoint
import os
from . import register_loader
logger = get_logger(__name__)


@register_loader("load_mlp_projector")
def load_mlp_projector(
        fc_args: "FCArguments",
        in_dim = 3074, 
        is_trainable: Optional[bool] = False,
        resume_from_checkpoint=None,
        output_dir=None
):
    if fc_args.stage == "classify":
        mlp_model = LLM2EmbedProjector(
            in_dim = in_dim,
            out_dim = fc_args.embed_dim, 
            fp16=False
        )

        if resume_from_checkpoint:
            if os.path.exists(output_dir):
                new_model_name_or_path_proj = get_last_checkpoint(output_dir)
                logger.info("start to load pfc not from: {}, but from {}".format(
                    fc_args.model_name_or_path_proj, new_model_name_or_path_proj
                ))
            else:
                new_model_name_or_path_proj = None
                logger.info('not exists (output_dir) for pfc {}, load pfc from {}'.format(output_dir, fc_args.model_name_or_path_proj))

            if new_model_name_or_path_proj:
                logger.info("(resume process) start to load pfc from: {}".format(new_model_name_or_path_proj))
                mlp_model.maybe_load_state_dict_from_path(new_model_name_or_path_proj)
            else:
                logger.info("(resume failed) still start to load pfc from: {}".format(fc_args.model_name_or_path_proj))
                mlp_model.maybe_load_state_dict_from_path(fc_args.model_name_or_path_proj)
        else:
            logger.info("start to load pfc from: {}".format(fc_args.model_name_or_path_proj))
            mlp_model.maybe_load_state_dict_from_path(fc_args.model_name_or_path_proj)


        if is_trainable:
            mlp_model.train()
        else:
            mlp_model.eval()
    else:
        mlp_model = None
    return mlp_model


@register_loader("load_add_projector")
def load_add_projector(
        fc_args: "FCArguments",
        in_dim = 3074, 
        is_trainable: Optional[bool] = False,
        resume_from_checkpoint=None,
        output_dir=None
):
    if fc_args.stage == "classify":
        mlp_model = LLM2EmbedAddProjector(
            in_dim = in_dim,
            out_dim = fc_args.embed_dim, 
            fp16=False
        )
        logger.info("No weights for LLM2EmbedAddProjector")
        if is_trainable:
            mlp_model.train()
        else:
            mlp_model.eval()
    else:
        mlp_model = None
    return mlp_model