onnx / tests /test_clip_text_model.py
trixlyaisolutions's picture
Upload folder using huggingface_hub (part 2)
e0cba20 verified
Raw History Blame Contribute Delete
4.78 kB
"""Tests for library/clip_text_model.py (CLIPTextModel layout compatibility wrapper).
The tests run on any transformers version: with < 5.6 the wrapper is not applied by
`wrap_clip_text_model` (the model already has `text_model`), so the wrapper class itself is
exercised by wrapping the inner transformer directly.
"""
import torch
from transformers import CLIPTextConfig, CLIPTextModel
from library.clip_text_model import (
CLIPTextModelWrapper,
is_flattened_clip_text_model,
unwrap_clip_text_model,
wrap_clip_text_model,
)
TINY_CONFIG = dict(
vocab_size=64,
hidden_size=16,
intermediate_size=32,
num_hidden_layers=2,
num_attention_heads=2,
max_position_embeddings=8,
hidden_act="quick_gelu",
layer_norm_eps=1e-05,
pad_token_id=1,
bos_token_id=0,
eos_token_id=2,
model_type="clip_text_model",
projection_dim=16,
)
EXPECTED_KEYS = [
"text_model.embeddings.token_embedding.weight",
"text_model.embeddings.position_embedding.weight",
"text_model.encoder.layers.0.self_attn.q_proj.weight",
"text_model.encoder.layers.1.mlp.fc2.bias",
"text_model.final_layer_norm.weight",
]
def _make_model() -> torch.nn.Module:
torch.manual_seed(0)
return CLIPTextModel._from_config(CLIPTextConfig(**TINY_CONFIG))
def _force_wrap(model: CLIPTextModel) -> CLIPTextModelWrapper:
# transformers >= 5.6: the model itself is flat; < 5.6: use the inner CLIPTextTransformer
inner = model if is_flattened_clip_text_model(model) else model.text_model
return CLIPTextModelWrapper(inner)
def test_wrap_keeps_legacy_state_dict_keys_and_module_names():
model = wrap_clip_text_model(_make_model())
keys = set(model.state_dict().keys())
for k in EXPECTED_KEYS:
assert k in keys, k
assert all(k.startswith("text_model.") for k in keys)
names = dict(model.named_modules())
assert "text_model.encoder.layers.0.self_attn" in names
assert "text_model.encoder.layers.0.mlp" in names
assert names["text_model.encoder.layers.0.self_attn"].__class__.__name__ in ("CLIPAttention", "CLIPSdpaAttention")
assert names["text_model.encoder.layers.0.mlp"].__class__.__name__ == "CLIPMLP"
def test_wrap_is_idempotent_and_unwrap_returns_transformers_model():
model = wrap_clip_text_model(_make_model())
assert wrap_clip_text_model(model) is model
assert isinstance(unwrap_clip_text_model(model), CLIPTextModel)
assert unwrap_clip_text_model(unwrap_clip_text_model(model)) is unwrap_clip_text_model(model)
def test_wrapper_forward_and_attributes_match_inner():
model = _force_wrap(_make_model())
inner = model.text_model
input_ids = torch.tensor([[0, 5, 6, 7, 2, 1, 1, 1]])
model.eval()
with torch.no_grad():
out = model(input_ids, output_hidden_states=True, return_dict=True)
ref = inner(input_ids, output_hidden_states=True, return_dict=True)
assert torch.equal(out.last_hidden_state, ref.last_hidden_state)
assert torch.equal(out.pooler_output, ref.pooler_output)
assert len(out.hidden_states) == TINY_CONFIG["num_hidden_layers"] + 1
# the legacy attribute path used all over the training scripts
assert torch.equal(model.text_model.final_layer_norm(out.hidden_states[-1]), out.last_hidden_state)
assert model.config.hidden_size == TINY_CONFIG["hidden_size"]
assert model.dtype == torch.float32
assert model.device == next(inner.parameters()).device
assert model.get_input_embeddings() is inner.embeddings.token_embedding
assert model.state_dict().keys() == {"text_model." + k for k in inner.state_dict().keys()}
def test_wrapper_load_state_dict_with_legacy_keys():
src = wrap_clip_text_model(_make_model())
dst = _force_wrap(_make_model())
legacy_sd = {k: v.clone() for k, v in src.state_dict().items()}
info = dst.load_state_dict(legacy_sd, strict=True)
assert not info.missing_keys and not info.unexpected_keys
for k, v in legacy_sd.items():
assert torch.equal(dst.state_dict()[k], v)
def test_wrapper_dtype_device_follow_module_to():
model = _force_wrap(_make_model())
model.to(dtype=torch.bfloat16)
assert model.dtype == torch.bfloat16
assert model.text_model.embeddings.token_embedding.weight.dtype == torch.bfloat16
def test_is_flattened_detection():
raw = _make_model()
# on any version: a wrapped model is never reported as flattened
assert not is_flattened_clip_text_model(wrap_clip_text_model(raw))
# < 5.6 has text_model, >= 5.6 does not: exactly one of raw / inner looks flat
if hasattr(raw, "text_model"):
assert not is_flattened_clip_text_model(raw)
assert is_flattened_clip_text_model(raw.text_model)
else:
assert is_flattened_clip_text_model(raw)