File size: 2,601 Bytes
79df115
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
"""Execute the real vendored dual-stream model at tiny width, on CPU."""
import torch
from remedi import RemeDiUPMModelLM
from remedi.configuration_remedi import RemeDiConfig
from remedi.modelling_remedi_bitowel import DynamicCache
from engine import generate, Settings, POLICIES
from test_engine import Tokenizer

def tiny_model():
    # All 32 layer positions retained: the UPS stream attaches at 0,10,20,30.
    config = RemeDiConfig(d_model=32, n_heads=4, n_kv_heads=4, n_layers=32,
                          mlp_hidden_size=64, vocab_size=128, embedding_size=128,
                          max_sequence_length=128, block_type="llama", activation_type="silu", rope=True,
                          weight_tying=False, include_bias=False, include_qkv_bias=False,
                          mask_token_id=15, eos_token_id=127, init_device="cpu")
    torch.manual_seed(12)
    model = RemeDiUPMModelLM(config, init_params=True).eval()
    model.model._activation_checkpoint_fn = lambda fn, *a, **kw: fn(*a, **kw)
    return model

def test_real_forward_and_custom_cache():
    torch.set_num_threads(1)
    m = tiny_model()
    cache = DynamicCache()
    with torch.inference_mode():
        out = m(torch.tensor([[1, 2, 3]]), kv_cache=cache, update_kv_cache=True)
        assert out.logits.shape == (1, 3, 128)
        assert out.confidences.shape == (1, 3)
        assert len(cache) == 36
        before = cache.layers[0][0].shape[-2]
        out = m(torch.tensor([[15, 15]]), position_ids=torch.tensor([[3, 4]]), kv_cache=cache)
        assert torch.isfinite(out.logits).all() and torch.isfinite(out.confidences).all()
        assert cache.layers[0][0].shape[-2] == before
        m(torch.tensor([[4, 5]]), position_ids=torch.tensor([[3, 4]]), kv_cache=cache, update_kv_cache=True)
        assert cache.layers[0][0].shape[-2] == before + 2

def test_real_end_to_end_all_policies():
    torch.set_num_threads(1)
    m = tiny_model()
    for policy in POLICIES:
        result = generate(m, Tokenizer(), torch.tensor([[1, 2, 3]]), Settings(17, 16, 2, policy), "cpu")
        assert 0 < result["metrics"]["output_tokens"] <= 17
        assert 15 not in result["token_ids"]
        assert all(isinstance(i, int) for i in result["token_ids"])

def test_save_reload_strict(tmp_path):
    m = tiny_model()
    m.save_pretrained(tmp_path)
    loaded, info = RemeDiUPMModelLM.from_pretrained(tmp_path, output_loading_info=True, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True)
    assert not info["missing_keys"] and not info["unexpected_keys"]
    assert next(loaded.parameters()).dtype == torch.bfloat16