"""Execute the real vendored dual-stream model at tiny width, on CPU.""" import torch from remedi import RemeDiUPMModelLM from remedi.configuration_remedi import RemeDiConfig from remedi.modelling_remedi_bitowel import DynamicCache from engine import generate, Settings, POLICIES from test_engine import Tokenizer def tiny_model(): # All 32 layer positions retained: the UPS stream attaches at 0,10,20,30. config = RemeDiConfig(d_model=32, n_heads=4, n_kv_heads=4, n_layers=32, mlp_hidden_size=64, vocab_size=128, embedding_size=128, max_sequence_length=128, block_type="llama", activation_type="silu", rope=True, weight_tying=False, include_bias=False, include_qkv_bias=False, mask_token_id=15, eos_token_id=127, init_device="cpu") torch.manual_seed(12) model = RemeDiUPMModelLM(config, init_params=True).eval() model.model._activation_checkpoint_fn = lambda fn, *a, **kw: fn(*a, **kw) return model def test_real_forward_and_custom_cache(): torch.set_num_threads(1) m = tiny_model() cache = DynamicCache() with torch.inference_mode(): out = m(torch.tensor([[1, 2, 3]]), kv_cache=cache, update_kv_cache=True) assert out.logits.shape == (1, 3, 128) assert out.confidences.shape == (1, 3) assert len(cache) == 36 before = cache.layers[0][0].shape[-2] out = m(torch.tensor([[15, 15]]), position_ids=torch.tensor([[3, 4]]), kv_cache=cache) assert torch.isfinite(out.logits).all() and torch.isfinite(out.confidences).all() assert cache.layers[0][0].shape[-2] == before m(torch.tensor([[4, 5]]), position_ids=torch.tensor([[3, 4]]), kv_cache=cache, update_kv_cache=True) assert cache.layers[0][0].shape[-2] == before + 2 def test_real_end_to_end_all_policies(): torch.set_num_threads(1) m = tiny_model() for policy in POLICIES: result = generate(m, Tokenizer(), torch.tensor([[1, 2, 3]]), Settings(17, 16, 2, policy), "cpu") assert 0 < result["metrics"]["output_tokens"] <= 17 assert 15 not in result["token_ids"] assert all(isinstance(i, int) for i in result["token_ids"]) def test_save_reload_strict(tmp_path): m = tiny_model() m.save_pretrained(tmp_path) loaded, info = RemeDiUPMModelLM.from_pretrained(tmp_path, output_loading_info=True, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True) assert not info["missing_keys"] and not info["unexpected_keys"] assert next(loaded.parameters()).dtype == torch.bfloat16