File size: 1,729 Bytes
79df115
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
"""Check live pinned metadata and tokenizer without loading the 9B weights or CUDA."""
import json
from accelerate import init_empty_weights
from huggingface_hub import hf_hub_download
from transformers import AutoTokenizer
from remedi import RemeDiUPMModelLM
from remedi.configuration_remedi import RemeDiConfig
from engine import MODEL_ID, MODEL_REVISION, TOKENIZER_ID, prompt_ids
from runtime import TOKENIZER_REVISION, validate_tokenizer

def main():
    def download_json(name):
        with open(hf_hub_download(MODEL_ID, name, revision=MODEL_REVISION)) as f:
            return json.load(f)
    raw = download_json("config.json")
    raw.pop("auto_map", None)
    raw.pop("model_type", None)
    config = RemeDiConfig(**raw)
    with init_empty_weights():
        model = RemeDiUPMModelLM(config)
    expected = set(model.state_dict())
    actual = set(download_json("model.safetensors.index.json")["weight_map"])
    if expected != actual:
        raise RuntimeError(f"Key mismatch: missing={sorted(expected-actual)}, unexpected={sorted(actual-expected)}")
    tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_ID, revision=TOKENIZER_REVISION, trust_remote_code=False)
    validate_tokenizer(tokenizer, config)
    print(json.dumps({"status": "pass", "checkpoint_keys": len(actual),
                      "parameters": sum(p.numel() for p in model.parameters()),
                      "tokenizer_vocabulary": len(tokenizer), "mask_id": tokenizer.mask_token_id,
                      "eos_id": tokenizer.eos_token_id,
                      "prompt_shape": list(prompt_ids(tokenizer, "Hello").shape),
                      "weights_loaded": False, "cuda_tested": False}, indent=2))

if __name__ == "__main__":
    main()