File size: 3,005 Bytes
79df115
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
import json
import os
import platform
import torch
import transformers
from huggingface_hub import hf_hub_download
from transformers import AutoTokenizer
from remedi import RemeDiUPMModelLM
from remedi.configuration_remedi import RemeDiConfig
from engine import MODEL_ID, MODEL_REVISION, TOKENIZER_ID

TOKENIZER_REVISION = "08b83a6feb34df1a6011b80c3c00c7563e963b07"

def validate_tokenizer(tokenizer, config):
    # Upstream stores <|mdm_mask|> in the vocabulary but leaves mask_token unset.
    if tokenizer.convert_tokens_to_ids("<|mdm_mask|>") != config.mask_token_id:
        raise RuntimeError("Base tokenizer mask vocabulary ID does not match checkpoint")
    tokenizer.mask_token = "<|mdm_mask|>"
    if tokenizer.mask_token_id != config.mask_token_id or tokenizer.eos_token_id != config.eos_token_id:
        raise RuntimeError("Base tokenizer special IDs do not match checkpoint")
    if len(tokenizer) > config.vocab_size:
        raise RuntimeError("Tokenizer exceeds the model vocabulary")

def load_runtime():
    """Called at module startup, outside the ZeroGPU lease (CUDA emulation)."""
    token = os.environ.get("HF_TOKEN") or None
    path = hf_hub_download(MODEL_ID, "config.json", revision=MODEL_REVISION, token=token)
    with open(path) as f:
        raw = json.load(f)
    # The checkpoint's auto_map points at absent LLaDA files. Use the vendored class.
    raw.pop("auto_map", None)
    raw.pop("model_type", None)
    config = RemeDiConfig(**raw)
    tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_ID, revision=TOKENIZER_REVISION,
                                             token=token, trust_remote_code=False)
    validate_tokenizer(tokenizer, config)
    model, loading = RemeDiUPMModelLM.from_pretrained(
        MODEL_ID, revision=MODEL_REVISION, config=config, torch_dtype=torch.bfloat16,
        low_cpu_mem_usage=True, output_loading_info=True, token=token,
    )
    problems = {k: loading.get(k) for k in ("missing_keys", "unexpected_keys", "mismatched_keys", "error_msgs") if loading.get(k)}
    if problems:
        raise RuntimeError(f"Checkpoint does not strictly match vendored architecture: {problems}")
    model.eval().requires_grad_(False).to("cuda")
    # The inference-only policy path need not invoke activation checkpointing.
    model.model._activation_checkpoint_fn = lambda fn, *args, **kwargs: fn(*args, **kwargs)
    diagnostics = {"model": MODEL_ID, "model_revision": MODEL_REVISION,
                   "tokenizer": TOKENIZER_ID, "tokenizer_revision": TOKENIZER_REVISION,
                   "upstream_code_revision": "2e1b19d34db09e345898f6c5b6ed60418659821a",
                   "parameters": sum(p.numel() for p in model.parameters()),
                   "dtype": str(next(model.parameters()).dtype), "python": platform.python_version(),
                   "torch": torch.__version__, "transformers": transformers.__version__,
                   "strict_loading": "passed", "special_ids": "passed"}
    return model, tokenizer, diagnostics