from __future__ import annotations import logging import torch from .configuration_riemann import RiemannConfig from .modeling_riemann import RiemannForCausalLM logger = logging.getLogger("riemann.openpeer_weights") OPENPEER_REPO = "OpenPeerAI/OpenPeerLLM" OPENPEER_WEIGHTS_FILE = "model.safetensors" def load_openpeer_state_dict() -> dict: from huggingface_hub import hf_hub_download from safetensors.torch import load_file path = hf_hub_download(OPENPEER_REPO, OPENPEER_WEIGHTS_FILE) return load_file(path) def build_stage0_from_openpeer_pretrained(config: RiemannConfig) -> RiemannForCausalLM: expected = dict( vocab_size=50257, hidden_size=768, num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072, max_position_embeddings=1024, ) for key, value in expected.items(): if getattr(config, key) != value: raise ValueError( f"build_stage0_from_openpeer_pretrained needs {key}={value} to match the published OpenPeerLLM checkpoint shape, got {getattr(config, key)}" ) model = RiemannForCausalLM(config) state_dict = load_openpeer_state_dict() missing, unexpected = model.load_state_dict(state_dict, strict=False) if missing or unexpected: logger.warning( "loading OpenPeerLLM weights: missing=%s unexpected=%s", missing, unexpected ) else: logger.info( "loaded all %d tensors from %s straight from the Hub, no conversion needed", len(state_dict), OPENPEER_REPO, ) return model if __name__ == "__main__": logging.basicConfig( level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s" ) cfg = RiemannConfig() m = build_stage0_from_openpeer_pretrained(cfg) n_params = sum((p.numel() for p in m.parameters())) print(f"built stage0 from real OpenPeerLLM weights, {n_params:,} params") ids = torch.randint(0, cfg.vocab_size, (1, 16)) out = m(input_ids=ids, labels=ids) print("sanity forward pass loss:", out.loss.item())