|
Download README.md from klyang/CardioLLM: direct link, hf CLI and curl.
- Browser
- Download file 3.26 kB
-
https://huggingface.co/klyang/CardioLLM/resolve/main/README.md
- Command line
-
hf download hf://klyang/CardioLLM/README.md
-
curl -L -o README.md https://huggingface.co/klyang/CardioLLM/resolve/main/README.md
3.26 kB
| # CardioLLM | |
| This guide explains how to download the ECG-CoCa encoder and load the CardioLLM checkpoint from Hugging Face. | |
| ## 1. Download the ECG-CoCa checkpoint | |
| **[Download the ECG-CoCa checkpoint from Google Drive](https://drive.google.com/file/d/1wOKYfkb-Nep0WzYZz9-n66oTzp_4cky7/view?usp=sharing)** | |
| ## 2. Load CardioLLM from Hugging Face | |
| ```python | |
| from pathlib import Path | |
| import torch | |
| from transformers import AutoTokenizer | |
| from llava.model.language_model.llava_llama import ( | |
| LlavaConfig, | |
| LlavaLlamaForCausalLM, | |
| ) | |
| MODEL_ID = "klyang/CardioLLM" # Replace with your repository ID. | |
| REVISION = "main" # Use a full commit SHA to pin a specific model version. | |
| ECG_CHECKPOINT = Path("checkpoints/cpt_wfep_epoch_20.pt").resolve() | |
| device = "cuda:0" if torch.cuda.is_available() else "cpu" | |
| dtype = torch.float16 if device.startswith("cuda") else torch.float32 | |
| print(f"Loading {MODEL_ID} on {device} with {dtype}", flush=True) | |
| tokenizer = AutoTokenizer.from_pretrained( | |
| MODEL_ID, | |
| revision=REVISION, | |
| use_fast=False, | |
| ) | |
| config = LlavaConfig.from_pretrained(MODEL_ID, revision=REVISION) | |
| if config.model_type != "llava_llama": | |
| raise ValueError(f"Unexpected architecture: {config.model_type}") | |
| # Override the local encoder path without changing the published config.json. | |
| config.mm_ecg_tower = str(ECG_CHECKPOINT) | |
| # Instantiate the vision encoder before restoring its fine-tuned weights. | |
| config.unfreeze_mm_vision_tower = True | |
| model, loading_info = LlavaLlamaForCausalLM.from_pretrained( | |
| MODEL_ID, | |
| revision=REVISION, | |
| config=config, | |
| torch_dtype=dtype, | |
| device_map={"": device}, | |
| low_cpu_mem_usage=True, | |
| use_safetensors=True, | |
| attn_implementation="eager", | |
| output_loading_info=True, | |
| ) | |
| # Do not silently accept missing or randomly initialized parameters. | |
| for category in ("missing_keys", "unexpected_keys", "mismatched_keys", "error_msgs"): | |
| problems = loading_info.get(category, []) | |
| if problems: | |
| raise RuntimeError(f"Incomplete checkpoint load: {category}: {problems[:10]}") | |
| unloaded = [name for name, parameter in model.named_parameters() if parameter.is_meta] | |
| if unloaded: | |
| raise RuntimeError(f"Unloaded parameters: {unloaded[:10]}") | |
| for name, tower in ( | |
| ("ECG", model.get_ecg_tower()), | |
| ("vision", model.get_vision_tower()), | |
| ): | |
| loaded = getattr(tower, "is_loaded", False) | |
| loaded = loaded() if callable(loaded) else loaded | |
| if tower is None or not loaded: | |
| raise RuntimeError(f"The {name} encoder was not initialized.") | |
| model.eval() | |
| print("CardioLLM, tokenizer, ECG encoder, and vision encoder loaded successfully.") | |
| # A small text-only forward pass checks that the loaded language model runs. | |
| batch = tokenizer("Hello.", return_tensors="pt") | |
| inputs = { | |
| key: value.to(device) | |
| for key, value in batch.items() | |
| if key in {"input_ids", "attention_mask"} | |
| } | |
| with torch.inference_mode(): | |
| logits = model(**inputs, use_cache=False, return_dict=True).logits | |
| expected_shape = (*inputs["input_ids"].shape, config.vocab_size) | |
| if tuple(logits.shape) != expected_shape or not torch.isfinite(logits).all().item(): | |
| raise RuntimeError("The forward pass returned invalid logits.") | |
| print(f"Forward pass passed: logits shape = {tuple(logits.shape)}") | |
| ``` |