Image-Text-to-Text
Transformers
Safetensors
English
gemma4_unified
text-classification
multimodal
medical
healthcare
calibrated
typed-decision
pyhealth
lora
adapter
conversational
Instructions to use nevermindai/carejev-omni with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use nevermindai/carejev-omni with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="nevermindai/carejev-omni") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("nevermindai/carejev-omni") model = AutoModelForMultimodalLM.from_pretrained("nevermindai/carejev-omni", device_map="auto") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use nevermindai/carejev-omni with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "nevermindai/carejev-omni" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "nevermindai/carejev-omni", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/nevermindai/carejev-omni
- SGLang
How to use nevermindai/carejev-omni with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "nevermindai/carejev-omni" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "nevermindai/carejev-omni", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "nevermindai/carejev-omni" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "nevermindai/carejev-omni", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use nevermindai/carejev-omni with Docker Model Runner:
docker model run hf.co/nevermindai/carejev-omni
File size: 3,214 Bytes
19d52b2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | """Runnable example shipped inside the export (copied next to the loader shim as ``example.py``).
pip install -r requirements.txt
python example.py # uses this directory as the checkpoint
Prints one answer of each question type. Needs a GPU with room for the checkpoint; pass a directory
as the first argument to score a different export.
"""
import sys
from pathlib import Path
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
from carejev_omni_loader import load_carejev_omni # noqa: E402 (shipped next to this file)
STATE = (
"PATIENT: 68-year-old male. Race: white.\n"
"STUDY ENTRY: hospital day 2; primary disease category: acute respiratory failure or multiple "
"organ system failure with sepsis (ARF/MOSF).\n"
"COMORBIDITIES: 2 recorded (diabetes).\n"
"SEVERITY: SUPPORT coma score 0; APACHE III physiology score 61; SUPPORT physiology score 33.9.\n"
"FUNCTION: 1 activities of daily living impaired (0 = independent, 7 = fully dependent).\n"
"DAY-3 PHYSIOLOGY: mean arterial pressure 72 mmHg; heart rate 104 /min; respiratory rate 28 /min; "
"temperature 38.1 °C; white cell count 15.2 x10^9/L; creatinine 2.1 mg/dL; sodium 139 mmol/L; "
"arterial pH 7.33; blood urea nitrogen 44 mg/dL; urine output 900 mL/day."
)
LOS_LEVELS = [
"Less than 1 day",
"1 day",
"2 days",
"3 days",
"4 days",
"5 days",
"6 days",
"7 days",
"8 to 14 days",
"More than 14 days",
]
def main() -> None:
path = sys.argv[1] if len(sys.argv) > 1 else str(HERE)
clf = load_carejev_omni(path)
# noul: a yes/no question. The answer is P(yes); option order never changes it.
noul = clf.predict(
state=STATE,
question="Will this patient die before being discharged from this hospitalization?",
options=["Yes", "No"],
)
print("noul P(yes) =", round(noul["probabilities"]["Yes"], 3))
# choice: one of k options. Probabilities are averaged over rotations of a canonical order.
choice = clf.predict(
state=STATE,
question="Which single problem best explains this presentation?",
options=["Sepsis", "Heart failure", "Pulmonary embolism", "Diabetic ketoacidosis"],
)
print("choice ->", choice["prediction"], "| confidence", round(choice["confidence"], 3))
# score: ordered levels. `score` is the expected level, i.e. sum(index * probability).
score = clf.predict(
state=STATE,
question="How long will this patient's hospital stay be in total?",
options=LOS_LEVELS,
)
top = max(score["probabilities"], key=score["probabilities"].get)
expected = sum(i * p for i, p in enumerate(score["probabilities"].values()))
print(f"score -> {top} | expected level {expected:.2f} of {len(LOS_LEVELS) - 1}")
# Images, heart sounds and other media go in the same call:
# clf.predict(state=..., question=..., options=[...], media="ecg.png", modality="image")
# clf.predict(state=..., question=..., options=["Yes", "No"], media="pcg.wav", modality="audio")
print("\nResearch use only. Not a medical device. See README.md.")
if __name__ == "__main__":
main()
|