NoGPU 0.2

.gitignore

@@ -5,12 +5,6 @@ my-notes
 .env
 .venv
 
-# I did use vibe code 😉
-.claude
-CLAUDE.md
-AGENTS.md
-claude*
-
 # Caches
 __pycache__
 *.db

core/compressor.py

@@ -1,9 +1,14 @@
 import torch
-import spaces
 import config
 from core.tokenizer_utils import count_tokens
 from models.model_loader import get_llm
 
+try:
+    import spaces
+    _gpu = spaces.GPU
+except ImportError:
+    _gpu = lambda fn: fn  # no-op when running locally without the spaces package
+
 
 _PROMPT_TEMPLATE = """You are a lossless compression assistant. Compress the following text to at most {target} tokens.
 Preserve all key facts, decisions, and intent. Do not add commentary. Output only the compressed text.
@@ -13,10 +18,13 @@ TEXT:
 
 COMPRESSED:"""
 
-@spaces.GPU
+
+@_gpu
 def _generate(prompt: str) -> str:
     model, tokenizer = get_llm()
-    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    model.to(device)
+    inputs = tokenizer(prompt, return_tensors="pt").to(device)
     with torch.no_grad():
         output_ids = model.generate(
             **inputs,
@@ -32,7 +40,6 @@ def compress(text: str, target_tokens: int) -> tuple[str, int, int]:
     """Returns (compressed_text, input_token_count, output_token_count)."""
     input_tokens = count_tokens(text)
 
-    # If already within budget, return as-is
     if input_tokens <= target_tokens:
         return text, input_tokens, input_tokens
 

core/scorer.py

@@ -1,10 +1,19 @@
+import torch
 import numpy as np
 from models.model_loader import get_embedder
 
+try:
+    import spaces
+    _gpu = spaces.GPU
+except ImportError:
+    _gpu = lambda fn: fn  # no-op when running locally without the spaces package
 
+
+@_gpu
 def semantic_score(original: str, compressed: str) -> float:
     embedder = get_embedder()
-    vecs = embedder.encode([original, compressed], convert_to_numpy=True)
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    vecs = embedder.encode([original, compressed], device=device, convert_to_numpy=True)
     cos = float(
         np.dot(vecs[0], vecs[1]) / (np.linalg.norm(vecs[0]) * np.linalg.norm(vecs[1]))
     )

models/model_loader.py

@@ -43,11 +43,11 @@ def switch_llm(model_id: str) -> str:
 def _load_llm(model_id: str):
     """Load model + its paired tokenizer. Both come from the same model_id."""
     global _llm, _tokenizer, _current_model_id
-    _tokenizer = AutoTokenizer.from_pretrained(model_id) #.to("cuda")
+    _tokenizer = AutoTokenizer.from_pretrained(model_id)
     _llm = AutoModelForCausalLM.from_pretrained(
         model_id,
-        torch_dtype=torch.float32,
-        device_map="auto",
+        torch_dtype="auto",  # uses model's native dtype (bfloat16 for Qwen2.5)
+        device_map=None,     # load to CPU; @spaces.GPU functions move it on demand
     )
     _llm.eval()
     _current_model_id = model_id