syvb commited on
Commit
e8dc334
·
verified ·
1 Parent(s): 6823e3f

load adapter weights on CPU (torch_device); log attach tracebacks

Browse files
Files changed (2) hide show
  1. app.py +1 -0
  2. oracle.py +5 -2
app.py CHANGED
@@ -46,6 +46,7 @@ def refresh_registry() -> str:
46
  BANK.attach(o)
47
  except Exception as e: # keep serving the others
48
  print(f"[registry] failed to attach {o.run}: {e}")
 
49
  continue
50
  REGISTRY[o.run] = o
51
  new.append(o.name)
 
46
  BANK.attach(o)
47
  except Exception as e: # keep serving the others
48
  print(f"[registry] failed to attach {o.run}: {e}")
49
+ traceback.print_exc()
50
  continue
51
  REGISTRY[o.run] = o
52
  new.append(o.name)
oracle.py CHANGED
@@ -195,10 +195,13 @@ class Bank:
195
  back = o.base_model in self.on_device
196
  if back: # late attach: assemble on CPU, then re-pack
197
  m.to("cpu"); self.on_device.discard(o.base_model)
 
 
198
  if isinstance(m, PeftModel):
199
- m.load_adapter(o.local_dir, adapter_name=o.adapter_name)
200
  else:
201
- self.models[o.base_model] = PeftModel.from_pretrained(m, o.local_dir, adapter_name=o.adapter_name).eval()
 
202
  self.loaded.add(o.run)
203
  if back:
204
  self.to_device()
 
195
  back = o.base_model in self.on_device
196
  if back: # late attach: assemble on CPU, then re-pack
197
  m.to("cpu"); self.on_device.discard(o.base_model)
198
+ # torch_device="cpu": PEFT otherwise infers "cuda" from ZeroGPU's patched availability flag
199
+ # and maps the safetensors there, which is forbidden outside a GPU call
200
  if isinstance(m, PeftModel):
201
+ m.load_adapter(o.local_dir, adapter_name=o.adapter_name, torch_device="cpu")
202
  else:
203
+ self.models[o.base_model] = PeftModel.from_pretrained(m, o.local_dir, adapter_name=o.adapter_name,
204
+ torch_device="cpu").eval()
205
  self.loaded.add(o.run)
206
  if back:
207
  self.to_device()