Jarbas commited on
Commit
497a713
·
verified ·
1 Parent(s): a469aa7

Load on CPU, move to ZeroGPU

Browse files
Files changed (1) hide show
  1. app.py +6 -2
app.py CHANGED
@@ -33,8 +33,12 @@ for name, repo in MODELS.items():
33
  opts = LoadOptions.from_env()
34
  if opts.dtype is None:
35
  opts = replace(opts, dtype=torch.bfloat16)
36
- LOADED[name] = checkpoint.load("cuda", opts)
37
- tok, model = LOADED[name]
 
 
 
 
38
  print(f"[jevasio] {repo}: base={checkpoint.meta.base}@{checkpoint.meta.base_revision} dtype={model.dtype} "
39
  f"temperature={model.head.temperature:.2f} loaded in {time.perf_counter() - t0:.0f} s", flush=True)
40
 
 
33
  opts = LoadOptions.from_env()
34
  if opts.dtype is None:
35
  opts = replace(opts, dtype=torch.bfloat16)
36
+ # ZeroGPU has no GPU at start-up: load on the CPU, then move. ZeroGPU defers the move until a @spaces.GPU call, and
37
+ # Kev builds its input tensors on model.device.
38
+ tok, model = checkpoint.load("cpu", opts)
39
+ model.to("cuda")
40
+ model.device = torch.device("cuda")
41
+ LOADED[name] = tok, model
42
  print(f"[jevasio] {repo}: base={checkpoint.meta.base}@{checkpoint.meta.base_revision} dtype={model.dtype} "
43
  f"temperature={model.head.temperature:.2f} loaded in {time.perf_counter() - t0:.0f} s", flush=True)
44