Load on CPU, move to ZeroGPU
Browse files
app.py
CHANGED
|
@@ -33,8 +33,12 @@ for name, repo in MODELS.items():
|
|
| 33 |
opts = LoadOptions.from_env()
|
| 34 |
if opts.dtype is None:
|
| 35 |
opts = replace(opts, dtype=torch.bfloat16)
|
| 36 |
-
|
| 37 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
print(f"[jevasio] {repo}: base={checkpoint.meta.base}@{checkpoint.meta.base_revision} dtype={model.dtype} "
|
| 39 |
f"temperature={model.head.temperature:.2f} loaded in {time.perf_counter() - t0:.0f} s", flush=True)
|
| 40 |
|
|
|
|
| 33 |
opts = LoadOptions.from_env()
|
| 34 |
if opts.dtype is None:
|
| 35 |
opts = replace(opts, dtype=torch.bfloat16)
|
| 36 |
+
# ZeroGPU has no GPU at start-up: load on the CPU, then move. ZeroGPU defers the move until a @spaces.GPU call, and
|
| 37 |
+
# Kev builds its input tensors on model.device.
|
| 38 |
+
tok, model = checkpoint.load("cpu", opts)
|
| 39 |
+
model.to("cuda")
|
| 40 |
+
model.device = torch.device("cuda")
|
| 41 |
+
LOADED[name] = tok, model
|
| 42 |
print(f"[jevasio] {repo}: base={checkpoint.meta.base}@{checkpoint.meta.base_revision} dtype={model.dtype} "
|
| 43 |
f"temperature={model.head.temperature:.2f} loaded in {time.perf_counter() - t0:.0f} s", flush=True)
|
| 44 |
|