Spaces:
Sleeping
Sleeping
Factor Studios commited on
Update ai.py
Browse files
ai.py
CHANGED
|
@@ -499,22 +499,41 @@ class AIAccelerator:
|
|
| 499 |
total_ops = total_params * batch_size * ops_per_param
|
| 500 |
return (total_ops / inference_time) / 1e12 # Convert to TFLOPS
|
| 501 |
|
| 502 |
-
def load_model(self, model_id: str, model: Any, processor: Any):
|
| 503 |
"""Loads a model directly into WebSocket storage without CPU intermediary."""
|
| 504 |
try:
|
| 505 |
-
#
|
| 506 |
-
|
| 507 |
-
|
| 508 |
-
|
| 509 |
-
|
| 510 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 511 |
|
| 512 |
# Store model state in WebSocket storage
|
| 513 |
self.storage.store_state(f"models/{model_id}", "info", model_info)
|
| 514 |
|
| 515 |
-
# Map weight tensors directly to WebSocket storage
|
| 516 |
-
|
| 517 |
-
|
| 518 |
|
| 519 |
for name, param in model.state_dict().items():
|
| 520 |
tensor_id = f"{model_id}/weights/{name}"
|
|
@@ -535,12 +554,37 @@ class AIAccelerator:
|
|
| 535 |
self.storage.store_state(f"models/{model_id}", "state", model)
|
| 536 |
self.model_registry[model_id] = tensor_id
|
| 537 |
|
| 538 |
-
|
|
|
|
|
|
|
|
|
|
| 539 |
self.model_loaded = True
|
| 540 |
print(f"Model '{model_id}' loaded into WebSocket storage")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 541 |
except Exception as e:
|
| 542 |
print(f"Error loading model into WebSocket storage: {str(e)}")
|
| 543 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 544 |
|
| 545 |
def has_model(self, model_id: str) -> bool:
|
| 546 |
"""Checks if a model is loaded in the accelerator's registry."""
|
|
|
|
| 499 |
total_ops = total_params * batch_size * ops_per_param
|
| 500 |
return (total_ops / inference_time) / 1e12 # Convert to TFLOPS
|
| 501 |
|
| 502 |
+
def load_model(self, model_id: str, model: Any = None, processor: Any = None, model_config: Dict[str, Any] = None):
|
| 503 |
"""Loads a model directly into WebSocket storage without CPU intermediary."""
|
| 504 |
try:
|
| 505 |
+
# Use provided config or create default
|
| 506 |
+
config = model_config or {}
|
| 507 |
+
|
| 508 |
+
if model is not None:
|
| 509 |
+
# Extract model metadata if model is provided
|
| 510 |
+
model_info = {
|
| 511 |
+
"architecture": model.__class__.__name__,
|
| 512 |
+
"processor": processor.__class__.__name__ if processor else "None",
|
| 513 |
+
"config": model.config.to_dict() if hasattr(model, "config") else {},
|
| 514 |
+
"model_config": {
|
| 515 |
+
k: str(v) if not isinstance(v, (bool, int, float, str, list, dict)) else v
|
| 516 |
+
for k, v in config.items()
|
| 517 |
+
} # Ensure config is JSON serializable
|
| 518 |
+
}
|
| 519 |
+
else:
|
| 520 |
+
# Use provided config for zero-copy mode
|
| 521 |
+
model_info = {
|
| 522 |
+
"architecture": "ZeroCopy",
|
| 523 |
+
"processor": "None",
|
| 524 |
+
"config": {},
|
| 525 |
+
"model_config": {
|
| 526 |
+
k: str(v) if not isinstance(v, (bool, int, float, str, list, dict)) else v
|
| 527 |
+
for k, v in config.items()
|
| 528 |
+
} # Ensure config is JSON serializable
|
| 529 |
+
}
|
| 530 |
|
| 531 |
# Store model state in WebSocket storage
|
| 532 |
self.storage.store_state(f"models/{model_id}", "info", model_info)
|
| 533 |
|
| 534 |
+
# Map weight tensors directly to WebSocket storage if model is provided
|
| 535 |
+
model_weights = {}
|
| 536 |
+
if model is not None and hasattr(model, "state_dict"):
|
| 537 |
|
| 538 |
for name, param in model.state_dict().items():
|
| 539 |
tensor_id = f"{model_id}/weights/{name}"
|
|
|
|
| 554 |
self.storage.store_state(f"models/{model_id}", "state", model)
|
| 555 |
self.model_registry[model_id] = tensor_id
|
| 556 |
|
| 557 |
+
# Store processor if provided
|
| 558 |
+
if processor is not None:
|
| 559 |
+
self.tokenizer_registry[model_id] = processor
|
| 560 |
+
|
| 561 |
self.model_loaded = True
|
| 562 |
print(f"Model '{model_id}' loaded into WebSocket storage")
|
| 563 |
+
|
| 564 |
+
# Additional setup for zero-copy mode
|
| 565 |
+
if model_config and model_config.get("zero_copy"):
|
| 566 |
+
# Register empty tensors for zero-copy mode
|
| 567 |
+
tensor_id = f"{model_id}/zero_copy"
|
| 568 |
+
self.model_registry[model_id] = {
|
| 569 |
+
"mode": "zero_copy",
|
| 570 |
+
"config": model_config,
|
| 571 |
+
"tensor_id": tensor_id
|
| 572 |
+
}
|
| 573 |
+
|
| 574 |
+
return True
|
| 575 |
+
|
| 576 |
except Exception as e:
|
| 577 |
print(f"Error loading model into WebSocket storage: {str(e)}")
|
| 578 |
+
if "is not JSON serializable" in str(e):
|
| 579 |
+
print("Attempting to serialize config differently...")
|
| 580 |
+
try:
|
| 581 |
+
# Try again with string conversion for non-serializable types
|
| 582 |
+
if model_config:
|
| 583 |
+
model_config = {k: str(v) for k, v in model_config.items()}
|
| 584 |
+
return self.load_model(model_id, model, processor, model_config)
|
| 585 |
+
except Exception as e2:
|
| 586 |
+
print(f"Second attempt also failed: {e2}")
|
| 587 |
+
return False
|
| 588 |
|
| 589 |
def has_model(self, model_id: str) -> bool:
|
| 590 |
"""Checks if a model is loaded in the accelerator's registry."""
|