Factor Studios commited on
Commit
97c652e
·
verified ·
1 Parent(s): d19c131

Update ai.py

Browse files
Files changed (1) hide show
  1. ai.py +56 -12
ai.py CHANGED
@@ -499,22 +499,41 @@ class AIAccelerator:
499
  total_ops = total_params * batch_size * ops_per_param
500
  return (total_ops / inference_time) / 1e12 # Convert to TFLOPS
501
 
502
- def load_model(self, model_id: str, model: Any, processor: Any):
503
  """Loads a model directly into WebSocket storage without CPU intermediary."""
504
  try:
505
- # Extract model metadata
506
- model_info = {
507
- "architecture": model.__class__.__name__,
508
- "processor": processor.__class__.__name__,
509
- "config": model.config.to_dict() if hasattr(model, "config") else {}
510
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
511
 
512
  # Store model state in WebSocket storage
513
  self.storage.store_state(f"models/{model_id}", "info", model_info)
514
 
515
- # Map weight tensors directly to WebSocket storage
516
- if hasattr(model, "state_dict"):
517
- model_weights = {}
518
 
519
  for name, param in model.state_dict().items():
520
  tensor_id = f"{model_id}/weights/{name}"
@@ -535,12 +554,37 @@ class AIAccelerator:
535
  self.storage.store_state(f"models/{model_id}", "state", model)
536
  self.model_registry[model_id] = tensor_id
537
 
538
- self.tokenizer_registry[model_id] = processor
 
 
 
539
  self.model_loaded = True
540
  print(f"Model '{model_id}' loaded into WebSocket storage")
 
 
 
 
 
 
 
 
 
 
 
 
 
541
  except Exception as e:
542
  print(f"Error loading model into WebSocket storage: {str(e)}")
543
- raise
 
 
 
 
 
 
 
 
 
544
 
545
  def has_model(self, model_id: str) -> bool:
546
  """Checks if a model is loaded in the accelerator's registry."""
 
499
  total_ops = total_params * batch_size * ops_per_param
500
  return (total_ops / inference_time) / 1e12 # Convert to TFLOPS
501
 
502
+ def load_model(self, model_id: str, model: Any = None, processor: Any = None, model_config: Dict[str, Any] = None):
503
  """Loads a model directly into WebSocket storage without CPU intermediary."""
504
  try:
505
+ # Use provided config or create default
506
+ config = model_config or {}
507
+
508
+ if model is not None:
509
+ # Extract model metadata if model is provided
510
+ model_info = {
511
+ "architecture": model.__class__.__name__,
512
+ "processor": processor.__class__.__name__ if processor else "None",
513
+ "config": model.config.to_dict() if hasattr(model, "config") else {},
514
+ "model_config": {
515
+ k: str(v) if not isinstance(v, (bool, int, float, str, list, dict)) else v
516
+ for k, v in config.items()
517
+ } # Ensure config is JSON serializable
518
+ }
519
+ else:
520
+ # Use provided config for zero-copy mode
521
+ model_info = {
522
+ "architecture": "ZeroCopy",
523
+ "processor": "None",
524
+ "config": {},
525
+ "model_config": {
526
+ k: str(v) if not isinstance(v, (bool, int, float, str, list, dict)) else v
527
+ for k, v in config.items()
528
+ } # Ensure config is JSON serializable
529
+ }
530
 
531
  # Store model state in WebSocket storage
532
  self.storage.store_state(f"models/{model_id}", "info", model_info)
533
 
534
+ # Map weight tensors directly to WebSocket storage if model is provided
535
+ model_weights = {}
536
+ if model is not None and hasattr(model, "state_dict"):
537
 
538
  for name, param in model.state_dict().items():
539
  tensor_id = f"{model_id}/weights/{name}"
 
554
  self.storage.store_state(f"models/{model_id}", "state", model)
555
  self.model_registry[model_id] = tensor_id
556
 
557
+ # Store processor if provided
558
+ if processor is not None:
559
+ self.tokenizer_registry[model_id] = processor
560
+
561
  self.model_loaded = True
562
  print(f"Model '{model_id}' loaded into WebSocket storage")
563
+
564
+ # Additional setup for zero-copy mode
565
+ if model_config and model_config.get("zero_copy"):
566
+ # Register empty tensors for zero-copy mode
567
+ tensor_id = f"{model_id}/zero_copy"
568
+ self.model_registry[model_id] = {
569
+ "mode": "zero_copy",
570
+ "config": model_config,
571
+ "tensor_id": tensor_id
572
+ }
573
+
574
+ return True
575
+
576
  except Exception as e:
577
  print(f"Error loading model into WebSocket storage: {str(e)}")
578
+ if "is not JSON serializable" in str(e):
579
+ print("Attempting to serialize config differently...")
580
+ try:
581
+ # Try again with string conversion for non-serializable types
582
+ if model_config:
583
+ model_config = {k: str(v) for k, v in model_config.items()}
584
+ return self.load_model(model_id, model, processor, model_config)
585
+ except Exception as e2:
586
+ print(f"Second attempt also failed: {e2}")
587
+ return False
588
 
589
  def has_model(self, model_id: str) -> bool:
590
  """Checks if a model is loaded in the accelerator's registry."""