Buckets:

download
raw
20.6 kB
import asyncio
import json
import os
import uuid
import datetime
import random
import struct
import threading
from typing import Dict, List, Set, Optional, Any
from fastapi import WebSocket
from ascii_colors import trace_exception, ASCIIColors
from .db import session as db_session_module
from .db.models.broadcast import BroadcastMessage
from .db.models.connections import WebSocketConnection
from .db.models.user import User as DBUser, Friendship as DBFriendship
from .db.base import FriendshipStatus
from .session import user_sessions
from .settings import settings
from backend.config import SERVER_CONFIG
class ConnectionManager:
def __init__(self):
self.active_connections: Dict[int, Dict[str, WebSocket]] = {} # {user_id: {session_id: websocket}}
self.admin_user_ids: Set[int] = set()
self._loop: asyncio.AbstractEventLoop = None
self.cleanup_tasks: Dict[int, asyncio.Task] = {} # {user_id: Task}
self.is_ready = True # Guard for startup sync storms
# Hub Client State
self.hub_writer: Optional[asyncio.StreamWriter] = None
self.is_hub_connected = False
# Cache server config workers count for quick access
self.workers_count = SERVER_CONFIG.get("workers", 1)
def set_loop(self, loop: asyncio.AbstractEventLoop):
self._loop = loop
async def connect(self, user_id_raw: Any, websocket: WebSocket):
await websocket.accept()
session_id = str(uuid.uuid4())
websocket.session_id = session_id
# Ensure user_id is a clean integer for DB lookups
try:
user_id = int(user_id_raw)
except (ValueError, TypeError):
ASCIIColors.error(f"Invalid user_id type received in WebSocket connect: {user_id_raw}")
return
if user_id in self.cleanup_tasks:
ASCIIColors.info(f"User {user_id} reconnected. Cancelling session cleanup.")
self.cleanup_tasks[user_id].cancel()
del self.cleanup_tasks[user_id]
if user_id not in self.active_connections:
self.active_connections[user_id] = {}
self.active_connections[user_id][session_id] = websocket
db = None
try:
db = db_session_module.SessionLocal()
# Record active connection
new_connection = WebSocketConnection(user_id=user_id, session_id=session_id)
db.add(new_connection)
connecting_user = db.query(DBUser).filter(DBUser.id == user_id).first()
if connecting_user:
# Historical tracking: Increment total logins
connecting_user.total_logins += 1
if connecting_user.is_admin:
self.register_admin(user_id)
# Commit early to ensure connection and login count are saved
db.commit()
# --- CLIENT WARMUP WITH PROGRESS STREAMING ---
def init_callback(text, msg_type, meta):
# We pipe progress messages directly to the user's socket
self.send_personal_message_sync({
"type": "init_progress",
"data": {
"message": text,
"is_error": msg_type == 24 # MSG_TYPE_ERROR = 24
}
}, user_id)
return True
from backend.session import build_lollms_client_from_params
# Trigger warmup in background
threading.Thread(
target=build_lollms_client_from_params,
args=(connecting_user.username,),
kwargs={'callback': init_callback},
daemon=True
).start()
friendships = db.query(DBFriendship).filter(
((DBFriendship.user1_id == user_id) | (DBFriendship.user2_id == user_id)),
DBFriendship.status == FriendshipStatus.ACCEPTED
).all()
friend_ids = {f.user1_id if f.user2_id == user_id else f.user2_id for f in friendships}
if friend_ids:
notification_payload = {
"type": "friend_online",
"data": {
"username": connecting_user.username,
"icon": connecting_user.icon
}
}
for friend_id in friend_ids:
self.send_personal_message_sync(notification_payload, friend_id)
except Exception as e:
print(f"ERROR: Failed to create DB record or register user: {e}")
if db: db.rollback()
finally:
if db: db.close()
async def _delayed_cleanup(self, user_id: int):
try:
await asyncio.sleep(10)
db = db_session_module.SessionLocal()
try:
user = db.query(DBUser).filter(DBUser.id == user_id).first()
if user and user.username in user_sessions:
del user_sessions[user.username]
ASCIIColors.yellow(f"INFO: Session cache cleared for user '{user.username}' after disconnection timeout.")
finally:
db.close()
if user_id in self.cleanup_tasks:
del self.cleanup_tasks[user_id]
except asyncio.CancelledError:
pass
except Exception as e:
print(f"ERROR: Error in delayed cleanup for user {user_id}: {e}")
def disconnect(self, user_id: int, websocket: WebSocket):
session_id = getattr(websocket, 'session_id', None)
all_connections_closed = False
if user_id in self.active_connections and session_id in self.active_connections[user_id]:
del self.active_connections[user_id][session_id]
if not self.active_connections[user_id]:
del self.active_connections[user_id]
self.admin_user_ids.discard(user_id)
all_connections_closed = True
if session_id:
db = None
try:
db = db_session_module.SessionLocal()
db.query(WebSocketConnection).filter(WebSocketConnection.session_id == session_id).delete()
user = db.query(DBUser).filter(DBUser.id == user_id).first()
if user:
user.last_activity_at = datetime.datetime.now(datetime.timezone.utc)
if all_connections_closed:
if user_id in self.cleanup_tasks:
self.cleanup_tasks[user_id].cancel()
task = asyncio.create_task(self._delayed_cleanup(user_id))
self.cleanup_tasks[user_id] = task
ASCIIColors.info(f"User {user_id} disconnected completely. Scheduled session cleanup in 10s.")
db.commit()
except Exception as e:
print(f"ERROR: Failed to process disconnect: {e}")
if db: db.rollback()
finally:
if db: db.close()
async def disconnect_user(self, user_id: int):
"""Closes all active WebSockets for a specific user."""
if user_id in self.active_connections:
# Copy items to avoid modification during iteration
sessions = list(self.active_connections[user_id].values())
for websocket in sessions:
try:
await websocket.close(code=1000, reason="Disconnected by administrator")
except Exception:
pass
self.disconnect(user_id, websocket)
def disconnect_user_sync(self, user_id: int):
"""Synchronous wrapper to trigger user disconnection across the cluster."""
self.broadcast_sync({"type": "internal_event", "event_type": "user_disconnect", "data": {"user_id": user_id}})
def register_admin(self, user_id: int):
self.admin_user_ids.add(user_id)
def unregister_admin(self, user_id: int):
self.admin_user_ids.discard(user_id)
async def send_personal_message(self, message_data: dict, user_id: int):
if user_id in self.active_connections:
sockets_to_remove = set()
for websocket in list(self.active_connections[user_id].values()):
try:
await websocket.send_json(message_data)
except Exception:
sockets_to_remove.add(websocket)
for websocket in sockets_to_remove:
self.disconnect(user_id, websocket)
async def broadcast(self, message_data: dict):
users_to_cleanup = {}
for user_id, sockets_dict in list(self.active_connections.items()):
for websocket in list(sockets_dict.values()):
try:
await websocket.send_json(message_data)
except Exception:
if user_id not in users_to_cleanup:
users_to_cleanup[user_id] = set()
users_to_cleanup[user_id].add(websocket)
for user_id, sockets_to_remove in users_to_cleanup.items():
for websocket in sockets_to_remove:
self.disconnect(user_id, websocket)
async def broadcast_to_admins(self, message_data: dict):
connected_admins = [uid for uid in self.admin_user_ids if uid in self.active_connections]
if not connected_admins: return
for user_id in connected_admins:
await self.send_personal_message(message_data, user_id)
async def _handle_broadcast_payload(self, payload: dict):
"""Processes incoming messages from the Com Hub push stream."""
# Loopback protection
if payload.get("_pid") == os.getpid():
return
# Internal synchronization events
if payload.get("type") == "internal_event":
event_type = payload.get("event_type")
data = payload.get("data", {})
if event_type == "user_cache_invalidate":
username = data.get("username")
if username in user_sessions:
user_sessions[username]['lollms_clients_cache'] = {}
elif event_type == "global_model_cache_invalidate":
from backend.session import _global_client_registry, _registry_lock
with _registry_lock:
_global_client_registry.clear()
for username, session in user_sessions.items():
if 'lollms_clients_cache' in session:
session['lollms_clients_cache'] = {}
elif event_type == "user_disconnect":
user_id = data.get("user_id")
if user_id:
await self.disconnect_user(int(user_id))
return
# Global settings refresh
if payload.get("type") == "settings_updated":
refresh_db = db_session_module.SessionLocal()
try:
settings.refresh(refresh_db)
finally:
refresh_db.close()
# Dispatch to local client WebSockets
try:
if payload.get("type") == "personal":
uid = payload.get("user_id")
if uid is not None:
await self.send_personal_message(payload.get("data"), uid)
elif payload.get("type") == "admins":
await self.broadcast_to_admins(payload.get("data"))
else:
await self.broadcast(payload)
except Exception as e:
print(f"ERROR: Failed to route push broadcast payload from hub: {e}")
def broadcast_sync(self, message_data: dict):
"""
Broadcasts a message across the worker cluster.
STRICT SIZE GUARD: Prevents browser STATUS_BREAKPOINT crashes.
"""
if not self.is_ready or not isinstance(message_data, dict):
return
# --- CRITICAL FRONTEND PROTECTION TEST ---
try:
# Prepare the JSON string to measure exact payload size
payload_json = json.dumps(message_data)
payload_size = len(payload_json)
# 1MB Hard Limit. Chromium browsers crash when the heap is flooded
# with large objects via WebSockets during high-frequency updates.
if payload_size > 1048576:
msg_type = message_data.get("type", "unknown")
# EMERGENCY STRIP: If the payload is too big, try to strip images before dropping
if "data" in message_data and isinstance(message_data["data"], dict):
# Look for common image keys and nullify them to save the text part
for key in ["image_references", "images", "discussion_images"]:
if key in message_data["data"]:
message_data["data"][key] = []
# Re-calculate size after stripping images
new_payload_json = json.dumps(message_data)
if len(new_payload_json) < 1048576:
ASCIIColors.warning(f"✂️ STRIPPED IMAGES from [{msg_type}] to fit quota.")
return self.broadcast_sync(message_data)
ASCIIColors.warning(f"⚠️ DROPPED PAYLOAD [{msg_type}]: {payload_size//1024}KB. Size exceeds browser write limits.")
return # DROP THE MESSAGE
except Exception:
# If we can't even JSON encode it, it's definitely too dangerous to send
return
try:
encoded_payload = json.dumps(message_data)
encoded_len = len(encoded_payload)
if encoded_len > 1024 * 1024:
# Gather detailed statistics about the payload
msg_type = type(message_data).__name__
key_count = len(message_data.keys()) if isinstance(message_data, dict) else 'N/A'
# Create a safe snapshot (first 500 chars, escaped)
snapshot = repr(encoded_payload[:500])
# Calculate size breakdown if dict
size_breakdown = {}
if isinstance(message_data, dict):
for k, v in message_data.items():
try:
size_breakdown[k] = len(json.dumps({k: v}))
except Exception:
size_breakdown[k] = 'unmeasurable'
top_heavy = sorted(size_breakdown.items(), key=lambda x: x[1] if isinstance(x[1], int) else 0, reverse=True)[:3]
top_keys_str = ', '.join([f"{k}={v}b" for k, v in top_heavy])
else:
top_keys_str = 'N/A (not a dict)'
# Build warning panel
panel_content = (
f"\n{'='*60}\n"
f" WEBSOCKET PAYLOAD TOO LARGE - DROPPED TO PROTECT UI\n"
f"{'='*60}\n"
f" Total Size: {encoded_len:,} bytes ({encoded_len/1024/1024:.2f} MB)\n"
f" Limit: {1024*1024:,} bytes (1.0 MB)\n"
f" Data Type: {msg_type}\n"
f" Top-Level Keys: {key_count}\n"
f" Top 3 Heavy Keys: {top_keys_str}\n"
f"{'-'*60}\n"
f" Payload Snapshot (first 500 chars):\n"
f" {snapshot}\n"
f"{'='*60}"
)
ASCIIColors.warning(panel_content)
return
except Exception as e:
ASCIIColors.warning(f"WS Safety Check failed with error: {e}")
pass
# DEBUG LOGGING: Use this to see the "Storm" in your terminal
msg_type = message_data.get("type")
if msg_type == "personal":
inner_type = message_data.get("data", {}).get("type", "unknown")
ASCIIColors.debug(f"[WS-SEND] -> User {message_data.get('user_id')}: {inner_type}")
else:
ASCIIColors.debug(f"[WS-SEND] -> Global: {msg_type}")
message_data["_pid"] = os.getpid()
if self._loop and self._loop.is_running():
async def dispatch():
# [FIX] Wrap local delivery in try-except so it doesn't block Hub propagation
try:
# 1. Local delivery (this worker instance)
msg_type = message_data.get("type")
if msg_type == "personal":
uid = message_data.get("user_id")
if uid is not None: await self.send_personal_message(message_data.get("data"), uid)
elif msg_type == "admins":
await self.broadcast_to_admins(message_data.get("data"))
else:
await self.broadcast(message_data)
except Exception as e:
print(f"ERROR: Local WebSocket delivery failed, proceeding to Hub sync: {e}")
# 2. Push to Communication Hub (cross-worker synchronization)
# Rely on hub_writer availability rather than just worker count check
if self.hub_writer:
try:
encoded = json.dumps(message_data).encode('utf-8')
packet = struct.pack('!I', len(encoded)) + encoded
self.hub_writer.write(packet)
await self.hub_writer.drain()
except Exception:
self.hub_writer = None
self.is_hub_connected = False
asyncio.run_coroutine_threadsafe(dispatch(), self._loop)
# Persistence to DB as a fallback log for multi-worker synchronization audit
if self.workers_count > 1:
self._put_on_db_queue(message_data)
def _put_on_db_queue(self, payload: dict):
db = None
try:
db = db_session_module.SessionLocal()
db_message = BroadcastMessage(payload=payload)
db.add(db_message)
db.commit()
except Exception:
if db: db.rollback()
finally:
if db: db.close()
def send_personal_message_sync(self, message_data: dict, user_id: int):
self.broadcast_sync({"type": "personal", "user_id": user_id, "data": message_data})
def broadcast_to_admins_sync(self, message_data: dict):
self.broadcast_sync({"type": "admins", "data": message_data})
def broadcast_internal_event_sync(self, event_type: str, data: dict):
self.broadcast_sync({"type": "internal_event", "event_type": event_type, "data": data})
manager = ConnectionManager()
async def listen_for_broadcasts():
"""
Background worker task that maintains a persistent connection to the Hub.
Replaces the previous database polling logic with event-driven pushes.
"""
# Use dynamic setting with a fallback to the SERVER_CONFIG/default
hub_port = settings.get("com_hub_port", SERVER_CONFIG.get("com_hub_port", 8042))
print(f"INFO: Worker {os.getpid()} initializing Communication Hub client listener on port {hub_port}.")
while True:
try:
reader, writer = await asyncio.open_connection('127.0.0.1', hub_port)
manager.hub_writer = writer
manager.is_hub_connected = True
print(f"INFO: Worker {os.getpid()} successfully connected to Communication Hub.")
while True:
length_data = await reader.readexactly(4)
length = struct.unpack('!I', length_data)[0]
data = await reader.readexactly(length)
payload = json.loads(data.decode('utf-8'))
await manager._handle_broadcast_payload(payload)
except (asyncio.IncompleteReadError, ConnectionRefusedError, ConnectionResetError, BrokenPipeError):
manager.hub_writer = None
manager.is_hub_connected = False
await asyncio.sleep(2) # Backoff before retry
except asyncio.CancelledError:
break
except Exception as e:
print(f"Hub Client Error (Worker {os.getpid()}): {e}")
await asyncio.sleep(5)

Xet Storage Details

Size:
20.6 kB
·
Xet hash:
5e3a5c9f0e9de37c5f12524d237e2d22c190f0054235bb0a8cdfc5f1914e3364

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.