Buckets:
AnonymousSpottedToad/human_verified_bucket / codebase /python /ParisNeo_lollms /backend /migration_utils.py
| import os | |
| import shutil | |
| import base64 | |
| import io | |
| import json | |
| import time | |
| import yaml | |
| import uuid | |
| from pathlib import Path | |
| from datetime import datetime | |
| from ascii_colors import trace_exception | |
| from typing import List, Dict, Any, Optional | |
| from dataclasses import dataclass, field as dataclass_field | |
| import requests | |
| from sqlalchemy import create_engine, text | |
| from sqlalchemy.orm import sessionmaker | |
| from sqlalchemy import create_engine, inspect | |
| from sqlalchemy.exc import IntegrityError | |
| from PIL import Image | |
| from backend.db import get_db as get_lollms_db | |
| from backend.db.models.user import User as LollmsUser | |
| from lollms_client import LollmsDiscussion | |
| from backend.session import get_user_lollms_client | |
| from backend.discussion import get_user_discussion_manager | |
| # --- LEGACY DISCUSSION CLASSES FOR MIGRATION --- | |
| class _LegacyMessage: | |
| sender: str | |
| sender_type: str | |
| content: str | |
| id: str = dataclass_field(default_factory=lambda: str(uuid.uuid4())) | |
| parent_id: Optional[str] = None | |
| created_at: datetime = dataclass_field(default_factory=lambda: datetime.now(datetime.timezone.utc)) | |
| binding_name: Optional[str] = None | |
| model_name: Optional[str] = None | |
| token_count: Optional[int] = None | |
| sources: Optional[List[Any]] = None | |
| steps: Optional[List[Any]] = None | |
| image_references: Optional[List[str]] = dataclass_field(default_factory=list) | |
| def from_dict(cls, data: dict) -> "_LegacyMessage": | |
| created_at_val = data.get("created_at") | |
| if isinstance(created_at_val, str): | |
| try: created_at = datetime.fromisoformat(created_at_val) | |
| except ValueError: created_at = datetime.now(datetime.timezone.utc) | |
| else: created_at = datetime.now(datetime.timezone.utc) | |
| sender = data.get("sender", "unknown") | |
| sender_type = data.get("sender_type", "user" if sender not in ["lollms", "assistant"] else "assistant") | |
| return cls( | |
| id=data.get("id", str(uuid.uuid4())), | |
| sender=sender, | |
| sender_type=sender_type, | |
| content=data.get("content", ""), | |
| parent_id=data.get("parent_message_id"), | |
| created_at=created_at, | |
| binding_name=data.get("binding_name"), | |
| model_name=data.get("model_name"), | |
| token_count=data.get("token_count"), | |
| sources=data.get("sources",[]), | |
| steps=data.get("steps",[]), | |
| image_references=data.get("image_references", []) | |
| ) | |
| class LegacyDiscussion: | |
| def __init__(self, discussion_id: Optional[str] = None, title: Optional[str] = None): | |
| self.messages: List[_LegacyMessage] = [] | |
| self.discussion_id: str = discussion_id or str(uuid.uuid4()) | |
| self.title: str = title or f"Imported {self.discussion_id[:8]}" | |
| self.rag_datastore_ids: Optional[list] = None | |
| self.active_branch_id: Optional[str] = None | |
| def load_from_yaml(file_path: Path) -> Optional["LegacyDiscussion"]: | |
| if not file_path.exists(): return None | |
| try: | |
| with open(file_path, "r", encoding="utf-8") as file: data = yaml.safe_load(file) | |
| except Exception: return None | |
| if not isinstance(data, dict): | |
| discussion = LegacyDiscussion(discussion_id=file_path.stem) | |
| if isinstance(data, list): | |
| for msg_data in data: | |
| if isinstance(msg_data, dict): discussion.messages.append(_LegacyMessage.from_dict(msg_data)) | |
| return discussion | |
| discussion = LegacyDiscussion(discussion_id=data.get("discussion_id", file_path.stem), title=data.get("title")) | |
| discussion.rag_datastore_ids = data.get("rag_datastore_ids") | |
| discussion.active_branch_id = data.get("active_branch_id") | |
| for msg_data in data.get("messages", []): | |
| if isinstance(msg_data, dict): discussion.messages.append(_LegacyMessage.from_dict(msg_data)) | |
| return discussion | |
| # --- END LEGACY --- | |
| def _fetch_and_process_icon(image_url: str) -> str | None: | |
| if not image_url or not image_url.startswith(('http', '/')): | |
| return None | |
| try: | |
| if image_url.startswith('/'): | |
| image_url = f"http://localhost:8080{image_url}" | |
| print(f" - Fetching icon: {image_url}") | |
| response = requests.get(image_url, timeout=10, stream=True) | |
| response.raise_for_status() | |
| image = Image.open(io.BytesIO(response.content)) | |
| image.thumbnail((128, 128)) | |
| buffer = io.BytesIO() | |
| image.save(buffer, format="PNG") | |
| base64_encoded = base64.b64encode(buffer.getvalue()).decode("utf-8") | |
| return f"data:image/png;base64,{base64_encoded}" | |
| except Exception as e: | |
| print(f" - WARNING: Could not process icon from {image_url}. Error: {e}") | |
| return None | |
| def _migrate_discussions_for_user(user_id: str, temp_dir: Path): | |
| print(f" - Searching for discussions for user ID: {user_id}") | |
| db_path = find_database_files(temp_dir) | |
| if len(db_path) == 0: | |
| print("CRITICAL: 'database.db' not found in uploaded files. Aborting migration.") | |
| shutil.rmtree(temp_dir) | |
| return | |
| db_path = db_path[0] | |
| print(f" - Found database: {db_path.name}. Processing...") | |
| discussion_engine = create_engine(f"sqlite:///{db_path}") | |
| DiscussionSession = sessionmaker(bind=discussion_engine) | |
| discussion_session = DiscussionSession() | |
| result = discussion_session.execute( | |
| text("SELECT id, name FROM user WHERE id = :user_id"), | |
| {"user_id": user_id} | |
| ) | |
| source_user = result.first()[1] | |
| try: | |
| discussions_query = text( | |
| "SELECT id, chat FROM chat WHERE user_id = :user_id" | |
| ) | |
| discussions = discussion_session.execute(discussions_query, {"user_id": user_id}).fetchall() | |
| migrated_discussions_count = 0 | |
| for disc in discussions: | |
| discussion_id = disc.id | |
| chat_data = json.loads(disc.chat) | |
| if chat_data is None: | |
| print(f" - WARNING: No chat data found for discussion {discussion_id}. Skipping.") | |
| continue | |
| try: | |
| messages = chat_data.get("history",{}).get("messages", []) | |
| title = chat_data.get("title", f"Imported Discussion {discussion_id[:8]}") | |
| except AttributeError: | |
| print(f" - WARNING: Invalid chat data format for discussion {discussion_id}. Skipping.") | |
| continue | |
| dm = get_user_discussion_manager(source_user) | |
| lc = get_user_lollms_client(source_user) | |
| lollms_discussion = LollmsDiscussion.create_new( | |
| lollms_client=lc, | |
| db_manager=dm, | |
| id=discussion_id, | |
| autosave=True, | |
| discussion_metadata={"title": title}, | |
| ) | |
| for _, msg in messages.items(): | |
| try: | |
| content = msg.get("content", "") | |
| sender = msg.get("role", "unknown") | |
| message_type = "user" if sender.lower() == "user" else "assistant" | |
| parent_message_id = msg.get("parent_id", None) | |
| message_id = msg.get("id", None) | |
| lollms_discussion.add_message( | |
| sender=sender, | |
| sender_type=message_type, | |
| message_type=message_type, | |
| content=content, | |
| parent_message_id=parent_message_id, | |
| message_id=message_id, | |
| ) | |
| except Exception as e: | |
| print(f" - WARNING: Error processing message: {e}. Skipping.") | |
| lollms_discussion.touch() | |
| migrated_discussions_count += 1 | |
| print(f" - SUCCESS: Migrated {migrated_discussions_count} discussions for user '{source_user}'.") | |
| return migrated_discussions_count | |
| except Exception as e: | |
| print(f" - ERROR: Failed to migrate discussions for user '{source_user}'. Error: {e}") | |
| return 0 | |
| finally: | |
| discussion_session.close() | |
| def find_database_files(directory): | |
| if not os.path.exists(directory): | |
| print(f"The directory {directory} does not exist.") | |
| return | |
| db_files = list(Path(directory).glob('*.db')) | |
| sql3_files = list(Path(directory).glob('*.sql3')) | |
| database_files = db_files + sql3_files | |
| return database_files | |
| def discover_database_schema(db_path: str): | |
| try: | |
| engine = create_engine(f"sqlite:///{db_path}", | |
| pool_size=20, # base pool size | |
| max_overflow=30, # additional connections beyond pool_size | |
| pool_timeout=30, # seconds to wait before raising TimeoutError | |
| pool_pre_ping=True # helps detect stale connections | |
| ) | |
| inspector = inspect(engine) | |
| markdown_output = "# Database Schema\n\n" | |
| table_names = inspector.get_table_names() | |
| if not table_names: | |
| return "## No tables found in the database." | |
| for table_name in table_names: | |
| markdown_output += f"## Table: `{table_name}`\n\n" | |
| columns = inspector.get_columns(table_name) | |
| if not columns: | |
| markdown_output += " *No columns found.*\n\n" | |
| continue | |
| markdown_output += "| Column Name | Data Type | Nullable |\n" | |
| markdown_output += "|---|---|---|\n" | |
| for column in columns: | |
| markdown_output += f"| `{column['name']}` | `{column['type']}` | `{column['nullable']}` |\n" | |
| markdown_output += "\n" | |
| with open("database_schema.log","w",encoding="utf8", errors="ignore") as f: | |
| f.write(markdown_output) | |
| return markdown_output | |
| except Exception as e: | |
| print(f"Error connecting to the database: {e}") | |
| return None | |
| def timestamp_to_datetime(timestamp): | |
| try: | |
| dt_object = datetime.fromtimestamp(timestamp) | |
| return dt_object | |
| except (TypeError, ValueError) as e: | |
| print(f"Error converting timestamp: {e}") | |
| return None | |
| def run_openwebui_migration(temp_dir_str: str): | |
| temp_dir = Path(temp_dir_str) | |
| print("\n--- Background OpenWebUI Migration Process Started ---") | |
| db_path = find_database_files(temp_dir) | |
| if len(db_path) == 0: | |
| print("CRITICAL: 'database.db' not found in uploaded files. Aborting migration.") | |
| shutil.rmtree(temp_dir) | |
| return | |
| db_path = db_path[0] | |
| schema = discover_database_schema(db_path) | |
| source_engine = create_engine(f"sqlite:///{db_path}") | |
| SourceSession = sessionmaker(bind=source_engine) | |
| source_session = SourceSession() | |
| lollms_db_session = next(get_lollms_db()) | |
| max_retries = 5 | |
| retry_delay = 0.5 | |
| for attempt in range(max_retries): | |
| try: | |
| source_users_query = text(""" | |
| SELECT | |
| u.id, | |
| u.name, | |
| u.email, | |
| a.password, | |
| u.role, | |
| u.profile_image_url, | |
| u.created_at, | |
| u.last_active_at | |
| FROM user u | |
| JOIN auth a ON u.email = a.email | |
| """) | |
| source_users = source_session.execute(source_users_query).fetchall() | |
| print(f"INFO: Found {len(source_users)} users in 'database.db'.") | |
| migrated_users_count = 0 | |
| skipped_users_count = 0 | |
| for user in source_users: | |
| print(f"\nProcessing user: {user.name} ({user.email})") | |
| existing_user = lollms_db_session.query(LollmsUser).filter(LollmsUser.email == user.email).first() | |
| if existing_user: | |
| print(" - INFO: User already exists. Skipping user data migration.") | |
| skipped_users_count += 1 | |
| _migrate_discussions_for_user(user.id, temp_dir) | |
| continue | |
| icon_b64 = _fetch_and_process_icon(user.profile_image_url) | |
| password_hash_with_marker = f"argon2_hash:{user.password}" | |
| new_lollms_user = LollmsUser( | |
| username=user.name, | |
| email=user.email, | |
| hashed_password=password_hash_with_marker, | |
| is_admin=(user.role == 'admin'), | |
| is_active=True, | |
| created_at=timestamp_to_datetime(user.created_at) or datetime.utcnow(), | |
| last_activity_at=timestamp_to_datetime(user.last_active_at), | |
| icon=icon_b64, | |
| lollms_model_name = "gemma3:27b" | |
| ) | |
| lollms_db_session.add(new_lollms_user) | |
| try: | |
| lollms_db_session.commit() | |
| print(f" - SUCCESS: Migrated user '{user.name}'.") | |
| migrated_users_count += 1 | |
| _migrate_discussions_for_user( user.id, temp_dir) | |
| except IntegrityError as e: | |
| trace_exception(e) | |
| lollms_db_session.rollback() | |
| print(f" - ERROR: Failed to insert user '{user.name}'. Username might be a duplicate. Skipping. Error: {e.orig}") | |
| skipped_users_count += 1 | |
| time.sleep(0.1) | |
| print("\n--- Migration Task Finished ---") | |
| print(f"Migrated users: {migrated_users_count}") | |
| print(f"Skipped users: {skipped_users_count}") | |
| break | |
| except Exception as e: | |
| trace_exception(e) | |
| print(f"Attempt {attempt + 1} failed: {e}") | |
| if attempt < max_retries - 1: | |
| print(f"Retrying in {retry_delay} seconds...") | |
| time.sleep(retry_delay) | |
| else: | |
| print("CRITICAL: Migration failed after multiple retries.") | |
| return | |
| finally: | |
| source_session.close() | |
| lollms_db_session.close() | |
| try: | |
| shutil.rmtree(temp_dir) | |
| print(f"INFO: Cleaned up temporary directory: {temp_dir}") | |
| except Exception as e: | |
| print(f"Error cleaning up temporary directory: {e}") | |
Xet Storage Details
- Size:
- 14.3 kB
- Xet hash:
- 1e84d01f120ebffd2df4454b5e0853d13949e223781e7442b11b8ffe15ef77ed
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.