llm

Paused

github-actions[bot]

GitHub deploy: bccc6e08ccf3e06607a2c6e3ca5f59455b3ceaaf

a733dcb about 2 years ago

6.74 kB

import importlib.metadata
import json
import logging
import os
import pkgutil
import sys
from pathlib import Path

import markdown
from bs4 import BeautifulSoup
from constants import ERROR_MESSAGES

####################################
# Load .env file
####################################

BACKEND_DIR = Path(__file__).parent  # the path containing this file
BASE_DIR = BACKEND_DIR.parent  # the path containing the backend/

print(BASE_DIR)

try:
    from dotenv import find_dotenv, load_dotenv

    load_dotenv(find_dotenv(str(BASE_DIR / ".env")))
except ImportError:
    print("dotenv not installed, skipping...")


####################################
# LOGGING
####################################

log_levels = ["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"]

GLOBAL_LOG_LEVEL = os.environ.get("GLOBAL_LOG_LEVEL", "").upper()
if GLOBAL_LOG_LEVEL in log_levels:
    logging.basicConfig(stream=sys.stdout, level=GLOBAL_LOG_LEVEL, force=True)
else:
    GLOBAL_LOG_LEVEL = "INFO"

log = logging.getLogger(__name__)
log.info(f"GLOBAL_LOG_LEVEL: {GLOBAL_LOG_LEVEL}")

log_sources = [
    "AUDIO",
    "COMFYUI",
    "CONFIG",
    "DB",
    "IMAGES",
    "MAIN",
    "MODELS",
    "OLLAMA",
    "OPENAI",
    "RAG",
    "WEBHOOK",
]

SRC_LOG_LEVELS = {}

for source in log_sources:
    log_env_var = source + "_LOG_LEVEL"
    SRC_LOG_LEVELS[source] = os.environ.get(log_env_var, "").upper()
    if SRC_LOG_LEVELS[source] not in log_levels:
        SRC_LOG_LEVELS[source] = GLOBAL_LOG_LEVEL
    log.info(f"{log_env_var}: {SRC_LOG_LEVELS[source]}")

log.setLevel(SRC_LOG_LEVELS["CONFIG"])


WEBUI_NAME = os.environ.get("WEBUI_NAME", "Open WebUI")
if WEBUI_NAME != "Open WebUI":
    WEBUI_NAME += " (Open WebUI)"

WEBUI_URL = os.environ.get("WEBUI_URL", "http://localhost:3000")

WEBUI_FAVICON_URL = "https://openwebui.com/favicon.png"


####################################
# ENV (dev,test,prod)
####################################

ENV = os.environ.get("ENV", "dev")

try:
    PACKAGE_DATA = json.loads((BASE_DIR / "package.json").read_text())
except Exception:
    try:
        PACKAGE_DATA = {"version": importlib.metadata.version("open-webui")}
    except importlib.metadata.PackageNotFoundError:
        PACKAGE_DATA = {"version": "0.0.0"}

VERSION = PACKAGE_DATA["version"]


# Function to parse each section
def parse_section(section):
    items = []
    for li in section.find_all("li"):
        # Extract raw HTML string
        raw_html = str(li)

        # Extract text without HTML tags
        text = li.get_text(separator=" ", strip=True)

        # Split into title and content
        parts = text.split(": ", 1)
        title = parts[0].strip() if len(parts) > 1 else ""
        content = parts[1].strip() if len(parts) > 1 else text

        items.append({"title": title, "content": content, "raw": raw_html})
    return items


try:
    changelog_path = BASE_DIR / "CHANGELOG.md"
    with open(str(changelog_path.absolute()), "r", encoding="utf8") as file:
        changelog_content = file.read()

except Exception:
    changelog_content = (pkgutil.get_data("open_webui", "CHANGELOG.md") or b"").decode()


# Convert markdown content to HTML
html_content = markdown.markdown(changelog_content)

# Parse the HTML content
soup = BeautifulSoup(html_content, "html.parser")

# Initialize JSON structure
changelog_json = {}

# Iterate over each version
for version in soup.find_all("h2"):
    version_number = version.get_text().strip().split(" - ")[0][1:-1]  # Remove brackets
    date = version.get_text().strip().split(" - ")[1]

    version_data = {"date": date}

    # Find the next sibling that is a h3 tag (section title)
    current = version.find_next_sibling()

    while current and current.name != "h2":
        if current.name == "h3":
            section_title = current.get_text().lower()  # e.g., "added", "fixed"
            section_items = parse_section(current.find_next_sibling("ul"))
            version_data[section_title] = section_items

        # Move to the next element
        current = current.find_next_sibling()

    changelog_json[version_number] = version_data


CHANGELOG = changelog_json

####################################
# SAFE_MODE
####################################

SAFE_MODE = os.environ.get("SAFE_MODE", "false").lower() == "true"

####################################
# WEBUI_BUILD_HASH
####################################

WEBUI_BUILD_HASH = os.environ.get("WEBUI_BUILD_HASH", "dev-build")

####################################
# DATA/FRONTEND BUILD DIR
####################################

DATA_DIR = Path(os.getenv("DATA_DIR", BACKEND_DIR / "data")).resolve()
FRONTEND_BUILD_DIR = Path(os.getenv("FRONTEND_BUILD_DIR", BASE_DIR / "build")).resolve()

RESET_CONFIG_ON_START = (
    os.environ.get("RESET_CONFIG_ON_START", "False").lower() == "true"
)
if RESET_CONFIG_ON_START:
    try:
        os.remove(f"{DATA_DIR}/config.json")
        with open(f"{DATA_DIR}/config.json", "w") as f:
            f.write("{}")
    except Exception:
        pass

try:
    CONFIG_DATA = json.loads((DATA_DIR / "config.json").read_text())
except Exception:
    CONFIG_DATA = {}


####################################
# Database
####################################

# Check if the file exists
if os.path.exists(f"{DATA_DIR}/ollama.db"):
    # Rename the file
    os.rename(f"{DATA_DIR}/ollama.db", f"{DATA_DIR}/webui.db")
    log.info("Database migrated from Ollama-WebUI successfully.")
else:
    pass

DATABASE_URL = os.environ.get("DATABASE_URL", f"sqlite:///{DATA_DIR}/webui.db")

# Replace the postgres:// with postgresql://
if "postgres://" in DATABASE_URL:
    DATABASE_URL = DATABASE_URL.replace("postgres://", "postgresql://")


####################################
# WEBUI_AUTH (Required for security)
####################################

WEBUI_AUTH = os.environ.get("WEBUI_AUTH", "True").lower() == "true"
WEBUI_AUTH_TRUSTED_EMAIL_HEADER = os.environ.get(
    "WEBUI_AUTH_TRUSTED_EMAIL_HEADER", None
)
WEBUI_AUTH_TRUSTED_NAME_HEADER = os.environ.get("WEBUI_AUTH_TRUSTED_NAME_HEADER", None)


####################################
# WEBUI_SECRET_KEY
####################################

WEBUI_SECRET_KEY = os.environ.get(
    "WEBUI_SECRET_KEY",
    os.environ.get(
        "WEBUI_JWT_SECRET_KEY", "t0p-s3cr3t"
    ),  # DEPRECATED: remove at next major version
)

WEBUI_SESSION_COOKIE_SAME_SITE = os.environ.get(
    "WEBUI_SESSION_COOKIE_SAME_SITE",
    os.environ.get("WEBUI_SESSION_COOKIE_SAME_SITE", "lax"),
)

WEBUI_SESSION_COOKIE_SECURE = os.environ.get(
    "WEBUI_SESSION_COOKIE_SECURE",
    os.environ.get("WEBUI_SESSION_COOKIE_SECURE", "false").lower() == "true",
)

if WEBUI_AUTH and WEBUI_SECRET_KEY == "":
    raise ValueError(ERROR_MESSAGES.ENV_VAR_NOT_FOUND)

	import importlib.metadata
	import json
	import logging
	import os
	import pkgutil
	import sys
	from pathlib import Path

	import markdown
	from bs4 import BeautifulSoup
	from constants import ERROR_MESSAGES

	####################################
	# Load .env file
	####################################

	BACKEND_DIR = Path(__file__).parent # the path containing this file
	BASE_DIR = BACKEND_DIR.parent # the path containing the backend/

	print(BASE_DIR)

	try:
	from dotenv import find_dotenv, load_dotenv

	load_dotenv(find_dotenv(str(BASE_DIR / ".env")))
	except ImportError:
	print("dotenv not installed, skipping...")


	####################################
	# LOGGING
	####################################

	log_levels = ["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"]

	GLOBAL_LOG_LEVEL = os.environ.get("GLOBAL_LOG_LEVEL", "").upper()
	if GLOBAL_LOG_LEVEL in log_levels:
	logging.basicConfig(stream=sys.stdout, level=GLOBAL_LOG_LEVEL, force=True)
	else:
	GLOBAL_LOG_LEVEL = "INFO"

	log = logging.getLogger(__name__)
	log.info(f"GLOBAL_LOG_LEVEL: {GLOBAL_LOG_LEVEL}")

	log_sources = [
	"AUDIO",
	"COMFYUI",
	"CONFIG",
	"DB",
	"IMAGES",
	"MAIN",
	"MODELS",
	"OLLAMA",
	"OPENAI",
	"RAG",
	"WEBHOOK",
	]

	SRC_LOG_LEVELS = {}

	for source in log_sources:
	log_env_var = source + "_LOG_LEVEL"
	SRC_LOG_LEVELS[source] = os.environ.get(log_env_var, "").upper()
	if SRC_LOG_LEVELS[source] not in log_levels:
	SRC_LOG_LEVELS[source] = GLOBAL_LOG_LEVEL
	log.info(f"{log_env_var}: {SRC_LOG_LEVELS[source]}")

	log.setLevel(SRC_LOG_LEVELS["CONFIG"])


	WEBUI_NAME = os.environ.get("WEBUI_NAME", "Open WebUI")
	if WEBUI_NAME != "Open WebUI":
	WEBUI_NAME += " (Open WebUI)"

	WEBUI_URL = os.environ.get("WEBUI_URL", "http://localhost:3000")

	WEBUI_FAVICON_URL = "https://openwebui.com/favicon.png"


	####################################
	# ENV (dev,test,prod)
	####################################

	ENV = os.environ.get("ENV", "dev")

	try:
	PACKAGE_DATA = json.loads((BASE_DIR / "package.json").read_text())
	except Exception:
	try:
	PACKAGE_DATA = {"version": importlib.metadata.version("open-webui")}
	except importlib.metadata.PackageNotFoundError:
	PACKAGE_DATA = {"version": "0.0.0"}

	VERSION = PACKAGE_DATA["version"]


	# Function to parse each section
	def parse_section(section):
	items = []
	for li in section.find_all("li"):
	# Extract raw HTML string
	raw_html = str(li)

	# Extract text without HTML tags
	text = li.get_text(separator=" ", strip=True)

	# Split into title and content
	parts = text.split(": ", 1)
	title = parts[0].strip() if len(parts) > 1 else ""
	content = parts[1].strip() if len(parts) > 1 else text

	items.append({"title": title, "content": content, "raw": raw_html})
	return items


	try:
	changelog_path = BASE_DIR / "CHANGELOG.md"
	with open(str(changelog_path.absolute()), "r", encoding="utf8") as file:
	changelog_content = file.read()

	except Exception:
	changelog_content = (pkgutil.get_data("open_webui", "CHANGELOG.md") or b"").decode()


	# Convert markdown content to HTML
	html_content = markdown.markdown(changelog_content)

	# Parse the HTML content
	soup = BeautifulSoup(html_content, "html.parser")

	# Initialize JSON structure
	changelog_json = {}

	# Iterate over each version
	for version in soup.find_all("h2"):
	version_number = version.get_text().strip().split(" - ")[0][1:-1] # Remove brackets
	date = version.get_text().strip().split(" - ")[1]

	version_data = {"date": date}

	# Find the next sibling that is a h3 tag (section title)
	current = version.find_next_sibling()

	while current and current.name != "h2":
	if current.name == "h3":
	section_title = current.get_text().lower() # e.g., "added", "fixed"
	section_items = parse_section(current.find_next_sibling("ul"))
	version_data[section_title] = section_items

	# Move to the next element
	current = current.find_next_sibling()

	changelog_json[version_number] = version_data


	CHANGELOG = changelog_json

	####################################
	# SAFE_MODE
	####################################

	SAFE_MODE = os.environ.get("SAFE_MODE", "false").lower() == "true"

	####################################
	# WEBUI_BUILD_HASH
	####################################

	WEBUI_BUILD_HASH = os.environ.get("WEBUI_BUILD_HASH", "dev-build")

	####################################
	# DATA/FRONTEND BUILD DIR
	####################################

	DATA_DIR = Path(os.getenv("DATA_DIR", BACKEND_DIR / "data")).resolve()
	FRONTEND_BUILD_DIR = Path(os.getenv("FRONTEND_BUILD_DIR", BASE_DIR / "build")).resolve()

	RESET_CONFIG_ON_START = (
	os.environ.get("RESET_CONFIG_ON_START", "False").lower() == "true"
	)
	if RESET_CONFIG_ON_START:
	try:
	os.remove(f"{DATA_DIR}/config.json")
	with open(f"{DATA_DIR}/config.json", "w") as f:
	f.write("{}")
	except Exception:
	pass

	try:
	CONFIG_DATA = json.loads((DATA_DIR / "config.json").read_text())
	except Exception:
	CONFIG_DATA = {}


	####################################
	# Database
	####################################

	# Check if the file exists
	if os.path.exists(f"{DATA_DIR}/ollama.db"):
	# Rename the file
	os.rename(f"{DATA_DIR}/ollama.db", f"{DATA_DIR}/webui.db")
	log.info("Database migrated from Ollama-WebUI successfully.")
	else:
	pass

	DATABASE_URL = os.environ.get("DATABASE_URL", f"sqlite:///{DATA_DIR}/webui.db")

	# Replace the postgres:// with postgresql://
	if "postgres://" in DATABASE_URL:
	DATABASE_URL = DATABASE_URL.replace("postgres://", "postgresql://")


	####################################
	# WEBUI_AUTH (Required for security)
	####################################

	WEBUI_AUTH = os.environ.get("WEBUI_AUTH", "True").lower() == "true"
	WEBUI_AUTH_TRUSTED_EMAIL_HEADER = os.environ.get(
	"WEBUI_AUTH_TRUSTED_EMAIL_HEADER", None
	)
	WEBUI_AUTH_TRUSTED_NAME_HEADER = os.environ.get("WEBUI_AUTH_TRUSTED_NAME_HEADER", None)


	####################################
	# WEBUI_SECRET_KEY
	####################################

	WEBUI_SECRET_KEY = os.environ.get(
	"WEBUI_SECRET_KEY",
	os.environ.get(
	"WEBUI_JWT_SECRET_KEY", "t0p-s3cr3t"
	), # DEPRECATED: remove at next major version
	)

	WEBUI_SESSION_COOKIE_SAME_SITE = os.environ.get(
	"WEBUI_SESSION_COOKIE_SAME_SITE",
	os.environ.get("WEBUI_SESSION_COOKIE_SAME_SITE", "lax"),
	)

	WEBUI_SESSION_COOKIE_SECURE = os.environ.get(
	"WEBUI_SESSION_COOKIE_SECURE",
	os.environ.get("WEBUI_SESSION_COOKIE_SECURE", "false").lower() == "true",
	)

	if WEBUI_AUTH and WEBUI_SECRET_KEY == "":
	raise ValueError(ERROR_MESSAGES.ENV_VAR_NOT_FOUND)