Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /MarkinHaus_ToolBoxV2 /toolboxv2 /utils /toolbox.py
"""Main module."""
import asyncio
import datetime
import inspect
import json
import logging
import os
import pkgutil
import sys
import threading
import time
from collections.abc import Callable
from concurrent.futures import Future, ThreadPoolExecutor
from enum import Enum
from functools import partial, wraps
from importlib import import_module, reload
from inspect import signature
from platform import node, system
from threading import Thread
from types import ModuleType
from typing import Any, Optional
from dotenv import load_dotenv
from .extras.db.mobile_db import create_mobile_db
from .extras.mkdocs import add_to_app
from ..utils.system.main_tool import get_version_from_pyproject
from .extras.Style import Spinner, Style, stram_print
from .singelton_class import Singleton
from .system.cache import FileCache, MemoryCache
from .system.file_handler import FileHandler
from .system.getting_and_closing_app import get_app
from .system.tb_logger import setup_logging, loggerNameOfToolboxv2, AuditLogger, set_log_db, LogSyncManager
from .system.types import (
ApiResult,
AppArgs,
AppType,
MainToolType,
RequestData,
Result,
ToolBoxInterfaces,
WebSocketContext,
)
load_dotenv()
class App(AppType, metaclass=Singleton):
def __init__(self, prefix: str = "", args=AppArgs().default()):
self.log_sync = None
if "test" not in prefix:
self.logger_prefix = self.REFIX = prefix
prefix = "main"
else:
self.logger_prefix = self.REFIX = "test"
super().__init__(prefix, args)
self._web_context = None
self._debug = args.debug
self.loop = None
t0 = time.perf_counter()
abspath = os.path.abspath(__file__)
self.system_flag = system() # Linux: Linux Mac: Darwin Windows: Windows
self.appdata = os.getenv('APPDATA') if os.name == 'nt' else os.getenv('XDG_CONFIG_HOME') or os.path.expanduser(
'~/.config') if os.name == 'posix' else None
if self.system_flag == "Darwin" or self.system_flag == "Linux":
dir_name = os.path.dirname(abspath).replace("/utils", "")
else:
dir_name = os.path.dirname(abspath).replace("\\utils", "")
self.start_dir = str(dir_name)
self.bg_tasks = []
# ponytail: prefix file follows TB_DATA_DIR like the other dirs.
lapp = os.path.join(os.getenv("TB_DATA_DIR") or dir_name, ".data")
_prefix_file = os.path.join(lapp, "last-app-prefix.txt")
if not prefix:
if not os.path.exists(_prefix_file):
os.makedirs(lapp, exist_ok=True)
open(_prefix_file, "a").close()
with open(_prefix_file) as prefix_file:
cont = prefix_file.read()
if cont:
prefix = cont.rstrip()
else:
if not os.path.exists(_prefix_file):
os.makedirs(lapp, exist_ok=True)
open(_prefix_file, "a").close()
with open(_prefix_file, "w") as prefix_file:
prefix_file.write(prefix)
self.prefix = prefix
node_ = node()
if 'localhost' in node_ and (host := os.getenv('HOSTNAME', 'localhost')) != 'localhost':
node_ = node_.replace('localhost', host)
self.id = prefix + '-' + node_
self.globals = {
"root": {**globals()},
}
self.locals = {
"user": {'app': self, **locals()},
}
identification = self.id
collective_identification = self.id
# ── data root: honor TB_DATA_DIR (writable, user-owned) over the
# package dir. ponytail: one base var, proper os.path.join. Default
# stays start_dir so existing installs are unchanged; the <id> leaf
# is preserved so downstream `info_dir.replace(app.id, '')` still works.
_data_root = os.getenv("TB_DATA_DIR") or self.start_dir
if "test" in prefix:
if self.system_flag == "Darwin" or self.system_flag == "Linux":
start_dir = self.start_dir.replace("ToolBoxV2/toolboxv2", "toolboxv2")
else:
start_dir = self.start_dir.replace("ToolBoxV2\\toolboxv2", "toolboxv2")
_root = os.getenv("TB_DATA_DIR") or start_dir
self.data_dir = os.path.join(_root, ".data", "test")
self.config_dir = os.path.join(_root, ".config", "test")
self.info_dir = os.path.join(_root, ".info", "test")
elif identification.startswith('collective-'):
collective_identification = identification.split('-')[1]
self.data_dir = os.path.join(_data_root, ".data", collective_identification)
self.config_dir = os.path.join(_data_root, ".config", collective_identification)
self.info_dir = os.path.join(_data_root, ".info", collective_identification)
self.id = collective_identification
else:
self.data_dir = os.path.join(_data_root, ".data", identification)
self.config_dir = os.path.join(_data_root, ".config", identification)
self.info_dir = os.path.join(_data_root, ".info", identification)
if self.appdata is None:
self.appdata = self.data_dir
else:
self.appdata += "/ToolBoxV2"
if not os.path.exists(self.appdata):
os.makedirs(self.appdata, exist_ok=True)
if not os.path.exists(self.data_dir):
os.makedirs(self.data_dir, exist_ok=True)
if not os.path.exists(self.config_dir):
os.makedirs(self.config_dir, exist_ok=True)
if not os.path.exists(self.info_dir):
os.makedirs(self.info_dir, exist_ok=True)
self.print(f"Starting ToolBox as {prefix} from :", Style.Bold(Style.CYAN(f"{os.getcwd()}")))
pid_file = os.path.join(f"{self.start_dir}",".info","pids",f"{args.modi}-{self.REFIX}.pid")
app_pid = str(os.getpid())
if not os.path.exists(os.path.join(f"{self.start_dir}",".info","pids")):
os.makedirs(os.path.join(f"{self.start_dir}",".info","pids"), exist_ok=True)
with open(pid_file, "w", encoding="utf8") as f:
f.write(app_pid)
self.manifest = None
from toolboxv2.utils.manifest.loader import ManifestLoader
loader = ManifestLoader(self.start_dir)
if not loader.exists():
self.print("No manifest found — creating minimal default (profile: local)")
try:
manifest = loader.load()
self.manifest = manifest
except Exception as e:
self.print("Error loading manifest",e)
from toolboxv2.utils.manifest.schema import TBManifest
self.manifest = TBManifest()
if args.init and args.init is not None and self.start_dir not in sys.path:
sys.path.append(self.start_dir)
__version__ = get_version_from_pyproject()
self.version = __version__
self.keys = {
"MACRO": "macro~~~~:",
"MACRO_C": "m_color~~:",
"HELPER": "helper~~~:",
"debug": "debug~~~~:",
"id": "name-spa~:",
"st-load": "mute~load:",
"comm-his": "comm-his~:",
"develop-mode": "dev~mode~:",
"provider::": "provider::",
}
defaults = {
"MACRO": ['Exit'],
"MACRO_C": {},
"HELPER": {},
"debug": args.debug,
"id": self.id,
"st-load": False,
"comm-his": [[]],
"develop-mode": False,
}
self.config_fh = FileHandler(collective_identification + ".config", keys=self.keys, defaults=defaults)
try:
self.config_fh.load_file_handler()
except Exception as e:
print(e)
self.flows = {}
self.dev_modi = self.config_fh.get_file_handler(self.keys["develop-mode"])
if self.config_fh.get_file_handler("provider::") is None:
self.config_fh.add_to_save_file_handler("provider::", "http://localhost:" + str(
self.args_sto.port) if os.environ.get("HOSTNAME","localhost") == "localhost" else "https://simplecore.app")
# Initialize the log database (encrypted, offline-first)
self.log_db = create_mobile_db(
path=os.path.join(f"{self.data_dir}", "system_logs.db"),
max_size_mb=200,
)
# Register globally so setup_logging() picks it up automatically
set_log_db(self.log_db, node_id=node())
# Now call set_logger (same signature as before)
logger_info_str, self.logger, self.logging_filename = self.set_logger(args.debug, self.logger_prefix)
from .system.session import Session
self.session: Session = Session(self.get_username())
self._ping_interval = None
if self.manifest:
self._ping_interval = self.manifest.app.ping_interval
#self._initialize_network() #+ 3.2s 7.4 s start up time Total
# Audit logger for business/compliance events
self.audit_logger = AuditLogger(self.logger, db=self.log_db, node_id=node())
self.logger.info("Logger initialized", extra={"event": "SYSTEM_STARTUP"})
self.print("================================")
self.logger.info("Logger initialized")
self.functions = {}
self.modules = {}
self.interface_type = ToolBoxInterfaces.native
self.PREFIX = Style.CYAN(f"~{node()}@>")
self.alive = True
self.called_exit = False, time.time()
self.print(f"Infos:\n {'Name':<8} -> {node()}"
f"\n {'ID':<8} -> {self.id}"
f"\n {'Version':<8} -> {self.version}"
f"\n {'PID':<8} -> {os.getpid()}"
f"\n {'Logger':<8} -> {logger_info_str}"
)
self.args_sto = args
if args.debug:
from toolboxv2.utils.extras.live_debugger import install as install_live_debugger
install_live_debugger(loop=self.loop_gard())
# ── Observability (from manifest) ─────────────────────────────
self._obs_adapter = None
self._obs_sync_manager = None
if len(sys.argv) > 2 and sys.argv[1] == "db":
return
from .clis.db_cli_manager import status as min_io_db_status, start as min_io_db_start, all_modes as min_io_db_all_modes
from .extras.blobs import create_server_storage, create_desktop_storage, create_offline_storage
if self.manifest.database.mode.value == "CB":
modes = min_io_db_all_modes()
for name, mode in modes:
if min_io_db_status(mode):
break
else:
if len(modes) == 0:
self.print("No minio db conferred run tb db -h")
elif len(modes) == 1:
min_io_db_start(modes[0][0])
self.print(f"starting db name={modes[0][0]} of mode={modes[0][1]}")
else:
try:
self.print(f"Starting default node {self.manifest.database.minio.default_name}")
min_io_db_start(self.manifest.database.minio.default_name)
except Exception as e:
self.debug_rains(e)
self.root_blob_storage = create_server_storage() if os.getenv("IS_OFFLINE_DB", "false")!="true" else create_offline_storage()
self.desktop_blob_storage = create_desktop_storage() if os.getenv("IS_OFFLINE_DB", "false")!="true" else create_offline_storage()
self.mkdocs = add_to_app(self)
# self._start_event_loop()
try:
self._setup_observability(self.manifest.observability.slow_on_init)
except Exception as e:
self.logger.debug(f"Observability setup skipped: {e}")
async def observability_health_check_and_anabel(self):
obs = self.manifest.observability
if not obs.dashboard.enabled or not obs.dashboard.password:
return
from toolboxv2.utils.system.tb_logger import enable_live_observability
res = self._obs_adapter.health_check()
if asyncio.iscoroutine(res):
res = await res
if res:
enable_live_observability(
self._obs_adapter,
system_stream=obs.dashboard.system_stream,
audit_stream=obs.dashboard.audit_stream,
flush_interval=obs.dashboard.flush_interval,
)
self.logger.info("Live observability connected",
extra={"endpoint": obs.dashboard.endpoint})
else:
self.logger.debug(f"OpenObserve not reachable at {obs.dashboard.endpoint}")
self._obs_adapter = None
def _setup_observability(self, do_slow=False):
"""Configure observability from manifest (dashboard + sync + cleanup)."""
from toolboxv2.utils.system.tb_logger import enable_live_observability
obs = self.manifest.observability
# ── Live Dashboard ──
if obs.dashboard.enabled and obs.dashboard.password:
try:
from toolboxv2.utils.system.observability_adapter import OpenObserveAdapter
self._obs_adapter = OpenObserveAdapter(
endpoint=obs.dashboard.endpoint,
org=obs.dashboard.org,
credentials=(obs.dashboard.user, obs.dashboard.password),
stream=obs.dashboard.system_stream,
audit_stream=obs.dashboard.audit_stream,
verify_ssl=obs.dashboard.verify_ssl,
)
if do_slow:
if self._obs_adapter.health_check():
enable_live_observability(
self._obs_adapter,
system_stream=obs.dashboard.system_stream,
audit_stream=obs.dashboard.audit_stream,
flush_interval=obs.dashboard.flush_interval,
)
self.logger.info("Live observability connected",
extra={"endpoint": obs.dashboard.endpoint})
else:
self.logger.debug(f"OpenObserve not reachable at {obs.dashboard.endpoint}")
self._obs_adapter = None
except Exception as e:
self.logger.debug(f"Dashboard setup failed: {e}")
# ── MinIO Log Sync ──
if obs.sync.enabled:
try:
from toolboxv2.utils.system.tb_logger import LogSyncManager
from toolboxv2.utils.extras.db.minio_helper import create_minio_client
if obs.sync.target == "minio":
mc = create_minio_client(
self.manifest.database.minio.endpoint,
access_key=self.manifest.database.minio.access_key,
secret_key=self.manifest.database.minio.secret_key,
secure=self.manifest.database.minio.use_ssl,
)
else:
mc = create_minio_client(
obs.sync.remote_endpoint,
access_key=obs.sync.remote_access_key,
secret_key=obs.sync.remote_secret_key,
secure=obs.sync.remote_secure,
)
self._obs_sync_manager = LogSyncManager(
db=self.log_db,
minio_client=mc,
endpoint=
self.manifest.database.minio.endpoint if obs.sync.target == "minio" else obs.sync.remote_endpoint,
bucket=obs.sync.remote_bucket,
app_id=self.id,
node_id=node(),
)
self._obs_sync_manager.ensure_bucket()
# Attach dashboard adapter to sync manager too (for historical catch-up)
if self._obs_adapter:
self._obs_sync_manager.set_observability_adapter(self._obs_adapter)
if obs.sync.interval_seconds > 0:
self._obs_sync_manager.start_auto_sync(
interval_seconds=obs.sync.interval_seconds
)
self.logger.info("Log auto-sync started",
extra={"interval": obs.sync.interval_seconds})
self.log_sync = self._obs_sync_manager
except ImportError:
self.logger.debug("minio package not installed, sync disabled")
except Exception as e:
self.logger.debug(f"Sync setup failed: {e}")
import traceback
traceback.print_exc()
# ── Log Cleanup ──
if obs.cleanup.enabled:
try:
self._schedule_log_cleanup(obs.cleanup)
except Exception as e:
self.logger.debug(f"Cleanup setup failed: {e}")
def _schedule_log_cleanup(self, cleanup_config):
"""Schedule automatic log cleanup based on manifest config."""
import threading
def _cleanup():
date_cutoff = (
datetime.datetime.now() - datetime.timedelta(days=cleanup_config.max_age_days)
).strftime("%Y-%m-%d")
prefix = "logs/"
blobs = self.log_db.list(prefix=prefix)
for meta in blobs:
parts = meta.path.split("/")
if len(parts) < 4:
continue
fname = parts[-1]
date_folder = parts[-2]
# Skip if newer than cutoff
if date_folder >= date_cutoff:
continue
# Skip audit if keep_audit
if cleanup_config.keep_audit and fname.startswith("audit_"):
continue
# Skip protected levels
if cleanup_config.keep_levels:
try:
data = self.log_db.get(meta.path)
if data:
first_line = data.decode("utf-8", errors="replace").split("\n")[0]
import json as _json
entry = _json.loads(first_line)
if entry.get("level") in cleanup_config.keep_levels:
continue
except Exception:
pass
try:
self.log_db.delete(meta.path)
except Exception:
pass
# Run cleanup once on startup (background thread)
t = threading.Thread(target=_cleanup, daemon=True, name="LogCleanup")
t.start()
def _start_event_loop(self):
"""Starts the asyncio event loop in a separate thread."""
if self.loop is None:
self.loop = asyncio.new_event_loop()
self.loop_thread = threading.Thread(target=self.loop.run_forever, daemon=True)
self.loop_thread.start()
def get_username(self, get_input=False, default="loot") -> str:
user_name = self.config_fh.get_file_handler("ac_user:::")
if get_input and user_name is None:
user_name = input("Input your username: ")
self.config_fh.add_to_save_file_handler("ac_user:::", user_name)
if user_name is None:
user_name = default
self.config_fh.add_to_save_file_handler("ac_user:::", user_name)
return user_name
def set_username(self, username):
return self.config_fh.add_to_save_file_handler("ac_user:::", username)
@staticmethod
def exit_main(*args, **kwargs):
"""proxi attr"""
@staticmethod
def hide_console(*args, **kwargs):
"""proxi attr"""
@staticmethod
def show_console(*args, **kwargs):
"""proxi attr"""
@staticmethod
def disconnect(*args, **kwargs):
"""proxi attr"""
def set_logger(self, debug=False, logger_prefix=None):
# Clear existing handlers
try:
logging.getLogger(loggerNameOfToolboxv2).handlers.clear()
except Exception:
print("No logger to clear or potential double logging")
if debug is None and os.getenv("TOOLBOX_LOGGING_LEVEL") is not None:
debug = True
if logger_prefix is None:
logger_prefix = self.logger_prefix
if "test" in self.logger_prefix and not debug:
logger, logging_filename = setup_logging(
logging.NOTSET,
name="toolbox-test",
interminal=True,
file_level=logging.NOTSET,
app_name=logger_prefix,
)
logger_info_str = "in Test Mode"
elif "live" in self.logger_prefix and not debug:
logger, logging_filename = setup_logging(
logging.DEBUG,
name="toolbox-live",
interminal=False,
file_level=logging.WARNING,
app_name=logger_prefix,
)
logger_info_str = "in Live Mode"
elif "debug" in self.logger_prefix or self.logger_prefix.endswith("D"):
self.logger_prefix = self.logger_prefix.replace("-debug", '').replace("debug", '')
logger, logging_filename = setup_logging(
logging.DEBUG,
name="toolbox-debug",
interminal=True,
file_level=logging.WARNING,
app_name=logger_prefix,
)
logger_info_str = "in debug Mode"
self.debug = True
elif debug:
if hasattr(logging, "getLevelNamesMapping"):
level = logging.getLevelNamesMapping().get(
os.getenv("TOOLBOX_LOGGING_LEVEL", "WARNING")
)
else:
level = logging.WARNING
logger, logging_filename = setup_logging(
level=level,
name=f"toolbox-{self.logger_prefix}-debug",
interminal=True,
file_level=level,
app_name=logger_prefix,
)
logger_info_str = "in args debug Mode"
else:
logger, logging_filename = setup_logging(
logging.ERROR,
name=f"toolbox-{self.logger_prefix}",
app_name=logger_prefix,
)
logger_info_str = "in Default"
return logger_info_str, logger, logging_filename
def setup_log_sync(self, minio_client, auto_interval: Optional[float] = None):
"""
Initialize log sync to the RYZEN root server.
Args:
minio_client: minio.Minio connected to the root server
auto_interval: If set, start auto-sync every N seconds.
None = manual sync only.
Usage:
# Manual sync
app.setup_log_sync(minio_client)
app.log_sync.sync_all() # everything
app.log_sync.sync_time_range("2026-02-20", "2026-02-23") # date range
app.log_sync.sync_audit_only(date_from="2026-02-23") # only audit
# Automatic (every 5 minutes)
app.setup_log_sync(minio_client, auto_interval=300)
# Check pending
print(app.log_sync.get_pending_stats())
# Stop auto sync
app.log_sync.stop_auto_sync()
"""
self.log_sync = LogSyncManager(
db=self.log_db,
minio_client=minio_client,
bucket="system-audit-logs",
app_id=self.id,
node_id=node(),
)
self.log_sync.ensure_bucket()
if auto_interval is not None:
self.log_sync.start_auto_sync(interval_seconds=auto_interval)
self.logger.info(
"Auto log sync started",
extra={"interval": auto_interval},
)
@property
def debug(self):
return self._debug
@debug.setter
def debug(self, value):
if not isinstance(value, bool):
self.logger.debug(f"Value must be an boolean. is : {value} type of {type(value)}")
raise ValueError("Value must be an boolean.")
# self.logger.info(f"Setting debug {value}")
self._debug = value
def debug_rains(self, e):
if self.debug:
import traceback
x = "="*5
x += " DEBUG "
x += "="*5
self.print(x)
self.print(traceback.format_exc())
self.print(x)
raise e
else:
self.logger.error(f"Error: {e}")
import traceback
x = "="*5
x += " DEBUG "
x += "="*5
self.print(x)
self.print(traceback.format_exc())
self.print(x)
def set_flows(self, r):
self.flows = r
async def run_flows(self, name, **kwargs):
from ..flows import flows_dict as flows_dict_func
if name not in self.flows:
self.flows = {**self.flows, **flows_dict_func(s=name, remote=True)}
if name in self.flows:
if asyncio.iscoroutinefunction(self.flows[name]):
return await self.flows[name](get_app(from_="runner"), self.args_sto, **kwargs)
else:
return self.flows[name](get_app(from_="runner"), self.args_sto, **kwargs)
else:
print("Flow not found, active flows:", len(self.flows.keys()))
def _coppy_mod(self, content, new_mod_dir, mod_name, file_type='py'):
mode = 'xb'
self.logger.info(f" coppy mod {mod_name} to {new_mod_dir} size : {sys.getsizeof(content) / 8388608:.3f} mb")
if not os.path.exists(new_mod_dir):
os.makedirs(new_mod_dir)
with open(f"{new_mod_dir}/__init__.py", "w") as nmd:
nmd.write(f"__version__ = '{self.version}'")
if os.path.exists(f"{new_mod_dir}/{mod_name}.{file_type}"):
mode = False
with open(f"{new_mod_dir}/{mod_name}.{file_type}", 'rb') as d:
runtime_mod = d.read() # Testing version but not efficient
if len(content) != len(runtime_mod):
mode = 'wb'
if mode:
with open(f"{new_mod_dir}/{mod_name}.{file_type}", mode) as f:
f.write(content)
def _pre_lib_mod(self, mod_name, path_to="./runtime", file_type='py'):
working_dir = self.id.replace(".", "_")
lib_mod_dir = f"toolboxv2.runtime.{working_dir}.mod_lib."
self.logger.info(f"pre_lib_mod {mod_name} from {lib_mod_dir}")
postfix = "_dev" if self.dev_modi else ""
mod_file_dir = f"./mods{postfix}/{mod_name}.{file_type}"
new_mod_dir = f"{path_to}/{working_dir}/mod_lib"
with open(mod_file_dir, "rb") as c:
content = c.read()
self._coppy_mod(content, new_mod_dir, mod_name, file_type=file_type)
return lib_mod_dir
def _copy_load(self, mod_name, file_type='py', **kwargs):
loc = self._pre_lib_mod(mod_name, file_type)
return self.inplace_load_instance(mod_name, loc=loc, **kwargs)
def helper_install_pip_module(self, module_name):
if 'main' in self.id:
return
self.print(f"Installing {module_name} GREEDY")
os.system(f"{sys.executable} -m pip install {module_name}")
def python_module_import_classifier(self, mod_name, error_message):
if error_message.startswith("No module named 'toolboxv2.utils"):
return Result.default_internal_error(f"404 {error_message.split('utils')[1]} not found")
if error_message.startswith("No module named 'toolboxv2.mods"):
if mod_name.startswith('.'):
return
return self.run_a_from_sync(self.a_run_any, ("CloudM", "install"), module_name=mod_name)
if error_message.startswith("No module named '"):
pip_requ = error_message.split("'")[1].replace("'", "").strip()
# if 'y' in input(f"\t\t\tAuto install {pip_requ} Y/n").lower:
return self.helper_install_pip_module(pip_requ)
# return Result.default_internal_error(f"404 {pip_requ} not found")
def inplace_load_instance(self, mod_name, loc="toolboxv2.mods.", spec='app', save=True, mfo=None):
if self.dev_modi and loc == "toolboxv2.mods.":
loc = "toolboxv2.mods_dev."
if spec=='app' and self.mod_online(mod_name):
self.logger.info(f"Reloading mod from : {loc + mod_name}")
self.remove_mod(mod_name, spec=spec, delete=False)
# Convert dotted module name to file path for existence check
# e.g., "CloudM.AuthHelper" -> "CloudM/AuthHelper"
mod_path = mod_name.replace('.', '/')
if (os.path.exists(self.start_dir + '/mods/' + mod_path) or os.path.exists(
self.start_dir + '/mods/' + mod_path + '.py')) and (
os.path.isdir(self.start_dir + '/mods/' + mod_path) or os.path.isfile(
self.start_dir + '/mods/' + mod_path + '.py')):
try:
if mfo is None:
modular_file_object = import_module(loc + mod_name)
else:
modular_file_object = mfo
self.modules[mod_name] = modular_file_object
except ModuleNotFoundError as e:
self.logger.error(Style.RED(f"module {loc + mod_name} not found is type sensitive {e}"))
self.print(Style.RED(f"module {loc + mod_name} not found is type sensitive {e}"))
if self.debug or self.args_sto.sysPrint:
self.python_module_import_classifier(mod_name, str(e))
self.debug_rains(e)
return None
else:
self.sprint(f"module {loc + mod_name} is not valid")
return None
if hasattr(modular_file_object, "Tools"):
tools_class = modular_file_object.Tools
else:
if hasattr(modular_file_object, "name"):
tools_class = modular_file_object
modular_file_object = import_module(loc + mod_name)
else:
tools_class = None
modular_id = None
instance = modular_file_object
app_instance_type = "file/application"
if tools_class is None:
modular_id = modular_file_object.Name if hasattr(modular_file_object, "Name") else mod_name
if tools_class is None and modular_id is None:
modular_id = str(modular_file_object.__name__)
self.logger.warning(f"Unknown instance loaded {mod_name}")
return modular_file_object
if tools_class is not None:
tools_class = self.save_initialized_module(tools_class, spec)
modular_id = tools_class.name
app_instance_type = "functions/class"
else:
instance.spec = spec
# if private:
# self.functions[modular_id][f"{spec}_private"] = private
if not save:
return instance if tools_class is None else tools_class
return self.save_instance(instance, modular_id, spec, app_instance_type, tools_class=tools_class)
def save_instance(self, instance, modular_id, spec='app', instance_type="file/application", tools_class=None):
if modular_id in self.functions and tools_class is None:
if self.functions[modular_id].get(f"{spec}_instance", None) is None:
self.functions[modular_id][f"{spec}_instance"] = instance
self.functions[modular_id][f"{spec}_instance_type"] = instance_type
else:
self.print("Firest instance stays use new spec to get new instance")
if modular_id in self.functions and self.functions[modular_id].get(f"{spec}_instance", None) is not None:
return self.functions[modular_id][f"{spec}_instance"]
else:
raise ImportError(f"Module already known {modular_id} and not avalabel reload using other spec then {spec}")
elif tools_class is not None:
if modular_id not in self.functions:
self.functions[modular_id] = {}
self.functions[modular_id][f"{spec}_instance"] = tools_class
self.functions[modular_id][f"{spec}_instance_type"] = instance_type
try:
if not hasattr(tools_class, 'tools'):
tools_class.tools = {"Version": tools_class.get_version, 'name': tools_class.name}
for function_name in list(tools_class.tools.keys()):
t_function_name = function_name.lower()
if t_function_name != "all" and t_function_name != "name":
self.tb(function_name, mod_name=modular_id)(tools_class.tools.get(function_name))
self.functions[modular_id][f"{spec}_instance_type"] += "/BC"
if hasattr(tools_class, 'on_exit'):
if "on_exit" in self.functions[modular_id]:
self.functions[modular_id]["on_exit"].append(tools_class.on_exit)
else:
self.functions[modular_id]["on_exit"] = [tools_class.on_exit]
except Exception as e:
self.logger.error(f"Starting Module {modular_id} compatibility failed with : {e}")
pass
elif modular_id not in self.functions and tools_class is None:
self.functions[modular_id] = {}
self.functions[modular_id][f"{spec}_instance"] = instance
self.functions[modular_id][f"{spec}_instance_type"] = instance_type
else:
raise ImportError(f"Modular {modular_id} is not a valid mod")
on_start = self.functions[modular_id].get("on_start")
if on_start is not None:
i = 1
for f in on_start:
try:
f_, e = self.get_function((modular_id, f), state=True, specification=spec)
if e == 0:
self.logger.info(Style.GREY(f"Running On start {f} {i}/{len(on_start)}"))
if asyncio.iscoroutinefunction(f_):
self.print(f"Async on start is only in Tool claas supported for {modular_id}.{f}" if tools_class is None else f"initialization starting soon for {modular_id}.{f}")
self.run_bg_task_advanced(f_)
else:
o = f_()
if o is not None:
self.print(f"Function {modular_id} On start result: {o}")
else:
self.logger.warning(f"starting function not found {e}")
except Exception as e:
self.logger.debug(Style.YELLOW(
Style.Bold(f"modular:{modular_id}.{f} on_start error {i}/{len(on_start)} -> {e}")))
self.debug_rains(e)
finally:
i += 1
return instance if tools_class is None else tools_class
def save_initialized_module(self, tools_class, spec):
tools_class.spec = spec
live_tools_class = tools_class(app=self)
return live_tools_class
def mod_online(self, mod_name, installed=False):
if installed and mod_name not in self.functions:
self.save_load(mod_name)
return mod_name in self.functions
def _get_function(self,
name: Enum or None,
state: bool = True,
specification: str = "app",
metadata=False, as_str: tuple or None = None, r=0, **kwargs):
if as_str is None and isinstance(name, Enum):
modular_id = str(name.NAME.value)
function_id = str(name.value)
elif as_str is None and isinstance(name, list):
modular_id, function_id = name[0], name[1]
else:
modular_id, function_id = as_str
self.logger.info(f"getting function : {specification}.{modular_id}.{function_id}")
if modular_id not in self.functions:
if r == 0:
self.save_load(modular_id, spec=specification)
return self.get_function(name=(modular_id, function_id),
state=state,
specification=specification,
metadata=metadata,
r=1)
self.logger.warning(f"function modular not found {modular_id} 404")
return "404", 404
if function_id not in self.functions[modular_id]:
self.logger.warning(f"function data not found {modular_id}.{function_id} 404")
return "404", 404
function_data = self.functions[modular_id][function_id]
if isinstance(function_data, list):
self.logger.info(f"functions {function_id} : {function_data}")
function_data = self.functions[modular_id][function_data[kwargs.get('i', -1)]]
self.logger.info(f"functions {modular_id} : {function_data}")
function = function_data.get("func")
params = function_data.get("params")
state_ = function_data.get("state")
if state_ is not None and state != state_:
state = state_
if function is None:
self.logger.warning("No function found")
return "404", 404
if params is None:
self.logger.warning("No function (params) found")
return "404", 301
if metadata and not state:
self.logger.info("returning metadata stateless")
return (function_data, function), 0
if not state: # mens a stateless function
self.logger.info("returning stateless function")
return function, 0
instance = self.functions[modular_id].get(f"{specification}_instance")
# instance_type = self.functions[modular_id].get(f"{specification}_instance_type", "functions/class")
if params[0] == 'app':
instance = get_app(from_=f"fuction {specification}.{modular_id}.{function_id}")
if instance is None and self.alive:
instance = self.inplace_load_instance(modular_id, spec=specification)
if instance is None:
self.logger.warning("No live Instance found")
return "404", 400
# if instance_type.endswith("/BC"): # for backwards compatibility functions/class/BC old modules
# # returning as stateless
# # return "422", -1
# self.logger.info(
# f"returning stateless function, cant find tools class for state handling found {instance_type}")
# if metadata:
# self.logger.info(f"returning metadata stateless")
# return (function_data, function), 0
# return function, 0
self.logger.info("wrapping in higher_order_function")
self.logger.info(f"returned fuction {specification}.{modular_id}.{function_id}")
higher_order_function = partial(function, instance)
if metadata:
self.logger.info("returning metadata stateful")
return (function_data, higher_order_function), 0
self.logger.info("returning stateful function")
return higher_order_function, 0
def save_exit(self):
self.logger.info(f"save exiting saving data to {self.config_fh.file_handler_filename} states of {self.debug=}")
self.config_fh.add_to_save_file_handler(self.keys["debug"], str(self.debug))
def init_mod(self, mod_name, spec='app'):
"""
Initializes a module in a thread-safe manner by submitting the
asynchronous initialization to the running event loop.
"""
if '.' in mod_name:
mod_name = mod_name.split('.')[0]
self.run_bg_task(self.a_init_mod, mod_name, spec)
# loop = self.loop_gard()
# if loop:
# # Create a future to get the result from the coroutine
# future: Future = asyncio.run_coroutine_threadsafe(
# self.a_init_mod(mod_name, spec), loop
# )
# # Block until the result is available
# return future.result()
# else:
# raise ValueError("Event loop is not running")
# # return self.loop_gard().run_until_complete(self.a_init_mod(mod_name, spec))
def run_bg_task(self, task: Callable, *args, **kwargs) -> asyncio.Task | None:
"""
Runs a coroutine in the background without blocking the caller.
This is the primary method for "fire-and-forget" async tasks. It schedules
the coroutine to run on the application's main event loop.
Args:
task: The coroutine function to run.
*args: Arguments to pass to the coroutine function.
**kwargs: Keyword arguments to pass to the coroutine function.
Returns:
An asyncio.Task object representing the scheduled task, or None if
the task could not be scheduled.
"""
if not callable(task):
self.logger.warning("Task passed to run_bg_task is not callable!")
return None
if not asyncio.iscoroutinefunction(task) and not asyncio.iscoroutine(task):
self.logger.warning(f"Task '{getattr(task, '__name__', 'unknown')}' is not a coroutine. "
f"Use run_bg_task_advanced for synchronous functions.")
# Fallback to advanced runner for convenience
return self.run_bg_task_advanced(task, *args, get_coro=True, **kwargs)
try:
loop = self.loop_gard()
if not loop.is_running():
# If the main loop isn't running, we can't create a task on it.
# This scenario is handled by run_bg_task_advanced.
self.logger.info("Main event loop not running. Delegating to advanced background runner.")
return self.run_bg_task_advanced(task, *args, **kwargs)
# Create the coroutine if it's a function
coro = task(*args, **kwargs) if asyncio.iscoroutinefunction(task) else task
# Create a task on the running event loop
bg_task = loop.create_task(coro)
# Add a callback to log exceptions from the background task
def _log_exception(the_task: asyncio.Task):
if not the_task.cancelled() and the_task.exception():
self.logger.error(f"Exception in background task '{the_task.get_name()}':",
exc_info=the_task.exception())
bg_task.add_done_callback(_log_exception)
self.bg_tasks.append(bg_task)
return bg_task
except Exception as e:
self.logger.error(f"Failed to schedule background task: {e}", exc_info=True)
return None
def run_bg_task_advanced(self, task: Callable, *args, get_coro=False, **kwargs) -> Thread | None:
"""
Runs a task in a separate, dedicated background thread with its own event loop.
This is ideal for:
1. Running an async task from a synchronous context.
2. Launching a long-running, independent operation that should not
interfere with the main application's event loop.
Args:
task: The function to run (can be sync or async).
*args: Arguments for the task.
**kwargs: Keyword arguments for the task.
Returns:
The threading.Thread object managing the background execution.
"""
if not callable(task):
self.logger.warning("Task for run_bg_task_advanced is not callable!")
return None
coro_0 = [None]
def thread_target():
# Each thread gets its own event loop.
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
result = None
try:
# Prepare the coroutine we need to run
if asyncio.iscoroutinefunction(task):
coro = task(*args, **kwargs)
elif asyncio.iscoroutine(task):
# It's already a coroutine object
coro = task
else:
result = task(*args, **kwargs)
if asyncio.iscoroutine(result):
result = loop.run_until_complete(result)
coro = None
# Run the coroutine to completion
coro_0[0] = coro
if coro is not None:
result = loop.run_until_complete(coro)
self.logger.debug(f"Advanced background task '{getattr(task, '__name__', 'unknown')}' completed.")
if result is not None:
self.logger.debug(f"Task result: {str(result)[:100]}")
except Exception as e:
self.logger.error(f"Error in advanced background task '{getattr(task, '__name__', 'unknown')}':",
exc_info=e)
finally:
# Cleanly shut down the event loop in this thread.
try:
all_tasks = asyncio.all_tasks(loop=loop)
if all_tasks:
for t in all_tasks:
t.cancel()
loop.run_until_complete(asyncio.gather(*all_tasks, return_exceptions=True))
finally:
loop.close()
asyncio.set_event_loop(None)
# Create, start, and return the thread.
# It's a daemon thread so it won't prevent the main app from exiting.
t = threading.Thread(target=thread_target, daemon=True, name=f"BGTask-{getattr(task, '__name__', 'unknown')}")
self.bg_tasks.append(t)
t.start()
if get_coro:
return coro_0[0]
return t
# Helper method to wait for background tasks to complete (optional)
def wait_for_bg_tasks(self, timeout=None):
"""
Wait for all background tasks to complete.
Args:
timeout: Maximum time to wait (in seconds) for all tasks to complete.
None means wait indefinitely.
Returns:
bool: True if all tasks completed, False if timeout occurred
"""
active_tasks = [t for t in self.bg_tasks if t.is_alive()]
for task in active_tasks:
task.join(timeout=timeout)
if task.is_alive():
return False
return True
def __call__(self, *args, **kwargs):
return self.run(*args, **kwargs)
def run(self, *args, mod_function_name=None, request=None, running_function_coro=None, **kwargs):
"""
Run a function with support for SSE streaming in both
threaded and non-threaded contexts.
"""
if mod_function_name is None:
mod_function_name = args[0]
if running_function_coro is None:
mn, fn = mod_function_name
if self.functions.get(mn, {}).get(fn, {}).get('request_as_kwarg', False):
kwargs["request"] = RequestData.from_dict(request)
if 'data' in kwargs and 'data' not in self.functions.get(mn, {}).get(fn, {}).get('params', []):
kwargs["request"].data = kwargs["request"].body = kwargs['data']
del kwargs['data']
if 'form_data' in kwargs and 'form_data' not in self.functions.get(mn, {}).get(fn, {}).get('params',
[]):
kwargs["request"].form_data = kwargs["request"].body = kwargs['form_data']
del kwargs['form_data']
else:
params = self.functions.get(mn, {}).get(fn, {}).get('params', [])
# auto pars data and form_data to kwargs by key
do = False
data = {}
if 'data' in kwargs and 'data' not in params:
do = True
data = kwargs['data']
del kwargs['data']
if 'form_data' in kwargs and 'form_data' not in params:
do = True
data = kwargs['form_data']
del kwargs['form_data']
if do:
for k in params:
if k in data:
kwargs[k] = data[k]
del data[k]
if 'spec' in kwargs and 'spec' not in self.functions.get(mn, {}).get(fn, {}).get('params',
[]):
if "tb_run_with_specification" in kwargs:
kwargs.pop('spec')
else:
kwargs['tb_run_with_specification'] = kwargs.pop('spec')
# Create the coroutine
coro = running_function_coro or self.a_run_any(*args,mod_function_name=mod_function_name, **kwargs)
# Get or create an event loop
try:
loop = asyncio.get_event_loop()
is_running = loop.is_running()
except RuntimeError:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
is_running = False
# If the loop is already running, run in a separate thread
if is_running:
# Create thread pool executor as needed
if not hasattr(self.__class__, '_executor'):
self.__class__._executor = ThreadPoolExecutor(max_workers=4)
def run_in_new_thread():
# Set up a new loop in this thread
new_loop = asyncio.new_event_loop()
asyncio.set_event_loop(new_loop)
try:
# Run the coroutine
return new_loop.run_until_complete(coro)
finally:
new_loop.close()
# Run in thread and get result
thread_result = self.__class__._executor.submit(run_in_new_thread).result()
# Handle streaming results from thread
if isinstance(thread_result, dict) and thread_result.get("is_stream"):
# Create a new SSE stream in the main thread
async def stream_from_function():
# Re-run the function with direct async access
stream_result = await self.a_run_any(*args, **kwargs)
if (isinstance(stream_result, Result) and
getattr(stream_result.result, 'data_type', None) == "stream"):
# Get and forward data from the original generator
original_gen = stream_result.result.data.get("generator")
if inspect.isasyncgen(original_gen):
async for item in original_gen:
yield item
# Return a new streaming Result
return Result.stream(
stream_generator=stream_from_function(),
headers=thread_result.get("headers", {})
)
result = thread_result
else:
# Direct execution when loop is not running
result = loop.run_until_complete(coro)
# Process the final result
if isinstance(result, Result):
if 'debug' in self.id:
result.print()
if getattr(result.result, 'data_type', None) == "stream":
return result
return result.to_api_result().model_dump(mode='json')
return result
def loop_gard(self):
if self.loop is None:
self._start_event_loop()
self.loop = asyncio.get_event_loop()
if self.loop.is_closed():
self.loop = asyncio.get_event_loop()
return self.loop
async def a_init_mod(self, mod_name, spec='app'):
mod = self.save_load(mod_name, spec=spec)
if hasattr(mod, "__initobj") and not mod.async_initialized:
await mod
return mod
def load_mod(self, mod_name: str, mlm='I', **kwargs):
action_list_helper = ['I (inplace load dill on error python)',
# 'C (coppy py file to runtime dir)',
# 'S (save py file to dill)',
# 'CS (coppy and save py file)',
# 'D (development mode, inplace load py file)'
]
action_list = {"I": lambda: self.inplace_load_instance(mod_name, **kwargs),
"C": lambda: self._copy_load(mod_name, **kwargs)
}
try:
if mlm in action_list:
return action_list.get(mlm)()
else:
self.logger.critical(
f"config mlm must be {' or '.join(action_list_helper)} is {mlm=}")
raise ValueError(f"config mlm must be {' or '.join(action_list_helper)} is {mlm=}")
except ValueError as e:
self.logger.warning(Style.YELLOW(f"Error Loading Module '{mod_name}', with error :{e}"))
self.debug_rains(e)
except ImportError as e:
self.logger.error(Style.YELLOW(f"Error Loading Module '{mod_name}', with error :{e}"))
self.debug_rains(e)
except Exception as e:
self.logger.critical(Style.RED(f"Error Loading Module '{mod_name}', with critical error :{e}"))
print(Style.RED(f"Error Loading Module '{mod_name}'"))
self.debug_rains(e)
return Result.default_internal_error(info="info's in logs.")
async def load_external_mods(self):
for mod_path in os.getenv("EXTERNAL_PATH_RUNNABLE", '').split(','):
if mod_path:
await self.load_all_mods_in_file(mod_path)
async def load_all_mods_in_file(self, working_dir="mods"):
"""
Load all modules from a directory.
NOTE: Module imports are done SEQUENTIALLY to avoid Python import deadlocks.
Python's import system uses a global lock (_ModuleLock) that can cause deadlocks
when multiple threads try to import interdependent modules simultaneously.
Only async initialization is done in parallel after all imports complete.
"""
self.logger.info(f"LOADING ALL MODS FROM FOLDER : {working_dir}")
t0 = time.perf_counter()
# Get the list of all modules
module_list = self.get_all_mods(working_dir)
open_modules = list(self.functions.keys())
start_len = len(open_modules)
# Remove already loaded modules
for om in open_modules:
if om in module_list:
module_list.remove(om)
# Phase 1: Load modules SEQUENTIALLY to avoid import deadlocks
# Python's import_module() uses a global lock that causes deadlocks
# when multiple threads import interdependent modules
loaded_results = []
for mod in module_list:
try:
result = self.save_load(mod, 'app')
if result:
loaded_results.append(result)
mod_name = getattr(result, 'Name', None) or getattr(result, 'name', None)
if mod_name:
self.print('Loaded :', mod_name)
except Exception as e:
self.logger.error(Style.RED(f"Error loading module {mod}: {str(e)}"))
self.debug_rains(e)
# Phase 2: Initialize async modules in parallel (safe - no imports here)
async_init_tasks = []
for result in loaded_results:
if hasattr(result, 'async_initialized') and not result.async_initialized:
async def init_module(mod_result):
try:
if asyncio.iscoroutine(mod_result):
await mod_result
mod_name = getattr(mod_result, 'Name', None) or getattr(mod_result, 'name', None)
if mod_name:
self.print('Initialized :', mod_name)
except Exception as e:
self.debug_rains(e)
mod_name = getattr(mod_result, 'Name', None) or getattr(mod_result, 'name', None)
if mod_name:
self.print('Error initializing :', mod_name)
async_init_tasks.append(asyncio.create_task(init_module(result)))
# Wait for all async initializations to complete
if async_init_tasks:
await asyncio.gather(*async_init_tasks, return_exceptions=True)
opened = len(self.functions.keys()) - start_len
elapsed = time.perf_counter() - t0
self.logger.info(f"Opened {opened} modules in {elapsed:.2f}s")
return f"Opened {opened} modules in {elapsed:.2f}s"
def get_all_mods(self, working_dir="mods", path_to="./runtime", use_wd=True):
self.logger.info(f"collating all mods in working directory {working_dir}")
pr = "_dev" if self.dev_modi else ""
if working_dir == "mods" and use_wd:
working_dir = f"{self.start_dir}/mods{pr}"
elif use_wd:
pass
else:
w_dir = self.id.replace(".", "_")
working_dir = f"{path_to}/{w_dir}/mod_lib{pr}/"
res = os.listdir(working_dir)
self.logger.info(f"found : {len(res)} files")
def do_helper(_mod):
if "mainTool" in _mod:
return False
# if not _mod.endswith(".py"):
# return False
if _mod.startswith("__"):
return False
if _mod.startswith("."):
return False
return not _mod.startswith("test_")
def r_endings(word: str):
if word.endswith(".py"):
return word[:-3]
return word
mods_list = list(map(r_endings, filter(do_helper, res)))
self.logger.info(f"found : {len(mods_list)} Modules")
return mods_list
def remove_all_modules(self, delete=False):
"""Remove all loaded modules synchronously.
For modules with async on_exit hooks, spawns them via
run_bg_task_advanced so they run concurrently, then joins
all exit threads with a shared timeout budget.
"""
exit_threads: list = []
for mod_name in list(self.functions.keys()):
self.logger.info(f"closing: {mod_name}")
on_exit = self.functions[mod_name].get("on_exit")
# Check for async on_exit hooks — run them in background threads
has_async_exit = False
if on_exit is not None:
for f in on_exit:
spec = 'app' # default spec
f_, e, f_name = self._get_exit_function(mod_name, f, spec, 0)
if e == 0 and asyncio.iscoroutinefunction(f_):
has_async_exit = True
t = self.run_bg_task_advanced(f_)
if t is not None:
exit_threads.append((mod_name, f_name, t))
if has_async_exit:
# Sync on_exit hooks + cleanup still handled by remove_mod
# but skip the async ones (already dispatched above)
self._remove_mod_sync_only(mod_name, delete=delete)
else:
# Pure sync module — normal path
self.remove_mod(mod_name, delete=delete)
# Join all async exit threads with shared 5s budget
if exit_threads:
deadline = time.time() + 5.0
for mod_name, f_name, thread in exit_threads:
remaining = max(deadline - time.time(), 0.05)
self.logger.info(f"Waiting for async exit {mod_name}.{f_name} ({remaining:.1f}s remaining)")
thread.join(timeout=remaining)
if thread.is_alive():
self.logger.warning(f"Async exit {mod_name}.{f_name} did not complete in time")
def _remove_mod_sync_only(self, mod_name, spec='app', delete=True):
"""Remove a module, running only its SYNC on_exit hooks.
Async hooks are assumed to be already dispatched via run_bg_task_advanced.
"""
if mod_name not in self.functions:
return
on_exit = self.functions[mod_name].get("on_exit")
# Handle BC instances (sync only)
if on_exit is None and self.functions[mod_name].get(f"{spec}_instance_type", "").endswith("/BC"):
instance = self.functions[mod_name].get(f"{spec}_instance")
if instance is not None and hasattr(instance, 'on_exit'):
if not asyncio.iscoroutinefunction(instance.on_exit):
try:
instance.on_exit()
except Exception as e:
self.logger.debug(f"BC on_exit error for {mod_name}: {e}")
# Run only sync exit handlers
if on_exit is not None:
for i, f in enumerate(on_exit, 1):
try:
f_, e, f_name = self._get_exit_function(mod_name, f, spec, i - 1)
if e == 0 and not asyncio.iscoroutinefunction(f_):
self.logger.info(Style.GREY(f"Running sync on_exit {f_name} {i}/{len(on_exit)}"))
o = f_()
if isinstance(o, Result):
o.log()
elif o is not None:
self.print(f"Function On Exit result: {o}")
except Exception as e:
self.logger.debug(f"on_exit error {mod_name}.{f}: {e}")
self.debug_rains(e)
self._cleanup_module_instances(mod_name, spec)
if delete:
self._delete_module(mod_name)
async def a_remove_all_modules(self, delete=False):
"""Remove all loaded modules asynchronously."""
for mod in list(self.functions.keys()):
self.logger.info(f"closing: {mod}")
await self.a_remove_mod(mod, delete=delete)
def _cleanup_module_instances(self, mod_name: str, spec: str):
"""Clean up module instances for a given spec."""
if f"{spec}_instance" in self.functions[mod_name]:
del self.functions[mod_name][f"{spec}_instance"]
if f"{spec}_instance_type" in self.functions[mod_name]:
del self.functions[mod_name][f"{spec}_instance_type"]
def _delete_module(self, mod_name: str):
"""Delete a module from the functions registry."""
self.functions[mod_name] = {}
del self.functions[mod_name]
def _get_exit_function(self, mod_name: str, f, spec: str, index: int = 0):
"""
Get the exit function and error code.
Returns (function, error_code, function_name).
"""
if isinstance(f, str):
f_, e = self.get_function((mod_name, f), state=True, specification=spec, i=index)
return f_, e, f
elif isinstance(f, Callable):
return f, 0, f.__name__
return None, 1, str(f)
def remove_mod(self, mod_name, spec='app', delete=True):
"""
Remove a module synchronously.
For async contexts, use a_remove_mod() instead.
"""
if mod_name not in self.functions:
self.logger.info(f"mod not active {mod_name}")
return
on_exit = self.functions[mod_name].get("on_exit")
self.logger.info(f"closing: {on_exit}")
# Handle BC (backwards compatible) instances
if on_exit is None and self.functions[mod_name].get(f"{spec}_instance_type", "").endswith("/BC"):
instance = self.functions[mod_name].get(f"{spec}_instance", None)
if instance is not None and hasattr(instance, 'on_exit'):
if asyncio.iscoroutinefunction(instance.on_exit):
self.exit_tasks.append(instance.on_exit)
else:
instance.on_exit()
# No exit handlers
if on_exit is None:
if delete:
self._delete_module(mod_name)
else:
self._cleanup_module_instances(mod_name, spec)
return
# Run exit handlers
for i, f in enumerate(on_exit, 1):
try:
f_, e, f_name = self._get_exit_function(mod_name, f, spec, i - 1)
if e == 0:
self.logger.info(Style.GREY(f"Running On exit {f_name} {i}/{len(on_exit)}"))
if asyncio.iscoroutinefunction(f_):
self.exit_tasks.append(f_)
o = None
else:
o = f_()
if o is not None:
self.print(f"Function On Exit result: {o}")
else:
self.logger.warning("closing function not found")
except Exception as e:
self.logger.debug(
Style.YELLOW(Style.Bold(f"modular:{mod_name}.{f} on_exit error {i}/{len(on_exit)} -> {e}")))
self.debug_rains(e)
self._cleanup_module_instances(mod_name, spec)
if delete:
self._delete_module(mod_name)
async def a_remove_mod(self, mod_name, spec='app', delete=True):
"""
Remove a module asynchronously.
Properly awaits async exit handlers.
"""
if mod_name not in self.functions:
self.logger.info(f"mod not active {mod_name}")
return
on_exit = self.functions[mod_name].get("on_exit")
self.logger.info(f"closing: {on_exit}")
# Handle BC (backwards compatible) instances
if on_exit is None and self.functions[mod_name].get(f"{spec}_instance_type", "").endswith("/BC"):
instance = self.functions[mod_name].get(f"{spec}_instance", None)
if instance is not None and hasattr(instance, 'on_exit'):
if asyncio.iscoroutinefunction(instance.on_exit):
await instance.on_exit()
else:
instance.on_exit()
# No exit handlers
if on_exit is None:
if delete:
self._delete_module(mod_name)
else:
self._cleanup_module_instances(mod_name, spec)
return
# Run exit handlers
for i, f in enumerate(on_exit, 1):
try:
f_, e, f_name = self._get_exit_function(mod_name, f, spec, i - 1)
if e == 0:
self.logger.info(Style.GREY(f"Running On exit {f_name} {i}/{len(on_exit)}"))
if asyncio.iscoroutinefunction(f_):
o = await f_()
else:
o = f_()
if isinstance(o, Result):
o.log()
elif o is not None:
self.print(f"Function On Exit result: {o}")
else:
self.logger.warning("closing function not found")
except Exception as e:
self.logger.debug(
Style.YELLOW(Style.Bold(f"modular:{mod_name}.{f} on_exit error {i}/{len(on_exit)} -> {e}")))
self.debug_rains(e)
self._cleanup_module_instances(mod_name, spec)
if delete:
self._delete_module(mod_name)
def _cleanup_threads(self, timeout: float = 5.0):
"""Clean up background threads on exit.
Default: only joins threads tracked in self.bg_tasks.
If args_sto.full_thread_cleanup is set, also joins ALL non-main threads
via threading.enumerate() (expensive, use for debugging).
Args:
timeout: Total wall-clock budget for all threads combined (default 5s).
"""
import threading
use_enumerate = getattr(self.args_sto, 'full_thread_cleanup', False)
if use_enumerate:
targets = [
t for t in threading.enumerate()
if t is not threading.current_thread() and t.name != "MainThread"
]
else:
# Only our own tracked tasks
targets = [t for t in self.bg_tasks if t.is_alive()]
if not targets:
return
deadline = time.time() + timeout
for thread in targets:
remaining = max(deadline - time.time(), 0.05)
if remaining <= 0.05:
self.logger.warning(f"Thread cleanup budget exhausted, {thread.name} still alive")
break
try:
thread.join(timeout=remaining)
if thread.is_alive():
self.logger.warning(f"Thread {thread.name} did not exit within budget")
except Exception as e:
self.logger.debug(f"Thread join error {thread.name}: {e}")
# Clear tracked tasks
self.bg_tasks = [t for t in self.bg_tasks if t.is_alive()]
def _cleanup_event_loop(self):
"""Clean up the event loop on exit."""
if not hasattr(self, 'loop') or self.loop is None:
return
try:
running_loop = asyncio.get_running_loop()
if running_loop is not self.loop:
with Spinner("closing Event loop:", symbols="+"):
self.loop.stop()
# If we're inside the loop, it will stop naturally when the coroutine completes
except RuntimeError:
# No running loop - safe to stop
with Spinner("closing Event loop:", symbols="+"):
self.loop.stop()
def exit(self, remove_all=True):
"""
Exit the ToolBox interface synchronously.
For async contexts, use a_exit() instead.
"""
if not self.alive:
return
with Spinner("Cleaning app sate"):
self._stop_event.set()
if self.args_sto.debug:
self.hide_console()
self.disconnect()
if remove_all:
self.remove_all_modules()
self.logger.info("Exiting ToolBox interface")
# ── Flush observability ──
try:
from toolboxv2.utils.system.tb_logger import disable_live_observability
disable_live_observability() # flushes remaining batch + closes worker
except Exception as e:
self.debug_rains(e)
import traceback
traceback.print_exc()
pass
to = time.time()
if self._obs_sync_manager:
try:
self._obs_sync_manager.stop_auto_sync()
stats = self._obs_sync_manager.sync_all() # final push
self.logger.info(f"AutoSync stop time: {time.time() - to}")
except Exception as e:
self.debug_rains(e)
self.alive = False
self.called_exit = True, time.time()
self.save_exit()
try:
self.config_fh.save_file_handler()
except SystemExit:
self.logger.warning("If you are testing this is fine, else ...")
self._cleanup_threads()
self._cleanup_event_loop()
async def a_exit(self):
"""
Exit the ToolBox interface asynchronously.
Properly cleans up async resources before calling sync exit.
"""
# Cleanup session before removing modules
with Spinner("Saving data"):
try:
if hasattr(self, 'session') and self.session is not None:
await self.session.cleanup()
except Exception as e:
self.logger.debug(f"Session cleanup error (ignored): {e}")
# Remove all modules asynchronously
await self.a_remove_all_modules(delete=True)
# Run async exit tasks
async_exit_tasks = [
asyncio.create_task(f())
for f in self.exit_tasks
if asyncio.iscoroutinefunction(f)
]
if async_exit_tasks:
results = await asyncio.gather(*async_exit_tasks)
for result in results:
self.print(f"Function On Exit result: {result}")
# Call sync exit without removing modules (already done)
self.exit(remove_all=False)
def save_load(self, modname, spec='app'):
self.logger.debug(f"Save load module {modname}")
if not modname:
self.logger.warning("no filename specified")
return False
try:
return self.load_mod(modname, spec=spec)
except ModuleNotFoundError as e:
self.logger.error(Style.RED(f"Module {modname} not found"))
self.debug_rains(e)
return False
def get_function(self, name: Enum or tuple, **kwargs):
"""
Kwargs for _get_function
metadata:: return the registered function dictionary
stateless: (function_data, None), 0
stateful: (function_data, higher_order_function), 0
state::boolean
specification::str default app
"""
if isinstance(name, tuple):
return self._get_function(None, as_str=name, **kwargs)
else:
return self._get_function(name, **kwargs)
def _parse_mod_function_name(
self,
mod_function_name: Enum | tuple | list,
args_: tuple | None,
kwargs_: dict | None,
args: tuple,
kwargs: dict
) -> tuple[str, str, tuple, dict]:
"""
Parse mod_function_name into (modular_name, function_name) and merge args/kwargs.
Returns (modular_name, function_name, args, kwargs).
"""
if kwargs_ is not None and not kwargs:
kwargs = kwargs_
if args_ is not None and not args:
args = args_
if isinstance(mod_function_name, tuple):
modular_name, function_name = mod_function_name
elif isinstance(mod_function_name, list):
modular_name, function_name = mod_function_name[0], mod_function_name[1]
elif isinstance(mod_function_name, Enum):
modular_name, function_name = mod_function_name.__class__.NAME.value, mod_function_name.value
else:
raise TypeError("Unknown function type")
return modular_name, function_name, args, kwargs
def _handle_ws_internal(
self,
mod_function_name: tuple,
modular_name: str,
kwargs: dict,
is_async: bool = False
) -> Result | None:
"""
Handle WebSocket internal specification.
Returns Result if handled, None if not a ws_internal call.
"""
handler_id, event_name = mod_function_name
if handler_id in self.websocket_handlers and event_name in self.websocket_handlers[handler_id]:
handler_func = self.websocket_handlers[handler_id][event_name]
try:
res = None
if inspect.iscoroutinefunction(handler_func):
if is_async:
# Will be awaited by caller
return handler_func(self, **kwargs)
else:
res = self.loop.run_until_complete(handler_func(self, **kwargs))
else:
res = handler_func(self, **kwargs)
if isinstance(res, (Result, ApiResult)):
return res
return Result.ok(info=f"WS handler '{event_name}' executed.", data=res)
except Exception as e:
self.logger.error(f"Error in WebSocket handler '{handler_id}/{event_name}': {e}", exc_info=True)
return Result.default_internal_error(info=str(e))
else:
return Result.ok(info=f"No WS handler for '{event_name}'.")
def _handle_error_code(
self,
error_code: int,
mod_function_name: tuple,
modular_name: str,
function_name: str
) -> Result | None:
"""
Handle error codes from get_function.
Returns Result if error, None if no error.
Unified error codes: 404 (not found), 300 (no state), others (internal error).
"""
# Map legacy error codes to unified codes
# Legacy: 2 -> 404, -1 -> 300, 1/3/400 -> retry needed
if error_code in (2, 404):
self.logger.warning(Style.RED("Function Not Found"))
return (Result.default_user_error(
interface=self.interface_type,
exec_code=404,
info="function not found - function is not decorated"
).set_origin(mod_function_name))
if error_code in (-1, 300):
return Result.default_internal_error(
interface=self.interface_type,
info=f"module {modular_name} has no state (instance)"
).set_origin(mod_function_name)
if error_code != 0:
return Result.default_internal_error(
interface=self.interface_type,
exec_code=error_code,
info=f"Internal error {modular_name}.{function_name}"
).set_origin(mod_function_name)
return None
def _extract_function(
self,
function_data: tuple,
tb_run_function_with_state: bool,
mod_function_name: tuple,
function_name: str
) -> tuple[dict, callable] | Result:
"""
Extract function from function_data.
Returns (function_data_dict, function) or Result on error.
"""
if not tb_run_function_with_state:
function_data, _ = function_data
function = function_data.get('func')
else:
function_data, function = function_data
if not function:
self.logger.warning(Style.RED(f"Function {function_name} not found"))
return Result.default_internal_error(
interface=self.interface_type,
exec_code=404,
info="function not found"
).set_origin(mod_function_name)
return function_data, function
async def a_run_function(self, mod_function_name: Enum | tuple,
tb_run_function_with_state=True,
tb_run_with_specification='app',
args_=None,
kwargs_=None,
*args,
**kwargs) -> Result:
"""
Run a module function asynchronously.
"""
modular_name, function_name, args, kwargs = self._parse_mod_function_name(
mod_function_name, args_, kwargs_, args, kwargs
)
# Handle WebSocket internal
if tb_run_with_specification == 'ws_internal':
modular_name = modular_name.split('/')[0]
if not self.mod_online(modular_name, installed=True):
self.get_mod(modular_name)
result = self._handle_ws_internal(mod_function_name, modular_name, kwargs, is_async=True)
if inspect.iscoroutine(result):
result = await result
return result
# Ensure module is loaded
if not self.mod_online(modular_name, installed=True):
self.get_mod(modular_name)
# Get function
function_data, error_code = self.get_function(
mod_function_name, state=tb_run_function_with_state,
metadata=True, specification=tb_run_with_specification
)
self.logger.info(f"Received function: {mod_function_name}, with execode: {error_code}")
# Retry if module needs async initialization
if error_code in (1, 3, 400, 404):
mod = self.get_mod(modular_name)
if hasattr(mod, "async_initialized") and not mod.async_initialized:
await mod
function_data, error_code = self.get_function(
mod_function_name, state=tb_run_function_with_state,
metadata=True, specification=tb_run_with_specification
)
# Handle errors
error_result = self._handle_error_code(error_code, mod_function_name, modular_name, function_name)
if error_result:
return error_result
# Extract function
extract_result = self._extract_function(
function_data, tb_run_function_with_state, mod_function_name, function_name
)
if isinstance(extract_result, Result):
return extract_result
function_data, function = extract_result
# Run function
self.logger.info("Profiling function")
t0 = time.perf_counter()
if asyncio.iscoroutinefunction(function):
return await self.a_fuction_runner(function, function_data, args, kwargs, t0)
else:
return self.fuction_runner(function, function_data, args, kwargs, t0)
def run_function(self, mod_function_name: Enum | tuple,
tb_run_function_with_state=True,
tb_run_with_specification='app',
args_=None,
kwargs_=None,
*args,
**kwargs) -> Result:
"""
Run a module function synchronously.
For async functions, use a_run_function() instead.
"""
modular_name, function_name, args, kwargs = self._parse_mod_function_name(
mod_function_name, args_, kwargs_, args, kwargs
)
# Ensure module is loaded
if not self.mod_online(modular_name, installed=True):
self.get_mod(modular_name)
# Handle WebSocket internal
if tb_run_with_specification == 'ws_internal':
return self._handle_ws_internal(mod_function_name, modular_name, kwargs, is_async=False)
# Get function
function_data, error_code = self.get_function(
mod_function_name, state=tb_run_function_with_state,
metadata=True, specification=tb_run_with_specification
)
self.logger.info(f"Received function: {mod_function_name}, with execode: {error_code}")
# Retry if needed
if error_code in (1, 3, 400, 404):
self.get_mod(modular_name)
function_data, error_code = self.get_function(
mod_function_name, state=tb_run_function_with_state,
metadata=True, specification=tb_run_with_specification
)
# Handle errors
error_result = self._handle_error_code(error_code, mod_function_name, modular_name, function_name)
if error_result:
return error_result
# Extract function
extract_result = self._extract_function(
function_data, tb_run_function_with_state, mod_function_name, function_name
)
if isinstance(extract_result, Result):
return extract_result
function_data, function = extract_result
# Run function
self.logger.info("Profiling function")
t0 = time.perf_counter()
if asyncio.iscoroutinefunction(function):
try:
return asyncio.run(self.a_fuction_runner(function, function_data, args, kwargs, t0))
except RuntimeError:
try:
return self.loop.run_until_complete(
self.a_fuction_runner(function, function_data, args, kwargs, t0)
)
except RuntimeError:
pass
raise ValueError(f"Function {function_name} is async - use a_run_any instead")
else:
return self.fuction_runner(function, function_data, args, kwargs, t0)
def run_a_from_sync(self, function, *args, **kwargs):
# Initialize self.loop if not already set.
if self.loop is None:
try:
self.loop = asyncio.get_running_loop()
except RuntimeError:
self.loop = asyncio.new_event_loop()
# If the loop is running, offload the coroutine to a new thread.
if self.loop.is_running():
result_future = Future()
def run_in_new_loop():
new_loop = asyncio.new_event_loop()
asyncio.set_event_loop(new_loop)
try:
result = new_loop.run_until_complete(function(*args, **kwargs))
result_future.set_result(result)
except Exception as e:
result_future.set_exception(e)
finally:
new_loop.close()
thread = threading.Thread(target=run_in_new_loop)
thread.start()
thread.join() # Block until the thread completes.
return result_future.result()
else:
# If the loop is not running, schedule and run the coroutine directly.
future = self.loop.create_task(function(*args, **kwargs))
return self.loop.run_until_complete(future)
def fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):
parameters = function_data.get('params')
modular_name = function_data.get('module_name')
function_name = function_data.get('func_name')
row = function_data.get('row')
mod_function_name = f"{modular_name}.{function_name}"
if_self_state = 1 if 'self' in parameters else 0
try:
if len(parameters) == 0:
res = function()
elif len(parameters) == len(args) + if_self_state:
res = function(*args)
elif len(parameters) == len(kwargs.keys()) + if_self_state:
res = function(**kwargs)
else:
res = function(*args, **kwargs)
self.logger.info(f"Execution done in {time.perf_counter()-t0:.4f}")
if isinstance(res, Result):
formatted_result = res
if formatted_result.origin is None:
formatted_result.set_origin(mod_function_name)
elif isinstance(res, ApiResult):
formatted_result = res
if formatted_result.origin is None:
formatted_result.as_result().set_origin(mod_function_name).to_api_result()
elif row:
formatted_result = res
else:
# Wrap the result in a Result object
formatted_result = Result.ok(
interface=self.interface_type,
data_info="Auto generated result",
data=res,
info="Function executed successfully"
).set_origin(mod_function_name)
if not row:
self.logger.info(
f"Function Exec code: {formatted_result.info.exec_code} Info's: {formatted_result.info.help_text}")
else:
self.logger.info(
f"Function Exec data: {formatted_result}")
except Exception as e:
self.logger.error(
Style.YELLOW(Style.Bold(
f"! Function ERROR: in {modular_name}.{function_name}")))
# Wrap the exception in a Result object
formatted_result = Result.default_internal_error(info=str(e)).set_origin(mod_function_name)
# res = formatted_result
self.logger.error(
f"Function {modular_name}.{function_name}"
f" executed wit an error {str(e)}, {type(e)}")
self.debug_rains(e)
self.print(f"! Function ERROR: in {modular_name}.{function_name} ")
else:
self.print_ok()
self.logger.info(
f"Function {modular_name}.{function_name}"
f" executed successfully")
return formatted_result
async def a_fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):
parameters = function_data.get('params')
modular_name = function_data.get('module_name')
function_name = function_data.get('func_name')
row = function_data.get('row')
mod_function_name = f"{modular_name}.{function_name}"
if_self_state = 1 if 'self' in parameters else 0
try:
if len(parameters) == 0:
res = await function()
elif len(parameters) == len(args) + if_self_state:
res = await function(*args)
elif len(parameters) == len(kwargs.keys()) + if_self_state:
res = await function(**kwargs)
else:
res = await function(*args, **kwargs)
self.logger.info(f"Execution done in {time.perf_counter()-t0:.4f}")
if isinstance(res, Result):
formatted_result = res
if formatted_result.origin is None:
formatted_result.set_origin(mod_function_name)
elif isinstance(res, ApiResult):
formatted_result = res
if formatted_result.origin is None:
formatted_result.as_result().set_origin(mod_function_name).to_api_result()
elif row:
formatted_result = res
else:
# Wrap the result in a Result object
formatted_result = Result.ok(
interface=self.interface_type,
data_info="Auto generated result",
data=res,
info="Function executed successfully"
).set_origin(mod_function_name)
if not row:
self.logger.info(
f"Function Exec code: {formatted_result.info.exec_code} Info's: {formatted_result.info.help_text}")
else:
self.logger.info(
f"Function Exec data: {formatted_result}")
except Exception as e:
self.logger.error(
Style.YELLOW(Style.Bold(
f"! Function ERROR: in {modular_name}.{function_name}")))
# Wrap the exception in a Result object
formatted_result = Result.default_internal_error(info=str(e)).set_origin(mod_function_name)
# res = formatted_result
self.logger.error(
f"Function {modular_name}.{function_name}"
f" executed wit an error {str(e)}, {type(e)}")
self.debug_rains(e)
else:
self.print_ok()
self.logger.info(
f"Function {modular_name}.{function_name}"
f" executed successfully")
return formatted_result
async def run_http(self, mod_function_name: Enum or str or tuple, function_name=None,
args_=None,
kwargs_=None, method="GET",
*args, **kwargs):
if kwargs_ is not None and not kwargs:
kwargs = kwargs_
if args_ is not None and not args:
args = args_
modular_name = mod_function_name
function_name = function_name
if isinstance(mod_function_name, str) and isinstance(function_name, str):
mod_function_name = (mod_function_name, function_name)
if isinstance(mod_function_name, tuple):
modular_name, function_name = mod_function_name
elif isinstance(mod_function_name, list):
modular_name, function_name = mod_function_name[0], mod_function_name[1]
elif isinstance(mod_function_name, Enum):
modular_name, function_name = mod_function_name.__class__.NAME.value, mod_function_name.value
self.logger.info(f"getting function : {modular_name}.{function_name} from http {self.session.base}")
r = await self.session.fetch(f"/api/{modular_name}/{function_name}{'?' + args_ if args_ is not None else ''}",
data=kwargs, method=method)
try:
if not r:
self.logger.info(f"§ Session server Offline! {self.session.base}")
return Result.default_internal_error(info="Session fetch failed").as_dict()
content_type = r.headers.get('Content-Type', '').lower()
if 'application/json' in content_type:
try:
return r.json()
except Exception as e:
self.logger.error(f"⚠ JSON decode error: {e}")
# Fallback to text if JSON decoding fails
text = r.text
else:
text = r.text
if isinstance(text, Callable):
if asyncio.iscoroutinefunction(text):
text = await text()
else:
text = text()
# Attempt YAML
if 'yaml' in content_type or text.strip().startswith('---'):
try:
import yaml
return yaml.safe_load(text)
except Exception as e:
self.logger.error(f"⚠ YAML decode error: {e}")
# Attempt XML
if 'xml' in content_type or text.strip().startswith('<?xml'):
try:
import xmltodict
return xmltodict.parse(text)
except Exception as e:
self.logger.error(f"⚠ XML decode error: {e}")
# Fallback: return plain text
return Result.default_internal_error(data={'raw_text': text, 'content_type': content_type}).as_dict()
except Exception as e:
self.logger.critical("❌ Fatal error during API call:", e)
self.debug_rains(e)
return Result.default_internal_error(str(e)).as_dict()
def run_local(self, *args, **kwargs):
return self.run_any(*args, **kwargs)
async def a_run_local(self, *args, **kwargs):
return await self.a_run_any(*args, **kwargs)
def _prepare_mod_function_name(
self,
mod_function_name: Enum | str | tuple,
backwords_compability_variabel_string_holder: str | None,
args_: tuple | None,
kwargs_: dict | None,
args: tuple,
kwargs: dict
) -> tuple[tuple, tuple, dict]:
"""
Shared helper to normalize mod_function_name and merge args/kwargs.
Returns (normalized_mod_function_name, args, kwargs).
"""
if kwargs_ is not None and not kwargs:
kwargs = kwargs_
if args_ is not None and not args:
args = args_
if isinstance(mod_function_name, str) and backwords_compability_variabel_string_holder is None:
backwords_compability_variabel_string_holder = mod_function_name.split('.')[-1]
mod_function_name = mod_function_name.replace(f".{backwords_compability_variabel_string_holder}", "")
if isinstance(mod_function_name, str) and isinstance(backwords_compability_variabel_string_holder, str):
mod_function_name = (mod_function_name, backwords_compability_variabel_string_holder)
return mod_function_name, args, kwargs
def _process_run_result(self, res: Result, get_results: bool) -> Result:
"""
Shared helper to process the result of run_function/a_run_function.
Handles background tasks, debug logging, and result extraction.
"""
if isinstance(res, ApiResult):
res = res.as_result()
if isinstance(res, Result) and res.bg_task is not None:
self.run_bg_task(res.bg_task)
if self.debug and isinstance(res, Result):
res.log(show_data=False)
if not get_results and isinstance(res, Result):
return res.get()
if get_results and not isinstance(res, Result):
return Result.ok(data=res)
return res
def run_any(self, mod_function_name: Enum | str | tuple, backwords_compability_variabel_string_holder=None,
get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
kwargs_=None,
*args, **kwargs):
"""
Run a module function synchronously.
For async functions, use a_run_any() instead.
"""
mod_function_name, args, kwargs = self._prepare_mod_function_name(
mod_function_name, backwords_compability_variabel_string_holder,
args_, kwargs_, args, kwargs
)
res: Result = self.run_function(
mod_function_name,
tb_run_function_with_state=tb_run_function_with_state,
tb_run_with_specification=tb_run_with_specification,
args_=args, kwargs_=kwargs
).as_result()
return self._process_run_result(res, get_results)
async def a_run_any(self, mod_function_name: Enum | str | tuple,
backwords_compability_variabel_string_holder=None,
get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
kwargs_=None,
*args, **kwargs):
"""
Run a module function asynchronously.
Preferred method for async contexts.
"""
mod_function_name, args, kwargs = self._prepare_mod_function_name(
mod_function_name, backwords_compability_variabel_string_holder,
args_, kwargs_, args, kwargs
)
res: Result = await self.a_run_function(
mod_function_name,
tb_run_function_with_state=tb_run_function_with_state,
tb_run_with_specification=tb_run_with_specification,
args_=args, kwargs_=kwargs
)
return self._process_run_result(res, get_results)
def web_context(self):
if self._web_context is None:
try:
print(os.path.join(self.data_dir.split(".data")[0],"dist","helper.html"), self.data_dir)
from toolboxv2.utils.workers.fast.endpoint import dist_dir
helper_file = dist_dir()
if helper_file is None or not (helper_file/ "helper.html").exists():
raise FileNotFoundError(f"Helper file not found: {helper_file/ 'helper.html'}")
self._web_context = (helper_file/ "helper.html").read_text(encoding='utf-8') # open(os.path.join(self.data_dir.split(".data")[0],"dist","helper.html"), encoding="utf-8").read()
except Exception as e:
self.logger.error(f"Could not load web context: {e}")
self._web_context = "<div><h1>Web Context not found</h1></div>"
return self._web_context
def get_mod(self, name, spec='app') -> ModuleType or MainToolType:
if spec != "app":
self.print(f"Getting Module {name} spec: {spec}")
if name not in self.functions:
mod = self.save_load(name, spec=spec)
if mod is False or (isinstance(mod, Result) and mod.is_error()):
self.logger.warning(f"Could not find {name} in {list(self.functions.keys())}")
raise ValueError(f"Could not find {name} in {list(self.functions.keys())} pleas install the module, or its posibly broken use --debug for infos")
# private = self.functions[name].get(f"{spec}_private")
# if private is not None:
# if private and spec != 'app':
# raise ValueError("Module is private")
if name not in self.functions:
self.logger.warning(f"Module '{name}' is not found")
return None
instance = self.functions[name].get(f"{spec}_instance")
if instance is None:
return self.load_mod(name, spec=spec)
return self.functions[name].get(f"{spec}_instance")
def _print(self, text="", *args, **kwargs):
print(text, *args, **kwargs)
def print(self, text="", *args, **kwargs):
# self.logger.info(f"Output : {text}")
if 'live' in self.id:
return
flush = kwargs.pop('flush', True)
if self.sprint(None):
self._print(Style.CYAN(f"System${self.id}:"), end=" ", flush=flush)
if 'color' in kwargs:
text = Style.style_dic[kwargs.pop('color')] + text + Style.style_dic["END"]
self._print(text, *args, **kwargs, flush=flush)
def sprint(self, text="", show_system=True, *args, **kwargs):
if text is None:
return True
if 'live' in self.id:
return
flush = kwargs.pop('flush', True)
# self.logger.info(f"Output : {text}")
if show_system:
self._print(Style.CYAN(f"System${self.id}:"), end=" ", flush=flush)
if isinstance(text, str) and kwargs == {} and text:
stram_print(text + ' '.join(args))
self._print()
else:
self._print(text, *args, **kwargs, flush=flush)
# ----------------------------------------------------------------
# Decorators for the toolbox
def reload_mod(self, mod_name, spec='app', is_file=True, loc="toolboxv2.mods."):
self.remove_mod(mod_name, delete=True)
if mod_name not in self.modules:
self.logger.warning(f"Module '{mod_name}' is not found")
return
if hasattr(self.modules[mod_name], 'reload_save') and self.modules[mod_name].reload_save:
def reexecute_module_code(x):
return x
else:
def reexecute_module_code(module_name):
if isinstance(module_name, str):
module = import_module(module_name)
else:
module = module_name
# Get the source code of the module
try:
source = inspect.getsource(module)
except Exception:
# print(f"No source for {str(module_name).split('from')[0]}: {e}")
return module
# Compile the source code
try:
code = compile(source, module.__file__, 'exec')
# Execute the code in the module's namespace
exec(code, module.__dict__)
except Exception:
# print(f"No source for {str(module_name).split('from')[0]}: {e}")
pass
return module
if not is_file:
mods = self.get_all_mods("./mods/" + mod_name)
def recursive_reload(package_name):
package = import_module(package_name)
# First, reload all submodules
if hasattr(package, '__path__'):
for _finder, name, _ispkg in pkgutil.walk_packages(package.__path__, package.__name__ + "."):
try:
mod = import_module(name)
reexecute_module_code(mod)
reload(mod)
except Exception as e:
self.logger.error(f"Error reloading module {name}: {e}")
break
# Finally, reload the package itself
reexecute_module_code(package)
reload(package)
for mod in mods:
if mod.endswith(".txt") or mod.endswith(".yaml"):
continue
try:
recursive_reload(loc + mod_name + '.' + mod)
self.print(f"Reloaded {mod_name}.{mod}")
except ImportError:
self.print(f"Could not load {mod_name}.{mod}")
reexecute_module_code(self.modules[mod_name])
if mod_name in self.functions:
if "on_exit" in self.functions[mod_name]:
self.functions[mod_name]["on_exit"] = []
if "on_start" in self.functions[mod_name]:
self.functions[mod_name]["on_start"] = []
self.inplace_load_instance(mod_name, spec=spec, mfo=reload(self.modules[mod_name]) if mod_name in self.modules else None)
def watch_mod(self, mod_name, spec='app', loc="toolboxv2.mods.", use_thread=True, path_name=None, on_reload=None):
if path_name is None:
path_name = mod_name
is_file = os.path.isfile(self.start_dir + '/mods/' + path_name + '.py')
import watchfiles
def helper():
paths = f'mods/{path_name}' + ('.py' if is_file else '')
self.logger.info(f'Watching Path: {paths}')
try:
for changes in watchfiles.watch(paths):
if not changes:
continue
self.reload_mod(mod_name, spec, is_file, loc)
if on_reload:
on_reload()
except FileNotFoundError:
self.logger.warning(f"Path {paths} not found")
if not use_thread:
helper()
else:
threading.Thread(target=helper, daemon=True).start()
def _register_function(self, module_name, func_name, data):
if module_name not in self.functions:
self.functions[module_name] = {}
if func_name in self.functions[module_name]:
self.functions[module_name][func_name] = data
else:
self.functions[module_name][func_name] = data
def _create_decorator(self, type_: str,
name: str = "",
mod_name: str = "",
level: int = -1,
restrict_in_virtual_mode: bool = False,
api: bool = False,
helper: str = "",
version: str or None = None,
initial: bool=False,
exit_f: bool=False,
test: bool=True,
samples:list[dict[str, Any]] | None=None,
state:bool | None=None,
pre_compute:Callable | None=None,
post_compute:Callable[[], Result] | None=None,
api_methods:list[str] | None=None,
memory_cache: bool=False,
file_cache: bool=False,
request_as_kwarg: bool=False,
row: bool=False,
memory_cache_max_size:int=100,
memory_cache_ttl:int=300,
websocket_handler: str | None = None,
websocket_context: bool=False,
):
if isinstance(type_, Enum):
type_ = type_.value
if memory_cache and file_cache:
raise ValueError("Don't use both cash at the same time for the same fuction")
use_cache = memory_cache or file_cache
cache = {}
if file_cache:
_cache_dir = os.path.join(self.data_dir, "cache", mod_name)
cache = FileCache(folder=_cache_dir + os.sep,
filename=os.path.join(_cache_dir, f"{name}cache.db"))
if memory_cache:
cache = MemoryCache(maxsize=memory_cache_max_size, ttl=memory_cache_ttl)
version = self.version if version is None else self.version + ':' + version
def _args_kwargs_helper(args_, kwargs_, parms, api=False):
if websocket_context and "request" in kwargs_:
# Prüfen ob es ein WebSocket-Request ist
request_data = kwargs_.get("request", {})
if isinstance(request_data, dict) and "websocket" in request_data:
# WebSocket-Kontext erstellen
ws_ctx = WebSocketContext.from_kwargs(kwargs_)
kwargs_["ws_context"] = ws_ctx
if "session" in parms and "session" not in kwargs_:
kwargs_["session"] = (
ws_ctx.user
) # oder ws_ctx.session, je nach Implementierung
if "conn_id" in parms and "conn_id" not in kwargs_:
kwargs_["conn_id"] = ws_ctx.conn_id
# Wenn der Parameter erwartet wird, Request-Object erstellen
if "request" in parms or request_as_kwarg:
kwargs_["request"] = RequestData.from_dict(request_data)
if request_as_kwarg and "request" in kwargs_:
kwargs_["request"] = (
RequestData.from_dict(kwargs_["request"])
if isinstance(kwargs_["request"], dict)
else kwargs_["request"]
)
if "data" in kwargs_ and "data" not in parms:
kwargs_["request"].data = kwargs_["request"].body = kwargs_["data"]
del kwargs_["data"]
if "form_data" in kwargs_ and "form_data" not in parms:
kwargs_["request"].form_data = kwargs_["request"].body = kwargs_[
"form_data"
]
del kwargs_["form_data"]
if not request_as_kwarg and "request" in kwargs_:
del kwargs_["request"]
if (
api
and "data" in kwargs_
and "data" not in parms
):
for k in kwargs_["data"]:
if k in parms:
kwargs_[k] = kwargs_["data"][k]
del kwargs_["data"]
if "app" not in parms and args_ and args_[0] is self and len(args_) == 1:
args_ = ()
args_ += (kwargs_.pop("args_"),) if "args_" in kwargs_ else ()
args_ += (kwargs_.pop("args"),) if "args" in kwargs_ else ()
return args_, kwargs_
def a_additional_process(func):
def args_kwargs_helper(args_, kwargs_):
module_name = mod_name if mod_name else func.__module__.split('.')[-1]
func_name = name if name else func.__name__
parms = self.functions.get(module_name, {}).get(func_name, {}).get('params', [])
return _args_kwargs_helper(args_, kwargs_, parms, api=self.functions.get(module_name, {}).get(func_name, {}).get('api', False))
async def executor(*args, **kwargs):
args, kwargs = args_kwargs_helper(args, kwargs)
if pre_compute is not None:
args, kwargs = await pre_compute(*args, **kwargs)
if asyncio.iscoroutinefunction(func):
result = await func(*args, **kwargs)
else:
result = func(*args, **kwargs)
if post_compute is not None:
result = await post_compute(result)
if row:
return result
if not isinstance(result, Result):
result = Result.ok(data=result)
if result.origin is None:
result.set_origin((mod_name if mod_name else func.__module__.split('.')[-1]
, name if name else func.__name__
, type_))
if result.result.data_to == ToolBoxInterfaces.native.name:
result.result.data_to = ToolBoxInterfaces.remote if api else ToolBoxInterfaces.native
# Wenden Sie die to_api_result Methode auf das Ergebnis an, falls verfügbar
if api and hasattr(result, 'to_api_result'):
return result.to_api_result()
return result
@wraps(func)
async def wrapper(*args, **kwargs):
if not use_cache:
return await executor(*args, **kwargs)
try:
cache_key = (f"{mod_name if mod_name else func.__module__.split('.')[-1]}"
f"-{func.__name__}-{str(args)},{str(kwargs.items())}")
except ValueError:
cache_key = (f"{mod_name if mod_name else func.__module__.split('.')[-1]}"
f"-{func.__name__}-{bytes(args)},{str(kwargs.items())}")
result = cache.get(cache_key)
if result is not None:
return result
result = await executor(*args, **kwargs)
cache.set(cache_key, result)
return result
return wrapper
def additional_process(func):
def args_kwargs_helper(args_, kwargs_):
module_name = mod_name if mod_name else func.__module__.split('.')[-1]
func_name = name if name else func.__name__
parms = self.functions.get(module_name, {}).get(func_name, {}).get('params', [])
return _args_kwargs_helper(args_, kwargs_, parms, api=self.functions.get(module_name, {}).get(func_name, {}).get('api', False))
def executor(*args, **kwargs):
args, kwargs = args_kwargs_helper(args, kwargs)
if pre_compute is not None:
args, kwargs = pre_compute(*args, **kwargs)
if asyncio.iscoroutinefunction(func):
result = func(*args, **kwargs)
else:
result = func(*args, **kwargs)
if post_compute is not None:
result = post_compute(result)
if row:
return result
if not isinstance(result, Result):
result = Result.ok(data=result)
if result.origin is None:
result.set_origin((mod_name if mod_name else func.__module__.split('.')[-1]
, name if name else func.__name__
, type_))
if result.result.data_to == ToolBoxInterfaces.native.name:
result.result.data_to = ToolBoxInterfaces.remote if api else ToolBoxInterfaces.native
# Wenden Sie die to_api_result Methode auf das Ergebnis an, falls verfügbar
if api and hasattr(result, 'to_api_result'):
return result.to_api_result()
return result
@wraps(func)
def wrapper(*args, **kwargs):
if not use_cache:
return executor(*args, **kwargs)
try:
cache_key = (f"{mod_name if mod_name else func.__module__.split('.')[-1]}"
f"-{func.__name__}-{str(args)},{str(kwargs.items())}")
except ValueError:
cache_key = (f"{mod_name if mod_name else func.__module__.split('.')[-1]}"
f"-{func.__name__}-{bytes(args)},{str(kwargs.items())}")
result = cache.get(cache_key)
if result is not None:
return result
result = executor(*args, **kwargs)
cache.set(cache_key, result)
return result
return wrapper
def decorator(func):
sig = signature(func)
params = list(sig.parameters)
module_name = mod_name if mod_name else func.__module__.split('.')[-1]
func_name = name if name else func.__name__
if func_name == 'on_start':
func_name = 'on_startup'
if func_name == 'on_exit':
func_name = 'on_close'
if api or pre_compute is not None or post_compute is not None or memory_cache or file_cache:
if asyncio.iscoroutinefunction(func):
func = a_additional_process(func)
else:
func = additional_process(func)
if api and str(sig.return_annotation) == 'Result':
raise ValueError(f"Fuction {module_name}.{func_name} registered as "
f"Api fuction but uses {str(sig.return_annotation)}\n"
f"Please change the sig from ..)-> Result to ..)-> ApiResult")
data = {
"type": type_,
"module_name": module_name,
"func_name": func_name,
"level": level,
"restrict_in_virtual_mode": restrict_in_virtual_mode,
"func": func,
"api": api,
"helper": helper,
"version": version,
"initial": initial,
"exit_f": exit_f,
"api_methods": api_methods if api_methods is not None else ["AUTO"],
"__module__": func.__module__,
"signature": sig,
"params": params,
"row": row,
"state": (
False if len(params) == 0 else params[0] in ["self", "state", "app"]
)
if state is None
else state,
"do_test": test,
"samples": samples,
"request_as_kwarg": request_as_kwarg,
"websocket_context": websocket_context,
}
if websocket_handler:
# Die dekorierte Funktion sollte ein Dict mit den Handlern zurückgeben
try:
handler_config = func(self) # Rufe die Funktion auf, um die Konfiguration zu erhalten
if not isinstance(handler_config, dict):
raise TypeError(
f"WebSocket handler function '{func.__name__}' must return a dictionary of handlers.")
# Handler-Identifikator, z.B. "ChatModule/room_chat"
handler_id = f"{module_name}/{websocket_handler}"
self.websocket_handlers[handler_id] = {}
for event_name, handler_func in handler_config.items():
if event_name in ["on_connect", "on_message", "on_disconnect"] and callable(handler_func):
if asyncio.iscoroutinefunction(handler_func):
handler_func = a_additional_process(handler_func)
else:
handler_func = additional_process(handler_func)
self.websocket_handlers[handler_id][event_name] = handler_func
else:
self.logger.warning(f"Invalid WebSocket handler event '{event_name}' in '{handler_id}'.")
self.logger.info(f"Registered WebSocket handlers for '{handler_id}'.")
except Exception as e:
self.logger.error(f"Failed to register WebSocket handlers for '{func.__name__}': {e}",
exc_info=True)
else:
self._register_function(module_name, func_name, data)
if exit_f:
if "on_exit" not in self.functions[module_name]:
self.functions[module_name]["on_exit"] = []
self.functions[module_name]["on_exit"].append(func_name)
if initial:
if "on_start" not in self.functions[module_name]:
self.functions[module_name]["on_start"] = []
self.functions[module_name]["on_start"].append(func_name)
return func
decorator.tb_init = True
return decorator
def export(self, *args, **kwargs):
return self.tb(*args, **kwargs)
def tb(self, name=None,
mod_name: str = "",
helper: str = "",
version: str | None = None,
test: bool = True,
restrict_in_virtual_mode: bool = False,
api: bool = False,
initial: bool = False,
exit_f: bool = False,
test_only: bool = False,
memory_cache: bool = False,
file_cache: bool = False,
request_as_kwarg: bool = False,
row: bool = False,
state: bool | None = None,
level: int = -1,
memory_cache_max_size: int = 100,
memory_cache_ttl: int = 300,
samples: list or dict or None = None,
interface: ToolBoxInterfaces or None or str = None,
pre_compute=None,
post_compute=None,
api_methods=None,
websocket_handler: str | None = None,
websocket_context: bool=False,
):
"""
A decorator for registering and configuring functions within a module.
This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.
Args:
name (str, optional): The name to register the function under. Defaults to the function's own name.
mod_name (str, optional): The name of the module the function belongs to.
helper (str, optional): A helper string providing additional information about the function.
version (str or None, optional): The version of the function or module.
test (bool, optional): Flag to indicate if the function is for testing purposes.
restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
api (bool, optional): Flag to indicate if the function is part of an API.
initial (bool, optional): Flag to indicate if the function should be executed at initialization.
exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
test_only (bool, optional): Flag to indicate if the function should only be used for testing.
memory_cache (bool, optional): Flag to enable memory caching for the function.
request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
file_cache (bool, optional): Flag to enable file caching for the function.
row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
state (bool or None, optional): Flag to indicate if the function maintains state.
level (int, optional): The level of the function, used for prioritization or categorization.
memory_cache_max_size (int, optional): Maximum size of the memory cache.
memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
samples (list or dict or None, optional): Samples or examples of function usage.
interface (str, optional): The interface type for the function.
pre_compute (callable, optional): A function to be called before the main function.
post_compute (callable, optional): A function to be called after the main function.
api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.
websocket_handler (str, optional): The name of the websocket handler to use.
websocket_context (bool, optional): Flag to indicate if the function should receive the websocket context.
Returns:
function: The decorated function with additional processing and registration capabilities.
"""
if interface is None:
interface = "tb"
if test_only and 'test' not in self.id:
return lambda *args, **kwargs: args
return self._create_decorator(interface,
name,
mod_name,
level=level,
restrict_in_virtual_mode=restrict_in_virtual_mode,
helper=helper,
api=api,
version=version,
initial=initial,
exit_f=exit_f,
test=test,
samples=samples,
state=state,
pre_compute=pre_compute,
post_compute=post_compute,
memory_cache=memory_cache,
file_cache=file_cache,
request_as_kwarg=request_as_kwarg,
row=row,
api_methods=api_methods,
memory_cache_max_size=memory_cache_max_size,
memory_cache_ttl=memory_cache_ttl,
websocket_handler=websocket_handler,
websocket_context=websocket_context,
)
def save_autocompletion_dict(self):
autocompletion_dict = {}
for module_name, _module in self.functions.items():
data = {}
for function_name, function_data in self.functions[module_name].items():
if not isinstance(function_data, dict):
continue
data[function_name] = {arg: None for arg in
function_data.get("params", [])}
if len(data[function_name].keys()) == 0:
data[function_name] = None
autocompletion_dict[module_name] = data if len(data.keys()) > 0 else None
self.config_fh.add_to_save_file_handler("auto~~~~~~", str(autocompletion_dict))
def get_autocompletion_dict(self):
return self.config_fh.get_file_handler("auto~~~~~~")
def save_registry_as_enums(self, directory: str, filename: str):
# Ordner erstellen, falls nicht vorhanden
if not os.path.exists(directory):
os.makedirs(directory)
# Dateipfad vorbereiten
filepath = os.path.join(directory, filename)
# Enum-Klassen als Strings generieren
enum_classes = [f'"""Automatic generated by ToolBox v = {self.version}"""'
f'\nfrom enum import Enum\nfrom dataclasses import dataclass'
f'\n\n\n']
for module, functions in self.functions.items():
if module.startswith("APP_INSTANCE"):
continue
class_name = module
enum_members = "\n ".join(
[
f"{func_name.upper().replace('-', '')}"
f" = '{func_name}' "
f"# Input: ({fuction_data['params'] if isinstance(fuction_data, dict) else ''}),"
f" Output: {fuction_data['signature'].return_annotation if isinstance(fuction_data, dict) else 'None'}"
for func_name, fuction_data in functions.items()])
enum_class = (f'@dataclass\nclass {class_name.upper().replace(".", "_").replace("-", "")}(Enum):'
f"\n NAME = '{class_name}'\n {enum_members}")
enum_classes.append(enum_class)
# Enums in die Datei schreiben
data = "\n\n\n".join(enum_classes)
if len(data) < 12:
raise ValueError(
"Invalid Enums Loosing content pleas delete it ur self in the (utils/system/all_functions_enums.py) or add mor new stuff :}")
with open(filepath, 'w') as file:
file.write(data)
self.logger.info(Style.Bold(Style.BLUE(f"Enums gespeichert in {filepath}")))
# WS logic
def _set_rust_ws_bridge(self, bridge_object: Any):
"""
Diese Methode wird von Rust aufgerufen, um die Kommunikationsbrücke zu setzen.
Sie darf NICHT manuell von Python aus aufgerufen werden.
"""
self.print(f"Rust WebSocket bridge has been set for instance {self.id}.")
self._rust_ws_bridge = bridge_object
async def ws_send(self, conn_id: str, payload: dict):
"""
Sendet eine Nachricht asynchron an eine einzelne WebSocket-Verbindung.
Args:
conn_id: Die eindeutige ID der Zielverbindung.
payload: Ein Dictionary, das als JSON gesendet wird.
"""
if self._rust_ws_bridge is None:
self.logger.error("Cannot send WebSocket message: Rust bridge is not initialized.")
return
try:
# Ruft die asynchrone Rust-Methode auf und wartet auf deren Abschluss
await self._rust_ws_bridge.send_message(conn_id, json.dumps(payload))
except Exception as e:
self.logger.error(f"Failed to send WebSocket message to {conn_id}: {e}", exc_info=True)
async def ws_broadcast(self, channel_id: str, payload: dict, source_conn_id: str = "python_broadcast"):
"""
Sendet eine Nachricht asynchron an alle Clients in einem Kanal/Raum.
Args:
channel_id: Der Kanal, an den gesendet werden soll.
payload: Ein Dictionary, das als JSON gesendet wird.
source_conn_id (optional): Die ID der ursprünglichen Verbindung, um Echos zu vermeiden.
"""
if self._rust_ws_bridge is None:
self.logger.error("Cannot broadcast WebSocket message: Rust bridge is not initialized.")
return
try:
# Ruft die asynchrone Rust-Broadcast-Methode auf
await self._rust_ws_bridge.broadcast_message(channel_id, json.dumps(payload), source_conn_id)
except Exception as e:
self.logger.error(f"Failed to broadcast WebSocket message to channel {channel_id}: {e}", exc_info=True)
"""Main module."""
import asyncio
import datetime
import inspect
import json
import logging
import os
import pkgutil
import sys
import threading
import time
from collections.abc import Callable
from concurrent.futures import Future, ThreadPoolExecutor
from enum import Enum
from functools import partial, wraps
from importlib import import_module, reload
from inspect import signature
from platform import node, system
from threading import Thread
from types import ModuleType
from typing import Any, Optional
from dotenv import load_dotenv
from .extras.db.mobile_db import create_mobile_db
from .extras.mkdocs import add_to_app
from ..utils.system.main_tool import get_version_from_pyproject
from .extras.Style import Spinner, Style, stram_print
from .singelton_class import Singleton
from .system.cache import FileCache, MemoryCache
from .system.file_handler import FileHandler
from .system.getting_and_closing_app import get_app
from .system.tb_logger import setup_logging, loggerNameOfToolboxv2, AuditLogger, set_log_db, LogSyncManager
from .system.types import (
ApiResult,
AppArgs,
AppType,
MainToolType,
RequestData,
Result,
ToolBoxInterfaces,
WebSocketContext,
)
load_dotenv()
class App(AppType, metaclass=Singleton):
def __init__(self, prefix: str = "", args=AppArgs().default()):
self.log_sync = None
if "test" not in prefix:
self.logger_prefix = self.REFIX = prefix
prefix = "main"
else:
self.logger_prefix = self.REFIX = "test"
super().__init__(prefix, args)
self._web_context = None
self._debug = args.debug
self.loop = None
t0 = time.perf_counter()
abspath = os.path.abspath(__file__)
self.system_flag = system() # Linux: Linux Mac: Darwin Windows: Windows
self.appdata = os.getenv('APPDATA') if os.name == 'nt' else os.getenv('XDG_CONFIG_HOME') or os.path.expanduser(
'~/.config') if os.name == 'posix' else None
if self.system_flag == "Darwin" or self.system_flag == "Linux":
dir_name = os.path.dirname(abspath).replace("/utils", "")
else:
dir_name = os.path.dirname(abspath).replace("\\utils", "")
self.start_dir = str(dir_name)
self.bg_tasks = []
# ponytail: prefix file follows TB_DATA_DIR like the other dirs.
lapp = os.path.join(os.getenv("TB_DATA_DIR") or dir_name, ".data")
_prefix_file = os.path.join(lapp, "last-app-prefix.txt")
if not prefix:
if not os.path.exists(_prefix_file):
os.makedirs(lapp, exist_ok=True)
open(_prefix_file, "a").close()
with open(_prefix_file) as prefix_file:
cont = prefix_file.read()
if cont:
prefix = cont.rstrip()
else:
if not os.path.exists(_prefix_file):
os.makedirs(lapp, exist_ok=True)
open(_prefix_file, "a").close()
with open(_prefix_file, "w") as prefix_file:
prefix_file.write(prefix)
self.prefix = prefix
node_ = node()
if 'localhost' in node_ and (host := os.getenv('HOSTNAME', 'localhost')) != 'localhost':
Xet Storage Details
- Size:
- 131 kB
- Xet hash:
- 561393311e09dabd6c4b21f1f6876b14fd55f0606073ae3da127a2012e5ac77e
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.