Buckets:

download
raw
18 kB
"""
Feature Loader - Automatisches Entpacken von Feature-ZIPs bei Installation
Dieses Modul wird beim Import von toolboxv2 ausgeführt und entpackt
die benötigten Features basierend auf den installierten optional-dependencies.
Flow:
1. pip install toolboxv2 → Alle ZIPs werden mitgeliefert, nur Core entpackt
2. pip install toolboxv2[web] → Core + Web werden entpackt
3. pip install toolboxv2[all] → Alle Features werden entpackt
Die Feature-ZIPs liegen in toolboxv2/features_packed/ und werden nach
toolboxv2/features/{name}/ und toolboxv2/{target_dirs}/ entpackt.
"""
import importlib.util
import os, sys
import shutil
import sys
import zipfile
from pathlib import Path
from typing import Dict, List, Optional, Set
# Mapping: pip extra name → feature names
EXTRA_TO_FEATURES: Dict[str, List[str]] = {
"cli": ["core","cli"],
"web": ["core","web"],
"desktop": ["core","desktop"],
"exotic": ["core","exotic"],
"isaa": ["core","isaa"],
"all": ["core", "cli", "web", "desktop", "exotic", "isaa"],
"production": ["core", "cli", "web"],
"dev": ["core", "cli", "web", "desktop", "exotic", "isaa"],
}
# Feature detection via importable packages
# Wenn eines dieser Packages installiert ist, wird das Feature benötigt
FEATURE_DETECTION: Dict[str, List[str]] = {
"core":[
"radon",
"qrcode",
"watchfiles",
"gitpython",
"redis",
"xmltodict",
"minio",
"prompt_toolkit",
"prompt-toolkit",
"opentelemetry-api",
"opentelemetry-sdk",
"docker",
],
"cli": [
"prompt-toolkit",
"ipython",
"rich",
"google-auth-oauthlib",
"google-api-python-client",
"pyperclip",
"dateparser",
],
"web": [
"websockets",
"webauthn",
"waitress",
],
"desktop": [
"pygetwindow",
"pyautogui",
"google-auth-oauthlib",
"google-api-python-client",
"dateparser",
"pyperclip",
"webrtcvad",
"customtkinter",
],
"exotic": [
"langchain-community",
"spacy",
"matplotlib",
],
"isaa": [
"networkx",
"numpy",
"litellm",
"dill",
"groq",
"pypdf2",
"nest-asyncio",
"pyvis",
"mcp",
"tenacity",
"html2text",
"chardet",
"faiss-cpu",
"playwright",
],
}
# Core ist immer dabei
CORE_FEATURE = "core"
def get_package_root() -> Path:
"""Ermittle das Root-Verzeichnis des toolboxv2 Packages."""
return Path(__file__).parent
def get_features_packed_dir() -> Path:
"""Verzeichnis mit gepackten Features."""
return get_package_root() / "features_packed"
def get_features_dir() -> Path:
"""Verzeichnis für entpackte Feature-Configs."""
return get_package_root() / "features"
def is_feature_installed(feature_name: str) -> bool:
"""
Prüfe ob ein Feature bereits entpackt/installiert ist.
Ein Feature gilt als installiert wenn:
1. Das Feature-Verzeichnis existiert UND
2. Eine feature.yaml UND .installed Marker darin liegt
"""
feature_dir = get_features_dir() / feature_name
feature_yaml = feature_dir / "feature.yaml"
installed_marker = feature_dir / ".installed"
return feature_yaml.exists() and installed_marker.exists()
def is_feature_available(feature_name: str) -> bool:
"""
Prüfe ob ein Feature als Source oder ZIP verfügbar ist.
"""
# Als Source vorhanden?
feature_yaml = get_features_dir() / feature_name / "feature.yaml"
if feature_yaml.exists():
return True
# Als ZIP vorhanden?
return is_feature_packed(feature_name)
def is_feature_packed(feature_name: str) -> bool:
"""Prüfe ob ein Feature als ZIP verfügbar ist."""
packed_dir = get_features_packed_dir()
if not packed_dir.exists():
return False
# Suche nach tbv2-feature-{name}-*.zip
for _ in packed_dir.glob(f"tbv2-feature-{feature_name}-*.zip"):
return True
return False
def get_packed_feature_path(feature_name: str) -> Optional[Path]:
"""Finde den Pfad zum gepackten Feature ZIP."""
packed_dir = get_features_packed_dir()
if not packed_dir.exists():
return None
# Finde neueste Version
candidates = list(packed_dir.glob(f"tbv2-feature-{feature_name}-*.zip"))
if not candidates:
return None
# Sortiere nach Dateiname (enthält Version)
candidates.sort(reverse=True)
return candidates[0]
try:
from .feature_loader_registry import download_feature_from_registry # noqa: F401
except ImportError:
# Fallback wenn feature_loader_registry noch nicht vorhanden
def download_feature_from_registry(feature_name: str, version=None, force=False):
"""Minimal urllib fallback."""
import urllib.request, json as _json
registry_url = os.environ.get("TB_REGISTRY_URL", "https://registry.simplecore.app")
packed_dir = get_features_packed_dir()
packed_dir.mkdir(parents=True, exist_ok=True)
zip_path = packed_dir / f"tbv2-feature-{feature_name}-latest.zip"
try:
meta_url = f"{registry_url}/api/v1/features/tbv2-feature-{feature_name}/latest"
with urllib.request.urlopen(meta_url, timeout=8) as r:
meta = _json.loads(r.read())
download_url = meta.get("download_url", f"{registry_url}/features/tbv2-feature-{feature_name}-latest.zip")
except Exception:
download_url = f"{registry_url}/features/tbv2-feature-{feature_name}-latest.zip"
try:
print(f"Downloading feature '{feature_name}' from registry...", file=sys.stderr)
urllib.request.urlretrieve(download_url, zip_path)
return zip_path
except Exception as e:
print(f"Warning: Registry download failed for '{feature_name}': {e}", file=sys.stderr)
if zip_path.exists():
zip_path.unlink()
return None
def detect_installed_extras() -> Set[str]:
"""
Erkenne welche optional-dependencies installiert sind.
Prüft ob die Marker-Packages für jedes Feature importierbar sind.
"""
installed_extras = set()
for extra_name, marker_packages in FEATURE_DETECTION.items():
# Prüfe ob mindestens ein Marker-Package installiert ist
for pkg in marker_packages:
if importlib.util.find_spec(pkg) is not None:
installed_extras.add(extra_name)
break
return installed_extras
def get_required_features() -> Set[str]:
"""
Ermittle welche Features basierend auf installierten Extras benötigt werden.
Returns:
Set von Feature-Namen die entpackt werden sollen
"""
required = {CORE_FEATURE} # Core ist immer dabei
installed_extras = detect_installed_extras()
for extra in installed_extras:
if extra in EXTRA_TO_FEATURES:
required.update(EXTRA_TO_FEATURES[extra])
return required
def unpack_feature(feature_name: str, force: bool = False) -> bool:
"""
Entpacke ein Feature aus seinem ZIP.
Args:
feature_name: Name des Features
force: Überschreibe existierendes Feature
Returns:
True wenn erfolgreich entpackt
"""
if is_feature_installed(feature_name) and not force:
return True # Bereits installiert
zip_path = get_packed_feature_path(feature_name)
if not zip_path:
zip_path = download_feature_from_registry(feature_name)
if not zip_path:
# Kein ZIP vorhanden - Feature ist vielleicht schon im Source
# Erstelle .installed Marker falls feature.yaml existiert
feature_yaml = get_features_dir() / feature_name / "feature.yaml"
if feature_yaml.exists():
(get_features_dir() / feature_name / ".installed").touch()
return True
return False
try:
package_root = get_package_root()
features_dir = get_features_dir()
feature_target = features_dir / feature_name
# Erstelle Features-Verzeichnis falls nicht vorhanden
features_dir.mkdir(parents=True, exist_ok=True)
feature_target.mkdir(parents=True, exist_ok=True)
with zipfile.ZipFile(zip_path, 'r') as zf:
# Entpacke feature.yaml nach features/{name}/
if "feature.yaml" in zf.namelist():
with zf.open("feature.yaml") as src:
(feature_target / "feature.yaml").write_bytes(src.read())
# Extrahiere requirements.txt falls vorhanden
if "requirements.txt" in zf.namelist():
with zf.open("requirements.txt") as src:
(feature_target / "requirements.txt").write_bytes(src.read())
# Entpacke files/ nach entsprechenden Verzeichnissen
for name in zf.namelist():
if name.startswith("files/") and not name.endswith("/"):
# Relativer Pfad ohne "files/" Prefix
rel_path = name[6:] # Remove "files/"
target_file = package_root / rel_path
# Erstelle Verzeichnis falls nötig
target_file.parent.mkdir(parents=True, exist_ok=True)
# Extrahiere Datei
with zf.open(name) as src:
target_file.write_bytes(src.read())
# Erstelle .installed Marker
(feature_target / ".installed").touch()
return True
except Exception as e:
print(f"Warning: Failed to unpack feature '{feature_name}': {e}", file=sys.stderr)
return False
def ensure_features_loaded() -> Dict[str, bool]:
"""
Stelle sicher dass alle benötigten Features geladen sind.
Wird beim Import von toolboxv2 aufgerufen.
Returns:
Dict mit {feature_name: success}
"""
results = {}
required = get_required_features()
for feature_name in required:
if is_feature_installed(feature_name):
results[feature_name] = True
elif is_feature_packed(feature_name):
results[feature_name] = unpack_feature(feature_name)
else:
# Feature weder installiert noch gepackt
# Prüfe ob Source vorhanden (Development Mode)
feature_yaml = get_features_dir() / feature_name / "feature.yaml"
if feature_yaml.exists():
# Erstelle .installed Marker
(get_features_dir() / feature_name / ".installed").touch()
results[feature_name] = True
else:
results[feature_name] = False
return results
def get_feature_status() -> Dict[str, dict]:
"""
Zeige Status aller Features.
Returns:
Dict mit Feature-Infos
"""
status = {}
# Alle möglichen Features
all_features = {CORE_FEATURE}
for features in EXTRA_TO_FEATURES.values():
all_features.update(features)
required = get_required_features()
for feature_name in sorted(all_features):
status[feature_name] = {
"required": feature_name in required,
"installed": is_feature_installed(feature_name),
"packed": is_feature_packed(feature_name),
"available": is_feature_available(feature_name),
}
return status
def list_available_features() -> List[str]:
"""Liste alle verfügbaren Features (installiert oder gepackt)."""
available = set()
# Installierte Features
features_dir = get_features_dir()
if features_dir.exists():
for feature_dir in features_dir.iterdir():
if feature_dir.is_dir() and (feature_dir / "feature.yaml").exists():
available.add(feature_dir.name)
# Gepackte Features
packed_dir = get_features_packed_dir()
if packed_dir.exists():
for zip_file in packed_dir.glob("tbv2-feature-*.zip"):
# Extract name from tbv2-feature-{name}-{version}.zip
name = zip_file.stem
if name.startswith("tbv2-feature-"):
parts = name[13:].rsplit("-", 2) # Remove prefix, split by -
if parts:
available.add(parts[0])
return sorted(available)
def cleanup_unpacked_features(keep_features: Optional[Set[str]] = None):
"""
Lösche entpackte Features die nicht mehr benötigt werden.
Args:
keep_features: Features die behalten werden sollen (Default: required features)
"""
if keep_features is None:
keep_features = get_required_features()
features_dir = get_features_dir()
package_root = get_package_root()
if not features_dir.exists():
return
for feature_dir in features_dir.iterdir():
if feature_dir.is_dir() and feature_dir.name not in keep_features:
# Prüfe ob es ein gepacktes Backup gibt
if is_feature_packed(feature_dir.name):
# Lösche entpackte Dateien
# Lade feature.yaml um files patterns zu bekommen
feature_yaml = feature_dir / "feature.yaml"
if feature_yaml.exists():
try:
import yaml
with open(feature_yaml) as f:
config = yaml.safe_load(f) or {}
# Lösche Feature-Dateien
for pattern in config.get("files", []):
if pattern.endswith("/*"):
dir_path = package_root / pattern[:-2]
if dir_path.exists():
shutil.rmtree(dir_path)
else:
file_path = package_root / pattern
if file_path.exists():
file_path.unlink()
except Exception:
pass
# Lösche Feature-Verzeichnis
shutil.rmtree(feature_dir)
# CLI für manuelle Feature-Verwaltung
def main():
"""CLI für Feature-Loader."""
import argparse
parser = argparse.ArgumentParser(
description="ToolBoxV2 Feature Loader",
prog="tb fl"
)
subparsers = parser.add_subparsers(dest="command")
# --- Bestehende Subcommands ---
subparsers.add_parser("status", help="Show feature status")
subparsers.add_parser("list", help="List available features")
p_load = subparsers.add_parser("load", help="Load/unpack required features")
p_load.add_argument("--all", action="store_true", help="Load all features")
p_load.add_argument("--force", action="store_true", help="Force reload")
p_unpack = subparsers.add_parser("unpack", help="Unpack specific feature")
p_unpack.add_argument("feature", help="Feature name")
p_unpack.add_argument("--force", action="store_true", help="Force unpack")
subparsers.add_parser("cleanup", help="Remove non-required features")
# --- NEU: Registry Subcommands ---
try:
from toolboxv2.feature_loader_registry import (
extend_feature_loader_cli,
handle_registry_cli_command,
)
extend_feature_loader_cli(subparsers)
_registry_cli = handle_registry_cli_command
except ImportError:
_registry_cli = lambda x:None
args = parser.parse_args()
if args.command == "status":
print("\n=== Feature Status ===\n")
status = get_feature_status()
required = get_required_features()
installed_extras = detect_installed_extras()
print(f"Detected extras: {installed_extras or 'none'}")
print(f"Required features: {required}")
print()
for name, info in status.items():
req = "✓" if info["required"] else " "
inst = "✓" if info["installed"] else "✗"
pack = "✓" if info["packed"] else " "
avail = "✓" if info["available"] else "✗"
print(f" [{req}] {name:12} installed={inst} packed={pack} available={avail}")
elif args.command == "list":
print("\n=== Available Features ===\n")
for feature in list_available_features():
installed = "✓" if is_feature_installed(feature) else " "
packed = "(packed)" if is_feature_packed(feature) else "(source)"
print(f" [{installed}] {feature} {packed}")
elif args.command == "load":
if args.all:
features = set(list_available_features())
else:
features = get_required_features()
print(f"\nLoading features: {features}\n")
for feature in features:
success = unpack_feature(feature, force=args.force)
status = "✓" if success else "✗"
print(f" {status} {feature}")
elif args.command == "unpack":
if not is_feature_available(args.feature):
print(f"✗ Feature not available: {args.feature}")
print(f" Available: {list_available_features()}")
sys.exit(1)
success = unpack_feature(args.feature, force=args.force)
if success:
print(f"✓ Unpacked: {args.feature}")
else:
print(f"✗ Failed to unpack: {args.feature}")
sys.exit(1)
elif args.command == "cleanup":
print("\nCleaning up non-required features...\n")
required = get_required_features()
cleanup_unpacked_features(required)
print("Done.")
elif args.command in ("upload", "check-updates", "update"):
if _registry_cli:
sys.exit(_registry_cli(args))
else:
print("Error: feature_loader_registry.py not installed.")
sys.exit(1)
else:
parser.print_help()
if __name__ == "__main__":
main()

Xet Storage Details

Size:
18 kB
·
Xet hash:
a960b4d598b15d7517174b3a3dac5d0aa4274017c80e48cb9b41576c6b743c2a

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.