Buckets:

download
raw
31.9 kB
import yaml
import json
import time
from pathlib import Path
import base64
from typing import List, Dict, Any, Literal, Optional
from sqlalchemy.orm import Session
from filelock import FileLock, Timeout
from backend.db import get_db
from backend.db.models.service import AppZooRepository, MCPZooRepository, PromptZooRepository, PersonalityZooRepository, SkillZooRepository
from backend.db.models.prompt import SavedPrompt
from backend.db.models.personality import Personality
from backend.db.models.skill import Skill as DBSkill
from backend.config import APPS_ZOO_ROOT_PATH, MCPS_ZOO_ROOT_PATH, PROMPTS_ZOO_ROOT_PATH, PERSONALITIES_ZOO_ROOT_PATH, SKILLS_ZOO_ROOT_PATH, APP_DATA_DIR
import datetime
from ascii_colors import ASCIIColors
import os
import re
ITEM_TYPES = Literal['app', 'mcp', 'prompt', 'personality', 'skill']
CACHE_FILE = APP_DATA_DIR / "zoo_cache.json"
CACHE_LOCK_FILE = APP_DATA_DIR / "zoo_cache.lock"
CACHE_LOCK_TIMEOUT = 5 # 5 seconds max timeout
_cache: Dict[str, Any] = {"timestamp": 0, "data": {}}
def _sanitize_for_json(data: Any) -> Any:
"""Recursively sanitizes data to ensure it's JSON serializable."""
if isinstance(data, (datetime.datetime, datetime.date)):
return data.isoformat()
if isinstance(data, dict):
return {k: _sanitize_for_json(v) for k, v in data.items()}
if isinstance(data, list):
return [_sanitize_for_json(v) for v in data]
return data
def get_zoo_root_path(item_type: ITEM_TYPES) -> Path:
if item_type == 'app': return APPS_ZOO_ROOT_PATH
if item_type == 'mcp': return MCPS_ZOO_ROOT_PATH
if item_type == 'prompt': return PROMPTS_ZOO_ROOT_PATH
if item_type == 'personality': return PERSONALITIES_ZOO_ROOT_PATH
if item_type == 'skill': return SKILLS_ZOO_ROOT_PATH
raise ValueError(f"Invalid item type: {item_type}")
def get_db_repo_model(item_type: ITEM_TYPES):
if item_type == 'app': return AppZooRepository
if item_type == 'mcp': return MCPZooRepository
if item_type == 'prompt': return PromptZooRepository
if item_type == 'personality': return PersonalityZooRepository
if item_type == 'skill': return SkillZooRepository
raise ValueError(f"Invalid item type: {item_type}")
def get_installed_items(db: Session, item_type: ITEM_TYPES) -> set:
if item_type in ['app', 'mcp']:
from backend.db.models.service import App as DBApp
return {item.name for item in db.query(DBApp.name).filter(DBApp.is_installed == True, DBApp.app_metadata['item_type'].as_string() == item_type).all()}
if item_type == 'prompt':
return {item.name for item in db.query(SavedPrompt.name).filter(SavedPrompt.owner_user_id.is_(None)).all()}
if item_type == 'personality':
return {item.name for item in db.query(Personality.name).filter(Personality.owner_user_id.is_(None)).all()}
if item_type == 'skill':
return {item.name for item in db.query(DBSkill.name).all()}
return set()
def _extract_license_from_path(item_path: Path, repo_path: Optional[Path] = None) -> Optional[str]:
"""Finds and identifies license information in a skill folder or repo."""
lic_candidates = ["LICENSE", "LICENSE.txt", "LICENSE.md", "license", "license.txt", "license.md"]
# Check item directory first
if item_path.is_dir():
for cand in lic_candidates:
lic_file = item_path / cand
if lic_file.is_file():
try:
text = lic_file.read_text(encoding='utf-8', errors='replace').strip()
if text:
lines = [l.strip() for l in text.splitlines() if l.strip()]
if lines:
first_line = lines[0]
# Detect common licenses
for known in ["MIT", "Apache-2.0", "Apache 2.0", "GPL-3.0", "GPL", "BSD", "CC0", "Unlicense"]:
if known.lower() in text[:200].lower():
return known
if len(first_line) <= 30 and not first_line.lower().startswith("copyright"):
return first_line
return "Standard License"
except Exception:
pass
# Check parent category or repo root
if repo_path and repo_path.is_dir():
for cand in lic_candidates:
lic_file = repo_path / cand
if lic_file.is_file():
try:
text = lic_file.read_text(encoding='utf-8', errors='replace').strip()
for known in ["MIT", "Apache-2.0", "Apache 2.0", "GPL-3.0", "GPL", "BSD", "CC0", "Unlicense"]:
if known.lower() in text[:200].lower():
return known
return "Repository License"
except Exception:
pass
return None
def _parse_skill_metadata(item_path: Path, repo_path: Optional[Path] = None) -> Dict[str, Any]:
"""
Parses metadata, content, and license for a skill from either:
1. A folder containing description.yaml, SKILL.md, skill.md, or skill.xml
2. A standalone .md or .xml skill file
"""
metadata: Dict[str, Any] = {}
content = ""
raw_text = ""
# Case 1: item_path is a directory
if item_path.is_dir():
# Check description.yaml
for yaml_candidate in ["description.yaml", "description.yml", "DESCRIPTION.YAML"]:
c_path = item_path / yaml_candidate
if c_path.exists():
try:
with open(c_path, 'r', encoding='utf-8') as f:
metadata = yaml.safe_load(f) or {}
break
except Exception:
pass
# Check for skill content files (case-insensitive search)
skill_file = None
for cand in ["SKILL.md", "skill.md", "Skill.md", "skill.xml", "SKILL.XML", "Skill.xml"]:
if (item_path / cand).exists():
skill_file = item_path / cand
break
if not skill_file:
# Check for any .xml or .md excluding readmes & licenses
for f in item_path.iterdir():
f_name_lower = f.name.lower()
if f.is_file() and f_name_lower not in ["readme.md", "license", "license.txt", "license.md", "contributing.md", "description.yaml", "description.yml"]:
if f.suffix.lower() in [".md", ".xml"]:
skill_file = f
break
if skill_file and skill_file.exists():
try:
raw_text = skill_file.read_text(encoding='utf-8', errors='replace')
content = raw_text
except Exception as e:
print(f"Warning: Could not read skill file at {skill_file}: {e}")
# Case 2: item_path is a standalone file (.md or .xml)
elif item_path.is_file():
try:
raw_text = item_path.read_text(encoding='utf-8', errors='replace')
content = raw_text
except Exception as e:
print(f"Warning: Could not read standalone skill file {item_path}: {e}")
# Parse metadata out of markdown frontmatter (--- ... ---)
if raw_text.strip().startswith("---"):
try:
parts = re.split(r'^---[ \t]*$', raw_text.replace('\r\n', '\n'), maxsplit=2, flags=re.MULTILINE)
if len(parts) >= 3:
frontmatter = yaml.safe_load(parts[1])
if isinstance(frontmatter, dict):
for k, v in frontmatter.items():
if not metadata.get(k):
metadata[k] = v
content = parts[2].strip()
except Exception:
pass
# Parse metadata out of XML tags if present
if "<skill" in raw_text.lower():
m_name = re.search(r'<name>(.*?)</name>', raw_text, re.IGNORECASE | re.DOTALL) or re.search(r'name=["\'](.*?)["\']', raw_text)
if m_name and not metadata.get('name'):
metadata['name'] = m_name.group(1).strip()
m_desc = re.search(r'<description>(.*?)</description>', raw_text, re.IGNORECASE | re.DOTALL) or re.search(r'description=["\'](.*?)["\']', raw_text)
if m_desc and not metadata.get('description'):
metadata['description'] = m_desc.group(1).strip()
m_cat = re.search(r'<category>(.*?)</category>', raw_text, re.IGNORECASE | re.DOTALL)
if m_cat and not metadata.get('category'):
metadata['category'] = m_cat.group(1).strip()
# Markdown # Title heuristic if name is still missing
if not metadata.get('name') and raw_text:
m_head = re.search(r'^#\s+(.+)$', raw_text, re.MULTILINE)
if m_head:
metadata['name'] = m_head.group(1).strip()
# Heuristic for description if still missing
if not metadata.get('description') and raw_text:
text_lines = [
l.strip() for l in raw_text.splitlines()
if l.strip() and not l.strip().startswith(('#', '---', '<', '```'))
]
if text_lines:
metadata['description'] = text_lines[0][:220]
# Category heuristic: infer from directory hierarchy (e.g. repo/Category/SkillFolder/SKILL.md)
if not metadata.get('category') and repo_path:
try:
rel_parts = item_path.relative_to(repo_path).parts
if len(rel_parts) > 1:
metadata['category'] = rel_parts[0].replace('_', ' ').replace('-', ' ').title()
except Exception:
pass
# Extract license
detected_license = _extract_license_from_path(item_path, repo_path=repo_path)
if detected_license:
metadata['license'] = detected_license
# Safe defaults
base_name = item_path.stem if item_path.is_file() else item_path.name
if not metadata.get('name'):
metadata['name'] = base_name.replace('_', ' ').replace('-', ' ').title()
if not metadata.get('category'):
metadata['category'] = 'Development'
if not metadata.get('description'):
metadata['description'] = f"AI skill for {metadata['name']}"
if not metadata.get('version'):
metadata['version'] = '1.0.0'
if not metadata.get('author'):
metadata['author'] = 'Community'
metadata['content'] = content or raw_text
return metadata
def parse_item_metadata(item_path: Path, item_type: ITEM_TYPES, repo_path: Optional[Path] = None) -> Dict[str, Any]:
metadata = {}
if item_type == 'personality':
desc_path = item_path / "description.yaml"
conf_path = item_path / "config.yaml"
if desc_path.exists():
with open(desc_path, 'r', encoding='utf-8') as f:
metadata = yaml.safe_load(f) or {}
if 'description' not in metadata and 'personality_description' in metadata:
metadata['description'] = metadata['personality_description']
if 'prompt_text' not in metadata and 'personality_conditioning' in metadata:
metadata['prompt_text'] = metadata['personality_conditioning']
elif conf_path.exists():
with open(conf_path, 'r', encoding='utf-8') as f:
legacy_data = yaml.safe_load(f) or {}
metadata = {
'name': legacy_data.get('name'), 'version': str(legacy_data.get('version', 'N/A')),
'author': legacy_data.get('author'), 'category': legacy_data.get('category'),
'description': legacy_data.get('personality_description') or legacy_data.get('description'),
'prompt_text': legacy_data.get('personality_conditioning') or legacy_data.get('prompt_text'),
'disclaimer': legacy_data.get('disclaimer'), 'tools': legacy_data.get('dependencies', [])
}
elif item_type == 'skill':
return _parse_skill_metadata(item_path, repo_path=repo_path)
else: # app, mcp, prompt
config_path = item_path / "description.yaml"
if config_path.exists():
with open(config_path, 'r', encoding='utf-8') as f:
metadata = yaml.safe_load(f) or {}
return metadata
DEFAULT_SEED_SKILLS = [
{
"folder": "python_refactoring_specialist",
"yaml": {
"name": "Python Refactoring Specialist",
"author": "ParisNeo",
"version": "1.0.0",
"category": "Coding",
"description": "Guides the model to refactor and optimize Python code using SOLID, DRY, KISS principles with clean type annotations."
},
"xml": """<skill name="python_refactoring_specialist">
<description>Refactor and optimize Python code adhering to PEP 8, clean architecture, and type safety.</description>
<instructions>
1. Always audit code for algorithmic complexity and memory leaks.
2. Enforce strict type compliance and comprehensive docstrings.
3. Eliminate dead code, unused imports, and mutable default arguments.
4. Provide production-ready, clean implementations without placeholders.
</instructions>
</skill>"""
},
{
"folder": "fastapi_security_auditor",
"yaml": {
"name": "FastAPI Security Auditor",
"author": "ParisNeo",
"version": "1.0.0",
"category": "Security",
"description": "Security directives to audit FastAPI endpoints against SSRF, IDOR, path traversal, injection, and broken authorization."
},
"xml": """<skill name="fastapi_security_auditor">
<description>Zero-trust security verification for FastAPI backend services.</description>
<instructions>
1. Check all user-controlled path parameters and filenames with secure_filename and is_relative_to().
2. Guard against SSRF by validating all outgoing request IPs against private/loopback/metadata blocks.
3. Enforce object-level permissions (IDOR) on all read/write endpoints.
4. Sanitize all markdown, HTML, and string outputs to eliminate stored XSS.
</instructions>
</skill>"""
},
{
"folder": "latex_typesetter",
"yaml": {
"name": "LaTeX Document Expert",
"author": "ParisNeo",
"version": "1.0.0",
"category": "Documentation",
"description": "Produces clean, compilable, publication-grade LaTeX documents, equations, and tables."
},
"xml": """<skill name="latex_typesetter">
<description>Publication-grade LaTeX authoring directives.</description>
<instructions>
1. Structure documents with standard preamble, babel/fontenc, and geometry configurations.
2. Write robust equation blocks using amsmath and cleveref for cross-referencing.
3. Output self-contained LaTeX blocks that compile without missing external packages.
</instructions>
</skill>"""
},
{
"folder": "code_reviewer",
"yaml": {
"name": "Senior Code Reviewer",
"author": "ParisNeo",
"version": "1.0.0",
"category": "Coding",
"description": "Performs methodical, senior-level code reviews analyzing correctness, edge cases, and performance bottlenecks."
},
"xml": """<skill name="code_reviewer">
<description>Senior engineering code review directives.</description>
<instructions>
1. Identify edge cases (empty inputs, race conditions, timeout handling).
2. Assess performance, memory consumption, and concurrency patterns.
3. Provide actionable suggestions with concrete code snippets.
</instructions>
</skill>"""
},
{
"folder": "markdown_technical_writer",
"yaml": {
"name": "Markdown Technical Writer",
"author": "ParisNeo",
"version": "1.0.0",
"category": "Writing",
"description": "Produces publication-ready technical manuals, API guides, and architectural design records."
},
"xml": """<skill name="markdown_technical_writer">
<description>High-density technical writing and architectural documentation.</description>
<instructions>
1. Structure content logically with executive summaries, visual diagrams, and code snippets.
2. Use GitHub Flavored Markdown tables, callouts, and clean formatting.
3. Avoid conversational filler and focus on actionable technical substance.
</instructions>
</skill>"""
}
]
def ensure_skills_zoo_seeded(repo_path: Path):
"""Seeds default skills if a skill repository directory is empty."""
try:
repo_path.mkdir(parents=True, exist_ok=True)
# Check if user already has skills (including SKILL.md or .xml)
existing_skills = (
list(repo_path.glob("**/description.yaml")) +
list(repo_path.glob("**/skill.xml")) +
list(repo_path.glob("**/*skill*.md")) +
list(repo_path.glob("**/*SKILL*.md"))
)
if not existing_skills and not any(repo_path.iterdir()):
for item in DEFAULT_SEED_SKILLS:
folder = repo_path / item["folder"]
folder.mkdir(parents=True, exist_ok=True)
with open(folder / "description.yaml", "w", encoding="utf-8") as f:
yaml.dump(item["yaml"], f)
with open(folder / "skill.xml", "w", encoding="utf-8") as f:
f.write(item["xml"])
with open(folder / "README.md", "w", encoding="utf-8") as f:
f.write(f"# {item['yaml']['name']}\n\n{item['yaml']['description']}\n")
except Exception as e:
print(f"Warning: Could not seed default skills: {e}")
def _resolve_repo_path(repo: Any, zoo_root: Path) -> Optional[Path]:
"""
Safely resolves the filesystem path for a repository, preventing Windows WinError 123
syntax errors when a URL is stored as repo.name.
"""
candidates = []
# 1. Local path if specified
if getattr(repo, 'type', '') == 'local' and repo.url:
try:
lp = Path(repo.url)
if lp.is_dir():
return lp
except Exception:
pass
# 2. Extract clean slug from name and url
for raw in [getattr(repo, 'name', None), getattr(repo, 'url', None)]:
if not raw:
continue
clean = raw.rstrip('/\\').split('/')[-1].split('\\')[-1]
if clean.endswith('.git'):
clean = clean[:-4]
if ':' not in clean and clean not in candidates:
candidates.append(clean)
candidates.append("lollms_skills_zoo")
for cand in candidates:
try:
target = zoo_root / cand
if target.is_dir():
return target
except OSError:
continue
# 3. Fallback: check any existing directory inside zoo_root
try:
existing_dirs = [d for d in zoo_root.iterdir() if d.is_dir() and not d.name.startswith('.')]
if existing_dirs:
return existing_dirs[0]
except Exception:
pass
return None
def _build_cache_for_type(db: Session, item_type: ITEM_TYPES) -> List[Dict[str, Any]]:
items = []
zoo_root = get_zoo_root_path(item_type)
repo_model = get_db_repo_model(item_type)
repositories = db.query(repo_model).all()
repos_to_scan = []
for repo in repositories:
resolved = _resolve_repo_path(repo, zoo_root)
if resolved:
repos_to_scan.append((repo.name, resolved))
# Also discover any unindexed directories directly inside zoo_root
try:
if zoo_root.is_dir():
for d in zoo_root.iterdir():
if d.is_dir() and not d.name.startswith('.') and not any(r[1] == d for r in repos_to_scan):
repos_to_scan.append((d.name, d))
except Exception:
pass
for repo_name, repo_path in repos_to_scan:
if item_type == 'skill':
if not repo_path.is_dir() or not any(repo_path.iterdir()):
ensure_skills_zoo_seeded(repo_path)
if not repo_path or not repo_path.is_dir():
continue
# Specialized discovery for skills (handles category directories, skill subfolders with SKILL.md, and raw root files)
if item_type == 'skill':
seen_rel_paths = set()
# 1. Discover folder-based skills (containing SKILL.md, skill.md, skill.xml, or description.yaml)
folder_candidates = set()
for root, dirs, files in os.walk(str(repo_path)):
dirs[:] = [d for d in dirs if not d.startswith('.')]
p_dir = Path(root)
lower_files = {f.lower(): f for f in files}
if any(m in lower_files for m in ['skill.md', 'skill.xml', 'description.yaml', 'description.yml']):
folder_candidates.add(p_dir)
for folder in sorted(list(folder_candidates), key=lambda x: str(x)):
rel_path = folder.relative_to(repo_path).as_posix()
if rel_path in seen_rel_paths:
continue
seen_rel_paths.add(rel_path)
try:
metadata = _parse_skill_metadata(folder, repo_path=repo_path)
icon_path = next((p for p in [folder / "icon.png", folder / "assets" / "logo.png"] if p.exists()), None)
icon_b64 = None
if icon_path:
with open(icon_path, 'rb') as f:
icon_b64 = f"data:image/png;base64,{base64.b64encode(f.read()).decode('utf-8')}"
items.append(_sanitize_for_json({
**metadata,
'item_type': 'skill',
'repository': repo.name,
'folder_name': rel_path,
'icon': icon_b64
}))
except Exception as e:
print(f"Warning: Could not process skill folder '{folder}' in repo '{repo.name}': {e}")
# 2. Discover loose / raw standalone skill files (.md or .xml) not inside a registered folder
ignore_names = {'readme.md', 'license.md', 'contributing.md', 'changelog.md', 'code_of_conduct.md', 'todo.md', 'pom.xml'}
for root, dirs, files in os.walk(str(repo_path)):
dirs[:] = [d for d in dirs if not d.startswith('.')]
p_dir = Path(root)
if any(folder == p_dir or folder in p_dir.parents for folder in folder_candidates):
continue
for f in sorted(files):
f_lower = f.lower()
if f_lower in ignore_names or f.startswith('.'):
continue
if f_lower.endswith('.md') or f_lower.endswith('.xml'):
file_path = p_dir / f
rel_file = file_path.relative_to(repo_path).as_posix()
if rel_file in seen_rel_paths:
continue
seen_rel_paths.add(rel_file)
try:
metadata = _parse_skill_metadata(file_path, repo_path=repo_path)
items.append(_sanitize_for_json({
**metadata,
'item_type': 'skill',
'repository': repo.name,
'folder_name': rel_file,
'icon': None
}))
except Exception as e:
print(f"Warning: Could not process raw skill file '{file_path}' in repo '{repo.name}': {e}")
continue
# Standard discovery for apps, mcps, personalities, prompts
candidate_folders = set()
for p in repo_path.glob('**/description.yaml'):
candidate_folders.add(p.parent)
if item_type == 'personality':
for p in repo_path.glob('**/config.yaml'):
candidate_folders.add(p.parent)
for item_folder in sorted(list(candidate_folders), key=lambda x: x.name):
try:
metadata = parse_item_metadata(item_folder, item_type)
if not metadata or not metadata.get('name'):
metadata['name'] = item_folder.name.replace('_', ' ').title()
if 'category' in metadata and isinstance(metadata['category'], list):
metadata['category'] = metadata['category'][0] if metadata['category'] else 'Generic'
if item_type == 'personality' and (item_folder / "scripts" / "processor.py").exists():
metadata['is_legacy_scripted'] = True
if item_type in ['app', 'mcp']:
metadata['has_dot_env_config'] = (item_folder / ".env.example").exists()
icon_path = next((p for p in [item_folder / "icon.png", item_folder / "assets" / "logo.png"] if p.exists()), None)
icon_b64 = None
if icon_path:
with open(icon_path, 'rb') as f:
icon_b64 = f"data:image/png;base64,{base64.b64encode(f.read()).decode('utf-8')}"
folder_name_rel = item_folder.relative_to(repo_path).as_posix()
items.append(_sanitize_for_json({
**metadata,
'item_type': item_type,
'repository': repo.name,
'folder_name': folder_name_rel,
'icon': icon_b64
}))
except Exception as e:
print(f"Warning: Could not process {item_type} item at '{item_folder}' in repo '{repo.name}': {e}")
return items
def force_build_full_cache():
"""Builds the cache from scratch, typically for a manual refresh."""
global _cache
lock = FileLock(str(CACHE_LOCK_FILE), timeout=CACHE_LOCK_TIMEOUT)
try:
with lock:
ASCIIColors.info("INFO: Forcing a full rebuild of the Zoo cache...")
db = next(get_db())
try:
_cache["data"] = {
'app': _build_cache_for_type(db, 'app'),
'mcp': _build_cache_for_type(db, 'mcp'),
'prompt': _build_cache_for_type(db, 'prompt'),
'personality': _build_cache_for_type(db, 'personality'),
'skill': _build_cache_for_type(db, 'skill'),
}
_cache["timestamp"] = time.time()
with open(CACHE_FILE, 'w') as f:
json.dump(_cache, f)
ASCIIColors.green("INFO: Zoo cache rebuild complete.")
finally:
db.close()
except Timeout:
ASCIIColors.warning("Could not acquire lock for force cache build. Another process might be building it.")
def refresh_repo_cache(repo_name: str, item_type: ITEM_TYPES):
global _cache
lock = FileLock(str(CACHE_LOCK_FILE), timeout=60)
try:
with lock:
if not _cache.get("data"):
load_cache()
_cache["data"][item_type] = [item for item in _cache.get("data", {}).get(item_type, []) if item.get('repository') != repo_name]
db = next(get_db())
try:
repo = db.query(get_db_repo_model(item_type)).filter_by(name=repo_name).first()
if repo:
repo_path = Path(repo.url) if repo.type == 'local' else get_zoo_root_path(item_type) / repo.name
if repo_path.is_dir():
config_files = list(repo_path.glob('**/description.yaml'))
if item_type == 'personality': config_files.extend(list(repo_path.glob('**/config.yaml')))
elif item_type == 'skill': config_files.extend(list(repo_path.glob('**/skill.xml')))
for config_file in config_files:
item_folder = config_file.parent
try:
metadata = parse_item_metadata(item_folder, item_type)
if not metadata.get('name'): metadata['name'] = item_folder.name
if 'category' in metadata and isinstance(metadata['category'], list):
metadata['category'] = metadata['category'][0] if metadata['category'] else 'Uncategorized'
if item_type == 'personality' and (item_folder / "scripts" / "processor.py").exists():
metadata['is_legacy_scripted'] = True
if item_type in ['app', 'mcp']:
metadata['has_dot_env_config'] = (item_folder / ".env.example").exists()
icon_path = next((p for p in [item_folder / "icon.png", item_folder / "assets" / "logo.png"] if p.exists()), None)
icon_b64 = f"data:image/png;base64,{base64.b64encode(icon_path.read_bytes()).decode('utf-8')}" if icon_path else None
folder_name_rel = item_folder.relative_to(repo_path).as_posix()
_cache["data"][item_type].append(_sanitize_for_json({**metadata, 'item_type': item_type, 'repository': repo.name, 'folder_name': folder_name_rel, 'icon': icon_b64}))
except Exception as e:
print(f"Warning: Could not process item '{item_folder}' on refresh: {e}")
_cache["timestamp"] = time.time()
with open(CACHE_FILE, 'w') as f: json.dump(_cache, f)
finally:
db.close()
except Timeout:
ASCIIColors.warning(f"Could not acquire lock to refresh repo '{repo_name}'. Cache may be slightly stale.")
def load_cache():
"""Loads the cache from file if it exists, otherwise builds it."""
global _cache
required_keys = {'app', 'mcp', 'prompt', 'personality', 'skill'}
if _cache.get("data") and required_keys.issubset(_cache.get("data", {}).keys()):
return
if CACHE_FILE.exists():
try:
with open(CACHE_FILE, 'r') as f:
disk_cache = json.load(f)
if disk_cache.get("data") and required_keys.issubset(disk_cache.get("data", {}).keys()):
_cache = disk_cache
return
except (json.JSONDecodeError, TypeError):
ASCIIColors.warning("Cache file is corrupted, rebuilding.")
# Rebuild if not in memory, file doesn't exist, or missing required keys like 'skill'
force_build_full_cache()
def refresh_item_type_cache(item_type: ITEM_TYPES):
"""Targeted refresh of a single modality without locking the entire universe."""
global _cache
db = next(get_db())
try:
if not _cache.get("data"):
_cache["data"] = {}
_cache["data"][item_type] = _build_cache_for_type(db, item_type)
_cache["timestamp"] = time.time()
try:
with open(CACHE_FILE, 'w') as f:
json.dump(_cache, f)
except Exception:
pass
finally:
db.close()
def get_all_items(item_type: ITEM_TYPES) -> List[Dict[str, Any]]:
load_cache()
items = _cache.get("data", {}).get(item_type)
if items is None or (item_type == 'skill' and len(items) == 0):
refresh_item_type_cache(item_type)
items = _cache.get("data", {}).get(item_type, [])
return items or []
def get_all_categories(item_type: ITEM_TYPES) -> List[str]:
items = get_all_items(item_type)
categories = {'All'}
for item in items:
if item.get('category'):
categories.add(item['category'])
return sorted(list(categories))

Xet Storage Details

Size:
31.9 kB
·
Xet hash:
f05ed5e3a1c9fb5d59bb24ce47624d89e19d86b568a498bfe6711ebdb534fc76

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.