auto / database /seeds.py
rsnarsna
feat(alli-backend): Wave-1 standardization, standalone sys.env node & vault crypto, and DEC-014 security hardening
8628c98
Raw History Blame Contribute Delete
11 kB
"""
ALLI Backend - Database Seed Loaders
Loads canonical JSON specs from alli_backend/specs/ and idempotently seeds MongoDB collections.
"""
import copy
import json
from functools import lru_cache
from pathlib import Path
from typing import Any, Dict, List
BACKEND_ROOT = Path(__file__).resolve().parent.parent
SPECS_DIR = BACKEND_ROOT / "specs"
VALIDATION_REPORT_PATH = BACKEND_ROOT / "node_validation_lab" / "reports" / "phase3_validation_report.json"
INVENTORY_AUDIT_PATH = BACKEND_ROOT / "node_inventory_audit.json"
@lru_cache(maxsize=16)
def _load_json_cached(path_str: str) -> Any:
path = Path(path_str)
if not path.is_file():
return None
with open(path, "r", encoding="utf-8") as f:
return json.load(f)
def _load_spec(filename: str, default: Any) -> Any:
data = _load_json_cached(str(SPECS_DIR / filename))
return copy.deepcopy(data) if data is not None else copy.deepcopy(default)
def get_default_packages() -> List[Dict[str, Any]]:
return _load_spec("packages.json", [])
def get_default_core_specs() -> List[Dict[str, Any]]:
return _load_spec("core_specs.json", [])
def get_default_studio_palette() -> List[Dict[str, Any]]:
return _load_spec("studio_palette_categories.json", [])
def get_default_starter_templates() -> List[Dict[str, Any]]:
return _load_spec("starter_templates.json", [])
def get_default_plans() -> List[Dict[str, Any]]:
return _load_spec("plans.json", [])
def get_default_python_fundamentals() -> Dict[str, Any]:
return _load_spec("python_fundamentals.json", {"version": "1.0.0", "name": "Python Fundamentals Specification", "constructs": []})
def get_raw_programming_fundamentals() -> List[Dict[str, Any]]:
return _load_spec("programming_fundamentals_11_categories.json", [])
def _palette_sig(cats: Any) -> tuple:
ids, ntypes, total = set(), set(), 0
def _walk(items: Any) -> None:
nonlocal total
for it in (items if isinstance(items, list) else ()):
if isinstance(it, dict):
total += 1
if it.get("id"): ids.add(str(it["id"]))
if it.get("nodeType"): ntypes.add(str(it["nodeType"]))
_walk(it.get("children"))
for c in (cats if isinstance(cats, list) else ()):
if isinstance(c, dict): _walk(c.get("items"))
return (len(cats) if isinstance(cats, list) else 0, total, frozenset(ids), frozenset(ntypes))
def is_studio_palette_stale(stored: Any, canonical: Any = None) -> bool:
"""Return True when persisted primary_studio_categories is stale vs specs/studio_palette_categories.json."""
canon = canonical if canonical is not None else get_default_studio_palette()
if not isinstance(stored, list):
return True
s_sig, c_sig = _palette_sig(stored), _palette_sig(canon)
return s_sig[0] != c_sig[0] or s_sig[1] != c_sig[1] or not c_sig[2].issubset(s_sig[2]) or not c_sig[3].issubset(s_sig[3]) or stored != canon
def consolidate_fundamentals_to_9(raw_categories: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""
Consolidate the 11 educational fundamentals categories into the 9 canonical Studio
categories while preserving 100% of the 53 topics, functions, rules, and quizzes.
"""
if not raw_categories:
return []
by_id = {c.get("id"): copy.deepcopy(c) for c in raw_categories if isinstance(c, dict)}
essentials = by_id.pop("cat_essentials", None)
modules = by_id.pop("cat_modules", None)
ordered_ids = [
"cat_variables", "cat_data_types", "cat_operators", "cat_math",
"cat_control_flow", "cat_functions", "cat_strings_collections", "cat_oop", "cat_files",
]
consolidated: List[Dict[str, Any]] = []
for idx, cid in enumerate(ordered_ids, start=1):
cat = by_id.pop(cid, None)
if not cat:
continue
if cid == "cat_variables" and essentials:
for k in ("topics", "functions", "learningObjectives"):
cat[k] = list(essentials.get(k, [])) + list(cat.get(k, []))
cat["mergedCategoryIds"] = ["cat_essentials", "cat_variables"]
elif cid == "cat_files" and modules:
for k in ("topics", "functions", "learningObjectives"):
cat[k] = list(cat.get(k, [])) + list(modules.get(k, []))
cat["mergedCategoryIds"] = ["cat_files", "cat_modules"]
for t_idx, topic in enumerate(cat.get("topics", []), start=1):
topic["topicNumber"] = t_idx
cat["categoryNumber"] = idx
cat["expectedTopicCount"] = len(cat.get("topics", []))
consolidated.append(cat)
return consolidated
def get_default_learning_modules() -> List[Dict[str, Any]]:
return _load_spec("learning_modules.json", [])
def get_default_blog_data() -> Dict[str, Any]:
return _load_spec("blog_posts.json", {"categories": [], "posts": []})
def get_default_validation_artifacts() -> Dict[str, Any]:
report = _load_json_cached(str(VALIDATION_REPORT_PATH)) or {}
inventory = _load_json_cached(str(INVENTORY_AUDIT_PATH)) or {}
return {"report": copy.deepcopy(report), "inventory_metadata": copy.deepcopy(inventory.get("audit_metadata", {})), "inventory": copy.deepcopy(inventory.get("inventory", []))}
def seed_all_collections(repo: Any) -> Dict[str, Any]:
"""
Idempotently seed MongoDB collections from backend specs when DB is connected.
Never blocks startup if DB is offline.
"""
if not getattr(repo, "is_connected", False):
return {"seeded": False, "reason": "offline_inmemory_fallback"}
seeded_counts: Dict[str, int] = {}
try:
from database.connection import get_utc_now
now = get_utc_now()
# 1. Core specs & primary studio palette
core_col = repo.get_collection("core_definitions")
if core_col is not None:
if not core_col.find_one({"key": "python_core_categories"}):
core_col.update_one(
{"key": "python_core_categories"},
{"$set": {"key": "python_core_categories", "categories": get_default_core_specs(), "updated_at": now}},
upsert=True,
)
canonical_palette = get_default_studio_palette()
existing_palette = core_col.find_one({"key": "primary_studio_categories"}, {"_id": 0})
if not existing_palette or is_studio_palette_stale(existing_palette.get("categories"), canonical_palette):
core_col.update_one(
{"key": "primary_studio_categories"},
{"$set": {"key": "primary_studio_categories", "categories": canonical_palette, "updated_at": now}},
upsert=True,
)
seeded_counts["core_definitions"] = 2
# 2. Python fundamentals (schema + 9 consolidated categories / 53 topics)
fund_col = repo.get_collection("python_fundamentals")
if fund_col is not None:
if not fund_col.find_one({"key": "python_fundamentals_schema"}):
py_fund = get_default_python_fundamentals()
fund_col.update_one(
{"key": "python_fundamentals_schema"},
{"$set": {**py_fund, "key": "python_fundamentals_schema", "updated_at": now}},
upsert=True,
)
raw_cats = get_raw_programming_fundamentals()
consolidated = consolidate_fundamentals_to_9(raw_cats)
if not fund_col.find_one({"key": "programming_fundamentals_categories"}):
fund_col.update_one(
{"key": "programming_fundamentals_categories"},
{
"$set": {
"key": "programming_fundamentals_categories",
"categories": consolidated,
"legacy_categories": raw_cats,
"total_categories": len(consolidated),
"total_topics": sum(len(c.get("topics", [])) for c in consolidated),
"updated_at": now,
}
},
upsert=True,
)
seeded_counts["python_fundamentals"] = len(consolidated)
# 3. Packages / default blueprints
bp_col = repo.get_collection("blueprints")
if bp_col is not None:
pkgs = get_default_packages()
for pkg in pkgs:
if not bp_col.find_one({"package": pkg["name"].lower()}):
bp_col.update_one(
{"package": pkg["name"].lower()},
{"$set": {"package": pkg["name"].lower(), "blueprint": pkg, "is_default_package": True, "updated_at": now}},
upsert=True,
)
seeded_counts["packages"] = len(pkgs)
# 4. Templates, Plans, Learning Modules, Blog Posts
tmpl_col = repo.get_collection("templates")
if tmpl_col is not None:
tmpls = get_default_starter_templates()
for t in tmpls:
if not tmpl_col.find_one({"id": t["id"]}):
tmpl_col.update_one({"id": t["id"]}, {"$set": t}, upsert=True)
seeded_counts["templates"] = len(tmpls)
plans_col = repo.get_collection("plans")
if plans_col is not None:
plans = get_default_plans()
for p in plans:
plans_col.update_one({"id": p["id"]}, {"$set": p}, upsert=True)
seeded_counts["plans"] = len(plans)
learn_col = repo.get_collection("learning_modules")
if learn_col is not None:
lessons = get_default_learning_modules()
for lesson in lessons:
if not learn_col.find_one({"id": lesson["id"]}):
learn_col.update_one({"id": lesson["id"]}, {"$set": lesson}, upsert=True)
seeded_counts["learning_modules"] = len(lessons)
blog_col = repo.get_collection("blog_posts")
if blog_col is not None:
blog_data = get_default_blog_data()
for post in blog_data.get("posts", []):
if not blog_col.find_one({"slug": post["slug"]}):
blog_col.update_one({"slug": post["slug"]}, {"$set": post}, upsert=True)
seeded_counts["blog_posts"] = len(blog_data.get("posts", []))
# 5. Canonical node schemas
from language_core.node_schema import default_registry
schemas_col = repo.get_collection("node_schemas")
if schemas_col is not None:
schemas = default_registry.to_json_dict()
for s in schemas:
schemas_col.update_one({"type": s["type"]}, {"$set": s}, upsert=True)
seeded_counts["node_schemas"] = len(schemas)
return {"seeded": True, "counts": seeded_counts}
except Exception as e:
print(f"[ALLI Seeds] Non-fatal seed error: {e}")
return {"seeded": False, "error": str(e)}