File size: 2,362 Bytes
501bc8c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | import json
import logging
from pathlib import Path
from typing import List, Dict, Any, Callable, Optional
from .config import DATA_DIR
logger = logging.getLogger("DataLoader")
INTERNAL_REPORT_PATH = DATA_DIR / 'research_report.json'
BAYMARD_PATH = DATA_DIR / 'baymard_unified_database_v2.json'
NNG_PATH = DATA_DIR / 'nng_articles.json'
def _load_json_file(
path: Path,
label: str,
flatten_fn: Optional[Callable[[Any], List[Dict[str, Any]]]] = None,
) -> List[Dict[str, Any]]:
"""Load a JSON file with standard error handling.
Args:
path: Path to the JSON file.
label: Human-readable label for log messages.
flatten_fn: Optional function to transform raw data into a flat list.
If None, the raw data is returned as-is (must already be a list).
"""
try:
with open(path, 'r', encoding='utf-8') as f:
data = json.load(f)
items = flatten_fn(data) if flatten_fn else data
logger.info(f"Loaded {len(items)} {label} items.")
return items
except FileNotFoundError:
logger.error(f"{label} file not found: {path}")
return []
except json.JSONDecodeError as e:
logger.error(f"Invalid JSON in {label} file: {e}")
return []
def _flatten_collections(data: Any) -> List[Dict[str, Any]]:
"""Flatten {collections: {name: {guidelines: [...]}}} structure."""
items: List[Dict[str, Any]] = []
if 'collections' in data:
for _name, collection_data in data['collections'].items():
if 'guidelines' in collection_data:
items.extend(collection_data['guidelines'])
return items
def load_internal_research() -> List[Dict[str, Any]]:
"""Loads internal research reports from JSON."""
return _load_json_file(INTERNAL_REPORT_PATH, "internal research")
def load_baymard_guidelines() -> List[Dict[str, Any]]:
"""Loads Baymard guidelines from JSON."""
return _load_json_file(BAYMARD_PATH, "Baymard", flatten_fn=_flatten_collections)
def load_nng_articles() -> List[Dict[str, Any]]:
"""Loads NN/g articles from JSON."""
return _load_json_file(NNG_PATH, "NN/g", flatten_fn=_flatten_collections)
if __name__ == "__main__":
# Test loading
internal = load_internal_research()
baymard = load_baymard_guidelines()
nng = load_nng_articles()
|