Download core/utils/json_helpers.py from Kyrlos11/nniback: direct link, hf CLI and curl.
- Browser
- Download file 11.7 kB
-
https://huggingface.co/spaces/Kyrlos11/nniback/resolve/main/core/utils/json_helpers.py
- Command line
-
hf download hf://spaces/Kyrlos11/nniback/core/utils/json_helpers.py
-
curl -L -o json_helpers.py https://huggingface.co/spaces/Kyrlos11/nniback/resolve/main/core/utils/json_helpers.py
11.7 kB
| """ | |
| JSON helper utilities for handling both legacy (string) and new (dict/list) formats. | |
| These utilities help with the transition from storing JSON as strings to storing | |
| them as proper JSONB objects in the database. | |
| """ | |
| import json | |
| import re | |
| import logging | |
| from typing import Any, Union, Dict, List, Tuple | |
| # Use orjson for hot paths (3-5x faster than stdlib json) | |
| try: | |
| import orjson | |
| _HAS_ORJSON = True | |
| except ImportError: | |
| _HAS_ORJSON = False | |
| logger = logging.getLogger(__name__) | |
| def repair_json(json_string: str) -> Tuple[str, bool]: | |
| """ | |
| Attempt to repair malformed JSON that LLMs sometimes produce. | |
| Common LLM JSON mistakes this handles: | |
| - Missing closing brackets/braces (e.g., `[{"a": 1}, {"b": 2]` -> `[{"a": 1}, {"b": 2}]`) | |
| - Trailing commas (e.g., `[1, 2, 3,]` -> `[1, 2, 3]`) | |
| - Missing commas between array elements (e.g., `[{"a":1} {"b":2}]` -> `[{"a":1}, {"b":2}]`) | |
| Args: | |
| json_string: The potentially malformed JSON string | |
| Returns: | |
| Tuple of (repaired_string, was_repaired) | |
| """ | |
| if not isinstance(json_string, str): | |
| return json_string, False | |
| original = json_string | |
| repaired = json_string.strip() | |
| was_repaired = False | |
| # First, try parsing as-is | |
| try: | |
| json.loads(repaired) | |
| return repaired, False | |
| except json.JSONDecodeError: | |
| pass | |
| # Fix 1: Remove trailing commas before ] or } | |
| # Pattern: comma followed by optional whitespace, then ] or } | |
| trailing_comma_pattern = r',(\s*[\]\}])' | |
| if re.search(trailing_comma_pattern, repaired): | |
| repaired = re.sub(trailing_comma_pattern, r'\1', repaired) | |
| was_repaired = True | |
| # Fix 2: Add missing commas between array elements or object properties | |
| # Pattern: } followed by optional whitespace, then { (missing comma) | |
| missing_comma_pattern = r'\}(\s*)\{' | |
| if re.search(missing_comma_pattern, repaired): | |
| repaired = re.sub(missing_comma_pattern, r'},\1{', repaired) | |
| was_repaired = True | |
| # Pattern: ] followed by optional whitespace, then [ (missing comma between arrays) | |
| missing_comma_array_pattern = r'\](\s*)\[' | |
| if re.search(missing_comma_array_pattern, repaired): | |
| repaired = re.sub(missing_comma_array_pattern, r'],\1[', repaired) | |
| was_repaired = True | |
| # Fix 3: Balance brackets/braces - count and add missing ones | |
| # This handles the specific case: `[{...}, {...], {...]` -> `[{...}, {...}]` | |
| open_braces = repaired.count('{') | |
| close_braces = repaired.count('}') | |
| open_brackets = repaired.count('[') | |
| close_brackets = repaired.count(']') | |
| # Add missing closing braces | |
| if open_braces > close_braces: | |
| # Find position to insert - look for patterns like `], {` that should be `}], {` | |
| # or at the end of the string | |
| missing_braces = open_braces - close_braces | |
| # Try to find where braces are missing by checking for ], or ] at end | |
| # Common pattern: `["task1", "task2"], {` should be `["task1", "task2"]}, {` | |
| # Look for `],` or `]` followed by `,` or `}` or end - missing `}` before | |
| fixed = False | |
| for i in range(len(repaired) - 1, 0, -1): | |
| if repaired[i] == ']': | |
| # Check if there's a missing } before this ] | |
| # Count braces up to this point | |
| temp_open = repaired[:i+1].count('{') | |
| temp_close = repaired[:i+1].count('}') | |
| if temp_open > temp_close: | |
| # Insert missing }s before the ] | |
| needed = temp_open - temp_close | |
| repaired = repaired[:i] + '}' * needed + repaired[i:] | |
| was_repaired = True | |
| fixed = True | |
| break | |
| if not fixed: | |
| # Just append at end | |
| repaired = repaired + '}' * missing_braces | |
| was_repaired = True | |
| # Add missing closing brackets | |
| if open_brackets > close_brackets: | |
| missing_brackets = open_brackets - close_brackets | |
| repaired = repaired + ']' * missing_brackets | |
| was_repaired = True | |
| # Validate the repair worked | |
| try: | |
| json.loads(repaired) | |
| if was_repaired: | |
| logger.info(f"🔧 [JSON REPAIR] Successfully repaired malformed JSON") | |
| logger.debug(f"🔧 [JSON REPAIR] Original (first 200 chars): {original[:200]}") | |
| logger.debug(f"🔧 [JSON REPAIR] Repaired (first 200 chars): {repaired[:200]}") | |
| return repaired, was_repaired | |
| except json.JSONDecodeError as e: | |
| # Repair failed, return original | |
| logger.debug(f"🔧 [JSON REPAIR] Repair attempt failed: {e}") | |
| return original, False | |
| def safe_json_parse_with_repair(value: Union[str, Dict, List, Any], default: Any = None) -> Any: | |
| """ | |
| Safely parse JSON with automatic repair of common LLM mistakes. | |
| This is an enhanced version of safe_json_parse that attempts to repair | |
| malformed JSON before giving up. | |
| Args: | |
| value: The value to parse | |
| default: Default value if parsing fails | |
| Returns: | |
| Parsed value or default | |
| """ | |
| if value is None: | |
| return default | |
| # If it's already a dict or list, return as-is | |
| if isinstance(value, (dict, list)): | |
| return value | |
| # If it's a string, try to parse it | |
| if isinstance(value, str): | |
| # First, try normal parsing | |
| try: | |
| return json.loads(value) | |
| except json.JSONDecodeError: | |
| pass | |
| # Try repair | |
| repaired, was_repaired = repair_json(value) | |
| if was_repaired: | |
| try: | |
| return json.loads(repaired) | |
| except json.JSONDecodeError: | |
| pass | |
| # If repair didn't work, return the string itself or default | |
| return value if default is None else default | |
| # For any other type, return as-is | |
| return value | |
| def ensure_dict(value: Union[str, Dict[str, Any], None], default: Dict[str, Any] = None) -> Dict[str, Any]: | |
| """ | |
| Ensure a value is a dictionary. | |
| Handles: | |
| - None -> returns default or {} | |
| - Dict -> returns as-is | |
| - JSON string -> parses and returns dict | |
| - Other -> returns default or {} | |
| Args: | |
| value: The value to ensure is a dict | |
| default: Default value if conversion fails | |
| Returns: | |
| A dictionary | |
| """ | |
| if default is None: | |
| default = {} | |
| if value is None: | |
| return default | |
| if isinstance(value, dict): | |
| return value | |
| if isinstance(value, str): | |
| try: | |
| parsed = json.loads(value) | |
| if isinstance(parsed, dict): | |
| return parsed | |
| return default | |
| except (json.JSONDecodeError, TypeError): | |
| return default | |
| return default | |
| def ensure_list(value: Union[str, List[Any], None], default: List[Any] = None) -> List[Any]: | |
| """ | |
| Ensure a value is a list. | |
| Handles: | |
| - None -> returns default or [] | |
| - List -> returns as-is | |
| - JSON string -> parses and returns list | |
| - Other -> returns default or [] | |
| Args: | |
| value: The value to ensure is a list | |
| default: Default value if conversion fails | |
| Returns: | |
| A list | |
| """ | |
| if default is None: | |
| default = [] | |
| if value is None: | |
| return default | |
| if isinstance(value, list): | |
| return value | |
| if isinstance(value, str): | |
| try: | |
| parsed = json.loads(value) | |
| if isinstance(parsed, list): | |
| return parsed | |
| return default | |
| except (json.JSONDecodeError, TypeError): | |
| return default | |
| return default | |
| def safe_json_parse(value: Union[str, Dict, List, Any], default: Any = None) -> Any: | |
| """ | |
| Safely parse a value that might be JSON string or already parsed. | |
| This handles the transition period where some data might be stored as | |
| JSON strings (old format) and some as proper objects (new format). | |
| Args: | |
| value: The value to parse | |
| default: Default value if parsing fails | |
| Returns: | |
| Parsed value or default | |
| """ | |
| if value is None: | |
| return default | |
| # If it's already a dict or list, return as-is | |
| if isinstance(value, (dict, list)): | |
| return value | |
| # If it's a string, try to parse it | |
| if isinstance(value, str): | |
| try: | |
| return json.loads(value) | |
| except (json.JSONDecodeError, TypeError): | |
| # If it's not valid JSON, return the string itself | |
| return value | |
| # For any other type, return as-is | |
| return value | |
| def to_json_string(value: Any) -> str: | |
| """ | |
| Convert a value to a JSON string if needed. | |
| This is used for backwards compatibility when yielding data that | |
| expects JSON strings. | |
| Uses ensure_ascii=False to preserve Unicode characters without escaping, | |
| preventing double-escaping issues when JSON strings are nested. | |
| Args: | |
| value: The value to convert | |
| Returns: | |
| JSON string representation | |
| """ | |
| if isinstance(value, str): | |
| # If it's already a string, check if it's valid JSON | |
| try: | |
| json.loads(value) | |
| return value # It's already a JSON string | |
| except (json.JSONDecodeError, TypeError): | |
| # It's a plain string, encode it as JSON | |
| return json.dumps(value, ensure_ascii=False) | |
| # For all other types, convert to JSON | |
| return json.dumps(value, ensure_ascii=False) | |
| def to_json_string_fast(value: Any) -> str: | |
| """ | |
| Fast path for converting to JSON string - no validation. | |
| Use this when you KNOW the value is a dict/list that needs serialization. | |
| This is optimized for the streaming hot path where we serialize every chunk. | |
| Uses orjson when available (3-5x faster than stdlib json) for hot paths. | |
| Falls back to stdlib json if orjson is not available. | |
| Args: | |
| value: The value to convert (must be JSON-serializable) | |
| Returns: | |
| JSON string representation | |
| """ | |
| if _HAS_ORJSON: | |
| # orjson is 3-5x faster and handles Unicode correctly by default | |
| return orjson.dumps(value).decode('utf-8') | |
| else: | |
| # Fallback to stdlib json with compact format | |
| return json.dumps(value, separators=(',', ':'), ensure_ascii=False) | |
| def format_for_yield(message_object: Dict[str, Any]) -> Dict[str, Any]: | |
| """ | |
| Format a message object for yielding, ensuring content and metadata are JSON strings. | |
| This maintains backward compatibility with clients expecting JSON strings | |
| while the database now stores proper objects. | |
| Args: | |
| message_object: The message object from the database | |
| Returns: | |
| Message object with content and metadata as JSON strings | |
| """ | |
| if not message_object: | |
| return message_object | |
| # Create a copy to avoid modifying the original | |
| formatted = message_object.copy() | |
| # Ensure content is a JSON string | |
| if 'content' in formatted and not isinstance(formatted['content'], str): | |
| formatted['content'] = json.dumps(formatted['content']) | |
| # Ensure metadata is a JSON string | |
| if 'metadata' in formatted and not isinstance(formatted['metadata'], str): | |
| formatted['metadata'] = json.dumps(formatted['metadata']) | |
| return formatted |