nniback / core /utils /json_helpers.py
Kyrlos11's picture
Upload 842 files
22087d0 verified
Raw History Blame Contribute Delete
11.7 kB
"""
JSON helper utilities for handling both legacy (string) and new (dict/list) formats.
These utilities help with the transition from storing JSON as strings to storing
them as proper JSONB objects in the database.
"""
import json
import re
import logging
from typing import Any, Union, Dict, List, Tuple
# Use orjson for hot paths (3-5x faster than stdlib json)
try:
import orjson
_HAS_ORJSON = True
except ImportError:
_HAS_ORJSON = False
logger = logging.getLogger(__name__)
def repair_json(json_string: str) -> Tuple[str, bool]:
"""
Attempt to repair malformed JSON that LLMs sometimes produce.
Common LLM JSON mistakes this handles:
- Missing closing brackets/braces (e.g., `[{"a": 1}, {"b": 2]` -> `[{"a": 1}, {"b": 2}]`)
- Trailing commas (e.g., `[1, 2, 3,]` -> `[1, 2, 3]`)
- Missing commas between array elements (e.g., `[{"a":1} {"b":2}]` -> `[{"a":1}, {"b":2}]`)
Args:
json_string: The potentially malformed JSON string
Returns:
Tuple of (repaired_string, was_repaired)
"""
if not isinstance(json_string, str):
return json_string, False
original = json_string
repaired = json_string.strip()
was_repaired = False
# First, try parsing as-is
try:
json.loads(repaired)
return repaired, False
except json.JSONDecodeError:
pass
# Fix 1: Remove trailing commas before ] or }
# Pattern: comma followed by optional whitespace, then ] or }
trailing_comma_pattern = r',(\s*[\]\}])'
if re.search(trailing_comma_pattern, repaired):
repaired = re.sub(trailing_comma_pattern, r'\1', repaired)
was_repaired = True
# Fix 2: Add missing commas between array elements or object properties
# Pattern: } followed by optional whitespace, then { (missing comma)
missing_comma_pattern = r'\}(\s*)\{'
if re.search(missing_comma_pattern, repaired):
repaired = re.sub(missing_comma_pattern, r'},\1{', repaired)
was_repaired = True
# Pattern: ] followed by optional whitespace, then [ (missing comma between arrays)
missing_comma_array_pattern = r'\](\s*)\['
if re.search(missing_comma_array_pattern, repaired):
repaired = re.sub(missing_comma_array_pattern, r'],\1[', repaired)
was_repaired = True
# Fix 3: Balance brackets/braces - count and add missing ones
# This handles the specific case: `[{...}, {...], {...]` -> `[{...}, {...}]`
open_braces = repaired.count('{')
close_braces = repaired.count('}')
open_brackets = repaired.count('[')
close_brackets = repaired.count(']')
# Add missing closing braces
if open_braces > close_braces:
# Find position to insert - look for patterns like `], {` that should be `}], {`
# or at the end of the string
missing_braces = open_braces - close_braces
# Try to find where braces are missing by checking for ], or ] at end
# Common pattern: `["task1", "task2"], {` should be `["task1", "task2"]}, {`
# Look for `],` or `]` followed by `,` or `}` or end - missing `}` before
fixed = False
for i in range(len(repaired) - 1, 0, -1):
if repaired[i] == ']':
# Check if there's a missing } before this ]
# Count braces up to this point
temp_open = repaired[:i+1].count('{')
temp_close = repaired[:i+1].count('}')
if temp_open > temp_close:
# Insert missing }s before the ]
needed = temp_open - temp_close
repaired = repaired[:i] + '}' * needed + repaired[i:]
was_repaired = True
fixed = True
break
if not fixed:
# Just append at end
repaired = repaired + '}' * missing_braces
was_repaired = True
# Add missing closing brackets
if open_brackets > close_brackets:
missing_brackets = open_brackets - close_brackets
repaired = repaired + ']' * missing_brackets
was_repaired = True
# Validate the repair worked
try:
json.loads(repaired)
if was_repaired:
logger.info(f"🔧 [JSON REPAIR] Successfully repaired malformed JSON")
logger.debug(f"🔧 [JSON REPAIR] Original (first 200 chars): {original[:200]}")
logger.debug(f"🔧 [JSON REPAIR] Repaired (first 200 chars): {repaired[:200]}")
return repaired, was_repaired
except json.JSONDecodeError as e:
# Repair failed, return original
logger.debug(f"🔧 [JSON REPAIR] Repair attempt failed: {e}")
return original, False
def safe_json_parse_with_repair(value: Union[str, Dict, List, Any], default: Any = None) -> Any:
"""
Safely parse JSON with automatic repair of common LLM mistakes.
This is an enhanced version of safe_json_parse that attempts to repair
malformed JSON before giving up.
Args:
value: The value to parse
default: Default value if parsing fails
Returns:
Parsed value or default
"""
if value is None:
return default
# If it's already a dict or list, return as-is
if isinstance(value, (dict, list)):
return value
# If it's a string, try to parse it
if isinstance(value, str):
# First, try normal parsing
try:
return json.loads(value)
except json.JSONDecodeError:
pass
# Try repair
repaired, was_repaired = repair_json(value)
if was_repaired:
try:
return json.loads(repaired)
except json.JSONDecodeError:
pass
# If repair didn't work, return the string itself or default
return value if default is None else default
# For any other type, return as-is
return value
def ensure_dict(value: Union[str, Dict[str, Any], None], default: Dict[str, Any] = None) -> Dict[str, Any]:
"""
Ensure a value is a dictionary.
Handles:
- None -> returns default or {}
- Dict -> returns as-is
- JSON string -> parses and returns dict
- Other -> returns default or {}
Args:
value: The value to ensure is a dict
default: Default value if conversion fails
Returns:
A dictionary
"""
if default is None:
default = {}
if value is None:
return default
if isinstance(value, dict):
return value
if isinstance(value, str):
try:
parsed = json.loads(value)
if isinstance(parsed, dict):
return parsed
return default
except (json.JSONDecodeError, TypeError):
return default
return default
def ensure_list(value: Union[str, List[Any], None], default: List[Any] = None) -> List[Any]:
"""
Ensure a value is a list.
Handles:
- None -> returns default or []
- List -> returns as-is
- JSON string -> parses and returns list
- Other -> returns default or []
Args:
value: The value to ensure is a list
default: Default value if conversion fails
Returns:
A list
"""
if default is None:
default = []
if value is None:
return default
if isinstance(value, list):
return value
if isinstance(value, str):
try:
parsed = json.loads(value)
if isinstance(parsed, list):
return parsed
return default
except (json.JSONDecodeError, TypeError):
return default
return default
def safe_json_parse(value: Union[str, Dict, List, Any], default: Any = None) -> Any:
"""
Safely parse a value that might be JSON string or already parsed.
This handles the transition period where some data might be stored as
JSON strings (old format) and some as proper objects (new format).
Args:
value: The value to parse
default: Default value if parsing fails
Returns:
Parsed value or default
"""
if value is None:
return default
# If it's already a dict or list, return as-is
if isinstance(value, (dict, list)):
return value
# If it's a string, try to parse it
if isinstance(value, str):
try:
return json.loads(value)
except (json.JSONDecodeError, TypeError):
# If it's not valid JSON, return the string itself
return value
# For any other type, return as-is
return value
def to_json_string(value: Any) -> str:
"""
Convert a value to a JSON string if needed.
This is used for backwards compatibility when yielding data that
expects JSON strings.
Uses ensure_ascii=False to preserve Unicode characters without escaping,
preventing double-escaping issues when JSON strings are nested.
Args:
value: The value to convert
Returns:
JSON string representation
"""
if isinstance(value, str):
# If it's already a string, check if it's valid JSON
try:
json.loads(value)
return value # It's already a JSON string
except (json.JSONDecodeError, TypeError):
# It's a plain string, encode it as JSON
return json.dumps(value, ensure_ascii=False)
# For all other types, convert to JSON
return json.dumps(value, ensure_ascii=False)
def to_json_string_fast(value: Any) -> str:
"""
Fast path for converting to JSON string - no validation.
Use this when you KNOW the value is a dict/list that needs serialization.
This is optimized for the streaming hot path where we serialize every chunk.
Uses orjson when available (3-5x faster than stdlib json) for hot paths.
Falls back to stdlib json if orjson is not available.
Args:
value: The value to convert (must be JSON-serializable)
Returns:
JSON string representation
"""
if _HAS_ORJSON:
# orjson is 3-5x faster and handles Unicode correctly by default
return orjson.dumps(value).decode('utf-8')
else:
# Fallback to stdlib json with compact format
return json.dumps(value, separators=(',', ':'), ensure_ascii=False)
def format_for_yield(message_object: Dict[str, Any]) -> Dict[str, Any]:
"""
Format a message object for yielding, ensuring content and metadata are JSON strings.
This maintains backward compatibility with clients expecting JSON strings
while the database now stores proper objects.
Args:
message_object: The message object from the database
Returns:
Message object with content and metadata as JSON strings
"""
if not message_object:
return message_object
# Create a copy to avoid modifying the original
formatted = message_object.copy()
# Ensure content is a JSON string
if 'content' in formatted and not isinstance(formatted['content'], str):
formatted['content'] = json.dumps(formatted['content'])
# Ensure metadata is a JSON string
if 'metadata' in formatted and not isinstance(formatted['metadata'], str):
formatted['metadata'] = json.dumps(formatted['metadata'])
return formatted