#!/usr/bin/env python3 """ dataset_divider.py Splits a dataset JSON file into modular JSON files organized by: - majortopic/ - topic/ - subtopics/ Also exports an index_summary.json containing lists of all unique categories. """ import argparse from collections import defaultdict import json import os import re from typing import Any, Dict, List def sanitize_filename(name: str) -> str: """Sanitizes strings to create safe filenames across OS environments.""" if not name: return "unspecified" # Replace illegal/problematic characters with underscores sanitized = re.sub(r'[\\/*?:"<>|]', "_", str(name)).strip() return sanitized if sanitized else "unspecified" def load_dataset(input_path: str) -> List[Dict[str, Any]]: """Loads dataset from either standard JSON (list/dict wrapper) or JSONL.""" records = [] with open(input_path, "r", encoding="utf-8") as f: # Check by extension first if explicitly .jsonl if input_path.lower().endswith(".jsonl"): for line_num, line in enumerate(f, 1): line = line.strip() if line: records.append(json.loads(line)) return records # Otherwise attempt reading as JSON, falling back to JSONL content = f.read() try: data = json.loads(content) if isinstance(data, list): return data elif isinstance(data, dict): for key in ["data", "entries", "rows"]: if key in data and isinstance(data[key], list): return data[key] return [data] except json.JSONDecodeError: # Fallback parsing line by line as JSONL for line in content.splitlines(): line = line.strip() if line: records.append(json.loads(line)) return records def main(): parser = argparse.ArgumentParser( description="Divide a dataset JSON into modular directories by major topic, topic, and subtopic." ) parser.add_argument( "--input", "-i", default="dataset.jsonl", help="Path to the input JSON file (default: dataset.jsonl)", ) parser.add_argument( "--output-dir", "-o", default="output_split", help="Base directory for outputs (default: output_split)", ) parser.add_argument( "--output-format", "-f", choices=["json", "jsonl"], default="jsonl", help="Format for partitioned output files: 'json' or 'jsonl' (default: jsonl)", ) args = parser.parse_args() print(f"Loading data from: {args.input}") records = load_dataset(args.input) print(f"Total entries loaded: {len(records)}") # Containers for grouping by_majortopic = defaultdict(list) by_topic = defaultdict(list) by_subtopics = defaultdict(list) # Track unique values for the final index summary unique_majortopics = set() unique_topics = set() unique_subtopics = set() for item in records: major = item.get("majortopic", "Uncategorized") topic = item.get("topic", "Uncategorized") # Handle field naming variations (subtopics vs subtopic) subtopic = item.get("subtopics") or item.get( "subtopic", "Uncategorized" ) unique_majortopics.add(major) unique_topics.add(topic) unique_subtopics.add(subtopic) by_majortopic[major].append(item) by_topic[topic].append(item) by_subtopics[subtopic].append(item) # Define target subdirectories subdirs = { "majortopic": by_majortopic, "topic": by_topic, "subtopics": by_subtopics, } # Write modular files (.json or .jsonl based on --output-format) ext = args.output_format for folder_name, grouped_data in subdirs.items(): folder_path = os.path.join(args.output_dir, folder_name) os.makedirs(folder_path, exist_ok=True) for category_name, items in grouped_data.items(): safe_name = sanitize_filename(category_name) file_path = os.path.join(folder_path, f"{safe_name}.{ext}") with open(file_path, "w", encoding="utf-8") as f: if ext == "jsonl": for item in items: f.write(json.dumps(item, ensure_ascii=False) + "\n") else: json.dump(items, f, indent=2, ensure_ascii=False) # Generate and export summary index summary = { "total_records": len(records), "majortopic_count": len(unique_majortopics), "majortopics": sorted(list(unique_majortopics)), "topic_count": len(unique_topics), "topics": sorted(list(unique_topics)), "subtopic_count": len(unique_subtopics), "subtopics": sorted(list(unique_subtopics)), } index_path = os.path.join(args.output_dir, "index_summary.json") with open(index_path, "w", encoding="utf-8") as f: json.dump(summary, f, indent=2, ensure_ascii=False) print("\nProcessing completed successfully!") print(f"Generated folders inside: {os.path.abspath(args.output_dir)}/") print( f" - majortopic/ ({len(unique_majortopics)} files generated, e.g., Chemistry.json)" ) print( f" - topic/ ({len(unique_topics)} files generated, e.g., Chemical Composition.json)" ) print( f" - subtopics/ ({len(unique_subtopics)} files generated, e.g., Composition of Actnides.json)" ) print(f" - index_summary.json generated.") if __name__ == "__main__": main()