Spaces:
Running
Running
Download PMPF_dataset_divider.py from Naphula/model_tools: direct link, hf CLI and curl.
- Browser
- Download file 5.87 kB
-
https://huggingface.co/spaces/Naphula/model_tools/resolve/main/PMPF_dataset_divider.py
- Command line
-
hf download hf://spaces/Naphula/model_tools/PMPF_dataset_divider.py
-
curl -L -o PMPF_dataset_divider.py https://huggingface.co/spaces/Naphula/model_tools/resolve/main/PMPF_dataset_divider.py
5.87 kB
| #!/usr/bin/env python3 | |
| """ | |
| dataset_divider.py | |
| Splits a dataset JSON file into modular JSON files organized by: | |
| - majortopic/ | |
| - topic/ | |
| - subtopics/ | |
| Also exports an index_summary.json containing lists of all unique categories. | |
| """ | |
| import argparse | |
| from collections import defaultdict | |
| import json | |
| import os | |
| import re | |
| from typing import Any, Dict, List | |
| def sanitize_filename(name: str) -> str: | |
| """Sanitizes strings to create safe filenames across OS environments.""" | |
| if not name: | |
| return "unspecified" | |
| # Replace illegal/problematic characters with underscores | |
| sanitized = re.sub(r'[\\/*?:"<>|]', "_", str(name)).strip() | |
| return sanitized if sanitized else "unspecified" | |
| def load_dataset(input_path: str) -> List[Dict[str, Any]]: | |
| """Loads dataset from either standard JSON (list/dict wrapper) or JSONL.""" | |
| records = [] | |
| with open(input_path, "r", encoding="utf-8") as f: | |
| # Check by extension first if explicitly .jsonl | |
| if input_path.lower().endswith(".jsonl"): | |
| for line_num, line in enumerate(f, 1): | |
| line = line.strip() | |
| if line: | |
| records.append(json.loads(line)) | |
| return records | |
| # Otherwise attempt reading as JSON, falling back to JSONL | |
| content = f.read() | |
| try: | |
| data = json.loads(content) | |
| if isinstance(data, list): | |
| return data | |
| elif isinstance(data, dict): | |
| for key in ["data", "entries", "rows"]: | |
| if key in data and isinstance(data[key], list): | |
| return data[key] | |
| return [data] | |
| except json.JSONDecodeError: | |
| # Fallback parsing line by line as JSONL | |
| for line in content.splitlines(): | |
| line = line.strip() | |
| if line: | |
| records.append(json.loads(line)) | |
| return records | |
| def main(): | |
| parser = argparse.ArgumentParser( | |
| description="Divide a dataset JSON into modular directories by major topic, topic, and subtopic." | |
| ) | |
| parser.add_argument( | |
| "--input", | |
| "-i", | |
| default="dataset.jsonl", | |
| help="Path to the input JSON file (default: dataset.jsonl)", | |
| ) | |
| parser.add_argument( | |
| "--output-dir", | |
| "-o", | |
| default="output_split", | |
| help="Base directory for outputs (default: output_split)", | |
| ) | |
| parser.add_argument( | |
| "--output-format", | |
| "-f", | |
| choices=["json", "jsonl"], | |
| default="jsonl", | |
| help="Format for partitioned output files: 'json' or 'jsonl' (default: jsonl)", | |
| ) | |
| args = parser.parse_args() | |
| print(f"Loading data from: {args.input}") | |
| records = load_dataset(args.input) | |
| print(f"Total entries loaded: {len(records)}") | |
| # Containers for grouping | |
| by_majortopic = defaultdict(list) | |
| by_topic = defaultdict(list) | |
| by_subtopics = defaultdict(list) | |
| # Track unique values for the final index summary | |
| unique_majortopics = set() | |
| unique_topics = set() | |
| unique_subtopics = set() | |
| for item in records: | |
| major = item.get("majortopic", "Uncategorized") | |
| topic = item.get("topic", "Uncategorized") | |
| # Handle field naming variations (subtopics vs subtopic) | |
| subtopic = item.get("subtopics") or item.get( | |
| "subtopic", "Uncategorized" | |
| ) | |
| unique_majortopics.add(major) | |
| unique_topics.add(topic) | |
| unique_subtopics.add(subtopic) | |
| by_majortopic[major].append(item) | |
| by_topic[topic].append(item) | |
| by_subtopics[subtopic].append(item) | |
| # Define target subdirectories | |
| subdirs = { | |
| "majortopic": by_majortopic, | |
| "topic": by_topic, | |
| "subtopics": by_subtopics, | |
| } | |
| # Write modular files (.json or .jsonl based on --output-format) | |
| ext = args.output_format | |
| for folder_name, grouped_data in subdirs.items(): | |
| folder_path = os.path.join(args.output_dir, folder_name) | |
| os.makedirs(folder_path, exist_ok=True) | |
| for category_name, items in grouped_data.items(): | |
| safe_name = sanitize_filename(category_name) | |
| file_path = os.path.join(folder_path, f"{safe_name}.{ext}") | |
| with open(file_path, "w", encoding="utf-8") as f: | |
| if ext == "jsonl": | |
| for item in items: | |
| f.write(json.dumps(item, ensure_ascii=False) + "\n") | |
| else: | |
| json.dump(items, f, indent=2, ensure_ascii=False) | |
| # Generate and export summary index | |
| summary = { | |
| "total_records": len(records), | |
| "majortopic_count": len(unique_majortopics), | |
| "majortopics": sorted(list(unique_majortopics)), | |
| "topic_count": len(unique_topics), | |
| "topics": sorted(list(unique_topics)), | |
| "subtopic_count": len(unique_subtopics), | |
| "subtopics": sorted(list(unique_subtopics)), | |
| } | |
| index_path = os.path.join(args.output_dir, "index_summary.json") | |
| with open(index_path, "w", encoding="utf-8") as f: | |
| json.dump(summary, f, indent=2, ensure_ascii=False) | |
| print("\nProcessing completed successfully!") | |
| print(f"Generated folders inside: {os.path.abspath(args.output_dir)}/") | |
| print( | |
| f" - majortopic/ ({len(unique_majortopics)} files generated, e.g., Chemistry.json)" | |
| ) | |
| print( | |
| f" - topic/ ({len(unique_topics)} files generated, e.g., Chemical Composition.json)" | |
| ) | |
| print( | |
| f" - subtopics/ ({len(unique_subtopics)} files generated, e.g., Composition of Actnides.json)" | |
| ) | |
| print(f" - index_summary.json generated.") | |
| if __name__ == "__main__": | |
| main() |