model_tools / PMPF_dataset_divider.py
Naphula's picture
Upload PMPF_dataset_divider.py
4528f21 verified
Raw History Blame Contribute Delete
5.87 kB
#!/usr/bin/env python3
"""
dataset_divider.py
Splits a dataset JSON file into modular JSON files organized by:
- majortopic/
- topic/
- subtopics/
Also exports an index_summary.json containing lists of all unique categories.
"""
import argparse
from collections import defaultdict
import json
import os
import re
from typing import Any, Dict, List
def sanitize_filename(name: str) -> str:
"""Sanitizes strings to create safe filenames across OS environments."""
if not name:
return "unspecified"
# Replace illegal/problematic characters with underscores
sanitized = re.sub(r'[\\/*?:"<>|]', "_", str(name)).strip()
return sanitized if sanitized else "unspecified"
def load_dataset(input_path: str) -> List[Dict[str, Any]]:
"""Loads dataset from either standard JSON (list/dict wrapper) or JSONL."""
records = []
with open(input_path, "r", encoding="utf-8") as f:
# Check by extension first if explicitly .jsonl
if input_path.lower().endswith(".jsonl"):
for line_num, line in enumerate(f, 1):
line = line.strip()
if line:
records.append(json.loads(line))
return records
# Otherwise attempt reading as JSON, falling back to JSONL
content = f.read()
try:
data = json.loads(content)
if isinstance(data, list):
return data
elif isinstance(data, dict):
for key in ["data", "entries", "rows"]:
if key in data and isinstance(data[key], list):
return data[key]
return [data]
except json.JSONDecodeError:
# Fallback parsing line by line as JSONL
for line in content.splitlines():
line = line.strip()
if line:
records.append(json.loads(line))
return records
def main():
parser = argparse.ArgumentParser(
description="Divide a dataset JSON into modular directories by major topic, topic, and subtopic."
)
parser.add_argument(
"--input",
"-i",
default="dataset.jsonl",
help="Path to the input JSON file (default: dataset.jsonl)",
)
parser.add_argument(
"--output-dir",
"-o",
default="output_split",
help="Base directory for outputs (default: output_split)",
)
parser.add_argument(
"--output-format",
"-f",
choices=["json", "jsonl"],
default="jsonl",
help="Format for partitioned output files: 'json' or 'jsonl' (default: jsonl)",
)
args = parser.parse_args()
print(f"Loading data from: {args.input}")
records = load_dataset(args.input)
print(f"Total entries loaded: {len(records)}")
# Containers for grouping
by_majortopic = defaultdict(list)
by_topic = defaultdict(list)
by_subtopics = defaultdict(list)
# Track unique values for the final index summary
unique_majortopics = set()
unique_topics = set()
unique_subtopics = set()
for item in records:
major = item.get("majortopic", "Uncategorized")
topic = item.get("topic", "Uncategorized")
# Handle field naming variations (subtopics vs subtopic)
subtopic = item.get("subtopics") or item.get(
"subtopic", "Uncategorized"
)
unique_majortopics.add(major)
unique_topics.add(topic)
unique_subtopics.add(subtopic)
by_majortopic[major].append(item)
by_topic[topic].append(item)
by_subtopics[subtopic].append(item)
# Define target subdirectories
subdirs = {
"majortopic": by_majortopic,
"topic": by_topic,
"subtopics": by_subtopics,
}
# Write modular files (.json or .jsonl based on --output-format)
ext = args.output_format
for folder_name, grouped_data in subdirs.items():
folder_path = os.path.join(args.output_dir, folder_name)
os.makedirs(folder_path, exist_ok=True)
for category_name, items in grouped_data.items():
safe_name = sanitize_filename(category_name)
file_path = os.path.join(folder_path, f"{safe_name}.{ext}")
with open(file_path, "w", encoding="utf-8") as f:
if ext == "jsonl":
for item in items:
f.write(json.dumps(item, ensure_ascii=False) + "\n")
else:
json.dump(items, f, indent=2, ensure_ascii=False)
# Generate and export summary index
summary = {
"total_records": len(records),
"majortopic_count": len(unique_majortopics),
"majortopics": sorted(list(unique_majortopics)),
"topic_count": len(unique_topics),
"topics": sorted(list(unique_topics)),
"subtopic_count": len(unique_subtopics),
"subtopics": sorted(list(unique_subtopics)),
}
index_path = os.path.join(args.output_dir, "index_summary.json")
with open(index_path, "w", encoding="utf-8") as f:
json.dump(summary, f, indent=2, ensure_ascii=False)
print("\nProcessing completed successfully!")
print(f"Generated folders inside: {os.path.abspath(args.output_dir)}/")
print(
f" - majortopic/ ({len(unique_majortopics)} files generated, e.g., Chemistry.json)"
)
print(
f" - topic/ ({len(unique_topics)} files generated, e.g., Chemical Composition.json)"
)
print(
f" - subtopics/ ({len(unique_subtopics)} files generated, e.g., Composition of Actnides.json)"
)
print(f" - index_summary.json generated.")
if __name__ == "__main__":
main()