import json import pyarrow.parquet as pq def load_dataset(file_path): if file_path.endswith(".parquet"): table = pq.read_table(file_path) dataset = table.to_pandas() elif file_path.endswith(".json"): with open(file_path, "r", encoding="utf-8") as file: dataset = json.load(file) elif file_path.endswith(".jsonl"): dataset = [] with open(file_path, "r", encoding="utf-8") as file: for line in file: dataset.append(json.loads(line)) else: raise ValueError( "Unsupported file format. Please provide a parquet, json, or jsonl file." ) return dataset