File size: 1,559 Bytes
46c2986 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 | # -*- coding: utf-8 -*-
import sys
from huggingface_hub import HfFileSystem
import pyarrow.parquet as pq
sys.stdout.reconfigure(encoding='utf-8')
fs = HfFileSystem()
target_files = [
'datasets/ViuAI/viu-mini-raw-pretrain/english/train-00000.parquet',
'datasets/ViuAI/viu-mini-raw-pretrain/wikipedia/train-00000-of-00041.parquet',
'datasets/ViuAI/viu-mini-raw-pretrain/hindi/train-00000.parquet',
'datasets/ViuAI/viu-mini-raw-pretrain/translation/samanantar-train-00000-of-00008.parquet',
'datasets/ViuAI/viu-mini-raw-pretrain/hinglish/alpaca-gpt4-hinglish.parquet',
'datasets/ViuAI/viu-mini-raw-pretrain/domains/train-00002.parquet',
]
print("=== ACTUAL REAL SAMPLES DIRECTLY FROM HUB PARQUET FILES ===")
for fpath in target_files:
parts = fpath.split('/')
folder = parts[-2]
fname = parts[-1]
try:
with fs.open(fpath, 'rb') as f:
pf = pq.ParquetFile(f)
total_rows = pf.metadata.num_rows
cols = pf.schema.names
# read only the first row group, slice first 1 row
rg = pf.read_row_group(0)
df = rg.slice(0, 1).to_pandas()
print(f"\n[{folder.upper()}] File: {folder}/{fname}")
print(f"Total Rows: {total_rows:,} | Columns: {cols}")
row_dict = df.iloc[0].to_dict()
for col, val in row_dict.items():
val_str = str(val).replace('\n', ' ')
print(f" * {col}: {val_str[:160]}...")
except Exception as e:
print(f"[{folder}] Error reading {fname}: {e}")
|