data/scripts/read_live_samples.py

# -*- coding: utf-8 -*-
import sys
from huggingface_hub import HfFileSystem
import pyarrow.parquet as pq

sys.stdout.reconfigure(encoding='utf-8')
fs = HfFileSystem()

target_files = [
    'datasets/ViuAI/viu-mini-raw-pretrain/english/train-00000.parquet',
    'datasets/ViuAI/viu-mini-raw-pretrain/wikipedia/train-00000-of-00041.parquet',
    'datasets/ViuAI/viu-mini-raw-pretrain/hindi/train-00000.parquet',
    'datasets/ViuAI/viu-mini-raw-pretrain/translation/samanantar-train-00000-of-00008.parquet',
    'datasets/ViuAI/viu-mini-raw-pretrain/hinglish/alpaca-gpt4-hinglish.parquet',
    'datasets/ViuAI/viu-mini-raw-pretrain/domains/train-00002.parquet',
]

print("=== ACTUAL REAL SAMPLES DIRECTLY FROM HUB PARQUET FILES ===")
for fpath in target_files:
    parts = fpath.split('/')
    folder = parts[-2]
    fname = parts[-1]
    try:
        with fs.open(fpath, 'rb') as f:
            pf = pq.ParquetFile(f)
            total_rows = pf.metadata.num_rows
            cols = pf.schema.names
            # read only the first row group, slice first 1 row
            rg = pf.read_row_group(0)
            df = rg.slice(0, 1).to_pandas()
            print(f"\n[{folder.upper()}] File: {folder}/{fname}")
            print(f"Total Rows: {total_rows:,} | Columns: {cols}")
            row_dict = df.iloc[0].to_dict()
            for col, val in row_dict.items():
                val_str = str(val).replace('\n', ' ')
                print(f"  * {col}: {val_str[:160]}...")
    except Exception as e:
        print(f"[{folder}] Error reading {fname}: {e}")