File size: 2,439 Bytes
0ab2566 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | """
Validate generated CoT data quality.
Usage:
python validate_cot.py --cot_path /path/to/cot.jsonl --num_samples 10
"""
import json
import argparse
import numpy as np
from collections import Counter
def load_cot_data(path: str) -> list:
data = []
with open(path, "r", encoding="utf-8") as f:
for line in f:
if line.strip():
try:
data.append(json.loads(line))
except json.JSONDecodeError:
continue
return data
def basic_stats(data: list):
print(f"\n{'='*60}")
print(f" Basic Statistics")
print(f"{'='*60}")
print(f"Total records: {len(data)}")
cot_lengths = [len(item["cot"]) for item in data]
print(f"\nCoT Length (chars):")
print(f" Mean: {np.mean(cot_lengths):.1f}")
print(f" Median: {np.median(cot_lengths):.1f}")
print(f" Min: {np.min(cot_lengths)}")
print(f" Max: {np.max(cot_lengths)}")
print(f" Std: {np.std(cot_lengths):.1f}")
short_threshold = 30
short_count = sum(1 for l in cot_lengths if l < short_threshold)
print(f"\n Short CoTs (<{short_threshold} chars): {short_count} ({short_count/len(data)*100:.2f}%)")
unique_rns = set(item["rn"] for item in data)
print(f"\nUnique query records covered: {len(unique_rns)}")
rn_counts = Counter(item["rn"] for item in data)
print(f"Instructions per record: mean={np.mean(list(rn_counts.values())):.1f}")
def show_samples(data: list, num_samples: int = 5):
print(f"\n{'='*60}")
print(f" Random Samples")
print(f"{'='*60}")
indices = np.random.choice(len(data), min(num_samples, len(data)), replace=False)
for idx in indices:
item = data[idx]
print(f"\n--- Sample (rn={item['rn'][:8]}..., idx={item['prompt_index']}) ---")
print(f"Instruction: {item['instruction']}")
print(f"CoT: {item['cot']}")
print()
def main(args):
print(f"Loading CoT data from: {args.cot_path}")
data = load_cot_data(args.cot_path)
if not data:
print("No data loaded!")
return
basic_stats(data)
show_samples(data, args.num_samples)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Validate CoT data quality")
parser.add_argument("--cot_path", type=str, required=True)
parser.add_argument("--num_samples", type=int, default=5)
args = parser.parse_args()
main(args)
|