team-qbits / training /run_data_pipeline.sh

Upload folder using huggingface_hub

e55c8a4 verified 18 days ago

2.46 kB

#!/usr/bin/env bash
# Rebuilds the SFT dataset from the raw PDF extractions. CPU only, a few minutes.
#
#   bash run_data_pipeline.sh
#
# Inputs expected in ./data :
#   candidates.jsonl                     bulk extraction (9,409 rows, O/L + Grade 10-11)
#   art.jsonl buddhim.jsonl dancing.jsonl sinhala.jsonl Eastern_music.jsonl drama.jsonl
#                                        later per-subject extractions
# Plus the public SinhalaMMLU TEST/easy split, used ONLY as a decontamination blocklist.
set -euo pipefail

# 0. public test split -> decontamination blocklist (never used for training or tuning)
hf download naist-nlp/SinhalaMMLU --repo-type dataset --include "TEST/easy/*" --local-dir .
python eval_sinhalammlu.py blocklist --data TEST/easy --out test_easy_block.jsonl

# 1. bulk extraction -> MCQ schema
python mcq_dataset.py convert data/candidates.jsonl data/converted.jsonl \
  --rejects data/latin_rejects.jsonl

# 2. merge the per-subject extractions
python add_new_data.py --base data/converted.jsonl \
  --new data/art.jsonl data/buddhim.jsonl data/dancing.jsonl data/sinhala.jsonl \
        data/Eastern_music.jsonl data/drama.jsonl \
  --out data/all_converted_v3.jsonl

# 3. align subject names with the official metadata.subject_original values
python normalize_subjects.py --inp data/all_converted_v3.jsonl --out data/all_converted_norm_v3.jsonl

# 4. clean, decontaminate, split
python mcq_dataset.py validate data/all_converted_norm_v3.jsonl
python mcq_dataset.py clean    data/all_converted_norm_v3.jsonl data/all_cleaned_v3.jsonl \
  --rejects data/all_clean_rejects_v3.jsonl
python mcq_dataset.py dedupe   data/all_cleaned_v3.jsonl data/all_deduped_v3.jsonl \
  --blocklist test_easy_block.jsonl --conflicts data/all_conflicts_v3.jsonl
python mcq_dataset.py stats    data/all_deduped_v3.jsonl
python mcq_dataset.py split    data/all_deduped_v3.jsonl \
  --train data/train3.jsonl --val data/val3.jsonl --val_frac 0.1 --seed 3407

# 5. build the prompt/completion files
python mcq_dataset.py build-sft data/train3.jsonl data/sft_train3.jsonl \
  --shuffle_copies 2 --templates si_instruction bare en_instruction --seed 3407
python mcq_dataset.py build-sft data/val3.jsonl data/sft_val3.jsonl \
  --shuffle_copies 0 --strip_zwj_prob 0 --templates si_instruction --seed 3407

echo
echo "== expected: 7,872 questions after dedupe (26 removed as test overlaps),"
echo "==           7,151 train -> 21,235 examples, 721 validation."

	#!/usr/bin/env bash
	# Rebuilds the SFT dataset from the raw PDF extractions. CPU only, a few minutes.
	#
	# bash run_data_pipeline.sh
	#
	# Inputs expected in ./data :
	# candidates.jsonl bulk extraction (9,409 rows, O/L + Grade 10-11)
	# art.jsonl buddhim.jsonl dancing.jsonl sinhala.jsonl Eastern_music.jsonl drama.jsonl
	# later per-subject extractions
	# Plus the public SinhalaMMLU TEST/easy split, used ONLY as a decontamination blocklist.
	set -euo pipefail

	# 0. public test split -> decontamination blocklist (never used for training or tuning)
	hf download naist-nlp/SinhalaMMLU --repo-type dataset --include "TEST/easy/*" --local-dir .
	python eval_sinhalammlu.py blocklist --data TEST/easy --out test_easy_block.jsonl

	# 1. bulk extraction -> MCQ schema
	python mcq_dataset.py convert data/candidates.jsonl data/converted.jsonl \
	--rejects data/latin_rejects.jsonl

	# 2. merge the per-subject extractions
	python add_new_data.py --base data/converted.jsonl \
	--new data/art.jsonl data/buddhim.jsonl data/dancing.jsonl data/sinhala.jsonl \
	data/Eastern_music.jsonl data/drama.jsonl \
	--out data/all_converted_v3.jsonl

	# 3. align subject names with the official metadata.subject_original values
	python normalize_subjects.py --inp data/all_converted_v3.jsonl --out data/all_converted_norm_v3.jsonl

	# 4. clean, decontaminate, split
	python mcq_dataset.py validate data/all_converted_norm_v3.jsonl
	python mcq_dataset.py clean data/all_converted_norm_v3.jsonl data/all_cleaned_v3.jsonl \
	--rejects data/all_clean_rejects_v3.jsonl
	python mcq_dataset.py dedupe data/all_cleaned_v3.jsonl data/all_deduped_v3.jsonl \
	--blocklist test_easy_block.jsonl --conflicts data/all_conflicts_v3.jsonl
	python mcq_dataset.py stats data/all_deduped_v3.jsonl
	python mcq_dataset.py split data/all_deduped_v3.jsonl \
	--train data/train3.jsonl --val data/val3.jsonl --val_frac 0.1 --seed 3407

	# 5. build the prompt/completion files
	python mcq_dataset.py build-sft data/train3.jsonl data/sft_train3.jsonl \
	--shuffle_copies 2 --templates si_instruction bare en_instruction --seed 3407
	python mcq_dataset.py build-sft data/val3.jsonl data/sft_val3.jsonl \
	--shuffle_copies 0 --strip_zwj_prob 0 --templates si_instruction --seed 3407

	echo
	echo "== expected: 7,872 questions after dedupe (26 removed as test overlaps),"
	echo "== 7,151 train -> 21,235 examples, 721 validation."