team-nena / training /prepare_curriculum_data.py
Chamith's picture
Team Nena: Backup submission package
ca24bf9 verified
Raw History Blame Contribute Delete
5.27 kB
"""
Sinhala MMLU Shared Task — Data Preparation & Cyclic Permutation Script
Team: Nena
This script demonstrates:
1. Sinhala Unicode text normalization (ZWJ/ZWNJ handling, virama consonant clusters).
2. 4-way cyclic option permutation to eliminate positional choice bias.
3. Micro-Chain-of-Thought (Micro-CoT) formatting with curriculum grounding.
"""
import json
import os
import re
import sys
if hasattr(sys.stdout, "reconfigure"):
try:
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
except Exception:
pass
VIRAMA = "\u0DCA"
ZWJ = "\u200D"
ZWNJ = "\u200C"
RA = "\u0DBB"
YA = "\u0DBA"
def normalize_sinhala_text(text: str) -> str:
"""
Normalizes Sinhala Unicode text, repairing zero-width joiners (ZWJ)
for yansaya / rakaransaya clusters and stripping invisible zero-width non-joiners.
"""
if not text or not isinstance(text, str):
return ""
t = text.replace("\ufeff", "").replace("\u200b", "")
t = t.replace(VIRAMA + ZWNJ, VIRAMA)
t = t.replace(ZWNJ, "")
t = re.sub(f"{VIRAMA}(?!{ZWJ}){RA}", f"{VIRAMA}{ZWJ}{RA}", t)
t = re.sub(f"{VIRAMA}(?!{ZWJ}){YA}", f"{VIRAMA}{ZWJ}{YA}", t)
t = re.sub(f"{ZWJ}+", ZWJ, t)
return re.sub(r"[ \t]+", " ", t).strip()
def generate_cyclic_permutations(item: dict) -> list[dict]:
"""
Generates 4 cyclic permutations of choices for an MCQ to ensure
the target model does not develop positional bias toward any option index.
Expected item schema:
- question: str
- choices: list of 4 str
- answer: int (1-based, 1..4)
- explanation: str
- subject: str (optional)
- grade: str (optional)
"""
q = normalize_sinhala_text(item["question"])
choices = [normalize_sinhala_text(c) for c in item["choices"]]
gold_1based = int(item.get("answer", item.get("correct_index", 1)))
gold_0based = gold_1based - 1
exp = normalize_sinhala_text(item.get("explanation", ""))
subject = item.get("subject", "General")
grade = item.get("grade", "General")
if len(choices) != 4:
return [item]
permutations = []
# 4 cyclic shifts: shift k in {0, 1, 2, 3}
for shift in range(4):
perm_choices = choices[shift:] + choices[:shift]
new_gold_0based = (gold_0based - shift) % 4
new_gold_1based = new_gold_0based + 1
permutations.append({
"question": q,
"choices": perm_choices,
"answer": new_gold_1based,
"explanation": exp,
"subject": subject,
"grade": grade,
"shift": shift,
})
return permutations
def format_micro_cot_sample(item: dict) -> dict:
"""
Formats a raw MCQ record into the exact prompt structure used for SFT training.
"""
system_prompt = (
"ඔබ ශ්‍රී ලංකා ජාතික විෂය නිර්දේශයේ ප්‍රවීණ ගුරුවරයෙකි. "
"පහත බහුවරණ ප්‍රශ්නයට කෙටි හේතු දැක්වීමක් සහිතව නිවැරදි පිළිතුර තෝරන්න."
)
subj = item.get("subject", "General")
grade = item.get("grade", "General")
q = normalize_sinhala_text(item["question"])
c = [normalize_sinhala_text(x) for x in item["choices"]]
ans_idx = int(item["answer"])
correct_text = c[ans_idx - 1]
exp = normalize_sinhala_text(item.get("explanation", ""))
choices_block = "\n".join(f"{i + 1}. {choice}" for i, choice in enumerate(c))
prompt = (
f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n"
f"{system_prompt}<|eot_id|>"
f"<|start_header_id|>user<|end_header_id|>\n\n"
f"<|subject: {subj}|> <|grade: {grade}|>\n\n"
f"ප්‍රශ්නය: {q}\n\n"
f"වරණ:\n"
f"{choices_block}\n\n"
f"පිළිතුර:<|eot_id|>"
f"<|start_header_id|>assistant<|end_header_id|>\n\n"
)
completion = (
f"හේතු දැක්වීම: {exp}\n\n"
f"නිවැරදි පිළිතුර: {ans_idx}. {correct_text}<|eot_id|>"
)
return {
"text": prompt + completion,
"prompt": prompt,
"completion": completion,
}
def main():
sample_file = os.path.join(os.path.dirname(__file__), "sample_data.jsonl")
if not os.path.exists(sample_file):
print(f"Sample data file {sample_file} not found.")
return
print(f"Loading sample questions from: {sample_file}")
with open(sample_file, "r", encoding="utf-8") as f:
records = [json.loads(line) for line in f if line.strip()]
print(f"Loaded {len(records)} sample questions.")
all_permuted = []
for r in records:
perms = generate_cyclic_permutations(r)
all_permuted.extend(perms)
print(f"Generated {len(all_permuted)} balanced permutations (4x expansion).")
first_formatted = format_micro_cot_sample(all_permuted[0])
print("\n--- Example Formatted Micro-CoT Training Text ---")
print(first_formatted["text"][:500] + "\n...")
if __name__ == "__main__":
main()