""" Sinhala MMLU Shared Task — Data Preparation & Cyclic Permutation Script Team: Nena This script demonstrates: 1. Sinhala Unicode text normalization (ZWJ/ZWNJ handling, virama consonant clusters). 2. 4-way cyclic option permutation to eliminate positional choice bias. 3. Micro-Chain-of-Thought (Micro-CoT) formatting with curriculum grounding. """ import json import os import re import sys if hasattr(sys.stdout, "reconfigure"): try: sys.stdout.reconfigure(encoding="utf-8", errors="replace") sys.stderr.reconfigure(encoding="utf-8", errors="replace") except Exception: pass VIRAMA = "\u0DCA" ZWJ = "\u200D" ZWNJ = "\u200C" RA = "\u0DBB" YA = "\u0DBA" def normalize_sinhala_text(text: str) -> str: """ Normalizes Sinhala Unicode text, repairing zero-width joiners (ZWJ) for yansaya / rakaransaya clusters and stripping invisible zero-width non-joiners. """ if not text or not isinstance(text, str): return "" t = text.replace("\ufeff", "").replace("\u200b", "") t = t.replace(VIRAMA + ZWNJ, VIRAMA) t = t.replace(ZWNJ, "") t = re.sub(f"{VIRAMA}(?!{ZWJ}){RA}", f"{VIRAMA}{ZWJ}{RA}", t) t = re.sub(f"{VIRAMA}(?!{ZWJ}){YA}", f"{VIRAMA}{ZWJ}{YA}", t) t = re.sub(f"{ZWJ}+", ZWJ, t) return re.sub(r"[ \t]+", " ", t).strip() def generate_cyclic_permutations(item: dict) -> list[dict]: """ Generates 4 cyclic permutations of choices for an MCQ to ensure the target model does not develop positional bias toward any option index. Expected item schema: - question: str - choices: list of 4 str - answer: int (1-based, 1..4) - explanation: str - subject: str (optional) - grade: str (optional) """ q = normalize_sinhala_text(item["question"]) choices = [normalize_sinhala_text(c) for c in item["choices"]] gold_1based = int(item.get("answer", item.get("correct_index", 1))) gold_0based = gold_1based - 1 exp = normalize_sinhala_text(item.get("explanation", "")) subject = item.get("subject", "General") grade = item.get("grade", "General") if len(choices) != 4: return [item] permutations = [] # 4 cyclic shifts: shift k in {0, 1, 2, 3} for shift in range(4): perm_choices = choices[shift:] + choices[:shift] new_gold_0based = (gold_0based - shift) % 4 new_gold_1based = new_gold_0based + 1 permutations.append({ "question": q, "choices": perm_choices, "answer": new_gold_1based, "explanation": exp, "subject": subject, "grade": grade, "shift": shift, }) return permutations def format_micro_cot_sample(item: dict) -> dict: """ Formats a raw MCQ record into the exact prompt structure used for SFT training. """ system_prompt = ( "ඔබ ශ්‍රී ලංකා ජාතික විෂය නිර්දේශයේ ප්‍රවීණ ගුරුවරයෙකි. " "පහත බහුවරණ ප්‍රශ්නයට කෙටි හේතු දැක්වීමක් සහිතව නිවැරදි පිළිතුර තෝරන්න." ) subj = item.get("subject", "General") grade = item.get("grade", "General") q = normalize_sinhala_text(item["question"]) c = [normalize_sinhala_text(x) for x in item["choices"]] ans_idx = int(item["answer"]) correct_text = c[ans_idx - 1] exp = normalize_sinhala_text(item.get("explanation", "")) choices_block = "\n".join(f"{i + 1}. {choice}" for i, choice in enumerate(c)) prompt = ( f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n" f"{system_prompt}<|eot_id|>" f"<|start_header_id|>user<|end_header_id|>\n\n" f"<|subject: {subj}|> <|grade: {grade}|>\n\n" f"ප්‍රශ්නය: {q}\n\n" f"වරණ:\n" f"{choices_block}\n\n" f"පිළිතුර:<|eot_id|>" f"<|start_header_id|>assistant<|end_header_id|>\n\n" ) completion = ( f"හේතු දැක්වීම: {exp}\n\n" f"නිවැරදි පිළිතුර: {ans_idx}. {correct_text}<|eot_id|>" ) return { "text": prompt + completion, "prompt": prompt, "completion": completion, } def main(): sample_file = os.path.join(os.path.dirname(__file__), "sample_data.jsonl") if not os.path.exists(sample_file): print(f"Sample data file {sample_file} not found.") return print(f"Loading sample questions from: {sample_file}") with open(sample_file, "r", encoding="utf-8") as f: records = [json.loads(line) for line in f if line.strip()] print(f"Loaded {len(records)} sample questions.") all_permuted = [] for r in records: perms = generate_cyclic_permutations(r) all_permuted.extend(perms) print(f"Generated {len(all_permuted)} balanced permutations (4x expansion).") first_formatted = format_micro_cot_sample(all_permuted[0]) print("\n--- Example Formatted Micro-CoT Training Text ---") print(first_formatted["text"][:500] + "\n...") if __name__ == "__main__": main()