Download training/prepare_curriculum_data.py from SinhalaMMLU-SharedTask/team-nena: direct link, hf CLI and curl.
- Browser
- Download file 5.27 kB
-
https://huggingface.co/SinhalaMMLU-SharedTask/team-nena/resolve/main/training/prepare_curriculum_data.py
- Command line
-
hf download hf://SinhalaMMLU-SharedTask/team-nena/training/prepare_curriculum_data.py
-
curl -L -o prepare_curriculum_data.py https://huggingface.co/SinhalaMMLU-SharedTask/team-nena/resolve/main/training/prepare_curriculum_data.py
5.27 kB
| """ | |
| Sinhala MMLU Shared Task — Data Preparation & Cyclic Permutation Script | |
| Team: Nena | |
| This script demonstrates: | |
| 1. Sinhala Unicode text normalization (ZWJ/ZWNJ handling, virama consonant clusters). | |
| 2. 4-way cyclic option permutation to eliminate positional choice bias. | |
| 3. Micro-Chain-of-Thought (Micro-CoT) formatting with curriculum grounding. | |
| """ | |
| import json | |
| import os | |
| import re | |
| import sys | |
| if hasattr(sys.stdout, "reconfigure"): | |
| try: | |
| sys.stdout.reconfigure(encoding="utf-8", errors="replace") | |
| sys.stderr.reconfigure(encoding="utf-8", errors="replace") | |
| except Exception: | |
| pass | |
| VIRAMA = "\u0DCA" | |
| ZWJ = "\u200D" | |
| ZWNJ = "\u200C" | |
| RA = "\u0DBB" | |
| YA = "\u0DBA" | |
| def normalize_sinhala_text(text: str) -> str: | |
| """ | |
| Normalizes Sinhala Unicode text, repairing zero-width joiners (ZWJ) | |
| for yansaya / rakaransaya clusters and stripping invisible zero-width non-joiners. | |
| """ | |
| if not text or not isinstance(text, str): | |
| return "" | |
| t = text.replace("\ufeff", "").replace("\u200b", "") | |
| t = t.replace(VIRAMA + ZWNJ, VIRAMA) | |
| t = t.replace(ZWNJ, "") | |
| t = re.sub(f"{VIRAMA}(?!{ZWJ}){RA}", f"{VIRAMA}{ZWJ}{RA}", t) | |
| t = re.sub(f"{VIRAMA}(?!{ZWJ}){YA}", f"{VIRAMA}{ZWJ}{YA}", t) | |
| t = re.sub(f"{ZWJ}+", ZWJ, t) | |
| return re.sub(r"[ \t]+", " ", t).strip() | |
| def generate_cyclic_permutations(item: dict) -> list[dict]: | |
| """ | |
| Generates 4 cyclic permutations of choices for an MCQ to ensure | |
| the target model does not develop positional bias toward any option index. | |
| Expected item schema: | |
| - question: str | |
| - choices: list of 4 str | |
| - answer: int (1-based, 1..4) | |
| - explanation: str | |
| - subject: str (optional) | |
| - grade: str (optional) | |
| """ | |
| q = normalize_sinhala_text(item["question"]) | |
| choices = [normalize_sinhala_text(c) for c in item["choices"]] | |
| gold_1based = int(item.get("answer", item.get("correct_index", 1))) | |
| gold_0based = gold_1based - 1 | |
| exp = normalize_sinhala_text(item.get("explanation", "")) | |
| subject = item.get("subject", "General") | |
| grade = item.get("grade", "General") | |
| if len(choices) != 4: | |
| return [item] | |
| permutations = [] | |
| # 4 cyclic shifts: shift k in {0, 1, 2, 3} | |
| for shift in range(4): | |
| perm_choices = choices[shift:] + choices[:shift] | |
| new_gold_0based = (gold_0based - shift) % 4 | |
| new_gold_1based = new_gold_0based + 1 | |
| permutations.append({ | |
| "question": q, | |
| "choices": perm_choices, | |
| "answer": new_gold_1based, | |
| "explanation": exp, | |
| "subject": subject, | |
| "grade": grade, | |
| "shift": shift, | |
| }) | |
| return permutations | |
| def format_micro_cot_sample(item: dict) -> dict: | |
| """ | |
| Formats a raw MCQ record into the exact prompt structure used for SFT training. | |
| """ | |
| system_prompt = ( | |
| "ඔබ ශ්රී ලංකා ජාතික විෂය නිර්දේශයේ ප්රවීණ ගුරුවරයෙකි. " | |
| "පහත බහුවරණ ප්රශ්නයට කෙටි හේතු දැක්වීමක් සහිතව නිවැරදි පිළිතුර තෝරන්න." | |
| ) | |
| subj = item.get("subject", "General") | |
| grade = item.get("grade", "General") | |
| q = normalize_sinhala_text(item["question"]) | |
| c = [normalize_sinhala_text(x) for x in item["choices"]] | |
| ans_idx = int(item["answer"]) | |
| correct_text = c[ans_idx - 1] | |
| exp = normalize_sinhala_text(item.get("explanation", "")) | |
| choices_block = "\n".join(f"{i + 1}. {choice}" for i, choice in enumerate(c)) | |
| prompt = ( | |
| f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n" | |
| f"{system_prompt}<|eot_id|>" | |
| f"<|start_header_id|>user<|end_header_id|>\n\n" | |
| f"<|subject: {subj}|> <|grade: {grade}|>\n\n" | |
| f"ප්රශ්නය: {q}\n\n" | |
| f"වරණ:\n" | |
| f"{choices_block}\n\n" | |
| f"පිළිතුර:<|eot_id|>" | |
| f"<|start_header_id|>assistant<|end_header_id|>\n\n" | |
| ) | |
| completion = ( | |
| f"හේතු දැක්වීම: {exp}\n\n" | |
| f"නිවැරදි පිළිතුර: {ans_idx}. {correct_text}<|eot_id|>" | |
| ) | |
| return { | |
| "text": prompt + completion, | |
| "prompt": prompt, | |
| "completion": completion, | |
| } | |
| def main(): | |
| sample_file = os.path.join(os.path.dirname(__file__), "sample_data.jsonl") | |
| if not os.path.exists(sample_file): | |
| print(f"Sample data file {sample_file} not found.") | |
| return | |
| print(f"Loading sample questions from: {sample_file}") | |
| with open(sample_file, "r", encoding="utf-8") as f: | |
| records = [json.loads(line) for line in f if line.strip()] | |
| print(f"Loaded {len(records)} sample questions.") | |
| all_permuted = [] | |
| for r in records: | |
| perms = generate_cyclic_permutations(r) | |
| all_permuted.extend(perms) | |
| print(f"Generated {len(all_permuted)} balanced permutations (4x expansion).") | |
| first_formatted = format_micro_cot_sample(all_permuted[0]) | |
| print("\n--- Example Formatted Micro-CoT Training Text ---") | |
| print(first_formatted["text"][:500] + "\n...") | |
| if __name__ == "__main__": | |
| main() | |