File size: 7,393 Bytes
682bee1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 | # === STANDALONE CELL 4: PASS 3 (4096 TOKENS + SANSKRIT/TECHNICAL HINT ON 720 TARGET QS) ===
import os, re, json, time, zipfile, shutil, subprocess, urllib.request
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor, as_completed
# 1. Ensure both Ollama servers (GPU 0 -> 11434, GPU 1 -> 11435) are running
for idx, port in [(0, 11434), (1, 11435)]:
try:
urllib.request.urlopen(f"http://127.0.0.1:{port}/api/tags", timeout=3)
except Exception:
env = {**os.environ, "CUDA_VISIBLE_DEVICES": str(idx), "OLLAMA_HOST": f"127.0.0.1:{port}", "OLLAMA_NUM_PARALLEL": "2", "OLLAMA_FLASH_ATTENTION": "1", "OLLAMA_KEEP_ALIVE": "24h"}
subprocess.Popen(["ollama", "serve"], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
time.sleep(3)
# 2. Locate /kaggle/working/work/data (or extract from /kaggle/input if needed)
work_dir = Path("/kaggle/working/work")
work_dir.mkdir(parents=True, exist_ok=True)
data_dir = work_dir / "data"
if not data_dir.exists():
for p in Path("/kaggle/input").rglob("*"):
if p.name == "kaggle_data.zip":
with zipfile.ZipFile(p, "r") as z:
z.extractall(work_dir)
break
elif p.name == "predictions_gemma4_cot.jsonl":
shutil.copytree(p.parent.parent, work_dir, dirs_exist_ok=True)
break
def load_all_questions(d_dir: Path):
qs = []
for lvl in ["easy", "medium", "hard"]:
for f in sorted((d_dir / lvl).glob("*.json")):
for item in json.load(open(f, encoding="utf-8")):
item["_level"] = lvl
qs.append(item)
return qs
def load_ckpt(path: Path):
m = {}
if path.exists():
for line in open(path, encoding="utf-8"):
if line.strip():
r = json.loads(line)
k = f"{str(r['level']).lower().strip()}_{str(r['subject']).strip()}_{str(r['q_no']).strip()}"
m[k] = str(r["answer"])
return m
# NOTE (archive edit): the original cell embedded the 2238 Pass-1 answers as a digit string (P1_DIGITS) to resume
# after a Kaggle session reset. They are identical to ../passes/pass1_think_2048_orig.jsonl and are loaded from there.
P1_DIGITS = ''.join(json.loads(l)['answer'] for l in open('/kaggle/working/pass1_think_2048_orig.jsonl', encoding='utf-8') if l.strip())
all_qs = load_all_questions(data_dir)
p1_map = {f"{q['_level'].lower()}_{q['subject'].strip()}_{q['q_no']}": P1_DIGITS[i] for i, q in enumerate(all_qs)}
p2_map = load_ckpt(Path("/kaggle/working/predictions_gemma4_think_pass2_ckpt.jsonl"))
cot_map = load_ckpt(data_dir / "predictions_gemma4_cot.jsonl")
dir_map = load_ckpt(data_dir / "predictions_gemma4_direct.jsonl")
# Target: (A) 348 questions where P1 != P2, plus (B) 372 questions not in P2 where P1 != CoT or P1 != Direct
target_qs = []
for q in all_qs:
k = f"{q['_level'].lower()}_{q['subject'].strip()}_{q['q_no']}"
if k in p2_map:
if p1_map.get(k) != p2_map.get(k):
target_qs.append(q)
else:
if p1_map.get(k) != cot_map.get(k) or p1_map.get(k) != dir_map.get(k):
target_qs.append(q)
p3_ckpt_path = Path("/kaggle/working/predictions_gemma4_think_pass3_ckpt.jsonl")
done_p3 = load_ckpt(p3_ckpt_path)
rem_p3 = [q for q in target_qs if f"{q['_level'].lower()}_{q['subject'].strip()}_{q['q_no']}" not in done_p3]
print(f"Pass 3 Target: {len(target_qs)} questions | Already done: {len(done_p3)} | Remaining: {len(rem_p3)}")
SYSTEM_THINK_PROMPT = (
"You are an expert teacher of the Sri Lankan national school curriculum (grades 6-13), "
"fluent in Sinhala. You will receive a multiple-choice question written in Sinhala with four "
"numbered options. Exactly one option is correct. Read the Sinhala carefully, including "
"negations such as 'නොවන' / 'නොවේ' (NOT). If the question or options contain Sanskrit, "
"Hindustani, Pali, Tamil, or technical terms, carefully map them to their standard curriculum "
"meaning. You may reason in English. Finish with the final answer in the form 'Answer: <number>'."
)
ANSWER_THINK_RE = re.compile(r"(?:Answer|පිළිතුර)\s*[::]?\s*\**\s*\(?([1-4])\)?", re.IGNORECASE)
def solve_p3(idx_q):
idx, q = idx_q
port = 11434 if (idx % 2 == 0) else 11435
lvl = q["_level"].lower()
subj = q.get("subject", "General")
q_no = q["q_no"]
q_key = f"{lvl}_{subj.strip()}_{q_no}"
grade = "6-9" if lvl == "easy" else ("10-11 (O/L)" if lvl == "medium" else "12-13 (A/L)")
opts = "\n".join(f"{i}. {c}" for i, c in enumerate(q["choices"], 1))
user_msg = (
f"විෂය (Subject): {subj} | ශ්රේණිය (Grade): {grade}\n\n"
f"ප්රශ්නය (Question):\n{q['question'].strip()}\n\n"
f"විකල්ප (Options):\n{opts}\n\n"
"Which option (1, 2, 3 or 4) is correct? End your reply with 'Answer: <number>'."
)
payload = {
"model": "gemma4:e4b",
"messages": [
{"role": "system", "content": SYSTEM_THINK_PROMPT},
{"role": "user", "content": user_msg},
],
"think": True,
"stream": False,
"options": {"temperature": 0.1, "top_p": 0.95, "num_predict": 4096},
}
for attempt in range(3):
try:
req = urllib.request.Request(
f"http://127.0.0.1:{port}/api/chat",
data=json.dumps(payload).encode("utf-8"),
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(req, timeout=300) as resp:
data = json.loads(resp.read().decode("utf-8"))
content = data.get("message", {}).get("content", "") or ""
thinking = data.get("message", {}).get("thinking", "") or ""
m = ANSWER_THINK_RE.findall(content) or ANSWER_THINK_RE.findall(thinking)
if m:
ans = m[-1]
else:
digits = re.findall(r"\b([1-4])\b", content if content.strip() else thinking)
ans = digits[-1] if digits else "1"
return {
"q_no": q_no,
"subject": subj,
"category": q.get("category", "general"),
"level": lvl,
"answer": str(ans),
}
except Exception:
time.sleep(2)
return {
"q_no": q_no,
"subject": subj,
"category": q.get("category", "general"),
"level": lvl,
"answer": p1_map.get(q_key, "1"),
}
t0 = time.time()
done_count = len(done_p3)
with open(p3_ckpt_path, "a", encoding="utf-8") as f_ckpt:
with ThreadPoolExecutor(max_workers=4) as ex:
futures = {ex.submit(solve_p3, (i, q)): q for i, q in enumerate(rem_p3)}
for fut in as_completed(futures):
res = fut.result()
f_ckpt.write(json.dumps(res, ensure_ascii=False) + "\n")
f_ckpt.flush()
done_count += 1
if done_count % 20 == 0 or done_count == len(target_qs):
el = time.time() - t0
rate = (done_count - len(done_p3)) / max(el, 1e-5)
eta = (len(target_qs) - done_count) / max(rate, 1e-5) / 60
print(f"[Pass 3: {done_count}/{len(target_qs)}] {rate:.2f} q/s | ETA: {eta:.1f} min", flush=True)
print("PASS 3 COMPLETE!")
|