{ "cells": [ { "cell_type": "markdown", "metadata": { "id": "sQUrZsAglHQz" }, "source": [ "# Import Module" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ES1ES_GVlEha" }, "outputs": [], "source": [ "import os\n", "import re\n", "import shutil\n", "import subprocess\n", "import tempfile\n", "import time\n", "from pathlib import Path\n", "from typing import List, Tuple\n", "\n", "import pandas as pd\n", "import torch\n", "from tqdm import tqdm\n", "from transformers import (\n", " AutoTokenizer,\n", " AutoModelForSeq2SeqLM,\n", " AutoModelForCausalLM,\n", " set_seed,\n", " StoppingCriteria,\n", " StoppingCriteriaList,\n", ")" ] }, { "cell_type": "markdown", "metadata": { "id": "6husnAQMlMgS" }, "source": [ "# Configuration" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "4lvmEKJMlMM1" }, "outputs": [], "source": [ "from google.colab import drive\n", "drive.mount('/content/drive')\n", "\n", "DETAIL_PATH = \"/content/drive/MyDrive/DETAIL_PLBART.xlsx\"\n", "STATS_PATH = \"/content/drive/MyDrive/Statistik_PLBART.xlsx\"\n", "\n", "JUNIT_JAR = \"/content/junit-4.12.jar\"\n", "HAMCREST_JAR = \"/content/hamcrest-core-1.3.jar\"\n", "\n", "\n", "DATASET_PATH = \"/content/decoded_databaru.xlsx\"\n", "MODEL_PATH = \"/content/drive/MyDrive/plbart-base-finetuned-apr-10k-final\"\n", "# MODEL_SIZE = \"base\"\n", "DEVICE = \"cuda\"\n", "NUM_PATCHES = 10\n", "TIMEOUT = 30\n", "KEEP_TMP = False\n", "OUTPUT_DIR = \"apr_results\"\n", "\n", "os.makedirs(OUTPUT_DIR, exist_ok=True)\n", "set_seed(42)" ] }, { "cell_type": "markdown", "metadata": { "id": "Ev4mOgMSlYy_" }, "source": [ "# Preprocessing" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "nXXKSeF8lcS-" }, "outputs": [], "source": [ "class StopOnTokens(StoppingCriteria):\n", " def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:\n", " return False\n", "\n", "\n", "def extract_public_class_name(java_text: str) -> str:\n", " m = re.search(r\"public\\s+class\\s+([A-Za-z_]\\w*)\", java_text)\n", " if m:\n", " return m.group(1)\n", " m2 = re.search(r\"class\\s+([A-Za-z_]\\w*)\", java_text)\n", " if m2:\n", " return m2.group(1)\n", " return \"PatchedClass\"\n", "\n", "\n", "def get_package_dir(java_text: str, base_dir: str) -> Tuple[str, str]:\n", " m = re.search(r\"package\\s+([a-zA-Z0-9_.]+)\\s*;\", java_text)\n", " if m:\n", " pkg = m.group(1)\n", " pkg_dir = os.path.join(base_dir, *pkg.split(\".\"))\n", " return pkg, pkg_dir\n", " return None, base_dir" ] }, { "cell_type": "markdown", "metadata": { "id": "h6uM0l6_ltnM" }, "source": [ "# Fungsi Java" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "TWwKlRkXluuG" }, "outputs": [], "source": [ "def save_java_file(java_text: str, base_tmp_dir: str) -> Tuple[str, str]:\n", " class_name = extract_public_class_name(java_text)\n", " pkg, pkg_dir = get_package_dir(java_text, base_tmp_dir)\n", " os.makedirs(pkg_dir, exist_ok=True)\n", " file_path = os.path.join(pkg_dir, f\"{class_name}.java\")\n", " with open(file_path, \"w\", encoding=\"utf-8\") as f:\n", " f.write(java_text)\n", " if pkg:\n", " full_classname = f\"{pkg}.{class_name}\"\n", " else:\n", " full_classname = class_name\n", " return file_path, full_classname\n", "\n", "\n", "def compile_all_java_files(tmp_dir: str, timeout: int = 30) -> Tuple[bool, str]:\n", " java_files = []\n", " for root, _, files in os.walk(tmp_dir):\n", " for fn in files:\n", " if fn.endswith(\".java\"):\n", " java_files.append(os.path.join(root, fn))\n", " if not java_files:\n", " return False, \"NO_JAVA_FILES\"\n", "\n", " classpath = f\".:{JUNIT_JAR}:{HAMCREST_JAR}\"\n", " cmd = [\"javac\", \"-cp\", classpath, \"-d\", tmp_dir] + java_files\n", "\n", " try:\n", " proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout, check=True)\n", " return True, \"\"\n", " except subprocess.CalledProcessError as e:\n", " stderr = e.stderr.decode(\"utf-8\", errors=\"ignore\")\n", " return False, f\"COMPILE_ERROR: {stderr}\"\n", " except subprocess.TimeoutExpired:\n", " return False, \"COMPILE_TIMEOUT\"\n", "\n", "\n", "def run_test_main(tmp_dir: str, main_class: str, timeout: int = 30) -> Tuple[bool, str]:\n", " classpath = f\"{tmp_dir}:{JUNIT_JAR}:{HAMCREST_JAR}\"\n", " cmd = [\"java\", \"-cp\", classpath, main_class]\n", "\n", " try:\n", " proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout, check=True)\n", " stdout = proc.stdout.decode(\"utf-8\", errors=\"ignore\").strip()\n", " if \"PASS\" in stdout:\n", " return True, \"PASS\"\n", " else:\n", " return False, f\"TEST_FAIL: {stdout or proc.stderr.decode('utf-8', errors='ignore')}\"\n", " except subprocess.CalledProcessError as e:\n", " stdout = e.stdout.decode(\"utf-8\", errors=\"ignore\")\n", " stderr = e.stderr.decode(\"utf-8\", errors=\"ignore\")\n", " return False, f\"RUNTIME_ERROR: stdout[{stdout}] stderr[{stderr}]\"\n", " except subprocess.TimeoutExpired:\n", " return False, \"RUN_TIMEOUT\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "I7Si63_cl699" }, "outputs": [], "source": [ "class APRWithCLM:\n", " def __init__(self, device=\"cpu\"):\n", " self.device = device if torch.cuda.is_available() and device == \"cuda\" else \"cpu\"\n", " set_seed(42)\n", "\n", " self.model_id = MODEL_PATH\n", " self.tokenizer = AutoTokenizer.from_pretrained(self.model_id, use_fast=True)\n", " self.model = AutoModelForSeq2SeqLM.from_pretrained(\n", " self.model_id,\n", " device_map=\"auto\" if self.device == \"cuda\" else None,\n", " torch_dtype=torch.float16 if self.device == \"cuda\" else None,\n", " )\n", " self.is_encoder_decoder = True\n", "\n", " self.model.eval()\n", " self.stop = StopOnTokens()\n", " self.eos_token_id = getattr(self.tokenizer, \"eos_token_id\", None)\n", " self.pad_token_id = getattr(self.tokenizer, \"pad_token_id\", self.eos_token_id)\n", " self.bos_token_id = getattr(self.tokenizer, \"bos_token_id\", None)\n", "\n", " def create_prompt(self, buggy_code: str, buggy_lines: str = None) -> str:\n", " if buggy_lines:\n", " return f\"Repair this Java code. The buggy lines are: {buggy_lines}\\n{buggy_code}\\nFixed version:\"\n", " return f\"Repair this Java code:\\n{buggy_code}\\nFixed version:\"\n", "\n", " def _clean_generated(self, text: str, prompt: str) -> str:\n", " if prompt and prompt in text:\n", " text = text.split(prompt, 1)[-1]\n", " text = text.strip()\n", " text = re.sub(r\"^.*?(public\\s+class\\s+)\", r\"\\1\", text, flags=re.S)\n", " idx = text.find(\"public class\")\n", " if idx != -1:\n", " text = text[idx:]\n", " m = re.search(r\"(public\\s+class\\s+\\w+\\s*\\{)\", text)\n", " if m:\n", " start = m.start()\n", " brace_count = 0\n", " for i in range(start, len(text)):\n", " if text[i] == \"{\":\n", " brace_count += 1\n", " elif text[i] == \"}\":\n", " brace_count -= 1\n", " if brace_count == 0:\n", " text = text[: i + 1]\n", " break\n", " return text.strip()\n", "\n", " def generate_single_patch(self, prompt: str, buggy_lines: str = None, max_length: int = 1024) -> str:\n", " inputs = self.tokenizer(prompt, return_tensors=\"pt\", truncation=True, max_length=1024)\n", " input_ids = inputs[\"input_ids\"].to(self.model.device)\n", " attention_mask = inputs.get(\"attention_mask\")\n", " if attention_mask is not None:\n", " attention_mask = attention_mask.to(self.model.device)\n", "\n", " generation_kwargs = {\n", " \"max_length\": max_length,\n", " \"do_sample\": True,\n", " \"top_k\": 50,\n", " \"top_p\": 0.95,\n", " \"temperature\": 0.7,\n", " \"eos_token_id\": self.eos_token_id,\n", " \"pad_token_id\": self.pad_token_id,\n", " }\n", " if self.is_encoder_decoder:\n", " decoder_start_id = None\n", " try:\n", " decoder_start_id = self.tokenizer.lang_code_to_id.get(\"java\")\n", " except Exception:\n", " decoder_start_id = None\n", " if decoder_start_id is None:\n", " decoder_start_id = self.bos_token_id or self.eos_token_id or 0\n", " generation_kwargs.update({\"decoder_start_token_id\": decoder_start_id})\n", "\n", " with torch.no_grad():\n", " try:\n", " outputs = self.model.generate(\n", " input_ids=input_ids,\n", " attention_mask=attention_mask,\n", " num_return_sequences=1,\n", " **generation_kwargs,\n", " )\n", " except Exception as e:\n", " print(\"Generation failed, retrying greedy. Error:\", e)\n", " outputs = self.model.generate(input_ids=input_ids, max_length=512, num_return_sequences=1)\n", "\n", " raw = self.tokenizer.decode(outputs[0], skip_special_tokens=False)\n", " cleaned = self._clean_generated(raw, prompt)\n", " return cleaned\n", "\n", " def generate_patches_with_progress(self, buggy_code: str, num_patches: int = 5, buggy_lines: str = None) -> List[str]:\n", " prompt = self.create_prompt(buggy_code, buggy_lines)\n", " patches = []\n", "\n", " for _ in tqdm(range(num_patches), desc=\" Generating\", leave=False, unit=\"patch\"):\n", " p = self.generate_single_patch(prompt, buggy_lines)\n", " if p and \"class \" in p and \"{\" in p and \"}\" in p:\n", " patches.append(p)\n", " else:\n", "\n", " patches.append(p)\n", " return patches[:num_patches]" ] }, { "cell_type": "markdown", "metadata": { "id": "69Fgzporl9uR" }, "source": [ "# Fungsi Simpan Hasil" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ayWKeU46l_SH" }, "outputs": [], "source": [ "def save_results_and_stats(results: List[dict], dataset_path: str):\n", "\n", " try:\n", " existing_results = pd.read_excel(DETAIL_PATH)\n", " results_df = pd.concat([existing_results, pd.DataFrame(results)], ignore_index=True)\n", " results_df = results_df.drop_duplicates(subset=[\"task_id\", \"patch_idx\"], keep=\"last\")\n", " except:\n", " df_original = pd.read_excel(dataset_path)\n", " results_df = pd.DataFrame(results)\n", " results_df = results_df.merge(\n", " df_original[[\"title\", \"bug\", \"correct\"]].rename(columns={\"title\": \"task_id\"}),\n", " on=\"task_id\", how=\"left\"\n", " )\n", "\n", "\n", " cols_order = [\"task_id\", \"patch_idx\", \"bug\", \"correct\", \"compiled\", \"passed\", \"validation_status\", \"message\", \"patch\"]\n", " results_df = results_df[cols_order]\n", "\n", "\n", " results_df.to_excel(DETAIL_PATH, index=False)\n", " print(f\"Saved DETAIL_HASILNYA.xlsx to {DETAIL_PATH}\")\n", "\n", " # ringkasan statistik\n", " task_stats = {}\n", " for _, row in results_df.iterrows():\n", " task = row[\"task_id\"]\n", " if task not in task_stats:\n", " task_stats[task] = {\n", " \"total_candidates\": 0,\n", " \"compilable\": 0,\n", " \"passed_test\": 0,\n", " \"valid\": 0,\n", " \"in_plausible\": 0,\n", " \"fail\": 0\n", " }\n", " task_stats[task][\"total_candidates\"] += 1\n", "\n", " if row[\"compiled\"]:\n", " task_stats[task][\"compilable\"] += 1\n", " if row[\"passed\"]:\n", " task_stats[task][\"passed_test\"] += 1\n", " task_stats[task][\"valid\"] += 1\n", " else:\n", " task_stats[task][\"in_plausible\"] += 1\n", " else:\n", " task_stats[task][\"fail\"] += 1\n", "\n", " stats_df = pd.DataFrame.from_dict(task_stats, orient=\"index\")\n", " stats_df[\"compilable_rate\"] = stats_df[\"compilable\"] / stats_df[\"total_candidates\"]\n", " stats_df[\"pass_rate\"] = stats_df[\"passed_test\"] / stats_df[\"total_candidates\"]\n", "\n", " stats_df.to_excel(STATS_PATH)\n", " print(f\"Saved BERAPA_YANG_BENAR.xlsx to {STATS_PATH}\")\n", "\n", " print(f\"\\nTotal tasks: {len(task_stats)}\")\n", " print(f\"Total compilable candidates: {stats_df['compilable'].sum()}\")\n", " print(f\"Total passed tests: {stats_df['passed_test'].sum()}\")\n", " total_correct_fixes = sum(1 for task in task_stats.values() if task[\"passed_test\"] > 0)\n", " print(\"*\" * 80)\n", " print(f\"Total tasks with at least 1 passed test: {total_correct_fixes}\")\n", "\n" ] }, { "cell_type": "markdown", "metadata": { "id": "sRX5Ln44mICR" }, "source": [ "# Training" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "XAal48chmH0t" }, "outputs": [], "source": [ "def run_dataset_and_eval(dataset_path: str,\n", " apr: APRWithCLM,\n", " num_patches: int = 5,\n", " timeout: int = 30,\n", " output_dir: str = OUTPUT_DIR,\n", " keep_tmp: bool = False) -> pd.DataFrame:\n", " df = pd.read_excel(dataset_path)\n", " required_cols = {\"bug\", \"correct\", \"test\", \"title\"}\n", " if not required_cols.issubset(set(df.columns)):\n", " raise ValueError(f\"Excel harus punya kolom: {required_cols}\")\n", "\n", "\n", " try:\n", " existing_results = pd.read_excel(DETAIL_PATH)\n", " completed_tasks = existing_results[\"task_id\"].unique()\n", " except:\n", " existing_results = None\n", " completed_tasks = []\n", "\n", " results = []\n", " start_ts = int(time.time())\n", "\n", " for idx, row in tqdm(df.iterrows(), total=len(df), desc=\"Tasks\", unit=\"task\"):\n", " task_id = row[\"title\"]\n", "\n", " if task_id in completed_tasks:\n", " print(f\"\\nSkipping already completed task: {task_id}\")\n", " continue\n", "\n", " buggy_code = row[\"bug\"]\n", " correct_code = row.get(\"correct\", \"\")\n", " test_code = row[\"test\"]\n", " tmp_dir = tempfile.mkdtemp(prefix=f\"apr_{task_id}_\")\n", "\n", " try:\n", " pkg, pkg_dir = get_package_dir(buggy_code, tmp_dir)\n", " os.makedirs(pkg_dir, exist_ok=True)\n", " buggy_class_name = extract_public_class_name(buggy_code)\n", " with open(os.path.join(pkg_dir, f\"{buggy_class_name}.java\"), \"w\", encoding=\"utf-8\") as f:\n", " f.write(buggy_code)\n", " except Exception:\n", " pass\n", "\n", " patches = apr.generate_patches_with_progress(buggy_code, num_patches=num_patches)\n", "\n", " for p_idx, patch in enumerate(tqdm(patches, desc=\" Patches\", leave=False, unit=\"p\")):\n", " if not patch or not patch.strip():\n", " results.append({\n", " \"task_id\": task_id,\n", " \"patch_idx\": p_idx + 1,\n", " \"compiled\": False,\n", " \"passed\": False,\n", " \"validation_status\": \"fail\",\n", " \"message\": \"EMPTY_PATCH\",\n", " \"patch\": \"\",\n", " })\n", " continue\n", "\n", " try:\n", " patched_file, full_classname = save_java_file(patch, tmp_dir)\n", " except Exception as e:\n", " results.append({\n", " \"task_id\": task_id,\n", " \"patch_idx\": p_idx + 1,\n", " \"compiled\": False,\n", " \"passed\": False,\n", " \"validation_status\": \"fail\",\n", " \"message\": f\"SAVE_ERROR: {e}\",\n", " \"patch\": patch[:500],\n", " })\n", " continue\n", "\n", " main_class, main_source = convert_junit_to_main(test_code, extract_public_class_name(patch))\n", " main_path = os.path.join(tmp_dir, f\"{main_class}.java\")\n", " with open(main_path, \"w\", encoding=\"utf-8\") as f:\n", " f.write(main_source)\n", "\n", " compiled, compile_msg = compile_all_java_files(tmp_dir, timeout=timeout)\n", " if not compiled:\n", " results.append({\n", " \"task_id\": task_id,\n", " \"patch_idx\": p_idx + 1,\n", " \"compiled\": False,\n", " \"passed\": False,\n", " \"validation_status\": \"fail\",\n", " \"message\": compile_msg,\n", " \"patch\": patch[:500],\n", " })\n", " continue\n", "\n", " passed, run_msg = run_test_main(tmp_dir, main_class, timeout=timeout)\n", "\n", " # Tentukan validation status\n", " if not compiled:\n", " val_status = \"fail\"\n", " elif passed:\n", " val_status = \"valid\"\n", " else:\n", " val_status = \"in-plausible\"\n", "\n", " results.append({\n", " \"task_id\": task_id,\n", " \"patch_idx\": p_idx + 1,\n", " \"compiled\": True,\n", " \"passed\": passed,\n", " \"validation_status\": val_status,\n", " \"message\": run_msg,\n", " \"patch\": patch[:10000],\n", " })\n", "\n", " save_results_and_stats(results, dataset_path)\n", " results = []\n", "\n", " if not keep_tmp:\n", " try:\n", " shutil.rmtree(tmp_dir)\n", " except Exception:\n", " pass\n", " else:\n", " try:\n", " dst = os.path.join(output_dir, f\"tmp_{task_id}_{start_ts}\")\n", " if os.path.exists(dst):\n", " shutil.rmtree(dst)\n", " shutil.move(tmp_dir, dst)\n", " except Exception:\n", " pass\n", "\n", " return pd.DataFrame(results)" ] }, { "cell_type": "markdown", "metadata": { "id": "EOFOm_3YmLX4" }, "source": [ "# Main" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "aJSyllYBmL7l" }, "outputs": [], "source": [ "if __name__ == \"__main__\":\n", " print(\"=\"*80)\n", " print(f\"{'Loading model & tokenizer':^80}\")\n", " print(\"=\"*80)\n", " apr = APRWithCLM(device=DEVICE)\n", " print(f\"\\nModel loaded: {apr.model_id} on device {apr.device}\\n\")\n", "\n", " print(\"=\"*80)\n", " print(f\"{'Running dataset':^80}\")\n", " print(\"=\"*80)\n", "\n", " # Buat progress bar utama untuk semua tasks\n", " with tqdm(total=len(pd.read_excel(DATASET_PATH)), desc=\"Overall Progress\") as pbar_main:\n", " res_df = run_dataset_and_eval(\n", " DATASET_PATH,\n", " apr,\n", " num_patches=NUM_PATCHES,\n", " timeout=TIMEOUT,\n", " keep_tmp=KEEP_TMP,\n", " )\n", "\n", " print(\"\\n\" + \"=\"*80)\n", " print(f\"{'Selesai. Ringkasan:':^80}\")\n", " print(\"=\"*80)\n", " print(res_df.groupby([\"compiled\", \"passed\"]).size())\n", " print(\"\\n\")" ] } ], "metadata": { "colab": { "provenance": [] }, "kernelspec": { "display_name": "Python 3", "name": "python3" }, "language_info": { "name": "python" } }, "nbformat": 4, "nbformat_minor": 0 }