generate_dataset.py

#!/usr/bin/env python3
"""
Harbour PRG/CH Dataset Generator for Fine-tuning qwen2.5-coder:14b
Generates structured JSONL dataset from .prg and .ch files with descriptions.
CLEANED VERSION - removes boilerplate, ensures code quality.
"""

import os
import re
import json
import random
from pathlib import Path
from typing import Dict, List, Tuple, Optional

# Configuration
HARBOUR_ROOT = Path("/home/fivetech/harbour")
OUTPUT_DIR = Path("/home/fivetech/finetune")
MAX_CODE_LENGTH = 8000
MIN_CODE_LENGTH = 80  # Minimum chars of actual code
TRAIN_RATIO = 0.9

# Module descriptions for contrib
MODULE_DESCRIPTIONS = {
    "hbhttpd": "Multithreaded HTTP/HTTPS server framework",
    "hbwin": "Windows API wrapper functions",
    "hbpgsql": "PostgreSQL database client library",
    "hbmysql": "MySQL database client library",
    "hbsqlit3": "SQLite3 database client library",
    "hbodbc": "ODBC database connectivity",
    "hbtip": "Internet protocol utilities (FTP, HTTP, SMTP, POP3)",
    "hbcurl": "libcurl wrapper for HTTP/FTP/SMTP operations",
    "hbssl": "OpenSSL wrapper for SSL/TLS encryption",
    "hbnf": "NanForum Toolkit - legacy Clipper compatibility functions",
    "hbct": "CA-Tools compatibility library",
    "hbmisc": "Miscellaneous utility functions",
    "hbgd": "Graphics drawing library (GD)",
    "hbcairo": "Cairo graphics library wrapper",
    "hbhpdf": "PDF generation library (libharu)",
    "hbbmp": "BMP image handling",
    "hbzebra": "Barcode generation library",
    "hbexpat": "XML parsing library (Expat)",
    "hbmxml": "XML generation library",
    "hbnetio": "Network I/O operations",
    "hbpipeio": "Process pipe I/O operations",
    "hbmemio": "Memory file I/O operations",
    "xhb": "Extended Harbour functions",
    "hbxpp": "xBase++ compatibility functions",
    "hbunix": "Unix-specific functions",
    "hbtpathy": "Telepath communication library",
    "hbblat": "Blat email sending utility",
    "hbblink": "Blinker function extender",
    "hbgs": "Ghostscript wrapper",
    "hbfship": "Fships library functions",
    "hbmzip": "ZIP file handling",
    "hbziparc": "ZIP archive handling",
    "hbxdiff": "File difference/patching",
    "hblzf": "LZF compression library",
    "hbmlzo": "LZO compression library",
    "hbbz2": "BZ2 compression library",
    "hbformat": "Text formatting utilities",
    "hbfoxpro": "FoxPro file format support",
    "hbplist": "Apple plist file format support",
    "hbcups": "CUPS printing system wrapper",
    "hbsms": "SMS sending via modem",
    "hbcomm": "Serial communication library",
    "hbfbird": "Firebird database client",
    "hbfimage": "FreeImage library wrapper",
    "hbdoc": "Documentation generation utilities",
    "hbtinymt": "Tiny Mersenne Twister PRNG",
    "hbtest": "Test framework utilities",
}


def remove_all_comments(code: str) -> str:
    """Remove ALL comments from Harbour code: //, /* */, *, and inline comments."""
    # Remove block comments /* ... */ (including multi-line)
    code = re.sub(r'/\*.*?\*/', '', code, flags=re.DOTALL)

    # Remove single-line comments // ...
    code = re.sub(r'//[^\n]*', '', code)

    # Remove lines that are only star-prefixed comments: * ...
    # And standalone star lines used in block comment formatting
    lines = code.split("\n")
    result_lines = []
    for line in lines:
        stripped = line.strip()
        # Skip lines that are ONLY comments (star-prefixed, standalone)
        if re.match(r'^\*\s', stripped) or stripped == '*' or stripped == '*/' or stripped == '/*':
            continue
        result_lines.append(line)
    code = "\n".join(result_lines)

    # Remove inline comments at end of lines: code  * comment
    # Pattern: something followed by whitespace then * ... (but not in strings)
    code = re.sub(r'([^\s"].*?)\s+\*[^"]*$', r'\1', code, flags=re.MULTILINE)

    return code


def clean_excessive_blank_lines(code: str) -> str:
    """Remove excessive blank lines (more than 2 consecutive)."""
    lines = code.split("\n")
    result_lines = []
    blank_count = 0

    for line in lines:
        if line.strip() == "":
            blank_count += 1
            if blank_count <= 2:
                result_lines.append(line)
        else:
            blank_count = 0
            result_lines.append(line)

    return "\n".join(result_lines)


def remove_disabled_code(code: str) -> str:
    """Remove #if 0 ... #endif blocks (disabled code)."""
    lines = code.split("\n")
    result_lines = []
    in_disabled = False
    disabled_depth = 0

    for line in lines:
        stripped = line.strip()

        if stripped.upper().startswith("#IF 0") or stripped.upper().startswith("#IFDEF _HARBOUR_DISABLE"):
            in_disabled = True
            disabled_depth += 1
            continue

        if in_disabled:
            if stripped.upper().startswith("#ENDIF"):
                disabled_depth -= 1
                if disabled_depth <= 0:
                    in_disabled = False
            continue

        result_lines.append(line)

    return "\n".join(result_lines)


def is_ch_file(filepath: Path) -> bool:
    """Check if file is a .ch (Clipper header) file."""
    return filepath.suffix.lower() == '.ch'


def has_real_ch_code(code: str) -> bool:
    """Check if .ch file contains actual preprocessor definitions."""
    upper = code.upper()

    # Must have at least one preprocessor construct
    ch_constructs = [
        "#XCOMMAND", "#XTRANSLATE", "#COMMAND", "#TRANSLATE",
        "#DEFINE", "#UNDEF", "#IFDEF", "#IFNDEF", "#IF ",
        "#INCLUDE", "#PRAGMA", "#ENDPROC",
    ]
    if not any(construct in upper for construct in ch_constructs):
        return False

    # Count actual definition lines
    lines = code.split("\n")
    def_lines = 0
    for line in lines:
        stripped = line.strip()
        if not stripped:
            continue
        if stripped.startswith("//") or stripped.startswith("/*") or stripped.startswith("*"):
            continue
        if stripped.startswith("#"):
            def_lines += 1

    return def_lines >= 3


def extract_ch_definitions(code: str) -> Dict:
    """Extract definitions from .ch file."""
    defines = []
    commands = []
    translates = []

    for line in code.split("\n"):
        stripped = line.strip()
        upper = stripped.upper()

        if upper.startswith("#DEFINE ") or upper.startswith("#UNDEF "):
            parts = stripped.split()
            if len(parts) >= 2:
                defines.append(parts[1])
        elif upper.startswith("#XCOMMAND") or upper.startswith("#COMMAND"):
            commands.append(stripped[:60])
        elif upper.startswith("#XTRANSLATE") or upper.startswith("#TRANSLATE"):
            translates.append(stripped[:60])

    return {
        "defines": defines[:10],
        "commands": commands[:5],
        "translates": translates[:5]
    }


def generate_ch_description(filepath: Path, code: str) -> str:
    """Generate description for .ch file."""
    rel_path = filepath.relative_to(HARBOUR_ROOT)
    filename = filepath.stem

    desc_parts = []

    # Determine location context
    if rel_path.parts[0] == "include":
        desc_parts.append(f"Harbour header file: {rel_path}")
    elif rel_path.parts[0] == "contrib":
        module = rel_path.parts[1] if len(rel_path.parts) > 1 else ""
        module_desc = MODULE_DESCRIPTIONS.get(module, "")
        if module_desc:
            desc_parts.append(f"Header file for contribution module '{module}' ({module_desc}): {rel_path}")
        else:
            desc_parts.append(f"Header file for contribution module '{module}': {rel_path}")
    else:
        desc_parts.append(f"Harbour header file: {rel_path}")

    # Extract and describe definitions
    defs = extract_ch_definitions(code)

    if defs["defines"]:
        if len(defs["defines"]) <= 5:
            desc_parts.append(f"Defines constants: {', '.join(defs['defines'])}")
        else:
            desc_parts.append(f"Defines {len(defs['defines'])} constants including: {', '.join(defs['defines'][:5])}")

    if defs["commands"]:
        desc_parts.append(f"Declares {len(defs['commands'])} preprocessor commands")

    if defs["translates"]:
        desc_parts.append(f"Declares {len(defs['translates'])} preprocessor translations")

    # Common header purposes
    upper = code.upper()
    if "ES_" in upper or "EG_" in upper or "ERROR" in filename.upper():
        desc_parts.append("Defines error handling constants and codes")
    elif "INKEY" in upper or "K_" in upper:
        desc_parts.append("Defines keyboard input constants")
    elif "SET" in upper and "CH" in filename.upper():
        desc_parts.append("Defines SET command options")
    elif "COLOR" in upper or "_SET_" in upper:
        desc_parts.append("Defines color and display constants")
    elif "HB_" in upper or "HBEXT" in upper:
        desc_parts.append("Defines Harbour internal constants and macros")
    elif "THREAD" in upper:
        desc_parts.append("Defines threading constants and macros")
    elif "FILE" in upper or "F_" in upper:
        desc_parts.append("Defines file I/O constants")
    elif "DB" in upper or "RDD" in upper:
        desc_parts.append("Defines database/RDD constants")
    elif "COM" in upper or "SERIAL" in upper:
        desc_parts.append("Defines communication constants")
    elif "GT" in upper:
        desc_parts.append("Defines graphics terminal constants")
    elif "BOX" in upper or "BORDER" in upper:
        desc_parts.append("Defines box and border drawing constants")
    elif "MEMO" in upper:
        desc_parts.append("Defines memo field constants")

    return ". ".join(desc_parts)


def has_real_code(code: str) -> bool:
    """Check if code contains actual Harbour code (not just comments/includes)."""
    upper = code.upper()

    # Must have at least one of these code constructs
    code_constructs = [
        "FUNCTION ", "PROCEDURE ", "CREATE CLASS", "ENDCLASS",
        "METHOD ", "RETURN ", "LOCAL ", "MEMVAR ",
        "THREAD STATIC", "IF ", "FOR ", "WHILE ", "DO CASE",
        "BEGIN SEQUENCE", "SWITCH ", "REQUEST ",
        "INIT PROCEDURE", "EXIT PROCEDURE",
    ]
    if not any(construct in upper for construct in code_constructs):
        return False

    # Count actual code lines (non-empty, non-preprocessor, non-blank)
    lines = code.split("\n")
    code_lines = 0
    for line in lines:
        stripped = line.strip()
        if not stripped:
            continue
        if stripped.startswith("#"):
            continue
        if stripped.startswith("//") or stripped.startswith("/*") or stripped.startswith("*"):
            continue
        code_lines += 1

    return code_lines >= 8


def is_code_complete(code: str) -> bool:
    """Check if code is complete (proper ENDCLASS, balanced structures)."""
    upper = code.upper()

    # Check class definitions have matching ENDCLASS
    class_count = upper.count("CREATE CLASS")
    endclass_count = upper.count("ENDCLASS")
    if class_count > 0 and endclass_count < class_count:
        return False

    # Check DO CASE has ENDDO CASE
    docase_count = upper.count("DO CASE")
    endcase_count = upper.count("ENDCASE") + upper.count("END CASE")
    if docase_count > 0 and endcase_count < docase_count:
        return False

    # Check FOR/NEXT balance
    for_count = len(re.findall(r'\bFOR\s+\w+', upper))
    next_count = upper.count("\nNEXT") + (1 if upper.endswith("NEXT") else 0)
    # Be lenient - some code uses EXIT in loops

    # Check DO WHILE / ENDDO balance
    dowhile_count = upper.count("DO WHILE")
    enddo_count = upper.count("ENDDO") + upper.count("END DO")
    if dowhile_count > 0 and enddo_count < dowhile_count:
        return False

    # Check BEGIN SEQUENCE / END / RECOVER balance
    seq_count = upper.count("BEGIN SEQUENCE")
    end_count = upper.count("\nEND\n") + upper.count("\nEND ") + (1 if upper.endswith("\nEND") or upper.endswith(" END") else 0)

    # Check parentheses balance (lenient)
    open_p = code.count('(')
    close_p = code.count(')')
    if abs(open_p - close_p) > 3:
        return False

    # Check BEGIN/END blocks
    begin_count = len(re.findall(r'\bBEGIN\b', upper))
    end_block_count = len(re.findall(r'\bEND\b', upper)) - upper.count("ENDCLASS") - upper.count("ENDCASE") - upper.count("END IF") - upper.count("ENDDO")
    # Very lenient check - just ensure it's not wildly unbalanced
    if begin_count > 0 and end_block_count > begin_count + 5:
        return False

    return True


def extract_classes_and_functions(code: str) -> Dict:
    """Extract class and function definitions from code."""
    classes = []
    functions = []
    procedures = []

    for line in code.split("\n"):
        line_stripped = line.strip()
        upper = line_stripped.upper()

        # Class definitions
        if upper.startswith("CREATE CLASS"):
            parts = line_stripped.split()
            if len(parts) >= 3:
                class_name = parts[2]
                classes.append(class_name)

        # Function definitions
        if upper.startswith("FUNCTION ") or (upper.startswith("STATIC FUNCTION ")):
            parts = line_stripped.split()
            idx = 2 if upper.startswith("STATIC") else 1
            if len(parts) >= idx + 1:
                func_name = parts[idx].split("(")[0]
                functions.append(func_name)

        # Procedure definitions
        if upper.startswith("PROCEDURE ") or upper.startswith("STATIC PROCEDURE "):
            parts = line_stripped.split()
            idx = 2 if upper.startswith("STATIC") else 1
            if len(parts) >= idx + 1:
                proc_name = parts[idx].split("(")[0]
                procedures.append(proc_name)

        # INIT/EXIT procedures
        if upper.startswith("INIT PROCEDURE") or upper.startswith("EXIT PROCEDURE"):
            parts = line_stripped.split()
            if len(parts) >= 3:
                proc_name = parts[2].split("(")[0]
                procedures.append(proc_name)

    return {
        "classes": classes,
        "functions": functions,
        "procedures": procedures
    }


def categorize_file(filepath: Path) -> Tuple[str, str]:
    """Categorize a PRG file into category and subcategory."""
    rel_path = filepath.relative_to(HARBOUR_ROOT)
    parts = rel_path.parts

    if parts[0] == "src":
        if parts[1] == "rtl":
            return "rtl", categorize_rtl_file(filepath)
        elif parts[1] == "rdd":
            return "rdd", "rdd_core"
        elif parts[1] == "debug":
            return "rtl", "utility"
        else:
            return "rtl", "utility"
    elif parts[0] == "contrib":
        module = parts[1] if len(parts) > 1 else "unknown"
        return "contrib", categorize_contrib_module(module)
    elif parts[0] == "tests":
        return "tests", categorize_test_file(filepath)
    elif parts[0] == "utils":
        return "utils", categorize_utils_file(filepath)
    elif parts[0] == "extras":
        return "extras", categorize_extras_file(filepath)
    else:
        return "rtl", "utility"


def categorize_rtl_file(filepath: Path) -> str:
    """Categorize RTL files into subcategories."""
    name = filepath.stem.lower()

    if name.startswith("t") and not name.startswith("text"):
        if any(x in name for x in ["get", "browse", "column", "editor", "scalar", "object", "class"]):
            return "oop_class"
        elif any(x in name for x in ["menu", "popup", "topbar"]):
            return "ui_menu"
        elif any(x in name for x in ["check", "radio", "push", "list", "label", "button"]):
            return "ui_widget"
        elif any(x in name for x in ["edit", "memo"]):
            return "text_edit"
        elif any(x in name for x in ["persist", "profile", "symbol"]):
            return "oop_class"
        else:
            return "oop_class"
    elif "get" in name or "read" in name:
        return "get_system"
    elif any(x in name for x in ["err", "alert"]):
        return "error_handling"
    elif any(x in name for x in ["file", "dir", "ini", "type"]):
        return "file_io"
    elif any(x in name for x in ["db", "memo"]):
        return "database"
    else:
        return "utility"


def categorize_contrib_module(module: str) -> str:
    """Categorize contrib modules."""
    db_modules = {"hbpgsql", "hbmysql", "hbsqlit3", "hbodbc", "hbfbird", "rddsql", "rddpg",
                  "rddmy", "rddfb", "rddads", "rddbm", "rddmisc", "sddpg", "sddmy",
                  "sddoci", "sddodbc", "sddsqlt3", "sddfb", "rddado"}
    net_modules = {"hbtip", "hbcurl", "hbhttpd", "hbnetio", "hbcomio", "hbtcpio", "hbpipeio"}
    sec_modules = {"hbssl", "hbmagic"}
    gfx_modules = {"hbbmp", "hbcairo", "hbhpdf", "hbgd", "hbzebra", "hbfimage", "hbformat"}
    fmt_modules = {"hbexpat", "hbmxml", "hbfoxpro", "hbplist", "hbmemio"}
    plat_modules = {"hbwin", "hbunix", "hboslib", "gtalleg", "gtwvg", "gtwvw", "gtwvb"}
    compat_modules = {"hbnf", "hbct", "xhb", "hbxpp", "hbtpathy", "hbfship"}

    if module in db_modules:
        return "database"
    elif module in net_modules:
        return "network"
    elif module in sec_modules:
        return "security"
    elif module in gfx_modules:
        return "graphics"
    elif module in fmt_modules:
        return "data_format"
    elif module in plat_modules:
        return "platform"
    elif module in compat_modules:
        return "compatibility"
    else:
        return "utility"


def categorize_test_file(filepath: Path) -> str:
    """Categorize test files."""
    name = filepath.stem.lower()

    if any(x in name for x in ["class", "oob", "inherit", "scope", "data"]):
        return "oop"
    elif any(x in name for x in ["db", "rdd", "browse"]):
        return "database"
    elif any(x in name for x in ["speed", "bench"]):
        return "performance"
    elif any(x in name for x in ["str", "math", "date", "array", "for", "while", "if", "case",
                                   "static", "mem", "gt", "regex", "file", "err", "hello"]):
        return "language_basics"
    else:
        return "function_api"


def categorize_utils_file(filepath: Path) -> str:
    """Categorize utility files."""
    name = filepath.stem.lower()

    if "hbmk" in name or "build" in name:
        return "build_system"
    elif "test" in name or "rt_" in name:
        return "test_framework"
    elif "i18n" in name or "lang" in name:
        return "i18n"
    else:
        return "build_system"


def categorize_extras_file(filepath: Path) -> str:
    """Categorize extras files."""
    parts = filepath.relative_to(HARBOUR_ROOT).parts

    if len(parts) > 1:
        module = parts[1].lower()
        if "pdf" in module or "vpdf" in module:
            return "pdf"
        elif "xls" in module or "excel" in module:
            return "spreadsheet"
        elif "srv" in module or "http" in module:
            return "server"
    return "utility"


def generate_description(filepath: Path, code: str, category: str, subcategory: str) -> str:
    """Generate a comprehensive description for a PRG file."""
    rel_path = filepath.relative_to(HARBOUR_ROOT)
    module_name = rel_path.parts[1] if len(rel_path.parts) > 1 else "rtl"

    # Get module description if contrib
    module_desc = ""
    if category == "contrib" and module_name in MODULE_DESCRIPTIONS:
        module_desc = MODULE_DESCRIPTIONS[module_name]

    # Extract code elements
    elements = extract_classes_and_functions(code)

    # Generate description based on category and content
    desc_parts = []

    # File location context
    if category == "rtl":
        desc_parts.append(f"Harbour Runtime Library file: {rel_path}")
    elif category == "contrib":
        desc_parts.append(f"Harbour contribution module '{module_name}' ({module_desc}): {rel_path}")
    elif category == "tests":
        desc_parts.append(f"Harbour test program: {rel_path}")
    elif category == "utils":
        desc_parts.append(f"Harbour utility program: {rel_path}")
    elif category == "extras":
        desc_parts.append(f"Harbour extra library: {rel_path}")
    else:
        desc_parts.append(f"Harbour source file: {rel_path}")

    # Add code structure information
    if elements["classes"]:
        desc_parts.append(f"Defines classes: {', '.join(elements['classes'][:5])}")

    if elements["functions"]:
        if len(elements["functions"]) <= 5:
            desc_parts.append(f"Provides functions: {', '.join(elements['functions'])}")
        else:
            desc_parts.append(f"Provides {len(elements['functions'])} functions including: {', '.join(elements['functions'][:5])}")

    if elements["procedures"]:
        if len(elements["procedures"]) <= 3:
            desc_parts.append(f"Contains procedures: {', '.join(elements['procedures'])}")
        else:
            desc_parts.append(f"Contains {len(elements['procedures'])} procedures")

    # Add subcategory context
    subcategory_descriptions = {
        "oop_class": "This file implements object-oriented classes using Harbour's class system",
        "ui_widget": "This file defines UI widget classes for graphical interfaces",
        "ui_menu": "This file implements menu system classes",
        "text_edit": "This file provides text editing functionality",
        "get_system": "This file implements the GET system for input field handling",
        "scalar_type": "This file defines scalar type wrapper classes",
        "error_handling": "This file implements error handling and reporting",
        "file_io": "This file provides file I/O operations",
        "database": "This file handles database operations",
        "utility": "This file provides utility functions",
        "rdd_core": "This file implements core Record Driver Driver functionality",
        "rdd_driver": "This file implements a database driver",
        "network": "This file provides network protocol implementations",
        "security": "This file implements security and encryption functions",
        "graphics": "This file provides graphics and image processing capabilities",
        "data_format": "This file handles data format parsing and generation",
        "platform": "This file provides platform-specific functionality",
        "compatibility": "This file provides legacy compatibility functions",
        "language_basics": "This test file exercises basic Harbour language features",
        "function_api": "This test file tests specific function APIs",
        "oop": "This test file tests object-oriented programming features",
        "performance": "This test file benchmarks performance characteristics",
        "build_system": "This file is part of the build system tooling",
        "test_framework": "This file is part of the test framework",
        "i18n": "This file provides internationalization support",
        "pdf": "This file provides PDF generation capabilities",
        "spreadsheet": "This file provides spreadsheet generation capabilities",
        "server": "This file implements server functionality",
    }

    if subcategory in subcategory_descriptions:
        desc_parts.append(subcategory_descriptions[subcategory])

    return ". ".join(desc_parts)


def create_training_entry(filepath: Path, code: str, description: str) -> Dict:
    """Create a training entry in the instruction format."""
    return {
        "instruction": f"Write Harbour (xBase/Clipper) code for: {description}",
        "input": "",
        "output": code,
        "metadata": {
            "file_path": str(filepath.relative_to(HARBOUR_ROOT)),
            "language": "harbour",
            "description": description
        }
    }


def create_completion_entry(filepath: Path, code: str, description: str) -> Dict:
    """Create a completion-style training entry with diverse instructions."""
    # Generate diverse user prompts based on code content
    import random
    random.seed(hash(filepath))  # Deterministic per file

    elements = extract_classes_and_functions(code)
    upper = code.upper()

    # Different prompt templates based on content
    templates = []

    if elements["classes"]:
        templates.append(f"Implement the following Harbour classes: {', '.join(elements['classes'][:3])}. {description}")
        templates.append(f"Create Harbour OOP classes for the functionality described: {description}")

    if elements["functions"]:
        templates.append(f"Write Harbour functions: {', '.join(elements['functions'][:3])}. {description}")
        templates.append(f"Implement these Harbour functions: {description}")

    if elements["procedures"]:
        templates.append(f"Write a Harbour program with procedures: {', '.join(elements['procedures'][:3])}. {description}")

    if "#DEFINE" in upper or "#XCOMMAND" in upper:
        templates.append(f"Create Harbour preprocessor definitions: {description}")
        templates.append(f"Define Harbour macros and constants: {description}")

    # General templates
    templates.append(f"Write the following Harbour (xBase/Clipper) code:\n\n{description}")
    templates.append(f"Implement this Harbour module: {description}")
    templates.append(f"Here is a Harbour (xBase/Clipper) implementation:\n\n{description}")
    templates.append(f"Generate Harbour code for: {description}")

    # Select a random template
    user_prompt = random.choice(templates)

    return {
        "messages": [
            {
                "role": "system",
                "content": "You are an expert Harbour (xBase/Clipper) programmer. Write clean, efficient code following Harbour conventions. Use proper Hungarian notation for variable names (c=character, n=numeric, l=logical, a=array, o=object, b=codeblock)."
            },
            {
                "role": "user",
                "content": user_prompt
            },
            {
                "role": "assistant",
                "content": code
            }
        ],
        "metadata": {
            "file_path": str(filepath.relative_to(HARBOUR_ROOT)),
            "language": "harbour",
            "description": description
        }
    }


def process_prg_file(filepath: Path) -> List[Dict]:
    """Process a single PRG file and generate training entries."""
    try:
        with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
            code = f.read()
    except Exception as e:
        print(f"Error reading {filepath}: {e}")
        return []

    # Skip empty files or very small files
    if len(code.strip()) < 50:
        return []

    # Step 1: Remove ALL comments (block, single-line, star-prefixed, inline)
    code = remove_all_comments(code)

    # Step 2: Remove disabled code blocks (#if 0)
    code = remove_disabled_code(code)

    # Step 3: Clean excessive blank lines
    code = clean_excessive_blank_lines(code)

    # Step 5: Strip leading/trailing whitespace
    code = code.strip()

    # Skip if no real code remains
    if not has_real_code(code):
        return []

    # Skip if code is too short
    if len(code) < MIN_CODE_LENGTH:
        return []

    # Skip if code is incomplete
    if not is_code_complete(code):
        return []

    # Truncate if too long
    if len(code) > MAX_CODE_LENGTH:
        lines = code.split("\n")
        truncated_lines = []
        current_length = 0
        for line in lines:
            if current_length + len(line) > MAX_CODE_LENGTH:
                break
            truncated_lines.append(line)
            current_length += len(line) + 1
        code = "\n".join(truncated_lines)

    # Categorize the file
    category, subcategory = categorize_file(filepath)

    # Generate description
    description = generate_description(filepath, code, category, subcategory)

    # Create training entry (chat format only - preferred for Qwen2.5-Coder)
    entry = create_completion_entry(filepath, code, description)
    entry["metadata"]["category"] = category
    entry["metadata"]["subcategory"] = subcategory

    return [entry]


def process_ch_file(filepath: Path) -> List[Dict]:
    """Process a single CH file and generate training entries."""
    try:
        with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
            code = f.read()
    except Exception as e:
        print(f"Error reading {filepath}: {e}")
        return []

    # Skip empty files
    if len(code.strip()) < 30:
        return []

    # Step 1: Remove ALL comments
    code = remove_all_comments(code)

    # Step 2: Remove disabled code blocks (#if 0)
    code = remove_disabled_code(code)

    # Step 3: Clean excessive blank lines
    code = clean_excessive_blank_lines(code)

    # Step 4: Strip leading/trailing whitespace
    code = code.strip()

    # Skip if no real code remains
    if not has_real_ch_code(code):
        return []

    # Skip if code is too short
    if len(code) < MIN_CODE_LENGTH:
        return []

    # Truncate if too long
    if len(code) > MAX_CODE_LENGTH:
        lines = code.split("\n")
        truncated_lines = []
        current_length = 0
        for line in lines:
            if current_length + len(line) > MAX_CODE_LENGTH:
                break
            truncated_lines.append(line)
            current_length += len(line) + 1
        code = "\n".join(truncated_lines)

    # Generate description
    description = generate_ch_description(filepath, code)

    # Determine category
    rel_path = filepath.relative_to(HARBOUR_ROOT)
    parts = rel_path.parts

    if parts[0] == "include":
        category = "include"
    elif parts[0] == "contrib":
        category = "contrib"
    elif parts[0] == "utils":
        category = "utils"
    elif parts[0] == "extras":
        category = "extras"
    else:
        category = "include"

    # Create training entry (chat format only)
    entry = create_completion_entry(filepath, code, description)
    entry["metadata"]["category"] = category
    entry["metadata"]["subcategory"] = "header"

    return [entry]


def main():
    """Main function to generate the dataset."""
    print("=" * 60)
    print("Harbour PRG/CH Dataset Generator (CLEANED)")
    print("=" * 60)

    # Find all PRG and CH files
    print("\n1. Finding all Harbour source files...")
    prg_files = list(HARBOUR_ROOT.rglob("*.prg"))
    ch_files = list(HARBOUR_ROOT.rglob("*.ch"))
    print(f"   Found {len(prg_files)} PRG files")
    print(f"   Found {len(ch_files)} CH files")

    # Process PRG files
    print("\n2. Processing PRG files...")
    all_entries = []
    category_counts = {}
    skipped_files = 0

    for i, filepath in enumerate(prg_files, 1):
        if i % 100 == 0:
            print(f"   Processing PRG file {i}/{len(prg_files)}...")

        entries = process_prg_file(filepath)
        if entries:
            all_entries.extend(entries)
            category = entries[0]["metadata"]["category"]
            category_counts[category] = category_counts.get(category, 0) + 1
        else:
            skipped_files += 1

    print(f"\n   PRG: Generated {len(all_entries)} entries, skipped {skipped_files} files")

    # Process CH files
    print("\n3. Processing CH files...")
    ch_entries = 0
    ch_skipped = 0

    for i, filepath in enumerate(ch_files, 1):
        if i % 20 == 0:
            print(f"   Processing CH file {i}/{len(ch_files)}...")

        entries = process_ch_file(filepath)
        if entries:
            all_entries.extend(entries)
            ch_entries += 1
            category = entries[0]["metadata"]["category"]
            category_counts[category] = category_counts.get(category, 0) + 1
        else:
            ch_skipped += 1

    print(f"\n   CH: Generated {ch_entries} file entries, skipped {ch_skipped} files")
    print(f"\n   Total: {len(all_entries)} training entries")

    # Print category statistics
    print("\n3. Category statistics:")
    for category, count in sorted(category_counts.items()):
        print(f"   {category}: {count} files")

    # Shuffle entries
    random.seed(42)
    random.shuffle(all_entries)

    # Split into train and validation
    print("\n4. Splitting into train/validation sets...")
    split_idx = int(len(all_entries) * TRAIN_RATIO)
    train_entries = all_entries[:split_idx]
    val_entries = all_entries[split_idx:]

    print(f"   Training set: {len(train_entries)} entries")
    print(f"   Validation set: {len(val_entries)} entries")

    # Save datasets
    print("\n5. Saving datasets...")

    # Save as JSONL (instruction format)
    train_jsonl_path = OUTPUT_DIR / "harbour_train.jsonl"
    val_jsonl_path = OUTPUT_DIR / "harbour_val.jsonl"

    with open(train_jsonl_path, 'w', encoding='utf-8') as f:
        for entry in train_entries:
            train_entry = {k: v for k, v in entry.items() if k != "metadata"}
            f.write(json.dumps(train_entry, ensure_ascii=False) + "\n")

    with open(val_jsonl_path, 'w', encoding='utf-8') as f:
        for entry in val_entries:
            val_entry = {k: v for k, v in entry.items() if k != "metadata"}
            f.write(json.dumps(val_entry, ensure_ascii=False) + "\n")

    print(f"   Saved training JSONL: {train_jsonl_path}")
    print(f"   Saved validation JSONL: {val_jsonl_path}")

    # Save full dataset with metadata
    full_dataset_path = OUTPUT_DIR / "harbour_dataset_full.jsonl"
    with open(full_dataset_path, 'w', encoding='utf-8') as f:
        for entry in all_entries:
            f.write(json.dumps(entry, ensure_ascii=False) + "\n")

    print(f"   Saved full dataset with metadata: {full_dataset_path}")

    # Generate statistics file
    total_files = len(prg_files) + len(ch_files)
    stats = {
        "total_prg_files": len(prg_files),
        "total_ch_files": len(ch_files),
        "total_files": total_files,
        "total_entries": len(all_entries),
        "skipped_prg": skipped_files,
        "skipped_ch": ch_skipped,
        "train_entries": len(train_entries),
        "val_entries": len(val_entries),
        "categories": category_counts,
        "files_per_category": {}
    }

    for entry in all_entries:
        cat = entry["metadata"]["category"]
        subcat = entry["metadata"]["subcategory"]
        if cat not in stats["files_per_category"]:
            stats["files_per_category"][cat] = {}
        stats["files_per_category"][cat][subcat] = stats["files_per_category"][cat].get(subcat, 0) + 1

    stats_path = OUTPUT_DIR / "dataset_stats.json"
    with open(stats_path, 'w', encoding='utf-8') as f:
        json.dump(stats, f, indent=2, ensure_ascii=False)

    print(f"   Saved statistics: {stats_path}")

    # Generate README
    readme_content = f"""# Harbour Fine-tuning Dataset

## Overview
This dataset contains {len(all_entries)} training entries extracted from:
- {len(prg_files)} Harbour PRG (.prg) source files
- {len(ch_files)} Harbour Header (.ch) files

{skipped_files} PRG files and {ch_skipped} CH files were skipped due to quality issues.

## Dataset Format
The dataset is provided in JSONL format with the following structure:

### Instruction Format (harbour_train.jsonl / harbour_val.jsonl)
```json
{{"instruction": "...", "input": "", "output": "..."}}
```

### Full Dataset (harbour_dataset_full.jsonl)
```json
{{"instruction": "...", "input": "", "output": "...", "metadata": {{"file_path": "...", "language": "harbour", "category": "...", "subcategory": "..."}}}}
```

## Categories
- **include**: Header files with constants/macros ({category_counts.get('include', 0)} files)
- **rtl**: Harbour Runtime Library ({category_counts.get('rtl', 0)} files)
- **contrib**: Contribution libraries ({category_counts.get('contrib', 0)} files)
- **tests**: Test programs ({category_counts.get('tests', 0)} files)
- **utils**: Utility programs ({category_counts.get('utils', 0)} files)
- **extras**: Extra libraries ({category_counts.get('extras', 0)} files)

## Cleaning Applied
- Copyright/license headers removed
- Disabled code blocks (#if 0) removed
- Excessive trailing comments removed
- Excessive blank lines removed
- Files without actual code filtered out
- Incomplete code (missing ENDCLASS, etc.) filtered out

## Usage for Fine-tuning
```bash
# Using Ollama with Modelfile
FROM qwen2.5-coder:14b

# Training command
ollama create harbour-coder -f Modelfile

# Or use with other training frameworks
# The JSONL format is compatible with:
# - OpenAI fine-tuning API
# - Hugging Face transformers
# - Axolotl
# - LLaMA-Factory
```

## File Structure
- `harbour_train.jsonl` - Training set ({len(train_entries)} entries)
- `harbour_val.jsonl` - Validation set ({len(val_entries)} entries)
- `harbour_dataset_full.jsonl` - Full dataset with metadata
- `dataset_stats.json` - Dataset statistics
- `generate_dataset.py` - This script

## Source
The source files are from the Harbour project (https://harbour.github.io/),
an open-source Clipper-compatible compiler.
"""

    readme_path = OUTPUT_DIR / "README.md"
    with open(readme_path, 'w', encoding='utf-8') as f:
        f.write(readme_content)

    print(f"   Saved README: {readme_path}")

    print("\n" + "=" * 60)
    print("Dataset generation complete!")
    print("=" * 60)


if __name__ == "__main__":
    main()