test_battery.py

#!/usr/bin/env python3
"""
Harbour Test Battery - Generates code, compiles with harbour, evaluates with qwen3.6:35b
"""

import json
import time
import subprocess
import requests
import tempfile
import os
from pathlib import Path
from datetime import datetime

OLLAMA_URL = "http://localhost:11434/api/generate"
MODEL = "qwen3.6:35b"
HARBOUR = "/home/fivetech/harbour/bin/linux/gcc/harbour"
WORK_DIR = Path("/home/fivetech/finetune/test_output")
WORK_DIR.mkdir(exist_ok=True)

def query_ollama(prompt, system="", timeout=300):
    payload = {
        "model": MODEL,
        "prompt": prompt,
        "stream": False,
        "options": {"temperature": 0.2, "num_predict": 3000, "top_p": 0.9}
    }
    if system:
        payload["system"] = system
    try:
        start = time.time()
        r = requests.post(OLLAMA_URL, json=payload, timeout=timeout)
        elapsed = time.time() - start
        data = r.json()
        return {
            "response": data.get("response", ""),
            "eval_count": data.get("eval_count", 0),
            "duration": elapsed,
            "tps": data.get("eval_count", 0) / max(data.get("eval_duration", 1) / 1e9, 0.001),
            "error": None
        }
    except Exception as e:
        return {"response": "", "error": str(e), "eval_count": 0, "duration": 0, "tps": 0}

def compile_harbour(code):
    """Compile code with harbour, return (success, error_msg, obj_exists)"""
    prg_file = WORK_DIR / "test.prg"
    prg_file.write_text(code)
    
    try:
        result = subprocess.run(
            [HARBOUR, str(prg_file), "-n", "-w"],
            capture_output=True, text=True, timeout=30
        )
        obj_file = WORK_DIR / "test.obj"
        success = result.returncode == 0
        obj_exists = obj_file.exists()
        error = result.stderr.strip() if result.stderr else ""
        if not success and not error:
            error = result.stdout.strip()
        return success, error, obj_exists
    except subprocess.TimeoutExpired:
        return False, "Compilation timeout", False
    except Exception as e:
        return False, str(e), False

def clean_code(response):
    """Extract code from model response, remove markdown."""
    lines = response.split('\n')
    in_code = False
    code_lines = []
    skip_explanation = True
    
    for line in lines:
        stripped = line.strip()
        
        # Skip markdown
        if stripped.startswith('```'):
            in_code = not in_code
            continue
        
        if in_code:
            code_lines.append(line)
            skip_explanation = False
        elif skip_explanation:
            # Detect start of code
            upper = stripped.upper()
            if any(upper.startswith(kw) for kw in [
                'FUNCTION', 'PROCEDURE', 'LOCAL', 'STATIC', 'PUBLIC',
                'PRIVATE', 'MEMVAR', '#DEFINE', '#INCLUDE', 'CLASS',
                'METHOD', 'RETURN', 'SET', 'REQUEST'
            ]):
                in_code = True
                code_lines.append(line)
                skip_explanation = False
    
    if not code_lines:
        # Fallback: take everything
        code_lines = response.split('\n')
    
    return '\n'.join(code_lines).strip()

# ============================================================
# TEST DEFINITIONS - Based on dataset patterns
# ============================================================

TESTS = [
    # ---- BASIC SYNTAX ----
    {
        "id": "SYNTAX_01", "category": "Basic Syntax", "name": "Variable types and declarations",
        "prompt": "Write a Harbour program that declares LOCAL variables of each type (numeric, character, logical, date, nil), prints them with ValType(), and uses proper Hungarian notation.",
        "expected_keywords": ["LOCAL", "ValType", "FUNCTION"],
        "min_lines": 8,
    },
    {
        "id": "SYNTAX_02", "category": "Basic Syntax", "name": "Preprocessor defines",
        "prompt": "Write Harbour preprocessor definitions for application constants: app name, version, max records, date format. Use #define and show conditional compilation with #ifdef.",
        "expected_keywords": ["#DEFINE", "#IFDEF", "#ENDIF"],
        "min_lines": 6,
    },
    {
        "id": "SYNTAX_03", "category": "Basic Syntax", "name": "String operations",
        "prompt": "Write a Harbour function that takes a full name string and returns initials. Use AllTrim, Upper, Left, At, SubStr, and Space functions.",
        "expected_keywords": ["FUNCTION", "AllTrim", "Upper", "Left", "At", "SubStr"],
        "min_lines": 6,
    },
    {
        "id": "SYNTAX_04", "category": "Basic Syntax", "name": "Date functions",
        "prompt": "Write a Harbour function that calculates the number of business days between two dates, excluding weekends. Use Date(), DOW(), and date arithmetic.",
        "expected_keywords": ["FUNCTION", "Date", "DOW"],
        "min_lines": 8,
    },
    {
        "id": "SYNTAX_05", "category": "Basic Syntax", "name": "Type conversion",
        "prompt": "Write Harbour code that converts between all types: Str, Val, CTOD, DTOC, ASC, Chr, Transform. Show edge cases.",
        "expected_keywords": ["Str", "Val", "CTOD", "DTOC"],
        "min_lines": 8,
    },

    # ---- CONTROL FLOW ----
    {
        "id": "CTRL_01", "category": "Control Flow", "name": "IF/ELSEIF/ENDIF",
        "prompt": "Write a Harbour function that classifies employee salary into tax brackets using IF/ELSEIF/ELSE/ENDIF. Include 5 brackets and error handling.",
        "expected_keywords": ["FUNCTION", "IF", "ELSEIF", "ELSE", "ENDIF"],
        "min_lines": 10,
    },
    {
        "id": "CTRL_02", "category": "Control Flow", "name": "DO CASE",
        "prompt": "Write a Harbour function using DO CASE to convert month number (1-12) to season name. Handle invalid input with OTHERWISE.",
        "expected_keywords": ["DO CASE", "CASE", "OTHERWISE", "ENDCASE"],
        "min_lines": 8,
    },
    {
        "id": "CTRL_03", "category": "Control Flow", "name": "FOR/NEXT loop",
        "prompt": "Write a Harbour function using FOR/NEXT to calculate the sum of all prime numbers below 100. Include STEP and EXIT.",
        "expected_keywords": ["FOR", "TO", "NEXT", "IF", "EXIT"],
        "min_lines": 10,
    },
    {
        "id": "CTRL_04", "category": "Control Flow", "name": "DO WHILE",
        "prompt": "Write a Harbour function using DO WHILE to implement the Euclidean algorithm for GCD. Include LOOP and EXIT.",
        "expected_keywords": ["DO WHILE", "ENDDO", "IF", "LOOP", "EXIT"],
        "min_lines": 6,
    },
    {
        "id": "CTRL_05", "category": "Control Flow", "name": "SCAN/ENDSCAN",
        "prompt": "Write Harbour code using SCAN/ENDSCAN to find the longest string in an array. Include NEXT clause.",
        "expected_keywords": ["SCAN", "ENDSCAN"],
        "min_lines": 6,
    },
    {
        "id": "CTRL_06", "category": "Control Flow", "name": "FOR EACH",
        "prompt": "Write Harbour code using FOR EACH to count word frequencies in a string. Use a hash for storage.",
        "expected_keywords": ["FOR EACH", "NEXT", ":="],
        "min_lines": 8,
    },

    # ---- FUNCTIONS ----
    {
        "id": "FUNC_01", "category": "Functions", "name": "Parameters and return",
        "prompt": "Write a Harbour function with default parameters, pass-by-reference using @, and return an array. Include proper Hungarian notation.",
        "expected_keywords": ["FUNCTION", "LOCAL", "RETURN"],
        "min_lines": 6,
    },
    {
        "id": "FUNC_02", "category": "Functions", "name": "Recursion",
        "prompt": "Write a recursive Harbour function for Fibonacci numbers with memoization using a hash. Include base case and error handling.",
        "expected_keywords": ["FUNCTION", "IF", "RETURN"],
        "min_lines": 8,
    },
    {
        "id": "FUNC_03", "category": "Functions", "name": "Variable scope",
        "prompt": "Write Harbour code demonstrating LOCAL, STATIC, PRIVATE, PUBLIC variables. Show scope differences with nested function calls.",
        "expected_keywords": ["LOCAL", "STATIC", "PRIVATE", "PUBLIC"],
        "min_lines": 8,
    },
    {
        "id": "FUNC_04", "category": "Functions", "name": "Code blocks",
        "prompt": "Write Harbour code using code blocks: AEval with {|x| x*2}, AScan, ASort with custom sort. Show evaluation with Eval().",
        "expected_keywords": ["AEval", "AScan", "ASort", "Eval"],
        "min_lines": 6,
    },
    {
        "id": "FUNC_05", "category": "Functions", "name": "Error handling",
        "prompt": "Write a Harbour function with BEGIN SEQUENCE/RECOVER/END SEQUENCE for file reading. Include DEFAULT and BREAK.",
        "expected_keywords": ["BEGIN SEQUENCE", "RECOVER", "END SEQUENCE"],
        "min_lines": 8,
    },

    # ---- ARRAYS ----
    {
        "id": "ARRAY_01", "category": "Arrays", "name": "Array operations",
        "prompt": "Write Harbour functions for: create 2D array, AAdd elements, ASort with custom order, AScan by value, ASize to resize. Include error handling.",
        "expected_keywords": ["ARRAY", "AAdd", "ASort", "AScan", "ASize"],
        "min_lines": 8,
    },
    {
        "id": "ARRAY_02", "category": "Arrays", "name": "Hash operations",
        "prompt": "Write Harbour code using hashes: create, add keys, iterate with FOR EACH, merge two hashes, check key existence with HB_HHasKey, convert to array.",
        "expected_keywords": [":=", "FOR EACH", "HB_HHasKey"],
        "min_lines": 8,
    },
    {
        "id": "ARRAY_03", "category": "Arrays", "name": "Sorting algorithm",
        "prompt": "Implement QuickSort in Harbour for an array of numbers. Include partition logic and proper recursion.",
        "expected_keywords": ["FUNCTION", "LOCAL", "IF", "RETURN"],
        "min_lines": 12,
    },

    # ---- OOP ----
    {
        "id": "OOP_01", "category": "OOP", "name": "Class definition",
        "prompt": "Write a Harbour class Person with DATA (name, age), METHOD (New constructor, GetName, SetAge), and CLASSDATA. Include validation in SetAge.",
        "expected_keywords": ["CLASS", "DATA", "METHOD", "RETURN"],
        "min_lines": 10,
    },
    {
        "id": "OOP_02", "category": "OOP", "name": "Inheritance",
        "prompt": "Write Harbour classes: Shape (base), Circle (derived) with area() method. Show inheritance syntax and method override.",
        "expected_keywords": ["CLASS", "METHOD", "INHERIT"],
        "min_lines": 10,
    },
    {
        "id": "OOP_03", "category": "OOP", "name": "Operator overloading",
        "prompt": "Write a Harbour class Vec2 for 2D vectors. Overload + and - operators. Include magnitude and normalize methods.",
        "expected_keywords": ["CLASS", "METHOD", "OPERATOR"],
        "min_lines": 12,
    },
    {
        "id": "OOP_04", "category": "OOP", "name": "Singleton pattern",
        "prompt": "Implement Singleton pattern in Harbour for a config manager. Ensure only one instance exists.",
        "expected_keywords": ["CLASS", "CLASSDATA", "METHOD"],
        "min_lines": 10,
    },

    # ---- DATABASE ----
    {
        "id": "DB_01", "category": "Database", "name": "Basic RDD",
        "prompt": "Write Harbour code that creates a DBF file, opens it, appends records, and closes properly. Use DBCreate and DBUseArea.",
        "expected_keywords": ["DBCreate", "DBUseArea", "DBAppend", "DBCLOSEALL"],
        "min_lines": 10,
    },
    {
        "id": "DB_02", "category": "Database", "name": "Indexing",
        "prompt": "Write Harbour code creating an index on a DBF field using RDD. Include ORDSCOPE for range queries.",
        "expected_keywords": ["ORDCREATE", "ORDSCOPE"],
        "min_lines": 8,
    },
    {
        "id": "DB_03", "category": "Database", "name": "DBEval",
        "prompt": "Write Harbour code using DBEval to process all records: count, sum field values, and mark records meeting a condition.",
        "expected_keywords": ["DBEval", "FOR", "WHILE"],
        "min_lines": 8,
    },

    # ---- FILE I/O ----
    {
        "id": "FILE_01", "category": "File I/O", "name": "Text file read/write",
        "prompt": "Write Harbour functions to read a text file line by line and write processed output. Use FCreate, FOpen, FRead, FWrite, FClose, FEof.",
        "expected_keywords": ["FCreate", "FOpen", "FRead", "FWrite", "FClose", "FEof"],
        "min_lines": 10,
    },
    {
        "id": "FILE_02", "category": "File I/O", "name": "Directory listing",
        "prompt": "Write Harbour code using Directory() to list files with a pattern, get file size and date, and process each file.",
        "expected_keywords": ["Directory", "LEN", "FOR"],
        "min_lines": 6,
    },

    # ---- COMPLEX ----
    {
        "id": "CMPX_01", "category": "Complex", "name": "CSV parser",
        "prompt": "Write a Harbour CSV parser that reads a CSV file, handles quoted fields, and returns an array of arrays. Include error handling.",
        "expected_keywords": ["FUNCTION", "LOCAL", "FClose", "FEof"],
        "min_lines": 15,
    },
    {
        "id": "CMPX_02", "category": "Complex", "name": "INI file reader",
        "prompt": "Write a Harbour INI file parser. Read sections, keys, and values into a hash. Handle comments and empty lines.",
        "expected_keywords": ["FUNCTION", "LOCAL", "HASH"],
        "min_lines": 12,
    },
    {
        "id": "CMPX_03", "category": "Complex", "name": "String template engine",
        "prompt": "Write a Harbour template engine replacing {{variable}} placeholders with hash values. Include error handling for missing keys.",
        "expected_keywords": ["FUNCTION", "LOCAL", "STRTRAN"],
        "min_lines": 8,
    },
    {
        "id": "CMPX_04", "category": "Complex", "name": "Logger",
        "prompt": "Write a Harbour logging system with DEBUG/INFO/WARN/ERROR levels, timestamp, file output, and configurable level filtering.",
        "expected_keywords": ["FUNCTION", "LOCAL", "FClose"],
        "min_lines": 12,
    },
    {
        "id": "CMPX_05", "category": "Complex", "name": "Base64 encoder",
        "prompt": "Write a Harbour Base64 encoder/decode function. Use Asc(), Chr(), and bit operations.",
        "expected_keywords": ["FUNCTION", "LOCAL", "Asc", "Chr"],
        "min_lines": 10,
    },
    {
        "id": "CMPX_06", "category": "Complex", "name": "JSON serializer",
        "prompt": "Write a Harbour function that serializes a hash to JSON string. Handle strings, numbers, booleans, arrays, and nested objects.",
        "expected_keywords": ["FUNCTION", "LOCAL", "HB_IsHash"],
        "min_lines": 15,
    },
    {
        "id": "CMPX_07", "category": "Complex", "name": "LRU Cache",
        "prompt": "Write a Harbour LRU cache class with get/set/delete, TTL expiration, and max size. Use a hash and an array for ordering.",
        "expected_keywords": ["CLASS", "DATA", "METHOD"],
        "min_lines": 15,
    },
    {
        "id": "CMPX_08", "category": "Complex", "name": "SQL-like query on arrays",
        "prompt": "Write a Harbour function that filters an array of hashes like SQL WHERE clause. Support =, <>, >, <, LIKE operators.",
        "expected_keywords": ["FUNCTION", "LOCAL", "FOR"],
        "min_lines": 12,
    },
    {
        "id": "CMPX_09", "category": "Complex", "name": "Rate limiter",
        "prompt": "Write a Harbour rate limiter class: max N requests per M seconds. Use timestamps and a queue.",
        "expected_keywords": ["CLASS", "METHOD", "LOCAL"],
        "min_lines": 12,
    },
    {
        "id": "CMPX_10", "category": "Complex", "name": "Config file writer",
        "prompt": "Write a Harbour config manager that saves/loads settings to JSON file. Include defaults, validation, and typed getters.",
        "expected_keywords": ["FUNCTION", "LOCAL", "FClose"],
        "min_lines": 12,
    },

    # ---- BUGGY CODE TO FIX ----
    {
        "id": "FIX_01", "category": "Bug Fix", "name": "Null pointer",
        "prompt": "Fix this Harbour code that crashes when array is empty:\nLOCAL a := {}\n? a[1]",
        "expected_keywords": ["IF", "LEN", "RETURN"],
        "min_lines": 3,
    },
    {
        "id": "FIX_02", "category": "Bug Fix", "name": "Wrong loop bounds",
        "prompt": "Fix this code that skips last element:\nLOCAL a := {10,20,30}\nFOR i := 1 TO LEN(a)-1\n   ? a[i]\nNEXT",
        "expected_keywords": ["FOR", "TO", "LEN"],
        "min_lines": 3,
    },
    {
        "id": "FIX_03", "category": "Bug Fix", "name": "String concat error",
        "prompt": "Fix this code that fails on nil values:\nLOCAL cName := NIL\n? 'Hello ' + cName",
        "expected_keywords": ["IF", "LOCAL", "RETURN"],
        "min_lines": 3,
    },

    # ---- HARBOUR-SPECIFIC ----
    {
        "id": "HARB_01", "category": "Harbour-Specific", "name": "HB_* functions",
        "prompt": "Write Harbour code using HB_IsString, HB_IsNumeric, HB_IsArray, HB_IsHash, HB_IsNil to validate function arguments. Include proper error messages.",
        "expected_keywords": ["HB_IsString", "HB_IsNumeric", "IF"],
        "min_lines": 6,
    },
    {
        "id": "HARB_02", "category": "Harbour-Specific", "name": "Regex",
        "prompt": "Write a Harbour function using HB_RegEx to validate email addresses. Use HB_RegExCompile and HB_RegExMatch.",
        "expected_keywords": ["HB_RegEx", "FUNCTION"],
        "min_lines": 6,
    },
    {
        "id": "HARB_03", "category": "Harbour-Specific", "name": "Serialization",
        "prompt": "Write Harbour code that serializes a hash to binary with HB_Serialize and deserializes with HB_Deserialize.",
        "expected_keywords": ["HB_Serialize", "HB_Deserialize"],
        "min_lines": 6,
    },
    {
        "id": "HARB_04", "category": "Harbour-Specific", "name": "File path operations",
        "prompt": "Write Harbour code using hb_DirBuild, hb_DirNameGet, hb_FileNameGet, hb_PathNormalize for cross-platform file handling.",
        "expected_keywords": ["hb_Dir", "hb_File", "hb_Path"],
        "min_lines": 6,
    },
]

# ============================================================
# MAIN
# ============================================================

def main():
    print("=" * 70)
    print("HARBOUR CODE GENERATION TEST BATTERY")
    print(f"Model: {MODEL}")
    print(f"Tests: {len(TESTS)}")
    print(f"Harbour: {HARBOUR}")
    print(f"Started: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    print("=" * 70)
    
    SYSTEM = """You are an expert Harbour programmer. Write clean, correct, COMPILABLE Harbour code.
Use Hungarian notation: n=numeric, c=character, l=logical, a=array, o=object, d=date.
Use 3-space indentation.
Do NOT include explanations or markdown. Only raw Harbour code.
End functions with RETURN and END FUNCTION."""

    results = []
    compile_pass = 0
    compile_fail = 0
    
    for i, test in enumerate(TESTS, 1):
        print(f"\n[{i:2d}/{len(TESTS)}] {test['id']}: {test['name']}")
        
        # Query model
        result = query_ollama(test["prompt"], SYSTEM)
        
        if result["error"]:
            print(f"  MODEL ERROR: {result['error']}")
            results.append({"test": test, "model_error": result["error"], "compile": False, "compile_error": ""})
            continue
        
        # Clean response
        code = clean_code(result["response"])
        
        # Check for expected keywords
        keywords_found = [kw for kw in test["expected_keywords"] if kw.upper() in code.upper()]
        keywords_missing = [kw for kw in test["expected_keywords"] if kw.upper() not in code.upper()]
        
        # Compile
        success, error, obj = compile_harbour(code)
        
        status = "PASS" if success else "FAIL"
        if success:
            compile_pass += 1
        else:
            compile_fail += 1
        
        print(f"  Compile: {status} | Keywords: {len(keywords_found)}/{len(test['expected_keywords'])} | TPS: {result['tps']:.0f}")
        if keywords_missing:
            print(f"  Missing keywords: {', '.join(keywords_missing)}")
        if not success and error:
            # Show first error only
            first_error = error.split('\n')[0][:120]
            print(f"  Error: {first_error}")
        
        results.append({
            "test": test,
            "code": code[:3000],
            "compile_success": success,
            "compile_error": error[:500] if error else "",
            "keywords_found": keywords_found,
            "keywords_missing": keywords_missing,
            "tokens": result["eval_count"],
            "tps": result["tps"],
            "duration": result["duration"],
            "lines": code.count('\n') + 1,
        })
    
    # Summary by category
    print("\n" + "=" * 70)
    print("RESULTS SUMMARY")
    print("=" * 70)
    
    categories = {}
    for r in results:
        cat = r["test"]["category"]
        if cat not in categories:
            categories[cat] = {"pass": 0, "fail": 0, "total": 0}
        categories[cat]["total"] += 1
        if r.get("compile_success"):
            categories[cat]["pass"] += 1
        else:
            categories[cat]["fail"] += 1
    
    print(f"\n{'Category':<20} {'Pass':<6} {'Fail':<6} {'Rate':<8}")
    print("-" * 45)
    for cat, data in sorted(categories.items()):
        rate = data["pass"] / data["total"] * 100 if data["total"] > 0 else 0
        print(f"{cat:<20} {data['pass']:<6} {data['fail']:<6} {rate:.0f}%")
    
    print(f"\n{'TOTAL':<20} {compile_pass:<6} {compile_fail:<6} {compile_pass/len(results)*100:.0f}%")
    print(f"Total tests: {len(results)}")
    
    total_tokens = sum(r.get("tokens", 0) for r in results)
    total_time = sum(r.get("duration", 0) for r in results)
    print(f"Total tokens: {total_tokens:,}")
    print(f"Total time: {total_time:.1f}s")
    
    # Save
    output = Path("/home/fivetech/finetune/test_baseline_qwen36.json")
    with open(output, "w") as f:
        json.dump({
            "model": MODEL,
            "timestamp": datetime.now().isoformat(),
            "compile_pass": compile_pass,
            "compile_fail": compile_fail,
            "compile_rate": compile_pass / len(results) * 100,
            "categories": categories,
            "results": results,
        }, f, indent=2, ensure_ascii=False)
    
    print(f"\nResults saved to: {output}")

if __name__ == "__main__":
    main()