scripts/112_quick_benchmark_14b10k.py

#!/usr/bin/env python3
import os,re,json,time,traceback
from pathlib import Path
from collections import defaultdict

ROOT=Path("/home/harness_user_1/dgx_ai_factory"); REP=ROOT/"reports"; REP.mkdir(exist_ok=True)
AD=Path(os.environ.get("ADAPTER_DIR",""))
MAX_NEW=int(os.environ.get("MAX_NEW_TOKENS","600"))
SYS="당신은 한국어 존댓말을 사용하는 실무형 개발 비서입니다. 중국어를 절대 쓰지 말고, 명령어와 설명을 분리해서 정확하게 답변합니다."
TESTS=[
("vllm","vLLM 서버가 켜져 있는지 확인하고 종료하는 명령어를 알려주세요.",["ps","grep","vllm"],[["pkill","kill"],["8000","ss -lntp"]]),
("systemd","systemd status=217/USER 오류의 원인과 확인 명령어를 알려주세요.",["217/USER","User","systemctl","journalctl"],[["getent passwd","id "],["daemon-reload","restart"]]),
("fastapi","FastAPI /health 엔드포인트와 curl 확인 예시를 보여주세요.",["FastAPI","/health","curl"],[["uvicorn","main:app"]]),
("jsonl","JSONL 파일이 깨졌는지 줄 번호와 함께 검사하는 파이썬 코드를 보여주세요.",["json","enumerate","json.loads"],[["JSONDecodeError","Exception"],["line_no","line","줄"]]),
("lora","LoRA adapter 폴더에서 확인해야 하는 파일과 stable/rejected 판단 순서를 알려주세요.",["adapter_config.json","adapter_model","smoke","benchmark"],[["stable","rejected"],["safetensors","PEFT"]]),
("docker","Docker 컨테이너 상태, 포트, 로그 확인 명령어를 정리해주세요.",["docker ps","docker logs"],[["Ports","port"],["docker compose","compose"]]),
("cuda","PyTorch에서 CUDA 사용 가능 여부와 GPU 메모리를 확인하는 코드를 보여주세요.",["torch.cuda","is_available","mem_get_info"],[["nvidia-smi","device_count"]]),
("dataset","code_python/linux 편향을 줄이는 balanced dataset 구성 기준을 설명해주세요.",["category","balanced","중복"],[["quota","분포"],["CJK","template"]]),
("linux","리눅스에서 8000 포트를 쓰는 프로세스를 확인하고 종료하는 명령어를 알려주세요.",["ss","grep","8000"],[["kill","pkill"],["ps","lsof"]]),
("workflow","현업식 파인튜닝 워크플로를 scripts/logs/reports/data, smoke, benchmark, stable/rejected 기준으로 정리해주세요.",["scripts","logs","reports","smoke","benchmark"],[["stable","rejected"],["Git","commit"]]),
]
BAD=["{problem}","{framework}","### Instruction","### Response","<|im_start|>","<|im_end|>","assistant:","user:"]

def cjk(x): return bool(re.search(r"[\u4e00-\u9fff]",x))
def leak(x): return any(b.lower() in x.lower() for b in BAD)
def honor(x): return any(m in x for m in ["습니다","합니다","됩니다","주세요","하세요","입니다"])
def score(out,must,anys):
    s=100; flags=[]; low=out.lower()
    if cjk(out): s-=60; flags.append("chinese_cjk")
    if leak(out): s-=35; flags.append("template_leak")
    if not honor(out): s-=12; flags.append("weak_honorific")
    if len(out.strip())<100: s-=12; flags.append("too_short")
    miss=[m for m in must if m.lower() not in low]
    if miss: s-=min(45,12*len(miss)); flags.append("missing:"+",".join(miss))
    for g in anys:
        if not any(x.lower() in low for x in g): s-=8; flags.append("missing_any:"+"|".join(g))
    s=max(0,s)
    return s,flags

def base_model(ad):
    cfg=json.loads((ad/"adapter_config.json").read_text(encoding="utf-8"))
    return cfg.get("base_model_name_or_path","Qwen/Qwen2.5-14B-Instruct")

def main():
    import torch
    from transformers import AutoTokenizer,AutoModelForCausalLM
    from peft import PeftModel
    if not AD.exists(): node-7.example.invalid SystemExit(f"[FAIL] ADAPTER_DIR not found: {AD}")
    bm=base_model(AD); ts=time.strftime("%Y%m%d_%H%M%S")
    print("==== 14B 10K AUTO QUICK BENCHMARK START ====")
    print("base_model:",bm); print("adapter_dir:",AD); print("max_new_tokens:",MAX_NEW)
    tok=AutoTokenizer.from_pretrained(bm,trust_remote_code=True)
    if tok.pad_token is None: tok.pad_token=<REDACTED_SECRET>
    model=AutoModelForCausalLM.from_pretrained(bm,torch_dtype=torch.bfloat16,device_map=None,low_cpu_mem_usage=False,trust_remote_code=True)
    model=model.to("cuda:0")
    model=PeftModel.from_pretrained(model,str(AD),is_trainable=False); model.eval()
    res=[]; cats=defaultdict(list); lat=[]
    for i,(cat,prompt,must,anys) in enumerate(TESTS,1):
        print("\n"+"-"*80); print("BENCH:",i,cat); print("-"*80)
        text=tok.apply_chat_template([{"role":"system","content":SYS},{"role":"user","content":prompt}],tokenize=False,add_generation_prompt=True)
        inp=tok(text,return_tensors="pt").to("cuda:0")
        t0=time.time()
        with torch.no_grad():
            out=model.generate(**inp,max_new_tokens=MAX_NEW,do_sample=False,repetition_penalty=1.08,pad_token_id=tok.eos_token_id)
        sec=time.time()-t0
        ans=tok.decode(out[0][inp["input_ids"].shape[-1]:],skip_special_tokens=True).strip()
        sc,flags=score(ans,must,anys)
        print(ans); print("\nSCORE:",sc); print("FLAGS:",flags); print("LATENCY_SEC:",round(sec,3))
        item={"id":i,"category":cat,"prompt":prompt,"output":ans,"score":sc,"flags":flags,"latency_sec":round(sec,3)}
        res.append(item); cats[cat].append(sc); lat.append(sec)
    avg=sum(r["score"] for r in res)/len(res); pass70=sum(r["score"]>=70 for r in res); strong85=sum(r["score"]>=85 for r in res)
    catavg={k:round(sum(v)/len(v),2) for k,v in sorted(cats.items())}
    decision="PASS_QUICK_BENCHMARK" if avg>=85 and pass70>=9 and strong85>=6 else ("REVIEW_QUICK_BENCHMARK" if avg>=80 and pass70>=8 else "REJECT_OR_REPAIR_NEEDED")
    report={"base_model":bm,"adapter_dir":str(AD),"average_score":round(avg,2),"pass_70_plus":f"{pass70}/{len(res)}","strong_85_plus":f"{strong85}/{len(res)}","category_avg":catavg,"average_latency_sec":round(sum(lat)/len(lat),3),"decision":decision,"results":res}
    rp=REP/f"14b10k_auto_quick_benchmark_{ts}.json"; rp.write_text(json.dumps(report,ensure_ascii=False,indent=2),encoding="utf-8")
    print("\n"+"="*80); print("FINAL 14B 10K AUTO QUICK BENCHMARK SUMMARY"); print("="*80)
    for k in ["base_model","adapter_dir","average_score","pass_70_plus","strong_85_plus","average_latency_sec","category_avg","decision"]:
        print(f"{k}:",report[k])
    print("report_path:",rp)
if __name__=="__main__":
    try: main()
    except Exception:
        print("\n[FATAL ERROR]"); traceback.print_exc(); node-7.example.invalid