| import os |
| import json |
| import subprocess |
| import pandas as pd |
| import shutil |
| from pathlib import Path |
|
|
| GLOBAL_MODEL_PATH = "../stage2_object_v8_1200" |
| GLOBAL_NORMAL_PATH = "正常-normal_data" |
|
|
| |
| TASK_CONFIGS = [ |
| ("rule_11_vllm.py", "文字占比"), |
| ("rule_12_vllm.py", "样式数量"), |
| |
| |
| ("rule_17_vllm.py", "信息量"), |
| ("rule_18_vllm.py", "排布间距"), |
| ("rule_19_vllm.py", "内容构图"), |
| ] |
|
|
| SUMMARY_OUTPUT = "vllm_audit_summary_normal.csv" |
|
|
| def run_vllm_task(script, label_name): |
| normal_path = Path(GLOBAL_NORMAL_PATH) |
| original_json = normal_path / f"audit_result_{normal_path.name}.json" |
| unique_json = normal_path / f"temp_{script.replace('.py', '')}.json" |
|
|
| if unique_json.exists(): |
| unique_json.unlink() |
|
|
| cmd = [ |
| "python", script, |
| "--input_dir", GLOBAL_NORMAL_PATH, |
| "--model_path", GLOBAL_MODEL_PATH |
| ] |
| |
| print(f"\n[EXEC] 正在运行规则: {label_name} ({script})") |
| |
| try: |
| subprocess.run(cmd, check=True) |
| |
| if original_json.exists(): |
| shutil.move(str(original_json), str(unique_json)) |
| print(f"[DONE] 结果已固化至: {unique_json.name}") |
| return unique_json |
| else: |
| print(f"[ERROR] 脚本运行完成但未找到生成文件: {original_json}") |
| return None |
| |
| except subprocess.CalledProcessError as e: |
| print(f"[ERROR] 脚本 {script} 崩溃: {e}") |
| return None |
|
|
| def extract_metrics(json_path, script, label_name): |
| """从重命名后的唯一 JSON 文件中读取数据""" |
| if not json_path or not json_path.exists(): |
| return {"规则名称": label_name, "状态": "失败"} |
|
|
| with open(json_path, 'r', encoding='utf-8') as f: |
| data = json.load(f) |
| |
| total = len(data) |
| unsuitable = sum(1 for item in data if item["label"] == "Unsuitable") |
| suitable = sum(1 for item in data if item["label"] == "Suitable") |
| fp_rate = (unsuitable / total) if total > 0 else 0 |
|
|
| return { |
| "规则脚本": script, |
| "规则维度": label_name, |
| "测试总数": total, |
| "误报数(Unsuitable)": unsuitable, |
| "正确通过数(Suitable)": suitable, |
| "误报率(FP Rate)": f"{fp_rate:.2%}", |
| "结果文件": json_path.name |
| } |
|
|
| def main(): |
| final_results = [] |
| |
| for script, label_name in TASK_CONFIGS: |
| unique_json_path = run_vllm_task(script, label_name) |
| |
| if unique_json_path: |
| metrics = extract_metrics(unique_json_path, script, label_name) |
| final_results.append(metrics) |
| |
| if final_results: |
| df = pd.DataFrame(final_results) |
| df.to_csv(SUMMARY_OUTPUT, index=False, encoding='utf-8-sig') |
| |
| print("\n" + "="*80) |
| print(f"测试完成！汇总已写入: {SUMMARY_OUTPUT}") |
| print("-" * 80) |
| print(df.to_string(index=False)) |
| print("="*80) |
| else: |
| print("[CRITICAL] 没有收集到任何有效数据。") |
|
|
| if __name__ == "__main__": |
| main() |
