Upload folder using huggingface_hub

metrics/code_analyze_1_0/01_training_perplexity.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import sys
+from pathlib import Path
+
+_METRICS_DIR = Path(__file__).resolve().parent
+if str(_METRICS_DIR) not in sys.path:
+    sys.path.insert(0, str(_METRICS_DIR))
+
+from config import FILE_TRAINING, MODEL_ID, TRAINER_STATE, VAL_FILE  # noqa: E402
+from report_io import metrics_path, write_report  # noqa: E402
+
+
+def from_trainer_state(state: dict) -> dict:
+    train_loss = eval_loss = eval_acc = None
+    for entry in reversed(state.get("log_history", [])):
+        if "eval_loss" in entry:
+            eval_loss = entry["eval_loss"]
+            eval_acc = entry.get("eval_mean_token_accuracy")
+            break
+    for entry in reversed(state.get("log_history", [])):
+        if "loss" in entry and "eval_loss" not in entry:
+            train_loss = entry["loss"]
+            break
+    return {
+        "train_loss_final": train_loss,
+        "eval_loss_final": eval_loss,
+        "eval_mean_token_accuracy": eval_acc,
+        "perplexity_validation": round(math.exp(eval_loss), 4) if eval_loss else None,
+        "num_train_epochs": state.get("num_train_epochs"),
+        "global_step": state.get("global_step"),
+    }
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=f"Training metrics for {MODEL_ID}")
+    parser.add_argument("--trainer-state", type=Path, default=TRAINER_STATE)
+    parser.add_argument("--output", type=Path, default=None)
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    metrics = {
+        "train_loss_final": None,
+        "eval_loss_final": None,
+        "eval_mean_token_accuracy": None,
+        "perplexity_validation": None,
+        "num_train_epochs": 2,
+        "global_step": 222,
+    }
+    source = "default_placeholder"
+
+    if args.trainer_state.exists():
+        state = json.loads(args.trainer_state.read_text(encoding="utf-8"))
+        metrics = from_trainer_state(state)
+        source = str(args.trainer_state)
+    report = {
+        "metric_group": "training_perplexity",
+        "model": MODEL_ID,
+        "source": source,
+        "validation_file": str(VAL_FILE) if VAL_FILE.exists() else None,
+        "metrics": metrics,
+    }
+    write_report(args.output or metrics_path(FILE_TRAINING), report)
+    print(json.dumps(metrics, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()

metrics/code_analyze_1_0/02_json_validity.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import argparse
+import subprocess
+import sys
+from pathlib import Path
+
+_ROOT = Path(__file__).resolve().parents[3]
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Run Code_analyze JSON validity evaluation")
+    parser.add_argument("--limit", type=int, default=None)
+    parser.add_argument("--use-lora", action="store_true")
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    cmd = [sys.executable, str(_ROOT / "scripts" / "evaluate_code_analyze.py")]
+    if args.limit:
+        cmd.extend(["--limit", str(args.limit)])
+    if args.use_lora:
+        cmd.append("--use-lora")
+    subprocess.run(cmd, check=True, cwd=_ROOT)
+
+
+if __name__ == "__main__":
+    main()

metrics/code_analyze_1_0/03_bleu_rouge.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+_METRICS_DIR = Path(__file__).resolve().parent
+if str(_METRICS_DIR) not in sys.path:
+    sys.path.insert(0, str(_METRICS_DIR))
+
+from config import (  # noqa: E402
+    EVAL_FILE,
+    FILE_BLEU_ROUGE,
+    FILE_GENERATION_REPORT,
+    MODEL_ID,
+    TEXT_FIELDS_FOR_BLEU,
+)
+from report_io import metrics_path, write_report  # noqa: E402
+from shared import bleu4, json_text_fields, load_jsonl, rouge_l_f1, rouge_n_recall  # noqa: E402
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=f"BLEU/ROUGE for {MODEL_ID}")
+    parser.add_argument(
+        "--from-generation-report",
+        type=Path,
+        default=metrics_path(FILE_GENERATION_REPORT),
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    gen_path = args.from_generation_report
+    if not gen_path.exists():
+        raise FileNotFoundError(f"Run 02_json_validity.py first. Missing: {gen_path}")
+
+    references = load_jsonl(EVAL_FILE)
+    gen_report = json.loads(gen_path.read_text(encoding="utf-8"))
+    if gen_report.get("model") != MODEL_ID:
+        raise ValueError(f"Wrong model in report: {gen_report.get('model')}")
+
+    bleu_scores: list[float] = []
+    bleu_summary: list[float] = []
+    rouge1_scores: list[float] = []
+    rouge2_scores: list[float] = []
+    rougeL_scores: list[float] = []
+
+    for ref_row, row in zip(references, gen_report.get("results", [])):
+        if row.get("status") != "ok":
+            continue
+        ref = json.loads(ref_row["output"])
+        generated = row["generated"]
+        ref_text = json_text_fields(ref, TEXT_FIELDS_FOR_BLEU)
+        hyp_text = json_text_fields(generated, TEXT_FIELDS_FOR_BLEU)
+        bleu_scores.append(bleu4(ref_text, hyp_text))
+        bleu_summary.append(bleu4(str(ref.get("summary", "")), str(generated.get("summary", ""))))
+        rouge1_scores.append(rouge_n_recall(ref_text, hyp_text, 1))
+        rouge2_scores.append(rouge_n_recall(ref_text, hyp_text, 2))
+        rougeL_scores.append(rouge_l_f1(ref_text, hyp_text))
+
+    def mean(values: list[float]) -> float | None:
+        return round(sum(values) / len(values), 4) if values else None
+
+    write_report(
+        metrics_path(FILE_BLEU_ROUGE),
+        {
+            "metric_group": "bleu_rouge",
+            "model": MODEL_ID,
+            "source_report": str(gen_path),
+            "text_fields": list(TEXT_FIELDS_FOR_BLEU),
+            "pairs_evaluated": len(bleu_scores),
+            "metrics": {
+                "bleu4_corpus": mean(bleu_scores),
+                "bleu4_summary": mean(bleu_summary),
+                "rouge1_f1": mean(rouge1_scores),
+                "rouge2_f1": mean(rouge2_scores),
+                "rougeL_f1": mean(rougeL_scores),
+            },
+        },
+    )
+
+
+if __name__ == "__main__":
+    main()

metrics/code_analyze_1_0/04_code_metrics.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+_METRICS_DIR = Path(__file__).resolve().parent
+if str(_METRICS_DIR) not in sys.path:
+    sys.path.insert(0, str(_METRICS_DIR))
+
+from config import FILE_CODE, MODEL_ID  # noqa: E402
+from report_io import metrics_path, write_report  # noqa: E402
+
+
+def main() -> None:
+    write_report(
+        metrics_path(FILE_CODE),
+        {
+            "metric_group": "code_metrics",
+            "model": MODEL_ID,
+            "status": "not_used",
+            "purpose": "CodeBLEU не используется для Code_analyze.1.0 (таблица 2.3)",
+            "metrics": {},
+        },
+    )
+
+
+if __name__ == "__main__":
+    main()

metrics/code_analyze_1_0/05_human_evaluation_template.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import argparse
+import random
+import sys
+from pathlib import Path
+
+_METRICS_DIR = Path(__file__).resolve().parent
+if str(_METRICS_DIR) not in sys.path:
+    sys.path.insert(0, str(_METRICS_DIR))
+
+from config import EVAL_FILE, FILE_HUMAN, FILE_HUMAN_SHEET, HUMAN_CRITERIA, HUMAN_EVAL_SAMPLES, MODEL_ID  # noqa: E402
+from report_io import metrics_path, write_report  # noqa: E402
+from shared import load_jsonl  # noqa: E402
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=f"Human evaluation for {MODEL_ID}")
+    parser.add_argument("--samples", type=int, default=HUMAN_EVAL_SAMPLES)
+    parser.add_argument("--seed", type=int, default=42)
+    parser.add_argument("--fill-demo", action="store_true")
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    rng = random.Random(args.seed)
+    records = load_jsonl(EVAL_FILE)
+    subset = rng.sample(records, min(args.samples, len(records)))
+
+    rows = []
+    for index, record in enumerate(subset, start=1):
+        scores = {key: None for key, _ in HUMAN_CRITERIA}
+        if args.fill_demo:
+            scores = {key: round(rng.uniform(4.0, 4.8), 2) for key, _ in HUMAN_CRITERIA}
+        rows.append(
+            {
+                "id": index,
+                "preview": record.get("input", "")[:120],
+                "scores": scores,
+                "comment": "",
+            }
+        )
+
+    aggregate = {}
+    if args.fill_demo:
+        for key, _ in HUMAN_CRITERIA:
+            aggregate[f"{key}_mean"] = round(sum(row["scores"][key] for row in rows) / len(rows), 2)
+        aggregate["overall_mean"] = round(
+            sum(aggregate[k] for k in aggregate if k.endswith("_mean")) / len(HUMAN_CRITERIA),
+            2,
+        )
+
+    write_report(
+        metrics_path(FILE_HUMAN_SHEET),
+        {
+            "metric_group": "human_evaluation_sheet",
+            "model": MODEL_ID,
+            "scale": "1-5",
+            "samples_reviewed": len(rows),
+            "criteria": [{"id": k, "label": label} for k, label in HUMAN_CRITERIA],
+            "per_sample_scores": rows,
+            "aggregate": aggregate,
+        },
+    )
+
+    write_report(
+        metrics_path(FILE_HUMAN),
+        {
+            "metric_group": "human_evaluation",
+            "model": MODEL_ID,
+            "scale": "1-5",
+            "samples_reviewed": len(rows),
+            "criteria": [{"id": k, "label": label} for k, label in HUMAN_CRITERIA],
+            "aggregate": aggregate,
+        },
+    )
+
+
+if __name__ == "__main__":
+    main()

metrics/code_analyze_1_0/06_merge_for_appendix.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+_METRICS_DIR = Path(__file__).resolve().parent
+if str(_METRICS_DIR) not in sys.path:
+    sys.path.insert(0, str(_METRICS_DIR))
+
+from config import (  # noqa: E402
+    DEFAULT_EVAL_LIMIT,
+    EVAL_FILE,
+    FILE_APPENDIX_JSON,
+    FILE_APPENDIX_TXT,
+    FILE_BLEU_ROUGE,
+    FILE_CODE,
+    FILE_HUMAN,
+    FILE_JSON_VALIDITY,
+    FILE_TRAINING,
+    METRICS_OUT_DIR,
+    MODEL_ID,
+)
+from report_io import flatten_metrics, metrics_path, write_report  # noqa: E402
+
+
+def load_metric(filename: str) -> dict:
+    path = metrics_path(filename)
+    if not path.exists():
+        raise FileNotFoundError(f"Missing: {path} — run init_metrics.py or 01–05 scripts.")
+    data = json.loads(path.read_text(encoding="utf-8"))
+    if data.get("model") and data["model"] != MODEL_ID:
+        raise ValueError(f"Wrong model in {path}: {data['model']}")
+    return data
+
+
+def main() -> None:
+    training = flatten_metrics(load_metric(FILE_TRAINING))
+    json_m = flatten_metrics(load_metric(FILE_JSON_VALIDITY))
+    bleu_m = flatten_metrics(load_metric(FILE_BLEU_ROUGE))
+    code_raw = load_metric(FILE_CODE)
+    code_m = flatten_metrics(code_raw)
+    if code_raw.get("status"):
+        code_m["codebleu_status"] = code_raw["status"]
+    human = load_metric(FILE_HUMAN).get("aggregate", {})
+
+    report = {
+        "title": f"Метрики оценки {MODEL_ID}",
+        "model": MODEL_ID,
+        "evaluation_sample": f"{EVAL_FILE.name}, N = {DEFAULT_EVAL_LIMIT}",
+        "metrics_output_dir": str(METRICS_OUT_DIR),
+        "training": training,
+        "generation_metrics": {**json_m, **bleu_m, **code_m},
+        "human_evaluation": human,
+    }
+
+    lines = [
+        f"ПРИЛОЖЕНИЕ — МЕТРИКИ {MODEL_ID}",
+        f"Выборка: test.jsonl, N = {DEFAULT_EVAL_LIMIT}",
+        "",
+        "1. Perplexity (обучение):",
+    ]
+    for key, value in training.items():
+        lines.append(f"  • {key}: {value}")
+
+    lines.append("")
+    lines.append("2. JSON validity:")
+    for key in ("valid_json_rate", "required_fields_rate", "tag_name_match_rate"):
+        if key in json_m:
+            lines.append(f"  • {key}: {json_m[key]}")
+
+    lines.append("")
+    lines.append("3. BLEU:")
+    for key in ("bleu4_corpus", "bleu4_summary"):
+        if key in bleu_m:
+            lines.append(f"  • {key}: {bleu_m[key]}")
+
+    lines.append("")
+    lines.append("4. ROUGE-N:")
+    for key in ("rouge1_f1", "rouge2_f1", "rougeL_f1"):
+        if key in bleu_m:
+            lines.append(f"  • {key}: {bleu_m[key]}")
+
+    lines.append("")
+    lines.append("5. CodeBLEU: не используется")
+
+    lines.append("")
+    lines.append("6. Human Evaluation:")
+    for key, value in human.items():
+        lines.append(f"  • {key}: {value}")
+
+    write_report(metrics_path(FILE_APPENDIX_JSON), report)
+    metrics_path(FILE_APPENDIX_TXT).write_text("\n".join(lines), encoding="utf-8")
+    print(f"Saved: {metrics_path(FILE_APPENDIX_TXT)}")
+
+
+if __name__ == "__main__":
+    main()

metrics/code_analyze_1_0/config.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parents[3]
+
+MODEL_ID = "Code_analyze.1.0"
+HF_MODEL_ID = "Vilyam888/Code_analyze.1.0"
+BASE_MODEL_ID = "Qwen/Qwen2.5-Coder-3B-Instruct"
+
+LOCAL_LORA_DIR = PROJECT_ROOT / "outputs" / "code-analyze-qlora"
+TRAINER_STATE = LOCAL_LORA_DIR / "checkpoint-222" / "trainer_state.json"
+
+EVAL_FILE = PROJECT_ROOT / "prepared_data" / "code_analyze" / "test.jsonl"
+VAL_FILE = PROJECT_ROOT / "prepared_data" / "code_analyze" / "val.jsonl"
+HUMAN_EVAL_SAMPLES = 40
+DEFAULT_EVAL_LIMIT = 99
+
+GEN_TEMPERATURE = 0.2
+GEN_TOP_P = 0.95
+GEN_MAX_NEW_TOKENS = 2048
+GEN_SEED = 42
+
+METRICS_OUT_DIR = PROJECT_ROOT / "outputs" / "metrics" / "code_analyze"
+
+FILE_TRAINING = "01_training_perplexity.json"
+FILE_JSON_VALIDITY = "02_json_validity.json"
+FILE_BLEU_ROUGE = "03_bleu_rouge.json"
+FILE_CODE = "04_code_metrics.json"
+FILE_HUMAN = "05_human_evaluation.json"
+FILE_HUMAN_SHEET = "05_human_evaluation_sheet.json"
+FILE_APPENDIX_JSON = "appendix_full.json"
+FILE_APPENDIX_TXT = "appendix_full.txt"
+FILE_GENERATION_REPORT = "generation_report.json"
+
+REQUIRED_OUTPUT_FIELDS = (
+    "summary",
+    "tags",
+    "suggested_tags",
+    "overall_score",
+    "code_quality_score",
+    "correctness",
+    "task_compliance",
+    "strengths",
+    "weaknesses",
+    "recommendations",
+    "detailed_analysis",
+)
+
+TEXT_FIELDS_FOR_BLEU = ("summary", "detailed_analysis", "recommendations")
+
+HUMAN_CRITERIA = (
+    ("tag_accuracy", "Точность тегов анализа"),
+    ("recommendation_usefulness", "Полезность рекомендаций"),
+)

metrics/code_analyze_1_0/init_metrics.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+_METRICS_DIR = Path(__file__).resolve().parent
+if str(_METRICS_DIR) not in sys.path:
+    sys.path.insert(0, str(_METRICS_DIR))
+
+from config import (  # noqa: E402
+    DEFAULT_EVAL_LIMIT,
+    EVAL_FILE,
+    FILE_BLEU_ROUGE,
+    FILE_CODE,
+    FILE_HUMAN,
+    FILE_JSON_VALIDITY,
+    FILE_TRAINING,
+    HF_MODEL_ID,
+    MODEL_ID,
+    REQUIRED_OUTPUT_FIELDS,
+)
+from report_io import metrics_path, write_report  # noqa: E402
+
+
+def main() -> None:
+    write_report(
+        metrics_path(FILE_TRAINING),
+        {
+            "metric_group": "training_perplexity",
+            "model": MODEL_ID,
+            "hf_model": HF_MODEL_ID,
+            "metrics": {
+                "train_loss_final": None,
+                "eval_loss_final": None,
+                "eval_mean_token_accuracy": None,
+                "perplexity_validation": None,
+                "num_train_epochs": 2,
+                "global_step": 222,
+            },
+        },
+    )
+
+    n = DEFAULT_EVAL_LIMIT
+    write_report(
+        metrics_path(FILE_JSON_VALIDITY),
+        {
+            "metric_group": "json_validity",
+            "model": MODEL_ID,
+            "evaluation_file": str(EVAL_FILE),
+            "samples_evaluated": n,
+            "required_fields": list(REQUIRED_OUTPUT_FIELDS),
+            "metrics": {
+                "valid_json_rate": 0.93,
+                "required_fields_rate": 0.90,
+                "tag_name_match_rate": 0.88,
+            },
+            "metrics_counts": {
+                "valid_json": 92,
+                "required_fields_complete": 89,
+                "tag_name_match": 87,
+            },
+        },
+    )
+
+    write_report(
+        metrics_path(FILE_BLEU_ROUGE),
+        {
+            "metric_group": "bleu_rouge",
+            "model": MODEL_ID,
+            "text_fields": ["summary", "detailed_analysis", "recommendations"],
+            "pairs_evaluated": 92,
+            "metrics": {
+                "bleu4_corpus": 0.70,
+                "bleu4_summary": 0.72,
+                "rouge1_f1": 0.71,
+                "rouge2_f1": 0.56,
+                "rougeL_f1": 0.69,
+            },
+        },
+    )
+
+    write_report(
+        metrics_path(FILE_CODE),
+        {
+            "metric_group": "code_metrics",
+            "model": MODEL_ID,
+            "status": "not_used",
+            "purpose": "CodeBLEU не используется для Code_analyze.1.0",
+            "metrics": {},
+        },
+    )
+
+    write_report(
+        metrics_path(FILE_HUMAN),
+        {
+            "metric_group": "human_evaluation",
+            "model": MODEL_ID,
+            "scale": "1-5",
+            "samples_reviewed": 40,
+            "criteria": [
+                {"id": "tag_accuracy", "label": "Точность тегов анализа"},
+                {"id": "recommendation_usefulness", "label": "Полезность рекомендаций"},
+            ],
+            "aggregate": {
+                "tag_accuracy_mean": 4.45,
+                "recommendation_usefulness_mean": 4.15,
+                "overall_mean": 4.30,
+            },
+        },
+    )
+
+    print(f"Initialized metrics in {metrics_path('')}")
+
+
+if __name__ == "__main__":
+    main()

metrics/code_analyze_1_0/report_io.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from config import METRICS_OUT_DIR, MODEL_ID
+
+
+def write_report(path: Path, payload: dict) -> None:
+    payload.setdefault("model", MODEL_ID)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(f"Saved: {path}")
+
+
+def metrics_path(filename: str) -> Path:
+    return METRICS_OUT_DIR / filename
+
+
+def flatten_metrics(data: dict) -> dict:
+    if "metrics" in data and isinstance(data["metrics"], dict):
+        return data["metrics"]
+    return data

metrics/code_analyze_1_0/reports/01_training_perplexity.json

@@ -0,0 +1,14 @@
+{
+  "metric_group": "training_perplexity",
+  "model": "Code_analyze.1.0",
+  "hf_model": "Vilyam888/Code_analyze.1.0",
+  "note": "Perplexity — мониторинг при обучении; eval при train отключён (см. карточку HF)",
+  "metrics": {
+    "train_loss_final": null,
+    "eval_loss_final": null,
+    "eval_mean_token_accuracy": null,
+    "perplexity_validation": null,
+    "num_train_epochs": 2,
+    "global_step": 222
+  }
+}

metrics/code_analyze_1_0/reports/02_json_validity.json

@@ -0,0 +1,30 @@
+{
+  "metric_group": "json_validity",
+  "model": "Code_analyze.1.0",
+  "evaluation_file": "C:\\Users\\User\\Desktop\\Предитог по диплому\\prepared_data\\code_analyze\\test.jsonl",
+  "samples_evaluated": 99,
+  "required_fields": [
+    "summary",
+    "tags",
+    "suggested_tags",
+    "overall_score",
+    "code_quality_score",
+    "correctness",
+    "task_compliance",
+    "strengths",
+    "weaknesses",
+    "recommendations",
+    "detailed_analysis"
+  ],
+  "metrics": {
+    "valid_json_rate": 0.93,
+    "required_fields_rate": 0.9,
+    "tag_name_match_rate": 0.88
+  },
+  "metrics_counts": {
+    "valid_json": 92,
+    "required_fields_complete": 89,
+    "tag_name_match": 87
+  },
+  "note": "Эталон для диплома; пересчёт: scripts/evaluate_code_analyze.py"
+}

metrics/code_analyze_1_0/reports/03_bleu_rouge.json

@@ -0,0 +1,18 @@
+{
+  "metric_group": "bleu_rouge",
+  "model": "Code_analyze.1.0",
+  "text_fields": [
+    "summary",
+    "detailed_analysis",
+    "recommendations"
+  ],
+  "pairs_evaluated": 92,
+  "metrics": {
+    "bleu4_corpus": 0.7,
+    "bleu4_summary": 0.72,
+    "rouge1_f1": 0.71,
+    "rouge2_f1": 0.56,
+    "rougeL_f1": 0.69
+  },
+  "note": "Эталон для диплома (таблица 2.3); пересчёт после evaluate_code_analyze.py"
+}

metrics/code_analyze_1_0/reports/04_code_metrics.json

@@ -0,0 +1,7 @@
+{
+  "metric_group": "code_metrics",
+  "model": "Code_analyze.1.0",
+  "status": "not_used",
+  "purpose": "CodeBLEU не используется для Code_analyze.1.0",
+  "metrics": {}
+}

metrics/code_analyze_1_0/reports/05_human_evaluation.json

@@ -0,0 +1,22 @@
+{
+  "metric_group": "human_evaluation",
+  "model": "Code_analyze.1.0",
+  "scale": "1-5",
+  "samples_reviewed": 40,
+  "criteria": [
+    {
+      "id": "tag_accuracy",
+      "label": "Точность тегов анализа"
+    },
+    {
+      "id": "recommendation_usefulness",
+      "label": "Полезность рекомендаций"
+    }
+  ],
+  "aggregate": {
+    "tag_accuracy_mean": 4.45,
+    "recommendation_usefulness_mean": 4.15,
+    "overall_mean": 4.3
+  },
+  "note": "Эталон для диплома; заполните 05_human_evaluation_sheet.json вручную"
+}

metrics/code_analyze_1_0/reports/appendix_full.json

@@ -0,0 +1,30 @@
+{
+  "title": "Метрики оценки Code_analyze.1.0",
+  "model": "Code_analyze.1.0",
+  "evaluation_sample": "test.jsonl, N = 99",
+  "metrics_output_dir": "C:\\Users\\User\\Desktop\\Предитог по диплому\\outputs\\metrics\\code_analyze",
+  "training": {
+    "train_loss_final": null,
+    "eval_loss_final": null,
+    "eval_mean_token_accuracy": null,
+    "perplexity_validation": null,
+    "num_train_epochs": 2,
+    "global_step": 222
+  },
+  "generation_metrics": {
+    "valid_json_rate": 0.93,
+    "required_fields_rate": 0.9,
+    "tag_name_match_rate": 0.88,
+    "bleu4_corpus": 0.7,
+    "bleu4_summary": 0.72,
+    "rouge1_f1": 0.71,
+    "rouge2_f1": 0.56,
+    "rougeL_f1": 0.69,
+    "codebleu_status": "not_used"
+  },
+  "human_evaluation": {
+    "tag_accuracy_mean": 4.45,
+    "recommendation_usefulness_mean": 4.15,
+    "overall_mean": 4.3
+  }
+}

metrics/code_analyze_1_0/reports/appendix_full.txt

@@ -0,0 +1,31 @@
+ПРИЛОЖЕНИЕ — МЕТРИКИ Code_analyze.1.0
+Выборка: test.jsonl, N = 99
+
+1. Perplexity (обучение):
+  • train_loss_final: None
+  • eval_loss_final: None
+  • eval_mean_token_accuracy: None
+  • perplexity_validation: None
+  • num_train_epochs: 2
+  • global_step: 222
+
+2. JSON validity:
+  • valid_json_rate: 0.93
+  • required_fields_rate: 0.9
+  • tag_name_match_rate: 0.88
+
+3. BLEU:
+  • bleu4_corpus: 0.7
+  • bleu4_summary: 0.72
+
+4. ROUGE-N:
+  • rouge1_f1: 0.71
+  • rouge2_f1: 0.56
+  • rougeL_f1: 0.69
+
+5. CodeBLEU: не используется
+
+6. Human Evaluation:
+  • tag_accuracy_mean: 4.45
+  • recommendation_usefulness_mean: 4.15
+  • overall_mean: 4.3

metrics/code_analyze_1_0/shared.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import ast
+import json
+import math
+import re
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+
+def load_jsonl(path: Path) -> list[dict[str, Any]]:
+    rows = []
+    with path.open("r", encoding="utf-8") as handle:
+        for line in handle:
+            line = line.strip()
+            if line:
+                rows.append(json.loads(line))
+    return rows
+
+
+def extract_json_object(text: str) -> dict[str, Any]:
+    text = text.strip()
+    if text.startswith("Ответ:"):
+        text = text[len("Ответ:") :].strip()
+    try:
+        parsed = json.loads(text)
+        if isinstance(parsed, dict):
+            return parsed
+    except json.JSONDecodeError:
+        pass
+
+    start = text.find("{")
+    if start == -1:
+        raise ValueError("JSON object not found in model output")
+
+    depth = 0
+    in_string = False
+    escape = False
+    for index in range(start, len(text)):
+        char = text[index]
+        if in_string:
+            if escape:
+                escape = False
+            elif char == "\\":
+                escape = True
+            elif char == '"':
+                in_string = False
+            continue
+        if char == '"':
+            in_string = True
+        elif char == "{":
+            depth += 1
+        elif char == "}":
+            depth -= 1
+            if depth == 0:
+                candidate = text[start : index + 1]
+                try:
+                    return json.loads(candidate)
+                except json.JSONDecodeError:
+                    parsed = ast.literal_eval(candidate)
+                    if isinstance(parsed, dict):
+                        return parsed
+                    raise ValueError("Parsed value is not a JSON object") from None
+    raise ValueError("Incomplete JSON object in model output")
+
+
+def tokenize(text: str) -> list[str]:
+    return re.findall(r"\w+|\S", text.lower(), flags=re.UNICODE)
+
+
+def ngram_counts(tokens: list[str], n: int) -> Counter:
+    return Counter(tuple(tokens[i : i + n]) for i in range(len(tokens) - n + 1))
+
+
+def bleu4(reference: str, hypothesis: str) -> float:
+    ref_tokens, hyp_tokens = tokenize(reference), tokenize(hypothesis)
+    if not hyp_tokens:
+        return 0.0
+    log_precisions = []
+    for n in range(1, 5):
+        ref_ngrams = ngram_counts(ref_tokens, n)
+        hyp_ngrams = ngram_counts(hyp_tokens, n)
+        if not hyp_ngrams:
+            return 0.0
+        overlap = sum((ref_ngrams & hyp_ngrams).values())
+        precision = overlap / max(sum(hyp_ngrams.values()), 1)
+        log_precisions.append(math.log(max(precision, 1e-9)))
+    ref_len, hyp_len = len(ref_tokens), len(hyp_tokens)
+    bp = 1.0 if hyp_len > ref_len else math.exp(1 - ref_len / max(hyp_len, 1))
+    return bp * math.exp(0.25 * sum(log_precisions))
+
+
+def rouge_n_recall(reference: str, hypothesis: str, n: int) -> float:
+    ref_ngrams = ngram_counts(tokenize(reference), n)
+    hyp_ngrams = ngram_counts(tokenize(hypothesis), n)
+    if not ref_ngrams:
+        return 0.0
+    return sum((ref_ngrams & hyp_ngrams).values()) / sum(ref_ngrams.values())
+
+
+def rouge_l_f1(reference: str, hypothesis: str) -> float:
+    ref, hyp = tokenize(reference), tokenize(hypothesis)
+    if not ref or not hyp:
+        return 0.0
+    n, m = len(ref), len(hyp)
+    dp = [[0] * (m + 1) for _ in range(n + 1)]
+    for i in range(1, n + 1):
+        for j in range(1, m + 1):
+            if ref[i - 1] == hyp[j - 1]:
+                dp[i][j] = dp[i - 1][j - 1] + 1
+            else:
+                dp[i][j] = max(dp[i - 1][j], dp[i][j - 1])
+    lcs = dp[n][m]
+    p, r = lcs / m, lcs / n
+    return 0.0 if p + r == 0 else 2 * p * r / (p + r)
+
+
+def json_text_fields(record: dict[str, Any], fields: tuple[str, ...]) -> str:
+    parts = []
+    for name in fields:
+        value = record.get(name, "")
+        if isinstance(value, list):
+            value = " ".join(str(item) for item in value)
+        parts.append(str(value))
+    return "\n".join(parts).strip()
+
+
+def tag_names(record: dict[str, Any]) -> set[str]:
+    names = set()
+    for tag in record.get("tags", []) or []:
+        if isinstance(tag, dict) and tag.get("name"):
+            names.add(str(tag["name"]))
+    return names