{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "title"
      },
      "source": [
        "# News Summarization Eval\n",
        "\n",
        "Colab-friendly notebook for running the news summarization benchmark on a strong open-weight Hugging Face model.\n",
        "\n",
        "This notebook pulls the benchmark code directly from the Hugging Face repo so it stays in sync with the HF page.\n",
        "\n",
        "Recommended first model:\n",
        "- `Qwen/Qwen2.5-3B-Instruct`\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "install"
      },
      "outputs": [],
      "source": [
        "!pip install -q transformers accelerate sentencepiece bert-score rouge-score huggingface_hub"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hf-login"
      },
      "outputs": [],
      "source": [
        "from google.colab import userdata\n",
        "from huggingface_hub import login\n",
        "import os\n",
        "\n",
        "token = userdata.get(\"HF_TOKEN\")\n",
        "os.environ[\"HF_TOKEN\"] = token\n",
        "login(token=token)\n",
        "print(\"HF token loaded from Colab Secrets\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "clone"
      },
      "outputs": [],
      "source": [
        "from huggingface_hub import snapshot_download\n",
        "\n",
        "snapshot_download(\n",
        "    repo_id=\"arach/training-lab\",\n",
        "    repo_type=\"model\",\n",
        "    local_dir=\"/content/training-lab-hf\",\n",
        "    allow_patterns=[\n",
        "        \"eval/news_summarization/*\",\n",
        "    ],\n",
        ")\n",
        "\n",
        "%cd /content/training-lab-hf"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "config"
      },
      "outputs": [],
      "source": [
        "MODEL = \"Qwen/Qwen2.5-3B-Instruct\"\n",
        "LIMIT = 10\n",
        "PROMPT_STYLE = \"simple\"\n",
        "DTYPE = \"bfloat16\"\n",
        "TRUST_REMOTE_CODE = False\n",
        "DISABLE_BERTSCORE = True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "run"
      },
      "outputs": [],
      "source": [
        "trust_flag = \"--trust-remote-code\" if TRUST_REMOTE_CODE else \"\"\n",
        "bertscore_flag = \"--disable-bertscore\" if DISABLE_BERTSCORE else \"\"\n",
        "!python eval/news_summarization/run_hf_transformers.py \\\n",
        "  --model {MODEL} \\\n",
        "  --limit {LIMIT} \\\n",
        "  --prompt-style {PROMPT_STYLE} \\\n",
        "  --dtype {DTYPE} \\\n",
        "  --verbose \\\n",
        "  {bertscore_flag} \\\n",
        "  {trust_flag}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "resume"
      },
      "outputs": [],
      "source": [
        "trust_flag = \"--trust-remote-code\" if TRUST_REMOTE_CODE else \"\"\n",
        "bertscore_flag = \"--disable-bertscore\" if DISABLE_BERTSCORE else \"\"\n",
        "!python eval/news_summarization/run_hf_transformers.py \\\n",
        "  --model {MODEL} \\\n",
        "  --limit {LIMIT} \\\n",
        "  --prompt-style {PROMPT_STYLE} \\\n",
        "  --dtype {DTYPE} \\\n",
        "  --resume \\\n",
        "  {bertscore_flag} \\\n",
        "  {trust_flag}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "inspect"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "from pathlib import Path\n",
        "\n",
        "results_dir = Path('eval/news_summarization/results')\n",
        "results_dir.mkdir(parents=True, exist_ok=True)\n",
        "files = sorted(results_dir.glob('hf-transformers-*.json'))\n",
        "print('Results dir:', results_dir)\n",
        "for path in files:\n",
        "    print(path.name, f'({path.stat().st_size} bytes)')\n",
        "\n",
        "if not files:\n",
        "    print('No result file yet')\n",
        "else:\n",
        "    latest = files[-1]\n",
        "    print('\\nLatest:', latest)\n",
        "    data = json.loads(latest.read_text())\n",
        "    print(json.dumps(data.get('summary', {}), indent=2))\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "A100"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}