Buckets:
| { | |
| "cells": [ | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "import json\n", | |
| "import copy as cp\n", | |
| "import numpy as np\n", | |
| "import matplotlib.pyplot as plt\n", | |
| "import matplotlib.font_manager as fm\n", | |
| "\n", | |
| "def download_file(url, filename=None):\n", | |
| " from urllib.request import urlretrieve\n", | |
| " if filename is None:\n", | |
| " filename = url.split('/')[-1]\n", | |
| " urlretrieve(url, filename)\n", | |
| "\n", | |
| "font_URL = 'http://opencompass.openxlab.space/utils/Fonts/segoepr.ttf'\n", | |
| "download_file(font_URL)\n", | |
| "\n", | |
| "font12 = fm.FontProperties(fname='segoepr.ttf', size=12)\n", | |
| "font15 = fm.FontProperties(fname='segoepr.ttf', size=15, weight='bold')\n", | |
| "font18 = fm.FontProperties(fname='segoepr.ttf', size=18, weight='bold')\n", | |
| "\n", | |
| "DATA_URL = 'http://opencompass.openxlab.space/utils/OpenVLM.json'\n", | |
| "download_file(DATA_URL)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "def pre_normalize(raw_data, labels):\n", | |
| " data_list = cp.deepcopy(raw_data)\n", | |
| " minimum, maximum, max_range, range_map = {}, {}, 0, {}\n", | |
| " for lb in labels:\n", | |
| " minimum[lb] = min([x[lb] for x in data_list])\n", | |
| " maximum[lb] = max([x[lb] for x in data_list])\n", | |
| " max_range = max(max_range, maximum[lb] - minimum[lb])\n", | |
| " max_range *= 1.25\n", | |
| " for lb in labels:\n", | |
| " mid = (minimum[lb] + maximum[lb]) / 2\n", | |
| " new_range = (mid - max_range / 2, mid + max_range / 2) if (mid + max_range / 2) < 100 else (100 - max_range, 100)\n", | |
| " range_map[lb] = new_range\n", | |
| " for item in data_list:\n", | |
| " assert new_range[0] <= item[lb] <= new_range[1]\n", | |
| " item[lb] = (item[lb] - new_range[0]) / max_range * 100\n", | |
| " return data_list, range_map\n", | |
| "\n", | |
| "# solve the problem that some benchmark score is too high and out of range\n", | |
| "def log_normalize(raw_data, labels):\n", | |
| " data_list = cp.deepcopy(raw_data)\n", | |
| " minimum, maximum, max_range, range_map = {}, {}, 0, {}\n", | |
| " for lb in labels:\n", | |
| " minimum[lb] = min([np.log(x[lb]) for x in data_list])\n", | |
| " maximum[lb] = max([np.log(x[lb]) for x in data_list])\n", | |
| " max_range = max(max_range, maximum[lb] - minimum[lb])\n", | |
| " max_range *= 1.005\n", | |
| " for lb in labels:\n", | |
| " mid = (minimum[lb] + maximum[lb]) / 2\n", | |
| " new_range = (mid - max_range / 2, mid + max_range / 2) if (mid + max_range / 2) < 100 else (100 - max_range, 100)\n", | |
| " range_map[lb] = new_range\n", | |
| " for item in data_list:\n", | |
| " assert new_range[0] <= np.log(item[lb]) <= new_range[1]\n", | |
| " item[lb] = (np.log(item[lb]) - new_range[0]) / max_range * 100\n", | |
| " return data_list, range_map" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "# Draw MMBench Radar Graph\n", | |
| "data = json.loads(open('OpenVLM.json').read())['results']\n", | |
| "models = list(data)\n", | |
| "print(models)\n", | |
| "\n", | |
| "# model2vis = [\n", | |
| "# 'GPT-4v (detail: low)', 'GeminiProVision', 'Qwen-VL-Plus', \n", | |
| "# 'InternLM-XComposer2-VL', 'LLaVA-v1.5-13B', 'CogVLM-17B-Chat',\n", | |
| "# 'mPLUG-Owl2', 'Qwen-VL-Chat', 'IDEFICS-80B-Instruct'\n", | |
| "# ]\n", | |
| "\n", | |
| "model2vis = [\n", | |
| " # 'GPT-4v (detail: low)', 'GeminiProVision', 'InternLM-XComposer2-VL', \n", | |
| " 'GPT-4v (1106, detail-low)', 'Gemini-1.0-Pro', 'Gemini-1.5-Pro', #'Gemini-1.5-Flash', 'Qwen-VL-Plus', \n", | |
| " 'InternLM-XComposer2', 'LLaVA-v1.5-13B', 'CogVLM-17B-Chat',\n", | |
| " 'mPLUG-Owl2', 'Qwen-VL-Chat', 'IDEFICS-80B-Instruct'\n", | |
| "]\n", | |
| "\n", | |
| "colors = [\n", | |
| " '#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', \n", | |
| " '#e377c2', '#7f7f7f', '#bcbd22'\n", | |
| "]" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "from collections import defaultdict\n", | |
| "\n", | |
| "split = 'MMBench_TEST_EN'\n", | |
| "# data_sub = {k: v[split] for k, v in data.items()}\n", | |
| "data_sub = {k: defaultdict(int, v)[split] for k, v in data.items()}\n", | |
| "# solve the problem that some model lack the evaluation of MMBench_TEST_EN\n", | |
| "\n", | |
| "labels = list(data_sub[model2vis[0]])\n", | |
| "labels.remove('Overall')\n", | |
| "num_vars = len(labels)\n", | |
| "\n", | |
| "raw_data = [data_sub[m] for m in model2vis]\n", | |
| "data_list, range_map = pre_normalize(raw_data, labels)\n", | |
| "\n", | |
| "alpha = 0.25\n", | |
| "angles = np.linspace(0, 2 * np.pi, num_vars, endpoint=False).tolist()\n", | |
| "angles_deg = np.linspace(0, 360, num_vars, endpoint=False).tolist()\n", | |
| "fig, ax_base = plt.subplots(nrows=1, ncols=1, figsize=(10, 10), subplot_kw=dict(polar=True))\n", | |
| "\n", | |
| "for i in range(len(data_list)):\n", | |
| " item = data_list[i]\n", | |
| " model_name = model2vis[i]\n", | |
| " color = colors[i]\n", | |
| " tmp_angles = angles[:] + [angles[0]]\n", | |
| " tmp_values = [item[lb] for lb in labels] + [item[labels[0]]]\n", | |
| " ax_base.plot(tmp_angles, tmp_values, color=color, linewidth=1, linestyle='solid', label=model_name)\n", | |
| " ax_base.fill(tmp_angles, tmp_values, color=color, alpha=alpha)\n", | |
| " \n", | |
| "angles += [angles[0]]\n", | |
| "ax_base.set_ylim(0, 100)\n", | |
| "ax_base.set_yticks([40, 60, 80, 100])\n", | |
| "ax_base.set_yticklabels([''] * 4)\n", | |
| "\n", | |
| "ax_base.tick_params(pad=25)\n", | |
| "ax_base.set_xticks(angles[:-1])\n", | |
| "ax_base.set_xticklabels(labels, fontproperties=font18)\n", | |
| "\n", | |
| "leg = ax_base.legend(loc='center right', bbox_to_anchor=(1.6, 0.5), prop=font15, ncol=1, frameon=True, labelspacing=1.2)\n", | |
| "for line in leg.get_lines():\n", | |
| " line.set_linewidth(2.5)\n", | |
| "\n", | |
| "cx, cy, sz = 0.44, 0.435, 0.34\n", | |
| "axes = [fig.add_axes([cx - sz, cy - sz, cx + sz, cy + sz], projection='polar', label='axes%d' % i) for i in range(num_vars)]\n", | |
| " \n", | |
| "for ax, angle, label in zip(axes, angles_deg, labels):\n", | |
| " ax.patch.set_visible(False)\n", | |
| " ax.grid(False)\n", | |
| " ax.xaxis.set_visible(False)\n", | |
| " cur_range = range_map[label]\n", | |
| " label_list = [cur_range[0] + (cur_range[1] - cur_range[0]) / 5 * i for i in range(2, 6)]\n", | |
| " label_list = [f'{x:.1f}' for x in label_list]\n", | |
| " ax.set_rgrids(range(40, 120, 20), angle=angle, labels=label_list, font_properties=font12)\n", | |
| " ax.spines['polar'].set_visible(False)\n", | |
| " ax.set_ylim(0, 100)\n", | |
| "\n", | |
| "title_text = f'{len(model2vis)} Representative VLMs on MMBench Test.'\n", | |
| "plt.figtext(.7, .95, title_text, fontproperties=font18, ha='center')\n", | |
| "plt.show()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "labels = ['SEEDBench_IMG', 'CCBench', 'MMBench_TEST_EN', 'MMBench_TEST_CN', 'MME', 'MMVet', 'MMMU_VAL', 'MathVista', 'HallusionBench', 'LLaVABench']\n", | |
| "num_vars = len(labels)\n", | |
| "\n", | |
| "raw_data = [{k: data[m][k]['Overall'] for k in labels} for m in model2vis]\n", | |
| "data_list, range_map = pre_normalize(raw_data, labels)\n", | |
| "\n", | |
| "alpha = 0.25\n", | |
| "angles = np.linspace(0, 2 * np.pi, num_vars, endpoint=False).tolist()\n", | |
| "angles_deg = np.linspace(0, 360, num_vars, endpoint=False).tolist()\n", | |
| "fig, ax_base = plt.subplots(nrows=1, ncols=1, figsize=(10, 10), subplot_kw=dict(polar=True))\n", | |
| "\n", | |
| "for i in range(len(data_list)):\n", | |
| " item = data_list[i]\n", | |
| " model_name = model2vis[i]\n", | |
| " color = colors[i]\n", | |
| " tmp_angles = angles[:] + [angles[0]]\n", | |
| " tmp_values = [item[lb] for lb in labels] + [item[labels[0]]]\n", | |
| " ax_base.plot(tmp_angles, tmp_values, color=color, linewidth=1, linestyle='solid', label=model_name)\n", | |
| " ax_base.fill(tmp_angles, tmp_values, color=color, alpha=alpha)\n", | |
| " \n", | |
| "angles += [angles[0]]\n", | |
| "ax_base.set_ylim(0, 100)\n", | |
| "ax_base.set_yticks([40, 60, 80, 100])\n", | |
| "ax_base.set_yticklabels([''] * 4)\n", | |
| "\n", | |
| "ax_base.tick_params(pad=15)\n", | |
| "ax_base.set_xticks(angles[:-1])\n", | |
| "ax_base.set_xticklabels(labels, fontproperties=font18)\n", | |
| "\n", | |
| "dataset_map = {\n", | |
| " 'MMBench_TEST_EN': 'MMBench (Test)', \n", | |
| " 'MMBench_TEST_CN': 'MMBenchCN (Test)', \n", | |
| " 'MathVista': 'MathVista (TestMini)', \n", | |
| " 'MMMU_VAL': 'MMMU (Val)'\n", | |
| "}\n", | |
| "for i, label in enumerate(ax_base.get_xticklabels()):\n", | |
| " x,y = label.get_position()\n", | |
| " text = label.get_text()\n", | |
| " text = dataset_map[text] if text in dataset_map else text\n", | |
| " lab = ax_base.text(x, y, text, transform=label.get_transform(),\n", | |
| " ha=label.get_ha(), va=label.get_va(), font_properties=font15)\n", | |
| " lab.set_rotation(360 / num_vars * i + 270)\n", | |
| " labels.append(lab)\n", | |
| "ax_base.set_xticklabels([])\n", | |
| "\n", | |
| "leg = ax_base.legend(loc='center right', bbox_to_anchor=(1.6, 0.5), prop=font15, ncol=1, frameon=True, labelspacing=1.2)\n", | |
| "for line in leg.get_lines():\n", | |
| " line.set_linewidth(2.5)\n", | |
| "\n", | |
| "cx, cy, sz = 0.44, 0.435, 0.34\n", | |
| "axes = [fig.add_axes([cx - sz, cy - sz, cx + sz, cy + sz], projection='polar', label='axes%d' % i) for i in range(num_vars)]\n", | |
| " \n", | |
| "for ax, angle, label in zip(axes, angles_deg, labels):\n", | |
| " ax.patch.set_visible(False)\n", | |
| " ax.grid(False)\n", | |
| " ax.xaxis.set_visible(False)\n", | |
| " cur_range = range_map[label]\n", | |
| " label_list = [cur_range[0] + (cur_range[1] - cur_range[0]) / 5 * i for i in range(2, 6)]\n", | |
| " label_list = [f'{x:.1f}' for x in label_list]\n", | |
| " ax.set_rgrids(range(40, 120, 20), angle=angle, labels=label_list, font_properties=font12)\n", | |
| " ax.spines['polar'].set_visible(False)\n", | |
| " ax.set_ylim(0, 100)\n", | |
| "\n", | |
| "title_text = f'{len(model2vis)} Representative VLMs on {num_vars} Benchmarks in OpenCompass Multi-Modal Leaderboard.'\n", | |
| "plt.figtext(.7, .95, title_text, fontproperties=font18, ha='center')\n", | |
| "plt.show()" | |
| ] | |
| } | |
| ], | |
| "metadata": { | |
| "kernelspec": { | |
| "display_name": "base", | |
| "language": "python", | |
| "name": "python3" | |
| }, | |
| "language_info": { | |
| "codemirror_mode": { | |
| "name": "ipython", | |
| "version": 3 | |
| }, | |
| "file_extension": ".py", | |
| "mimetype": "text/x-python", | |
| "name": "python", | |
| "nbconvert_exporter": "python", | |
| "pygments_lexer": "ipython3", | |
| "version": "3.8.5" | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 2 | |
| } | |
Xet Storage Details
- Size:
- 11 kB
- Xet hash:
- c607d82f8887dd6053335566659300336a179996493ea5093182dde8a0e35904
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.