Hugging Face's logo

latentcanon
/
code

Model card Files
xet
 Community
laura.wagner commited on
Commit
1feb807
·
1 Parent(s): 808cf81

added model download script, 2026 most popular models script

Browse files
Files changed (3) hide show
  1. jupyter_notebooks/Section_2-3-3_Download_top_models_2026.ipynb +333 -0
  2. jupyter_notebooks/Section_2-3-3_Figure_7_top_models_2026.ipynb +0 -0
  3. plots/Sec_2-3-3_top_models_2026.svg +0 -0
jupyter_notebooks/Section_2-3-3_Download_top_models_2026.ipynb ADDED
@@ -0,0 +1,333 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": 2,
6
+ "id": "a1000001",
7
+ "metadata": {},
8
+ "outputs": [
9
+ {
10
+ "name": "stdout",
11
+ "output_type": "stream",
12
+ "text": [
13
+ "Paths initialised.\n"
14
+ ]
15
+ }
16
+ ],
17
+ "source": [
18
+ "import os\n",
19
+ "import time\n",
20
+ "import requests\n",
21
+ "import pandas as pd\n",
22
+ "from itertools import cycle\n",
23
+ "from pathlib import Path\n",
24
+ "\n",
25
+ "current_dir = Path.cwd()\n",
26
+ "data_dir = current_dir.parent / 'data/CSV'\n",
27
+ "models_dir = current_dir.parent / 'data/models'\n",
28
+ "creds_file = current_dir.parent / 'misc/credentials/civit.txt'\n",
29
+ "\n",
30
+ "print('Paths initialised.')"
31
+ ]
32
+ },
33
+ {
34
+ "cell_type": "code",
35
+ "execution_count": 3,
36
+ "id": "a1000002",
37
+ "metadata": {},
38
+ "outputs": [],
39
+ "source": [
40
+ "# =============================================================================\n",
41
+ "# CONFIGURATION\n",
42
+ "# =============================================================================\n",
43
+ "\n",
44
+ "# Input CSV\n",
45
+ "input_csv = data_dir / 'Civiverse-Models_final_2026.csv'\n",
46
+ "\n",
47
+ "# How many top models (by downloadCount) to download\n",
48
+ "n_top = 30\n",
49
+ "\n",
50
+ "# Include adapter types (LORA, TextualInversion, Controlnet) alongside Checkpoints?\n",
51
+ "include_adapters = True\n",
52
+ "\n",
53
+ "# Date range filter on publishedAt ('YYYY-MM-DD' or None for no bound)\n",
54
+ "date_from = '2026-01-01'\n",
55
+ "date_to = None\n",
56
+ "\n",
57
+ "# Seconds to wait between downloads (be polite to the API)\n",
58
+ "delay_between_downloads = 2"
59
+ ]
60
+ },
61
+ {
62
+ "cell_type": "code",
63
+ "execution_count": 4,
64
+ "id": "a1000003",
65
+ "metadata": {},
66
+ "outputs": [
67
+ {
68
+ "name": "stdout",
69
+ "output_type": "stream",
70
+ "text": [
71
+ "Loaded 3 API key(s).\n"
72
+ ]
73
+ }
74
+ ],
75
+ "source": [
76
+ "# Load and rotate API keys from credentials file\n",
77
+ "with open(creds_file, 'r') as f:\n",
78
+ " api_keys = [line.strip() for line in f if line.strip()]\n",
79
+ "\n",
80
+ "if not api_keys:\n",
81
+ " raise ValueError(f'No API keys found in {creds_file}')\n",
82
+ "\n",
83
+ "print(f'Loaded {len(api_keys)} API key(s).')"
84
+ ]
85
+ },
86
+ {
87
+ "cell_type": "code",
88
+ "execution_count": 5,
89
+ "id": "a1000004",
90
+ "metadata": {},
91
+ "outputs": [
92
+ {
93
+ "name": "stdout",
94
+ "output_type": "stream",
95
+ "text": [
96
+ "Selected 30 models for download.\n",
97
+ " name type baseModel downloadCount\n",
98
+ "0 CyberRealistic Pony Checkpoint Pony 600714\n",
99
+ "1 PornMaster-色情大师 Checkpoint SDXL 1.0 295653\n",
100
+ "2 Hands XL + SD 1.5 + F1D + Pony + Illustrious + zit LORA ZImageTurbo 260122\n",
101
+ "3 WAN DR34ML4Y - All-In-One NSFW LORA Wan Video 2.2 I2V-A14B 210436\n",
102
+ "4 PerfectDeliberate Checkpoint Illustrious 204663\n",
103
+ "5 Copax TimeLess Checkpoint ZImageTurbo 186787\n",
104
+ "6 Nova Furry XL Checkpoint Illustrious 180843\n",
105
+ "7 CyberRealistic XL Checkpoint SDXL 1.0 177192\n",
106
+ "8 ✨ JANKU Trained + NoobAI + RouWei Illustrious XL ✨ Checkpoint Illustrious 147652\n",
107
+ "9 Prefect illustrious XL Checkpoint Illustrious 141922\n",
108
+ "10 Real Dream Checkpoint ZImageTurbo 141721\n",
109
+ "11 Big Love Checkpoint SDXL 1.0 138367\n",
110
+ "12 Stabilizer IL/NAI/CK LORA NoobAI 130365\n",
111
+ "13 WAN 2.2 Enhanced NSFW | SVI | camera prompt adherence (Lightning Edition) I2V and T2V fp8 GGUF Checkpoint Wan Video 2.2 I2V-A14B 127818\n",
112
+ "14 PornMaster-Pro 色情大师- Illustrious & noob Checkpoint Illustrious 123173\n",
113
+ "15 NSFW MASTER FLUX LORA ZImageTurbo 117967\n",
114
+ "16 CyberRealistic CyberIllustrious Checkpoint Illustrious 110474\n",
115
+ "17 Smooth Mix Wan 2.2 14B (I2V/T2V) Checkpoint Wan Video 2.2 I2V-A14B 94599\n",
116
+ "18 Detailed Perfection style (Hands + Feet + Face + Body + All in one) XL + F1D + SD1.5 + Pony + Illu + zit LORA ZImageTurbo 84797\n",
117
+ "19 Nova Orange XL Checkpoint Illustrious 78252\n",
118
+ "20 Fluxed Up [Flux NSFW Checkpoint] Checkpoint Flux.1 D 74682\n",
119
+ "21 薄塗り / USNR STYLE LORA NoobAI 67449\n",
120
+ "22 F.2 Midjourney V6.1 meets FLUX 🖼️ [+ILLUSTRIOUS / SDXL] LORA Flux.2 D 67378\n",
121
+ "23 Feet XL + SD 1.5 + F1D + Pony + Illustrious + zit LORA ZImageTurbo 66453\n",
122
+ "24 ⚜️💀 DaSiWa WAN 2.2 I2V 14B | Lightspeed | Safetensors 💀⚜️ Checkpoint Wan Video 2.2 I2V-A14B 59455\n",
123
+ "25 MRitual丨熟メス儀式 Illustrious Checkpoint Illustrious 58832\n",
124
+ "26 2DN Checkpoint Illustrious 58647\n",
125
+ "27 Wan2.2-Remix (T2V&I2V) Checkpoint Wan Video 2.2 I2V-A14B 55862\n",
126
+ "28 Cum/Facial - Wan2.2 LORA Wan Video 2.2 I2V-A14B 53205\n",
127
+ "29 Perfection Realistic [ILXL / Illustrious XL NSFW] Checkpoint Illustrious 50397\n"
128
+ ]
129
+ }
130
+ ],
131
+ "source": [
132
+ "# Load CSV, apply filters, select top N\n",
133
+ "\n",
134
+ "CHECKPOINT_TYPE = 'Checkpoint'\n",
135
+ "ADAPTER_TYPES = {'LORA', 'TextualInversion', 'Controlnet'}\n",
136
+ "\n",
137
+ "data = pd.read_csv(input_csv)\n",
138
+ "\n",
139
+ "# Date filter\n",
140
+ "if date_from or date_to:\n",
141
+ " data['publishedAt'] = pd.to_datetime(data['publishedAt'], utc=True)\n",
142
+ " if date_from:\n",
143
+ " data = data[data['publishedAt'] >= pd.Timestamp(date_from, tz='UTC')]\n",
144
+ " if date_to:\n",
145
+ " data = data[data['publishedAt'] <= pd.Timestamp(date_to, tz='UTC')]\n",
146
+ "\n",
147
+ "# Type filter\n",
148
+ "if not include_adapters:\n",
149
+ " data = data[data['type'] == CHECKPOINT_TYPE]\n",
150
+ "else:\n",
151
+ " data = data[data['type'].isin({CHECKPOINT_TYPE} | ADAPTER_TYPES)]\n",
152
+ "\n",
153
+ "if data.empty:\n",
154
+ " raise RuntimeError('No rows match the current filter settings.')\n",
155
+ "\n",
156
+ "data = data.sort_values('downloadCount', ascending=False).head(n_top).reset_index(drop=True)\n",
157
+ "\n",
158
+ "print(f'Selected {len(data)} models for download.')\n",
159
+ "print(data[['name', 'type', 'baseModel', 'downloadCount']].to_string())"
160
+ ]
161
+ },
162
+ {
163
+ "cell_type": "code",
164
+ "execution_count": 6,
165
+ "id": "a1000005",
166
+ "metadata": {},
167
+ "outputs": [],
168
+ "source": [
169
+ "def resolve_dest_folder(model_type, base_model):\n",
170
+ " \"\"\"\n",
171
+ " Checkpoints → data/models/checkpoints/{baseModel}/\n",
172
+ " LORA → data/models/adapters/LoRA/\n",
173
+ " TextualInv. → data/models/adapters/TextualInversion/\n",
174
+ " Controlnet → data/models/adapters/Controlnet/\n",
175
+ " \"\"\"\n",
176
+ " t = model_type.strip()\n",
177
+ " if t == CHECKPOINT_TYPE:\n",
178
+ " folder = models_dir / 'checkpoints' / (base_model or 'Unknown')\n",
179
+ " elif t == 'LORA':\n",
180
+ " folder = models_dir / 'adapters' / 'LoRA'\n",
181
+ " elif t == 'TextualInversion':\n",
182
+ " folder = models_dir / 'adapters' / 'TextualInversion'\n",
183
+ " elif t == 'Controlnet':\n",
184
+ " folder = models_dir / 'adapters' / 'Controlnet'\n",
185
+ " else:\n",
186
+ " folder = models_dir / 'other' / t\n",
187
+ " folder.mkdir(parents=True, exist_ok=True)\n",
188
+ " return folder\n",
189
+ "\n",
190
+ "\n",
191
+ "def get_filename(response, url):\n",
192
+ " cd = response.headers.get('content-disposition', '')\n",
193
+ " if 'filename=' in cd:\n",
194
+ " return cd.split('filename=')[-1].strip('\"').strip(\"'\")\n",
195
+ " return url.split('/')[-1].split('?')[0] or 'model_file'\n",
196
+ "\n",
197
+ "\n",
198
+ "def download_with_key_rotation(url, dest_folder, api_keys, delay=2):\n",
199
+ " \"\"\"\n",
200
+ " Try each API key in order until one succeeds or all fail.\n",
201
+ " Skips download if the file already exists.\n",
202
+ " \"\"\"\n",
203
+ " for key in api_keys:\n",
204
+ " headers = {'Authorization': f'Bearer {key}'}\n",
205
+ " try:\n",
206
+ " response = requests.get(url, headers=headers, stream=True, timeout=60)\n",
207
+ " if response.status_code == 401:\n",
208
+ " print(f' Key ending …{key[-6:]} → 401, trying next key.')\n",
209
+ " continue\n",
210
+ " response.raise_for_status()\n",
211
+ "\n",
212
+ " filename = get_filename(response, url)\n",
213
+ " dest_path = dest_folder / filename\n",
214
+ "\n",
215
+ " if dest_path.exists():\n",
216
+ " print(f' Already exists, skipping: {dest_path.name}')\n",
217
+ " return True\n",
218
+ "\n",
219
+ " with open(dest_path, 'wb') as f:\n",
220
+ " for chunk in response.iter_content(chunk_size=8192):\n",
221
+ " f.write(chunk)\n",
222
+ "\n",
223
+ " print(f' Saved → {dest_path}')\n",
224
+ " time.sleep(delay)\n",
225
+ " return True\n",
226
+ "\n",
227
+ " except requests.exceptions.RequestException as e:\n",
228
+ " print(f' Request error with key …{key[-6:]}: {e}')\n",
229
+ " continue\n",
230
+ "\n",
231
+ " print(f' FAILED: all API keys exhausted for {url}')\n",
232
+ " return False"
233
+ ]
234
+ },
235
+ {
236
+ "cell_type": "code",
237
+ "execution_count": 7,
238
+ "id": "a1000006",
239
+ "metadata": {},
240
+ "outputs": [
241
+ {
242
+ "name": "stdout",
243
+ "output_type": "stream",
244
+ "text": [
245
+ "[1/30] CyberRealistic Pony (Checkpoint / Pony)\n"
246
+ ]
247
+ },
248
+ {
249
+ "ename": "KeyboardInterrupt",
250
+ "evalue": "",
251
+ "output_type": "error",
252
+ "traceback": [
253
+ "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
254
+ "\u001b[31mKeyboardInterrupt\u001b[39m Traceback (most recent call last)",
255
+ "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[7]\u001b[39m\u001b[32m, line 20\u001b[39m\n\u001b[32m 17\u001b[39m \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[32m 19\u001b[39m dest_folder = resolve_dest_folder(model_type, base_model)\n\u001b[32m---> \u001b[39m\u001b[32m20\u001b[39m ok = \u001b[43mdownload_with_key_rotation\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdest_folder\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mapi_keys\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdelay\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdelay_between_downloads\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 21\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m ok:\n\u001b[32m 22\u001b[39m succeeded += \u001b[32m1\u001b[39m\n",
256
+ "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[6]\u001b[39m\u001b[32m, line 52\u001b[39m, in \u001b[36mdownload_with_key_rotation\u001b[39m\u001b[34m(url, dest_folder, api_keys, delay)\u001b[39m\n\u001b[32m 49\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[32m 51\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(dest_path, \u001b[33m'\u001b[39m\u001b[33mwb\u001b[39m\u001b[33m'\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[32m---> \u001b[39m\u001b[32m52\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mchunk\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m.\u001b[49m\u001b[43miter_content\u001b[49m\u001b[43m(\u001b[49m\u001b[43mchunk_size\u001b[49m\u001b[43m=\u001b[49m\u001b[32;43m8192\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m:\u001b[49m\n\u001b[32m 53\u001b[39m \u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m.\u001b[49m\u001b[43mwrite\u001b[49m\u001b[43m(\u001b[49m\u001b[43mchunk\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 55\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m'\u001b[39m\u001b[33m Saved → \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdest_path\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m)\n",
257
+ "\u001b[36mFile \u001b[39m\u001b[32m~/000_PHD/000_010_PUBLICATION/CODE/pm-paper/.venv/lib/python3.12/site-packages/requests/models.py:822\u001b[39m, in \u001b[36mResponse.iter_content.<locals>.generate\u001b[39m\u001b[34m()\u001b[39m\n\u001b[32m 820\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(\u001b[38;5;28mself\u001b[39m.raw, \u001b[33m\"\u001b[39m\u001b[33mstream\u001b[39m\u001b[33m\"\u001b[39m):\n\u001b[32m 821\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m822\u001b[39m \u001b[38;5;28;01myield from\u001b[39;00m \u001b[38;5;28mself\u001b[39m.raw.stream(chunk_size, decode_content=\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m 823\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m ProtocolError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m 824\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m ChunkedEncodingError(e)\n",
258
+ "\u001b[36mFile \u001b[39m\u001b[32m~/000_PHD/000_010_PUBLICATION/CODE/pm-paper/.venv/lib/python3.12/site-packages/urllib3/response.py:1257\u001b[39m, in \u001b[36mHTTPResponse.stream\u001b[39m\u001b[34m(self, amt, decode_content)\u001b[39m\n\u001b[32m 1251\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 1252\u001b[39m \u001b[38;5;28;01mwhile\u001b[39;00m (\n\u001b[32m 1253\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m is_fp_closed(\u001b[38;5;28mself\u001b[39m._fp)\n\u001b[32m 1254\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m._decoded_buffer) > \u001b[32m0\u001b[39m\n\u001b[32m 1255\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m (\u001b[38;5;28mself\u001b[39m._decoder \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m._decoder.has_unconsumed_tail)\n\u001b[32m 1256\u001b[39m ):\n\u001b[32m-> \u001b[39m\u001b[32m1257\u001b[39m data = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m=\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1259\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m data:\n\u001b[32m 1260\u001b[39m \u001b[38;5;28;01myield\u001b[39;00m data\n",
259
+ "\u001b[36mFile \u001b[39m\u001b[32m~/000_PHD/000_010_PUBLICATION/CODE/pm-paper/.venv/lib/python3.12/site-packages/urllib3/response.py:1112\u001b[39m, in \u001b[36mHTTPResponse.read\u001b[39m\u001b[34m(self, amt, decode_content, cache_content)\u001b[39m\n\u001b[32m 1109\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m._decoded_buffer) >= amt:\n\u001b[32m 1110\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._decoded_buffer.get(amt)\n\u001b[32m-> \u001b[39m\u001b[32m1112\u001b[39m data = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_raw_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1114\u001b[39m flush_decoder = amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m (amt != \u001b[32m0\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m data)\n\u001b[32m 1116\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[32m 1117\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m data\n\u001b[32m 1118\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m._decoded_buffer) == \u001b[32m0\u001b[39m\n\u001b[32m 1119\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m._decoder \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m._decoder.has_unconsumed_tail)\n\u001b[32m 1120\u001b[39m ):\n",
260
+ "\u001b[36mFile \u001b[39m\u001b[32m~/000_PHD/000_010_PUBLICATION/CODE/pm-paper/.venv/lib/python3.12/site-packages/urllib3/response.py:1028\u001b[39m, in \u001b[36mHTTPResponse._raw_read\u001b[39m\u001b[34m(self, amt, read1)\u001b[39m\n\u001b[32m 1025\u001b[39m fp_closed = \u001b[38;5;28mgetattr\u001b[39m(\u001b[38;5;28mself\u001b[39m._fp, \u001b[33m\"\u001b[39m\u001b[33mclosed\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[32m 1027\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m._error_catcher():\n\u001b[32m-> \u001b[39m\u001b[32m1028\u001b[39m data = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_fp_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mread1\u001b[49m\u001b[43m=\u001b[49m\u001b[43mread1\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m fp_closed \u001b[38;5;28;01melse\u001b[39;00m \u001b[33mb\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 1029\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m amt != \u001b[32m0\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m data:\n\u001b[32m 1030\u001b[39m \u001b[38;5;66;03m# Platform-specific: Buggy versions of Python.\u001b[39;00m\n\u001b[32m 1031\u001b[39m \u001b[38;5;66;03m# Close the connection when no data is returned\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 1036\u001b[39m \u001b[38;5;66;03m# not properly close the connection in all cases. There is\u001b[39;00m\n\u001b[32m 1037\u001b[39m \u001b[38;5;66;03m# no harm in redundantly calling close.\u001b[39;00m\n\u001b[32m 1038\u001b[39m \u001b[38;5;28mself\u001b[39m._fp.close()\n",
261
+ "\u001b[36mFile \u001b[39m\u001b[32m~/000_PHD/000_010_PUBLICATION/CODE/pm-paper/.venv/lib/python3.12/site-packages/urllib3/response.py:1011\u001b[39m, in \u001b[36mHTTPResponse._fp_read\u001b[39m\u001b[34m(self, amt, read1)\u001b[39m\n\u001b[32m 1008\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._fp.read1(amt) \u001b[38;5;28;01mif\u001b[39;00m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m._fp.read1()\n\u001b[32m 1009\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 1010\u001b[39m \u001b[38;5;66;03m# StringIO doesn't like amt=None\u001b[39;00m\n\u001b[32m-> \u001b[39m\u001b[32m1011\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_fp\u001b[49m\u001b[43m.\u001b[49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mif\u001b[39;00m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m._fp.read()\n",
262
+ "\u001b[36mFile \u001b[39m\u001b[32m/usr/lib/python3.12/http/client.py:484\u001b[39m, in \u001b[36mHTTPResponse.read\u001b[39m\u001b[34m(self, amt)\u001b[39m\n\u001b[32m 481\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.length \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m amt > \u001b[38;5;28mself\u001b[39m.length:\n\u001b[32m 482\u001b[39m \u001b[38;5;66;03m# clip the read to the \"end of response\"\u001b[39;00m\n\u001b[32m 483\u001b[39m amt = \u001b[38;5;28mself\u001b[39m.length\n\u001b[32m--> \u001b[39m\u001b[32m484\u001b[39m s = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mfp\u001b[49m\u001b[43m.\u001b[49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 485\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m s \u001b[38;5;129;01mand\u001b[39;00m amt:\n\u001b[32m 486\u001b[39m \u001b[38;5;66;03m# Ideally, we would raise IncompleteRead if the content-length\u001b[39;00m\n\u001b[32m 487\u001b[39m \u001b[38;5;66;03m# wasn't satisfied, but it might break compatibility.\u001b[39;00m\n\u001b[32m 488\u001b[39m \u001b[38;5;28mself\u001b[39m._close_conn()\n",
263
+ "\u001b[36mFile \u001b[39m\u001b[32m/usr/lib/python3.12/socket.py:707\u001b[39m, in \u001b[36mSocketIO.readinto\u001b[39m\u001b[34m(self, b)\u001b[39m\n\u001b[32m 705\u001b[39m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[32m 706\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m707\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_sock\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrecv_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 708\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m timeout:\n\u001b[32m 709\u001b[39m \u001b[38;5;28mself\u001b[39m._timeout_occurred = \u001b[38;5;28;01mTrue\u001b[39;00m\n",
264
+ "\u001b[36mFile \u001b[39m\u001b[32m/usr/lib/python3.12/ssl.py:1252\u001b[39m, in \u001b[36mSSLSocket.recv_into\u001b[39m\u001b[34m(self, buffer, nbytes, flags)\u001b[39m\n\u001b[32m 1248\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m flags != \u001b[32m0\u001b[39m:\n\u001b[32m 1249\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[32m 1250\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mnon-zero flags not allowed in calls to recv_into() on \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m\"\u001b[39m %\n\u001b[32m 1251\u001b[39m \u001b[38;5;28mself\u001b[39m.\u001b[34m__class__\u001b[39m)\n\u001b[32m-> \u001b[39m\u001b[32m1252\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnbytes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1253\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 1254\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28msuper\u001b[39m().recv_into(buffer, nbytes, flags)\n",
265
+ "\u001b[36mFile \u001b[39m\u001b[32m/usr/lib/python3.12/ssl.py:1104\u001b[39m, in \u001b[36mSSLSocket.read\u001b[39m\u001b[34m(self, len, buffer)\u001b[39m\n\u001b[32m 1102\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m 1103\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m buffer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1104\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_sslobj\u001b[49m\u001b[43m.\u001b[49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1105\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 1106\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._sslobj.read(\u001b[38;5;28mlen\u001b[39m)\n",
266
+ "\u001b[31mKeyboardInterrupt\u001b[39m: "
267
+ ]
268
+ }
269
+ ],
270
+ "source": [
271
+ "# Download loop\n",
272
+ "\n",
273
+ "succeeded = 0\n",
274
+ "failed = []\n",
275
+ "\n",
276
+ "for i, row in data.iterrows():\n",
277
+ " url = row.get('downloadUrl', '')\n",
278
+ " model_type = row.get('type', '')\n",
279
+ " base_model = row.get('baseModel', '')\n",
280
+ " name = row.get('name', f'model_{i}')\n",
281
+ "\n",
282
+ " print(f'[{i+1}/{len(data)}] {name} ({model_type} / {base_model})')\n",
283
+ "\n",
284
+ " if not url or pd.isna(url):\n",
285
+ " print(' No download URL, skipping.')\n",
286
+ " failed.append(name)\n",
287
+ " continue\n",
288
+ "\n",
289
+ " dest_folder = resolve_dest_folder(model_type, base_model)\n",
290
+ " ok = download_with_key_rotation(url, dest_folder, api_keys, delay=delay_between_downloads)\n",
291
+ " if ok:\n",
292
+ " succeeded += 1\n",
293
+ " else:\n",
294
+ " failed.append(name)\n",
295
+ "\n",
296
+ "print(f'\\nDone. {succeeded}/{len(data)} downloaded successfully.')\n",
297
+ "if failed:\n",
298
+ " print('Failed:')\n",
299
+ " for name in failed:\n",
300
+ " print(f' - {name}')"
301
+ ]
302
+ },
303
+ {
304
+ "cell_type": "code",
305
+ "execution_count": null,
306
+ "id": "15a7c5dc",
307
+ "metadata": {},
308
+ "outputs": [],
309
+ "source": []
310
+ }
311
+ ],
312
+ "metadata": {
313
+ "kernelspec": {
314
+ "display_name": "pm_paper",
315
+ "language": "python",
316
+ "name": "flux_pm_paper"
317
+ },
318
+ "language_info": {
319
+ "codemirror_mode": {
320
+ "name": "ipython",
321
+ "version": 3
322
+ },
323
+ "file_extension": ".py",
324
+ "mimetype": "text/x-python",
325
+ "name": "python",
326
+ "nbconvert_exporter": "python",
327
+ "pygments_lexer": "ipython3",
328
+ "version": "3.12.3"
329
+ }
330
+ },
331
+ "nbformat": 4,
332
+ "nbformat_minor": 5
333
+ }
jupyter_notebooks/Section_2-3-3_Figure_7_top_models_2026.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
plots/Sec_2-3-3_top_models_2026.svg ADDED