saliacoel commited on
Commit
bcb4a17
·
verified ·
1 Parent(s): 4fa8e25

Upload GPT_TO_HF_CityResponsesBatch.py

Browse files
Files changed (1) hide show
  1. GPT_TO_HF_CityResponsesBatch.py +2175 -0
GPT_TO_HF_CityResponsesBatch.py ADDED
@@ -0,0 +1,2175 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import base64
4
+ import hashlib
5
+ import json
6
+ import os
7
+ import re
8
+ import secrets
9
+ import shutil
10
+ import urllib.error
11
+ import urllib.parse
12
+ import urllib.request
13
+ from datetime import datetime, timezone
14
+ from io import BytesIO
15
+ from pathlib import Path
16
+ from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple
17
+
18
+ try:
19
+ import folder_paths
20
+ except ImportError: # Allows offline tests outside ComfyUI.
21
+ folder_paths = None
22
+
23
+ try:
24
+ from PIL import Image
25
+ except ImportError: # Raised with a useful message only when an image is decoded.
26
+ Image = None
27
+
28
+
29
+ NODE_VERSION = "2.1.0"
30
+ PURPOSE = "salia-city-responses-batch-waves"
31
+ OPENAI_BATCH_ENDPOINT = "/v1/responses"
32
+ OPENAI_MODEL = "gpt-5.6-sol"
33
+ OPENAI_IMAGE_MODEL = "gpt-image-2.5-sunburst"
34
+ OPENAI_REASONING: Dict[str, str] = {
35
+ "effort": "medium",
36
+ "mode": "standard",
37
+ "summary": "auto",
38
+ }
39
+ RESPONSE_MODERATION: Dict[str, Any] = {
40
+ "model": "omni-moderation-latest",
41
+ "policy": {
42
+ "input": {"mode": "score"},
43
+ "output": {"mode": "score"},
44
+ },
45
+ }
46
+ OPENAI_IMAGE_TOOL: Dict[str, Any] = {
47
+ "type": "image_generation",
48
+ "model": OPENAI_IMAGE_MODEL,
49
+ "action": "generate",
50
+ "size": "1536x1024",
51
+ "quality": "auto",
52
+ "output_format": "png",
53
+ "background": "opaque",
54
+ "moderation": "low",
55
+ "partial_images": 0,
56
+ }
57
+
58
+ SOURCE_REPO_ID = "saliacoel/maps_imgs"
59
+ GENERATED_REPO_ID = "saliacoel/map_imgs_gpt"
60
+ REFERENCE_REPO_ID = "saliacoel/x"
61
+ REPO_TYPE = "model"
62
+ REPO_REVISION = "main"
63
+ MANIFEST_FILE_NAME = "gpt_navigation_manifest.json"
64
+ STATE_DIRECTORY_NAME = "Salia_City_Responses_Batch"
65
+ EXPECTED_MANIFEST_VERSION = 5
66
+ EXPECTED_BATCH_PLAN_VERSION = 3
67
+ EXPECTED_BATCH_STRATEGY = (
68
+ "entrance_fronts_first_scene_tree_with_deferred_cycles_v3"
69
+ )
70
+ EXPECTED_PROMPT_VERSION = "city_responses_continuity_prompt_v2"
71
+ EXPECTED_REFERENCE_TYPES = (
72
+ "4_way_Junction",
73
+ "Left_Forward",
74
+ "Forward_Right",
75
+ "Left_Right",
76
+ "Forward_Only",
77
+ "Left_Only",
78
+ "Right_Only",
79
+ "Dead_End",
80
+ "straight_door",
81
+ "door_Left",
82
+ "door_Right",
83
+ "station_gate_Front_Direct",
84
+ "station_gate_End_Left",
85
+ "station_gate_End_Right",
86
+ "station_gate_Forward_Left",
87
+ "station_gate_Forward_Right",
88
+ "station_gate_Distant_Front",
89
+ "station_gate_Distant_Left",
90
+ "station_gate_Distant_Right",
91
+ )
92
+
93
+ ACTION_PREVIEW = "PREVIEW_PROMPTS_ONLY"
94
+ ACTION_RUN = "RUN_OR_CONTINUE"
95
+ _PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
96
+ _SAFE_BATCH_ID = re.compile(r"^batch_[A-Za-z0-9_-]{1,200}$")
97
+ _SAFE_FILE_ID = re.compile(r"^file-[A-Za-z0-9_-]{1,200}$")
98
+ _TERMINAL_BATCH_STATUSES = {"completed", "failed", "expired", "cancelled"}
99
+ _WAITING_BATCH_STATUSES = {
100
+ "validating",
101
+ "in_progress",
102
+ "finalizing",
103
+ "cancelling",
104
+ }
105
+
106
+
107
+ def _utc_now() -> str:
108
+ return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
109
+
110
+
111
+ def _output_root() -> Path:
112
+ if folder_paths is not None:
113
+ return Path(folder_paths.get_output_directory())
114
+ return Path.cwd() / "output"
115
+
116
+
117
+ def _optional_secret(value: str) -> Optional[str]:
118
+ cleaned = str(value or "").strip()
119
+ if cleaned.lower() in {"", "none", "xxx", "hf_xxx", "sk_xxx", "sk-xxx"}:
120
+ return None
121
+ return cleaned
122
+
123
+
124
+ def _required_secret(value: str, label: str) -> str:
125
+ cleaned = _optional_secret(value)
126
+ if not cleaned:
127
+ raise ValueError(f"A real {label} is required for RUN_OR_CONTINUE.")
128
+ return cleaned
129
+
130
+
131
+ def _safe_district_name(value: str) -> str:
132
+ name = str(value or "").strip().strip("\"'")
133
+ if (
134
+ not name
135
+ or name in {".", ".."}
136
+ or "/" in name
137
+ or "\\" in name
138
+ or len(name) > 180
139
+ or any(ord(character) < 32 or ord(character) == 127 for character in name)
140
+ ):
141
+ raise ValueError("DISTRICT_NAME must be one safe Hugging Face folder name.")
142
+ return name
143
+
144
+
145
+ def _safe_file_name(value: Any, label: str) -> str:
146
+ name = str(value or "").strip()
147
+ if (
148
+ not name
149
+ or name in {".", ".."}
150
+ or "/" in name
151
+ or "\\" in name
152
+ or len(name) > 240
153
+ or any(ord(character) < 32 or ord(character) == 127 for character in name)
154
+ ):
155
+ raise ValueError(f"Unsafe {label}: {name!r}.")
156
+ return name
157
+
158
+
159
+ def _write_json(path: Path, value: Any) -> None:
160
+ path.parent.mkdir(parents=True, exist_ok=True)
161
+ temporary = path.with_suffix(path.suffix + ".part")
162
+ temporary.write_text(
163
+ json.dumps(value, ensure_ascii=False, indent=2) + "\n",
164
+ encoding="utf-8",
165
+ )
166
+ os.replace(temporary, path)
167
+
168
+
169
+ def _read_json(path: Path) -> Dict[str, Any]:
170
+ parsed = json.loads(path.read_text(encoding="utf-8-sig"))
171
+ if not isinstance(parsed, dict):
172
+ raise ValueError(f"Expected a JSON object in {path}.")
173
+ return parsed
174
+
175
+
176
+ def _write_jsonl(path: Path, values: Sequence[Mapping[str, Any]]) -> None:
177
+ path.parent.mkdir(parents=True, exist_ok=True)
178
+ temporary = path.with_suffix(path.suffix + ".part")
179
+ with temporary.open("w", encoding="utf-8", newline="\n") as handle:
180
+ for value in values:
181
+ handle.write(
182
+ json.dumps(value, ensure_ascii=False, separators=(",", ":"))
183
+ + "\n"
184
+ )
185
+ os.replace(temporary, path)
186
+
187
+
188
+ def _load_hf_tools():
189
+ try:
190
+ from huggingface_hub import CommitOperationAdd, HfApi, hf_hub_download
191
+ except ImportError as exception:
192
+ raise RuntimeError(
193
+ "huggingface_hub is missing. Install it with ComfyUI's Python and "
194
+ "restart ComfyUI."
195
+ ) from exception
196
+ return CommitOperationAdd, HfApi, hf_hub_download
197
+
198
+
199
+ def _repo_commit(api: Any, repo_id: str, token: Optional[str]) -> str:
200
+ info = api.repo_info(
201
+ repo_id=repo_id,
202
+ repo_type=REPO_TYPE,
203
+ revision=REPO_REVISION,
204
+ token=token,
205
+ )
206
+ commit = str(getattr(info, "sha", "") or "").strip()
207
+ if not commit:
208
+ raise ValueError(f"Could not resolve the current commit of {repo_id}.")
209
+ return commit
210
+
211
+
212
+ def _hf_url(repo_id: str, revision: str, path_in_repo: str) -> str:
213
+ return (
214
+ "https://huggingface.co/"
215
+ + repo_id
216
+ + "/resolve/"
217
+ + urllib.parse.quote(str(revision), safe="")
218
+ + "/"
219
+ + urllib.parse.quote(str(path_in_repo), safe="/")
220
+ )
221
+
222
+
223
+ def _hf_blob_url(repo_id: str, revision: str, path_in_repo: str) -> str:
224
+ return (
225
+ "https://huggingface.co/"
226
+ + repo_id
227
+ + "/blob/"
228
+ + urllib.parse.quote(str(revision), safe="")
229
+ + "/"
230
+ + urllib.parse.quote(str(path_in_repo), safe="/")
231
+ )
232
+
233
+
234
+ def _repo_path(district: str, file_name: str) -> str:
235
+ return district + "/" + _safe_file_name(file_name, "repository filename")
236
+
237
+
238
+ def _download_manifest(
239
+ district: str,
240
+ hf_token: Optional[str],
241
+ ) -> Tuple[Dict[str, Any], str, str, Path]:
242
+ _, HfApi, hf_hub_download = _load_hf_tools()
243
+ api = HfApi(token=hf_token)
244
+ source_commit = _repo_commit(api, SOURCE_REPO_ID, hf_token)
245
+ path_in_repo = _repo_path(district, MANIFEST_FILE_NAME)
246
+ local = Path(
247
+ hf_hub_download(
248
+ repo_id=SOURCE_REPO_ID,
249
+ filename=path_in_repo,
250
+ repo_type=REPO_TYPE,
251
+ revision=source_commit,
252
+ token=hf_token,
253
+ force_download=True,
254
+ )
255
+ )
256
+ raw = local.read_bytes()
257
+ manifest_sha256 = hashlib.sha256(raw).hexdigest()
258
+ parsed = json.loads(raw.decode("utf-8-sig"))
259
+ if not isinstance(parsed, dict):
260
+ raise ValueError("The downloaded navigation manifest is not a JSON object.")
261
+ return parsed, source_commit, manifest_sha256, local
262
+
263
+
264
+ def _image_label(zero_based_index: int) -> str:
265
+ value = int(zero_based_index) + 1
266
+ if value <= 0:
267
+ raise ValueError("Image label index must be non-negative.")
268
+ letters = ""
269
+ while value:
270
+ value, remainder = divmod(value - 1, 26)
271
+ letters = chr(ord("A") + remainder) + letters
272
+ return "Image_" + letters
273
+
274
+
275
+ def _segmentation_file_name(reference_type: str) -> str:
276
+ return (
277
+ "4_way_Junction_Segmentation.png"
278
+ if reference_type == "4_way_Junction"
279
+ else reference_type + "_Segment.png"
280
+ )
281
+
282
+
283
+ def _require_mapping(value: Any, label: str) -> Mapping[str, Any]:
284
+ if not isinstance(value, Mapping):
285
+ raise ValueError(f"{label} must be a JSON object.")
286
+ return value
287
+
288
+
289
+ def _require_list(value: Any, label: str) -> List[Any]:
290
+ if not isinstance(value, list):
291
+ raise ValueError(f"{label} must be a JSON array.")
292
+ return value
293
+
294
+
295
+ def _validate_manifest(
296
+ manifest: Mapping[str, Any],
297
+ requested_district: str,
298
+ ) -> List[Dict[str, Any]]:
299
+ if int(manifest.get("formatVersion") or 0) != EXPECTED_MANIFEST_VERSION:
300
+ raise ValueError(
301
+ f"Unsupported navigation manifest formatVersion; expected "
302
+ f"{EXPECTED_MANIFEST_VERSION}. Recapture with Unity V6.0.22's v5 exporter."
303
+ )
304
+ if int(manifest.get("batchPlanVersion") or 0) != EXPECTED_BATCH_PLAN_VERSION:
305
+ raise ValueError(
306
+ f"Unsupported batchPlanVersion; expected {EXPECTED_BATCH_PLAN_VERSION}."
307
+ )
308
+ if str(manifest.get("districtName") or "") != requested_district:
309
+ raise ValueError(
310
+ "The manifest districtName does not equal the requested Hugging Face "
311
+ "folder name."
312
+ )
313
+ expected_top_level = {
314
+ "sourceImageRepoId": SOURCE_REPO_ID,
315
+ "generatedImageRepoId": GENERATED_REPO_ID,
316
+ "referenceImageRepoId": REFERENCE_REPO_ID,
317
+ "repoRevision": REPO_REVISION,
318
+ "batchStrategy": EXPECTED_BATCH_STRATEGY,
319
+ "promptVersion": EXPECTED_PROMPT_VERSION,
320
+ }
321
+ for key, expected in expected_top_level.items():
322
+ actual = str(manifest.get(key) or "")
323
+ if actual != expected:
324
+ raise ValueError(f"Manifest {key} is {actual!r}; expected {expected!r}.")
325
+
326
+ catalog = _require_list(manifest.get("referenceCatalog"), "referenceCatalog")
327
+ catalog_by_type: Dict[str, Mapping[str, Any]] = {}
328
+ for index, raw_entry in enumerate(catalog):
329
+ entry = _require_mapping(raw_entry, f"referenceCatalog[{index}]")
330
+ reference_type = str(entry.get("referenceType") or "")
331
+ if reference_type in catalog_by_type:
332
+ raise ValueError(f"Duplicate reference type {reference_type!r}.")
333
+ catalog_by_type[reference_type] = entry
334
+ if tuple(catalog_by_type.keys()) != EXPECTED_REFERENCE_TYPES:
335
+ raise ValueError(
336
+ f"referenceCatalog must contain the {len(EXPECTED_REFERENCE_TYPES)} "
337
+ "canonical types in canonical order."
338
+ )
339
+ for reference_type in EXPECTED_REFERENCE_TYPES:
340
+ entry = catalog_by_type[reference_type]
341
+ expected_names = {
342
+ "depthFileName": reference_type + "_Depth.png",
343
+ "segmentationFileName": _segmentation_file_name(reference_type),
344
+ "lineartFileName": reference_type + "_Lineart.png",
345
+ }
346
+ for key, expected in expected_names.items():
347
+ if str(entry.get(key) or "") != expected:
348
+ raise ValueError(
349
+ f"Reference {reference_type!r} has invalid {key}; expected "
350
+ f"{expected!r}."
351
+ )
352
+
353
+ raw_sets = _require_list(manifest.get("batchSets"), "batchSets")
354
+ if int(manifest.get("batchSetCount") or -1) != len(raw_sets) or not raw_sets:
355
+ raise ValueError("batchSetCount does not match the non-empty batchSets array.")
356
+
357
+ jobs: List[Dict[str, Any]] = []
358
+ jobs_by_id: Dict[str, Dict[str, Any]] = {}
359
+ output_names = set()
360
+ set_ids = set()
361
+ declared_dependencies_by_set: Dict[int, List[int]] = {}
362
+ for set_index, raw_set in enumerate(raw_sets):
363
+ batch_set = _require_mapping(raw_set, f"batchSets[{set_index}]")
364
+ set_number = int(batch_set.get("batchSetNumber") or 0)
365
+ if set_number != set_index + 1:
366
+ raise ValueError("Batch-set numbers must be consecutive and one-based.")
367
+ set_id = str(batch_set.get("id") or "").strip()
368
+ if not set_id or set_id in set_ids:
369
+ raise ValueError(f"Missing or duplicate Batch-set id {set_id!r}.")
370
+ set_ids.add(set_id)
371
+ raw_depends_on = _require_list(
372
+ batch_set.get("dependsOnBatchSets"),
373
+ f"batchSets[{set_index}].dependsOnBatchSets",
374
+ )
375
+ depends_on = [int(value) for value in raw_depends_on]
376
+ if (
377
+ depends_on != sorted(set(depends_on))
378
+ or any(value < 1 or value >= set_number for value in depends_on)
379
+ ):
380
+ raise ValueError(
381
+ f"Batch set {set_number} has invalid dependsOnBatchSets."
382
+ )
383
+ declared_dependencies_by_set[set_number] = depends_on
384
+ raw_images = _require_list(
385
+ batch_set.get("images"), f"batchSets[{set_index}].images"
386
+ )
387
+ if int(batch_set.get("imageCount") or -1) != len(raw_images) or not raw_images:
388
+ raise ValueError(f"Batch set {set_number} has an invalid imageCount.")
389
+ for image_index, raw_image in enumerate(raw_images):
390
+ image = dict(
391
+ _require_mapping(
392
+ raw_image,
393
+ f"batchSets[{set_index}].images[{image_index}]",
394
+ )
395
+ )
396
+ custom_id = str(image.get("customId") or "").strip()
397
+ if not custom_id or custom_id in jobs_by_id:
398
+ raise ValueError(f"Missing or duplicate image customId {custom_id!r}.")
399
+ if str(image.get("kind") or "").lower() == "bench" or "bench" in str(
400
+ image.get("topology") or ""
401
+ ).lower():
402
+ raise ValueError("Bench images are forbidden in the city Batch plan.")
403
+ source_name = _safe_file_name(
404
+ image.get("sourceScreenshotFileName"), "Unity screenshot filename"
405
+ )
406
+ target_name = _safe_file_name(
407
+ image.get("targetGeneratedFileName"), "generated filename"
408
+ )
409
+ if not target_name.lower().endswith(".png") or target_name in output_names:
410
+ raise ValueError(f"Invalid or duplicate generated filename {target_name!r}.")
411
+ output_names.add(target_name)
412
+ reference_type = str(image.get("referenceType") or "")
413
+ if reference_type not in catalog_by_type:
414
+ raise ValueError(
415
+ f"Image {custom_id!r} uses unknown referenceType {reference_type!r}."
416
+ )
417
+ reference_images = _require_mapping(
418
+ image.get("referenceImages"), f"referenceImages for {custom_id}"
419
+ )
420
+ if str(reference_images.get("referenceType") or "") != reference_type:
421
+ raise ValueError(
422
+ f"Image {custom_id!r} referenceImages.referenceType does not "
423
+ "match referenceType."
424
+ )
425
+ catalog_entry = catalog_by_type[reference_type]
426
+ for key in ("depthFileName", "segmentationFileName", "lineartFileName"):
427
+ if str(reference_images.get(key) or "") != str(
428
+ catalog_entry.get(key) or ""
429
+ ):
430
+ raise ValueError(
431
+ f"Image {custom_id!r} does not use the catalog's {key}."
432
+ )
433
+ dependencies = _require_list(
434
+ image.get("dependencies"), f"dependencies for {custom_id}"
435
+ )
436
+ if int(image.get("inputImageCount") or -1) != 4 + len(dependencies):
437
+ raise ValueError(f"Image {custom_id!r} has an invalid inputImageCount.")
438
+ if str(image.get("promptVersion") or "") != EXPECTED_PROMPT_VERSION:
439
+ raise ValueError(f"Image {custom_id!r} has the wrong promptVersion.")
440
+ # Do not normalize this string: Unity authored the complete request prompt,
441
+ # and the Batch request must receive it byte-for-byte as decoded JSON text.
442
+ prompt = str(image.get("prompt") or "")
443
+ if not prompt.strip():
444
+ raise ValueError(f"Image {custom_id!r} has an empty prompt.")
445
+ for required_text in (
446
+ "Image_A is the authoritative raw Unity render",
447
+ "Image_B is the",
448
+ "Image_C is the",
449
+ "Image_D is the",
450
+ "Do not swap left and right",
451
+ ):
452
+ if required_text not in prompt:
453
+ raise ValueError(
454
+ f"Image {custom_id!r} prompt is missing {required_text!r}."
455
+ )
456
+ image["_batchSetNumber"] = set_number
457
+ image["_batchSetId"] = set_id
458
+ image["_sourceName"] = source_name
459
+ image["_targetName"] = target_name
460
+ image["_referenceImages"] = dict(reference_images)
461
+ image["_dependencies"] = [
462
+ dict(_require_mapping(dep, f"dependency for {custom_id}"))
463
+ for dep in dependencies
464
+ ]
465
+ image["_prompt"] = prompt
466
+ jobs.append(image)
467
+ jobs_by_id[custom_id] = image
468
+
469
+ if int(manifest.get("batchImageCount") or -1) != len(jobs):
470
+ raise ValueError("batchImageCount does not match the Batch image total.")
471
+
472
+ actual_dependencies_by_set: Dict[int, set] = {
473
+ number: set() for number in declared_dependencies_by_set
474
+ }
475
+
476
+ for job in jobs:
477
+ custom_id = str(job["customId"])
478
+ dependency_ids = set()
479
+ for dependency_index, dependency in enumerate(job["_dependencies"]):
480
+ dependency_id = str(dependency.get("customId") or "")
481
+ if not dependency_id or dependency_id in dependency_ids:
482
+ raise ValueError(
483
+ f"Image {custom_id!r} has a missing or duplicate dependency."
484
+ )
485
+ dependency_ids.add(dependency_id)
486
+ source = jobs_by_id.get(dependency_id)
487
+ if source is None:
488
+ raise ValueError(
489
+ f"Image {custom_id!r} depends on unknown image {dependency_id!r}."
490
+ )
491
+ if int(source["_batchSetNumber"]) >= int(job["_batchSetNumber"]):
492
+ raise ValueError(
493
+ f"Image {custom_id!r} has a same-wave or forward dependency."
494
+ )
495
+ actual_dependencies_by_set[int(job["_batchSetNumber"])].add(
496
+ int(source["_batchSetNumber"])
497
+ )
498
+ expected_label = _image_label(4 + dependency_index)
499
+ if str(dependency.get("imageLabel") or "") != expected_label:
500
+ raise ValueError(
501
+ f"Image {custom_id!r} dependency {dependency_index} must be "
502
+ f"labeled {expected_label}."
503
+ )
504
+ if str(dependency.get("generatedFileName") or "") != str(
505
+ source["_targetName"]
506
+ ):
507
+ raise ValueError(
508
+ f"Image {custom_id!r} has a mismatched dependency filename."
509
+ )
510
+ if str(dependency.get("locationId") or "") != str(
511
+ source.get("locationId") or ""
512
+ ):
513
+ raise ValueError(
514
+ f"Image {custom_id!r} has a mismatched dependency locationId."
515
+ )
516
+ if str(dependency.get("sourceReferenceType") or "") != str(
517
+ source.get("referenceType") or ""
518
+ ):
519
+ raise ValueError(
520
+ f"Image {custom_id!r} has a mismatched dependency "
521
+ "sourceReferenceType."
522
+ )
523
+ if not str(dependency.get("relationship") or "").strip():
524
+ raise ValueError(
525
+ f"Image {custom_id!r} has a dependency without a relationship."
526
+ )
527
+ movement = str(dependency.get("movementInstruction") or "").strip()
528
+ if not movement or movement not in str(job["_prompt"]):
529
+ raise ValueError(
530
+ f"Image {custom_id!r} is missing its exact movement instruction."
531
+ )
532
+ expected_primary = (
533
+ str(job["_dependencies"][0].get("customId") or "")
534
+ if job["_dependencies"]
535
+ else ""
536
+ )
537
+ if str(job.get("primaryContinuityCustomId") or "") != expected_primary:
538
+ raise ValueError(
539
+ f"Image {custom_id!r} has an invalid primaryContinuityCustomId."
540
+ )
541
+ if int(job["_batchSetNumber"]) > 1 and not job["_dependencies"]:
542
+ raise ValueError(
543
+ f"Image {custom_id!r} is outside Batch set 1 but has no dependency."
544
+ )
545
+
546
+ for set_number, declared in declared_dependencies_by_set.items():
547
+ actual = sorted(actual_dependencies_by_set[set_number])
548
+ if declared != actual:
549
+ raise ValueError(
550
+ f"Batch set {set_number} declares dependencies {declared}, but its "
551
+ f"images actually require {actual}."
552
+ )
553
+
554
+ actual_reference_counts: Dict[str, int] = {
555
+ reference_type: 0 for reference_type in EXPECTED_REFERENCE_TYPES
556
+ }
557
+ for job in jobs:
558
+ actual_reference_counts[str(job["referenceType"])] += 1
559
+ for reference_type, entry in catalog_by_type.items():
560
+ if int(entry.get("usedByImageCount") or 0) != actual_reference_counts[
561
+ reference_type
562
+ ]:
563
+ raise ValueError(
564
+ f"Reference {reference_type!r} has an invalid usedByImageCount."
565
+ )
566
+ required_reference_types = sum(
567
+ 1 for count in actual_reference_counts.values() if count > 0
568
+ )
569
+ expected_reference_counts = {
570
+ "referenceTypeCount": len(EXPECTED_REFERENCE_TYPES),
571
+ "referenceFileCount": len(EXPECTED_REFERENCE_TYPES) * 3,
572
+ "requiredReferenceTypeCount": required_reference_types,
573
+ "requiredReferenceFileCount": required_reference_types * 3,
574
+ }
575
+ for key, expected in expected_reference_counts.items():
576
+ if int(manifest.get(key) or -1) != expected:
577
+ raise ValueError(f"Manifest {key} is invalid; expected {expected}.")
578
+
579
+ raw_locations = _require_list(manifest.get("locations"), "locations")
580
+ if len(raw_locations) != len(jobs):
581
+ raise ValueError("locations must contain exactly one entry per Batch image.")
582
+ locations_by_custom_id: Dict[str, Mapping[str, Any]] = {}
583
+ for index, raw_location in enumerate(raw_locations):
584
+ location_entry = _require_mapping(raw_location, f"locations[{index}]")
585
+ custom_id = str(location_entry.get("customId") or "")
586
+ if not custom_id or custom_id in locations_by_custom_id:
587
+ raise ValueError(f"Missing or duplicate location customId {custom_id!r}.")
588
+ locations_by_custom_id[custom_id] = location_entry
589
+ if set(locations_by_custom_id) != set(jobs_by_id):
590
+ raise ValueError("locations and batchSets do not contain the same customIds.")
591
+ for custom_id, job in jobs_by_id.items():
592
+ location_entry = locations_by_custom_id[custom_id]
593
+ location = _require_mapping(
594
+ location_entry.get("location"), f"location object for {custom_id}"
595
+ )
596
+ if str(location.get("id") or "") != str(job.get("locationId") or ""):
597
+ raise ValueError(f"Location id mismatch for image {custom_id!r}.")
598
+ if str(location_entry.get("sourceScreenshotFileName") or "") != str(
599
+ job["_sourceName"]
600
+ ):
601
+ raise ValueError(f"Screenshot filename mismatch for image {custom_id!r}.")
602
+
603
+ first_set = [job for job in jobs if int(job["_batchSetNumber"]) == 1]
604
+ if not first_set or any(job["_dependencies"] for job in first_set):
605
+ raise ValueError(
606
+ "Batch set 1 must contain dependency-free entrance identity seeds."
607
+ )
608
+ allowed_seed_types = {"straight_door", "station_gate_Front_Direct"}
609
+ if any(str(job.get("referenceType") or "") not in allowed_seed_types for job in first_set):
610
+ raise ValueError(
611
+ "Batch set 1 may contain only straight-door and station-front seeds."
612
+ )
613
+ if int(manifest.get("entranceSeedCount") or -1) != len(first_set):
614
+ raise ValueError("entranceSeedCount does not match Batch set 1.")
615
+ raw_seed_ids = _require_list(
616
+ manifest.get("entranceSeedCustomIds"), "entranceSeedCustomIds"
617
+ )
618
+ seed_ids = [str(value or "") for value in raw_seed_ids]
619
+ if len(seed_ids) != len(set(seed_ids)) or set(seed_ids) != {
620
+ str(job["customId"]) for job in first_set
621
+ }:
622
+ raise ValueError("entranceSeedCustomIds does not identify all Batch-set-1 seeds.")
623
+ root_id = str(manifest.get("batchRootCustomId") or "")
624
+ root = jobs_by_id.get(root_id)
625
+ if root is None or int(root["_batchSetNumber"]) != 1:
626
+ raise ValueError("batchRootCustomId must identify a Batch-set-1 entrance seed.")
627
+ if str(manifest.get("batchRootLocationId") or "") != str(
628
+ root.get("locationId") or ""
629
+ ):
630
+ raise ValueError("batchRootLocationId does not match batchRootCustomId.")
631
+ if int(manifest.get("excludedBenchCount", -1)) != 0:
632
+ raise ValueError("The v5 city Batch manifest must exclude all benches.")
633
+ return jobs
634
+
635
+
636
+ def _workflow_root(district: str, manifest_sha256: str) -> Path:
637
+ district_key = re.sub(r"[^A-Za-z0-9._-]+", "_", district).strip("._")
638
+ return (
639
+ _output_root()
640
+ / STATE_DIRECTORY_NAME
641
+ / (district_key or "district")
642
+ / manifest_sha256[:20]
643
+ )
644
+
645
+
646
+ def _preview_inputs(
647
+ manifest: Mapping[str, Any],
648
+ job: Mapping[str, Any],
649
+ district: str,
650
+ source_commit: str,
651
+ reference_commit: str,
652
+ generated_revision: str,
653
+ ) -> List[Dict[str, str]]:
654
+ references = _require_mapping(job.get("_referenceImages"), "referenceImages")
655
+ inputs: List[Dict[str, str]] = [
656
+ {
657
+ "label": "Image_A",
658
+ "role": "authoritative Unity target view",
659
+ "repoId": SOURCE_REPO_ID,
660
+ "revision": source_commit,
661
+ "path": _repo_path(district, str(job["_sourceName"])),
662
+ },
663
+ {
664
+ "label": "Image_B",
665
+ "role": "Depth structural reference",
666
+ "repoId": REFERENCE_REPO_ID,
667
+ "revision": reference_commit,
668
+ "path": str(references["depthFileName"]),
669
+ },
670
+ {
671
+ "label": "Image_C",
672
+ "role": "Segmentation structural reference",
673
+ "repoId": REFERENCE_REPO_ID,
674
+ "revision": reference_commit,
675
+ "path": str(references["segmentationFileName"]),
676
+ },
677
+ {
678
+ "label": "Image_D",
679
+ "role": "Lineart structural reference",
680
+ "repoId": REFERENCE_REPO_ID,
681
+ "revision": reference_commit,
682
+ "path": str(references["lineartFileName"]),
683
+ },
684
+ ]
685
+ for dependency in job["_dependencies"]:
686
+ inputs.append(
687
+ {
688
+ "label": str(dependency["imageLabel"]),
689
+ "role": "previously generated continuity view",
690
+ "repoId": GENERATED_REPO_ID,
691
+ "revision": generated_revision,
692
+ "path": _repo_path(district, str(dependency["generatedFileName"])),
693
+ }
694
+ )
695
+ for item in inputs:
696
+ item["url"] = _hf_url(item["repoId"], item["revision"], item["path"])
697
+ return inputs
698
+
699
+
700
+ def _write_prompt_preview(
701
+ manifest: Mapping[str, Any],
702
+ jobs: Sequence[Mapping[str, Any]],
703
+ district: str,
704
+ source_commit: str,
705
+ reference_commit: str,
706
+ manifest_sha256: str,
707
+ ) -> Tuple[Path, Path, str]:
708
+ root = _workflow_root(district, manifest_sha256)
709
+ entries: List[Dict[str, Any]] = []
710
+ text_parts: List[str] = []
711
+ for job in jobs:
712
+ inputs = _preview_inputs(
713
+ manifest,
714
+ job,
715
+ district,
716
+ source_commit,
717
+ reference_commit,
718
+ REPO_REVISION,
719
+ )
720
+ entry = {
721
+ "batchSetNumber": int(job["_batchSetNumber"]),
722
+ "batchSetId": str(job["_batchSetId"]),
723
+ "customId": str(job["customId"]),
724
+ "locationId": str(job.get("locationId") or ""),
725
+ "referenceType": str(job.get("referenceType") or ""),
726
+ "targetGeneratedFileName": str(job["_targetName"]),
727
+ "inputs": inputs,
728
+ "prompt": str(job["_prompt"]),
729
+ }
730
+ entries.append(entry)
731
+ text_parts.append(
732
+ "=" * 80
733
+ + f"\nBATCH SET {entry['batchSetNumber']:03d} | {entry['customId']}"
734
+ + f"\nTARGET: {entry['targetGeneratedFileName']}"
735
+ + "\n\nINPUTS\n"
736
+ + "\n".join(
737
+ f"{item['label']} = {item['url']} ({item['role']})"
738
+ for item in inputs
739
+ )
740
+ + "\n\nPROMPT\n"
741
+ + entry["prompt"]
742
+ + "\n"
743
+ )
744
+ preview = {
745
+ "formatVersion": 1,
746
+ "nodeVersion": NODE_VERSION,
747
+ "createdUtc": _utc_now(),
748
+ "districtName": district,
749
+ "manifestSha256": manifest_sha256,
750
+ "sourceCommit": source_commit,
751
+ "referenceCommit": reference_commit,
752
+ "generatedPreviewRevision": REPO_REVISION,
753
+ "batchSetCount": int(manifest.get("batchSetCount") or 0),
754
+ "imageCount": len(jobs),
755
+ "entries": entries,
756
+ }
757
+ json_path = root / "all_prompt_previews.json"
758
+ text_path = root / "all_prompt_previews.txt"
759
+ _write_json(json_path, preview)
760
+ full_text = "\n".join(text_parts)
761
+ text_path.parent.mkdir(parents=True, exist_ok=True)
762
+ temporary = text_path.with_suffix(".txt.part")
763
+ temporary.write_text(full_text, encoding="utf-8")
764
+ os.replace(temporary, text_path)
765
+ return json_path, text_path, full_text
766
+
767
+
768
+ def _openai_error(exception: urllib.error.HTTPError) -> str:
769
+ try:
770
+ body = exception.read().decode("utf-8", errors="replace")
771
+ parsed = json.loads(body)
772
+ error = parsed.get("error") if isinstance(parsed, dict) else None
773
+ if isinstance(error, Mapping):
774
+ return str(error.get("message") or body)[:1600]
775
+ return body[:1600]
776
+ except Exception:
777
+ return str(exception)
778
+
779
+
780
+ def _openai_json(
781
+ method: str,
782
+ path: str,
783
+ openai_key: str,
784
+ payload: Optional[Mapping[str, Any]] = None,
785
+ timeout: int = 300,
786
+ ) -> Dict[str, Any]:
787
+ data = None
788
+ headers = {
789
+ "Authorization": "Bearer " + openai_key,
790
+ "User-Agent": f"Salia-City-Responses-Batch/{NODE_VERSION}",
791
+ }
792
+ if payload is not None:
793
+ data = json.dumps(payload, separators=(",", ":")).encode("utf-8")
794
+ headers["Content-Type"] = "application/json"
795
+ request = urllib.request.Request(
796
+ "https://api.openai.com" + path,
797
+ data=data,
798
+ headers=headers,
799
+ method=method,
800
+ )
801
+ try:
802
+ with urllib.request.urlopen(request, timeout=timeout) as response:
803
+ parsed = json.loads(response.read().decode("utf-8"))
804
+ except urllib.error.HTTPError as exception:
805
+ raise RuntimeError(
806
+ f"OpenAI API {method} {path} failed: {_openai_error(exception)}"
807
+ ) from exception
808
+ if not isinstance(parsed, dict):
809
+ raise RuntimeError(f"OpenAI API {path} returned a non-object response.")
810
+ return parsed
811
+
812
+
813
+ def _upload_openai_batch_file(path: Path, openai_key: str) -> Dict[str, Any]:
814
+ boundary = "----SaliaCityResponses" + secrets.token_hex(16)
815
+ body = bytearray()
816
+ body.extend(f"--{boundary}\r\n".encode("ascii"))
817
+ body.extend(b'Content-Disposition: form-data; name="purpose"\r\n\r\n')
818
+ body.extend(b"batch\r\n")
819
+ body.extend(f"--{boundary}\r\n".encode("ascii"))
820
+ body.extend(
821
+ (
822
+ 'Content-Disposition: form-data; name="file"; filename="'
823
+ + path.name
824
+ + '"\r\n'
825
+ ).encode("utf-8")
826
+ )
827
+ body.extend(b"Content-Type: application/jsonl\r\n\r\n")
828
+ body.extend(path.read_bytes())
829
+ body.extend(b"\r\n")
830
+ body.extend(f"--{boundary}--\r\n".encode("ascii"))
831
+ request = urllib.request.Request(
832
+ "https://api.openai.com/v1/files",
833
+ data=bytes(body),
834
+ headers={
835
+ "Authorization": "Bearer " + openai_key,
836
+ "Content-Type": "multipart/form-data; boundary=" + boundary,
837
+ "User-Agent": f"Salia-City-Responses-Batch/{NODE_VERSION}",
838
+ },
839
+ method="POST",
840
+ )
841
+ try:
842
+ with urllib.request.urlopen(request, timeout=300) as response:
843
+ parsed = json.loads(response.read().decode("utf-8"))
844
+ except urllib.error.HTTPError as exception:
845
+ raise RuntimeError(
846
+ "OpenAI Batch file upload failed: " + _openai_error(exception)
847
+ ) from exception
848
+ file_id = str(parsed.get("id") or "") if isinstance(parsed, dict) else ""
849
+ if not _SAFE_FILE_ID.fullmatch(file_id):
850
+ raise RuntimeError("OpenAI did not return a valid Batch input file id.")
851
+ return parsed
852
+
853
+
854
+ def _download_openai_file(file_id: str, destination: Path, openai_key: str) -> Path:
855
+ if not _SAFE_FILE_ID.fullmatch(file_id):
856
+ raise ValueError(f"Invalid OpenAI file id {file_id!r}.")
857
+ destination.parent.mkdir(parents=True, exist_ok=True)
858
+ temporary = destination.with_suffix(destination.suffix + ".part")
859
+ request = urllib.request.Request(
860
+ "https://api.openai.com/v1/files/"
861
+ + urllib.parse.quote(file_id, safe="")
862
+ + "/content",
863
+ headers={
864
+ "Authorization": "Bearer " + openai_key,
865
+ "User-Agent": f"Salia-City-Responses-Batch/{NODE_VERSION}",
866
+ },
867
+ method="GET",
868
+ )
869
+ try:
870
+ with urllib.request.urlopen(request, timeout=3600) as response, temporary.open(
871
+ "wb"
872
+ ) as output:
873
+ shutil.copyfileobj(response, output, length=1024 * 1024)
874
+ os.replace(temporary, destination)
875
+ except Exception:
876
+ if temporary.exists():
877
+ temporary.unlink()
878
+ raise
879
+ return destination
880
+
881
+
882
+ def _batch_status(batch_id: str, openai_key: str) -> Dict[str, Any]:
883
+ if not _SAFE_BATCH_ID.fullmatch(batch_id):
884
+ raise ValueError("Saved OpenAI Batch id is invalid.")
885
+ return _openai_json(
886
+ "GET",
887
+ "/v1/batches/" + urllib.parse.quote(batch_id, safe=""),
888
+ openai_key,
889
+ )
890
+
891
+
892
+ def _request_counts(batch: Mapping[str, Any]) -> Tuple[int, int, int]:
893
+ counts = batch.get("request_counts")
894
+ if not isinstance(counts, Mapping):
895
+ return 0, 0, 0
896
+ return (
897
+ int(counts.get("total") or 0),
898
+ int(counts.get("completed") or 0),
899
+ int(counts.get("failed") or 0),
900
+ )
901
+
902
+
903
+ def _assistant_messages(body: Mapping[str, Any]) -> List[str]:
904
+ result: List[str] = []
905
+ output = body.get("output")
906
+ if not isinstance(output, list):
907
+ return result
908
+ for item in output:
909
+ if not isinstance(item, Mapping) or str(item.get("type") or "") != "message":
910
+ continue
911
+ content = item.get("content")
912
+ if not isinstance(content, list):
913
+ continue
914
+ for part in content:
915
+ if isinstance(part, Mapping):
916
+ text = str(part.get("text") or "").strip()
917
+ if text:
918
+ result.append(text)
919
+ return result
920
+
921
+
922
+ def _response_diagnostic(
923
+ record: Mapping[str, Any],
924
+ request_mapping: Mapping[str, Any],
925
+ line_number: int,
926
+ ) -> Dict[str, Any]:
927
+ response = record.get("response")
928
+ response_map = response if isinstance(response, Mapping) else {}
929
+ body_value = response_map.get("body")
930
+ body = body_value if isinstance(body_value, Mapping) else {}
931
+ output = body.get("output") if isinstance(body.get("output"), list) else []
932
+ calls: List[Dict[str, Any]] = []
933
+ reasoning_items: List[Dict[str, Any]] = []
934
+ for output_index, item in enumerate(output):
935
+ if not isinstance(item, Mapping):
936
+ continue
937
+ item_type = str(item.get("type") or "")
938
+ if item_type == "image_generation_call":
939
+ result_value = item.get("result")
940
+ calls.append(
941
+ {
942
+ "outputIndex": output_index,
943
+ "id": str(item.get("id") or ""),
944
+ "status": str(item.get("status") or "unknown"),
945
+ "revised_prompt": item.get("revised_prompt"),
946
+ "revisedPromptPresent": "revised_prompt" in item,
947
+ "resultPresent": isinstance(result_value, str)
948
+ and bool(result_value),
949
+ "resultBase64CharacterCount": len(result_value)
950
+ if isinstance(result_value, str)
951
+ else 0,
952
+ "availableFields": sorted(str(key) for key in item.keys()),
953
+ "allFieldsExceptResult": {
954
+ str(key): value
955
+ for key, value in item.items()
956
+ if str(key) != "result"
957
+ },
958
+ }
959
+ )
960
+ elif item_type == "reasoning":
961
+ reasoning_items.append(
962
+ {
963
+ "outputIndex": output_index,
964
+ "id": str(item.get("id") or ""),
965
+ "status": str(item.get("status") or ""),
966
+ "summary": item.get("summary"),
967
+ "content": item.get("content"),
968
+ "encryptedContentPresent": bool(item.get("encrypted_content")),
969
+ "availableFields": sorted(str(key) for key in item.keys()),
970
+ }
971
+ )
972
+ reasoning = body.get("reasoning") if isinstance(body.get("reasoning"), Mapping) else {}
973
+ usage = body.get("usage") if isinstance(body.get("usage"), Mapping) else {}
974
+ output_details = (
975
+ usage.get("output_tokens_details")
976
+ if isinstance(usage.get("output_tokens_details"), Mapping)
977
+ else {}
978
+ )
979
+ tool_usage = (
980
+ body.get("tool_usage") if isinstance(body.get("tool_usage"), Mapping) else {}
981
+ )
982
+ image_usage = (
983
+ tool_usage.get("image_gen")
984
+ if isinstance(tool_usage.get("image_gen"), Mapping)
985
+ else {}
986
+ )
987
+ messages = _assistant_messages(body)
988
+ completed_image_present = any(
989
+ call.get("status") == "completed" and call.get("resultPresent")
990
+ for call in calls
991
+ )
992
+ return {
993
+ "batchOutputLine": line_number,
994
+ "batchCustomId": str(record.get("custom_id") or ""),
995
+ "jobCustomId": str(request_mapping.get("jobCustomId") or ""),
996
+ "batchSetNumber": int(request_mapping.get("batchSetNumber") or 0),
997
+ "attempt": int(request_mapping.get("attempt") or 0),
998
+ "targetPath": str(request_mapping.get("targetPath") or ""),
999
+ "responseStatusCode": int(response_map.get("status_code") or 0),
1000
+ "batchRequestId": str(response_map.get("request_id") or ""),
1001
+ "responseId": str(body.get("id") or ""),
1002
+ "responseStatus": str(body.get("status") or ""),
1003
+ "responseModel": str(body.get("model") or ""),
1004
+ "responseError": body.get("error"),
1005
+ "responseIncompleteDetails": body.get("incomplete_details"),
1006
+ "responseModeration": body.get("moderation"),
1007
+ "responseReasoning": dict(reasoning),
1008
+ "reasoningEffort": reasoning.get("effort"),
1009
+ "reasoningMode": reasoning.get("mode"),
1010
+ "reasoningSummarySetting": reasoning.get("summary"),
1011
+ "reasoningTokenCount": int(output_details.get("reasoning_tokens") or 0),
1012
+ "responseUsage": dict(usage),
1013
+ "imageGenUsage": dict(image_usage),
1014
+ "imageGenerationCalls": calls,
1015
+ "reasoningOutputItems": reasoning_items,
1016
+ "assistantMessages": messages,
1017
+ "assistantFailureMessage": "" if completed_image_present else "\n".join(messages),
1018
+ "topLevelBatchError": record.get("error"),
1019
+ }
1020
+
1021
+
1022
+ def _extract_single_png(body: Mapping[str, Any]) -> bytes:
1023
+ if str(body.get("status") or "") != "completed" or body.get("error"):
1024
+ raise ValueError(
1025
+ "Responses request did not complete: "
1026
+ + str(body.get("error") or body.get("status") or "unknown")
1027
+ )
1028
+ output = body.get("output")
1029
+ if not isinstance(output, list):
1030
+ raise ValueError("Responses result has no output array.")
1031
+ encoded_results: List[str] = []
1032
+ statuses: List[str] = []
1033
+ for item in output:
1034
+ if not isinstance(item, Mapping) or str(item.get("type") or "") != "image_generation_call":
1035
+ continue
1036
+ statuses.append(str(item.get("status") or "unknown"))
1037
+ encoded = item.get("result")
1038
+ if str(item.get("status") or "") == "completed" and isinstance(encoded, str) and encoded:
1039
+ encoded_results.append(encoded)
1040
+ if len(encoded_results) != 1:
1041
+ raise ValueError(
1042
+ f"Expected exactly one completed image result; received "
1043
+ f"{len(encoded_results)}. Call statuses: {', '.join(statuses) or 'none'}."
1044
+ )
1045
+ raw = base64.b64decode(encoded_results[0], validate=True)
1046
+ if not raw.startswith(_PNG_SIGNATURE):
1047
+ raise ValueError("Generated result is not a PNG.")
1048
+ if Image is None:
1049
+ raise RuntimeError("Pillow is required to validate generated image dimensions.")
1050
+ with Image.open(BytesIO(raw)) as image:
1051
+ if image.size != (1536, 1024):
1052
+ raise ValueError(
1053
+ f"Generated PNG dimensions are {image.size}; expected (1536, 1024)."
1054
+ )
1055
+ image.verify()
1056
+ return raw
1057
+
1058
+
1059
+ def _validate_remote_inputs(
1060
+ api: Any,
1061
+ jobs: Sequence[Mapping[str, Any]],
1062
+ district: str,
1063
+ source_commit: str,
1064
+ reference_commit: str,
1065
+ generated_commit: Optional[str],
1066
+ hf_token: str,
1067
+ ) -> None:
1068
+ missing: List[str] = []
1069
+ checked = set()
1070
+ for job in jobs:
1071
+ candidates = [
1072
+ (SOURCE_REPO_ID, source_commit, _repo_path(district, str(job["_sourceName"]))),
1073
+ (
1074
+ REFERENCE_REPO_ID,
1075
+ reference_commit,
1076
+ str(job["_referenceImages"]["depthFileName"]),
1077
+ ),
1078
+ (
1079
+ REFERENCE_REPO_ID,
1080
+ reference_commit,
1081
+ str(job["_referenceImages"]["segmentationFileName"]),
1082
+ ),
1083
+ (
1084
+ REFERENCE_REPO_ID,
1085
+ reference_commit,
1086
+ str(job["_referenceImages"]["lineartFileName"]),
1087
+ ),
1088
+ ]
1089
+ for dependency in job["_dependencies"]:
1090
+ if not generated_commit:
1091
+ missing.append(
1092
+ GENERATED_REPO_ID
1093
+ + ":"
1094
+ + _repo_path(district, str(dependency["generatedFileName"]))
1095
+ + " (no generated repository commit)"
1096
+ )
1097
+ else:
1098
+ candidates.append(
1099
+ (
1100
+ GENERATED_REPO_ID,
1101
+ generated_commit,
1102
+ _repo_path(district, str(dependency["generatedFileName"])),
1103
+ )
1104
+ )
1105
+ for repo_id, revision, path_in_repo in candidates:
1106
+ key = (repo_id, revision, path_in_repo)
1107
+ if key in checked:
1108
+ continue
1109
+ checked.add(key)
1110
+ if not api.file_exists(
1111
+ repo_id=repo_id,
1112
+ filename=path_in_repo,
1113
+ repo_type=REPO_TYPE,
1114
+ revision=revision,
1115
+ token=hf_token,
1116
+ ):
1117
+ missing.append(f"{repo_id}@{revision}:{path_in_repo}")
1118
+ if missing:
1119
+ raise FileNotFoundError(
1120
+ "Cannot start this wave because required input files are missing: "
1121
+ + "; ".join(missing[:30])
1122
+ )
1123
+
1124
+
1125
+ def _batch_line(
1126
+ request_mapping: Mapping[str, Any],
1127
+ job: Mapping[str, Any],
1128
+ inputs: Sequence[Mapping[str, str]],
1129
+ ) -> Dict[str, Any]:
1130
+ content: List[Dict[str, Any]] = [
1131
+ {"type": "input_text", "text": str(job["_prompt"])}
1132
+ ]
1133
+ for item in inputs:
1134
+ content.append(
1135
+ {
1136
+ "type": "input_text",
1137
+ "text": str(item["label"])
1138
+ + " follows: "
1139
+ + str(item["role"])
1140
+ + ".",
1141
+ }
1142
+ )
1143
+ content.append({"type": "input_image", "image_url": str(item["url"])})
1144
+ return {
1145
+ "custom_id": str(request_mapping["batchCustomId"]),
1146
+ "method": "POST",
1147
+ "url": OPENAI_BATCH_ENDPOINT,
1148
+ "body": {
1149
+ "model": OPENAI_MODEL,
1150
+ "input": [{"role": "user", "content": content}],
1151
+ "reasoning": dict(OPENAI_REASONING),
1152
+ "moderation": json.loads(json.dumps(RESPONSE_MODERATION)),
1153
+ "tools": [dict(OPENAI_IMAGE_TOOL)],
1154
+ "tool_choice": {"type": "image_generation"},
1155
+ "store": False,
1156
+ },
1157
+ }
1158
+
1159
+
1160
+ def _build_wave_requests(
1161
+ state: Mapping[str, Any],
1162
+ jobs: Sequence[Mapping[str, Any]],
1163
+ district: str,
1164
+ source_commit: str,
1165
+ reference_commit: str,
1166
+ generated_commit: str,
1167
+ maximum_attempts: int,
1168
+ ) -> Tuple[int, Dict[str, int], List[Dict[str, Any]], List[Dict[str, Any]]]:
1169
+ if not jobs:
1170
+ raise ValueError("Cannot build an empty city Batch wave.")
1171
+ set_numbers = {int(job["_batchSetNumber"]) for job in jobs}
1172
+ if len(set_numbers) != 1:
1173
+ raise ValueError("One OpenAI Batch may contain only one logical Batch set.")
1174
+ set_number = next(iter(set_numbers))
1175
+ completed = set(str(value) for value in state.get("completedCustomIds") or [])
1176
+ for job in jobs:
1177
+ for dependency in job["_dependencies"]:
1178
+ if str(dependency["customId"]) not in completed:
1179
+ raise ValueError(
1180
+ f"Image {job['customId']!r} is not ready; dependency "
1181
+ f"{dependency['customId']!r} is unpublished."
1182
+ )
1183
+
1184
+ attempt_counts = dict(state.get("attemptCounts") or {})
1185
+ request_mappings: List[Dict[str, Any]] = []
1186
+ lines: List[Dict[str, Any]] = []
1187
+ for job_index, job in enumerate(jobs):
1188
+ job_id = str(job["customId"])
1189
+ attempt = int(attempt_counts.get(job_id) or 0) + 1
1190
+ if attempt > maximum_attempts:
1191
+ raise RuntimeError(
1192
+ f"Image {job_id!r} reached MAX_ATTEMPTS={maximum_attempts}. "
1193
+ "Inspect its saved diagnostics before trying again."
1194
+ )
1195
+ fingerprint = (
1196
+ str(state["manifestSha256"])
1197
+ + "\0"
1198
+ + job_id
1199
+ + "\0"
1200
+ + str(attempt)
1201
+ )
1202
+ batch_custom_id = (
1203
+ "city_"
1204
+ + str(set_number).zfill(3)
1205
+ + "_"
1206
+ + str(job_index).zfill(4)
1207
+ + "_a"
1208
+ + str(attempt).zfill(2)
1209
+ + "_"
1210
+ + hashlib.sha256(fingerprint.encode("utf-8")).hexdigest()[:12]
1211
+ )
1212
+ mapping = {
1213
+ "batchCustomId": batch_custom_id,
1214
+ "jobCustomId": job_id,
1215
+ "batchSetNumber": set_number,
1216
+ "attempt": attempt,
1217
+ "targetPath": _repo_path(district, str(job["_targetName"])),
1218
+ "prompt": str(job["_prompt"]),
1219
+ }
1220
+ inputs = _preview_inputs(
1221
+ state,
1222
+ job,
1223
+ district,
1224
+ source_commit,
1225
+ reference_commit,
1226
+ generated_commit or REPO_REVISION,
1227
+ )
1228
+ mapping["inputs"] = inputs
1229
+ request_mappings.append(mapping)
1230
+ lines.append(_batch_line(mapping, job, inputs))
1231
+ attempt_counts[job_id] = attempt
1232
+ return set_number, attempt_counts, request_mappings, lines
1233
+
1234
+
1235
+ def _start_wave(
1236
+ state: Dict[str, Any],
1237
+ jobs: Sequence[Mapping[str, Any]],
1238
+ district: str,
1239
+ hf_token: str,
1240
+ openai_key: str,
1241
+ maximum_attempts: int,
1242
+ ) -> Tuple[str, str, int, str]:
1243
+ _, HfApi, _ = _load_hf_tools()
1244
+ api = HfApi(token=hf_token)
1245
+ generated_commit: Optional[str] = _repo_commit(
1246
+ api, GENERATED_REPO_ID, hf_token
1247
+ )
1248
+ source_commit = str(state["sourceCommit"])
1249
+ reference_commit = str(state["referenceCommit"])
1250
+ (
1251
+ set_number,
1252
+ attempt_counts,
1253
+ request_mappings,
1254
+ lines,
1255
+ ) = _build_wave_requests(
1256
+ state,
1257
+ jobs,
1258
+ district,
1259
+ source_commit,
1260
+ reference_commit,
1261
+ generated_commit or REPO_REVISION,
1262
+ maximum_attempts,
1263
+ )
1264
+ _validate_remote_inputs(
1265
+ api,
1266
+ jobs,
1267
+ district,
1268
+ source_commit,
1269
+ reference_commit,
1270
+ generated_commit,
1271
+ hf_token,
1272
+ )
1273
+
1274
+ wave_sequence = int(state.get("openAiBatchSequence") or 0) + 1
1275
+ batches_root = Path(str(state["workflowRoot"])) / "openai_batches"
1276
+ while True:
1277
+ run_root = batches_root / (
1278
+ "batch_"
1279
+ + str(wave_sequence).zfill(3)
1280
+ + "_set_"
1281
+ + str(set_number).zfill(3)
1282
+ )
1283
+ if not run_root.exists():
1284
+ break
1285
+ # A prior upload/create attempt may have failed before its Batch id could
1286
+ # be persisted. Keep that evidence and use a fresh local run directory.
1287
+ wave_sequence += 1
1288
+ run_root.mkdir(parents=True, exist_ok=False)
1289
+ jsonl_path = run_root / "city_responses_batch.jsonl"
1290
+ request_map_path = run_root / "request_map.json"
1291
+ _write_jsonl(jsonl_path, lines)
1292
+ _write_json(
1293
+ request_map_path,
1294
+ {
1295
+ "formatVersion": 1,
1296
+ "nodeVersion": NODE_VERSION,
1297
+ "districtName": district,
1298
+ "manifestSha256": str(state["manifestSha256"]),
1299
+ "batchSetNumber": set_number,
1300
+ "requestCount": len(request_mappings),
1301
+ "responsesModel": OPENAI_MODEL,
1302
+ "reasoning": OPENAI_REASONING,
1303
+ "responseModeration": RESPONSE_MODERATION,
1304
+ "imageTool": OPENAI_IMAGE_TOOL,
1305
+ "requests": request_mappings,
1306
+ },
1307
+ )
1308
+ uploaded = _upload_openai_batch_file(jsonl_path, openai_key)
1309
+ input_file_id = str(uploaded["id"])
1310
+ batch = _openai_json(
1311
+ "POST",
1312
+ "/v1/batches",
1313
+ openai_key,
1314
+ {
1315
+ "input_file_id": input_file_id,
1316
+ "endpoint": OPENAI_BATCH_ENDPOINT,
1317
+ "completion_window": "24h",
1318
+ "metadata": {
1319
+ "purpose": PURPOSE,
1320
+ "node_version": NODE_VERSION,
1321
+ "batch_set": str(set_number),
1322
+ "manifest": str(state["manifestSha256"])[:32],
1323
+ },
1324
+ },
1325
+ )
1326
+ batch_id = str(batch.get("id") or "")
1327
+ if not _SAFE_BATCH_ID.fullmatch(batch_id):
1328
+ raise RuntimeError("OpenAI did not return a valid Batch id.")
1329
+ state["attemptCounts"] = attempt_counts
1330
+ state["openAiBatchSequence"] = wave_sequence
1331
+ state["activeBatch"] = {
1332
+ "batchId": batch_id,
1333
+ "inputFileId": input_file_id,
1334
+ "batchSetNumber": set_number,
1335
+ "requestCount": len(request_mappings),
1336
+ "requests": request_mappings,
1337
+ "runRoot": str(run_root.resolve()),
1338
+ "jsonlPath": str(jsonl_path.resolve()),
1339
+ "requestMapPath": str(request_map_path.resolve()),
1340
+ "createdUtc": _utc_now(),
1341
+ "generatedInputCommit": generated_commit or "",
1342
+ }
1343
+ _write_json(Path(str(state["statePath"])), state)
1344
+ return (
1345
+ f"Started OpenAI Responses Batch {batch_id}: logical Batch set "
1346
+ f"{set_number}, {len(jobs)} image request(s), attempts "
1347
+ + ", ".join(
1348
+ f"{mapping['jobCustomId']}={mapping['attempt']}"
1349
+ for mapping in request_mappings[:8]
1350
+ )
1351
+ + (" ..." if len(request_mappings) > 8 else "")
1352
+ + ".",
1353
+ batch_id,
1354
+ set_number,
1355
+ str(jsonl_path.resolve()),
1356
+ )
1357
+
1358
+
1359
+ def _upload_jsonl_artifact(
1360
+ jsonl_path: Path,
1361
+ district: str,
1362
+ set_number: int,
1363
+ manifest_sha256: str,
1364
+ hf_token: str,
1365
+ ) -> Tuple[str, str, str]:
1366
+ raw = jsonl_path.read_bytes()
1367
+ jsonl_sha256 = hashlib.sha256(raw).hexdigest()
1368
+ file_name = (
1369
+ "city_responses_"
1370
+ + manifest_sha256[:12]
1371
+ + "_set_"
1372
+ + str(set_number).zfill(3)
1373
+ + "_"
1374
+ + jsonl_sha256[:12]
1375
+ + ".jsonl"
1376
+ )
1377
+ path_in_repo = (
1378
+ district
1379
+ + "/openai_batch_jsonl/"
1380
+ + _safe_file_name(file_name, "JSONL artifact filename")
1381
+ )
1382
+ CommitOperationAdd, HfApi, _ = _load_hf_tools()
1383
+ api = HfApi(token=hf_token)
1384
+ parent_commit = _repo_commit(api, SOURCE_REPO_ID, hf_token)
1385
+ if api.file_exists(
1386
+ repo_id=SOURCE_REPO_ID,
1387
+ filename=path_in_repo,
1388
+ repo_type=REPO_TYPE,
1389
+ revision=parent_commit,
1390
+ token=hf_token,
1391
+ ):
1392
+ commit_oid = parent_commit
1393
+ else:
1394
+ commit = api.create_commit(
1395
+ repo_id=SOURCE_REPO_ID,
1396
+ repo_type=REPO_TYPE,
1397
+ revision=REPO_REVISION,
1398
+ operations=[
1399
+ CommitOperationAdd(
1400
+ path_in_repo=path_in_repo,
1401
+ path_or_fileobj=str(jsonl_path),
1402
+ )
1403
+ ],
1404
+ commit_message=(
1405
+ "City Responses JSONL only: Batch set "
1406
+ + str(set_number)
1407
+ + " ("
1408
+ + jsonl_sha256[:12]
1409
+ + ")"
1410
+ ),
1411
+ parent_commit=parent_commit,
1412
+ token=hf_token,
1413
+ )
1414
+ commit_oid = str(getattr(commit, "oid", "") or "").strip()
1415
+ if not commit_oid:
1416
+ raise RuntimeError("Hugging Face JSONL commit returned no oid.")
1417
+ return (
1418
+ _hf_blob_url(SOURCE_REPO_ID, commit_oid, path_in_repo),
1419
+ commit_oid,
1420
+ path_in_repo,
1421
+ )
1422
+
1423
+
1424
+ def _create_jsonl_only(
1425
+ state: Dict[str, Any],
1426
+ jobs: Sequence[Mapping[str, Any]],
1427
+ district: str,
1428
+ hf_token: str,
1429
+ maximum_attempts: int,
1430
+ ) -> Tuple[str, int, str, str]:
1431
+ _, HfApi, _ = _load_hf_tools()
1432
+ api = HfApi(token=hf_token)
1433
+ generated_commit = _repo_commit(api, GENERATED_REPO_ID, hf_token)
1434
+ source_commit = str(state["sourceCommit"])
1435
+ reference_commit = str(state["referenceCommit"])
1436
+ (
1437
+ set_number,
1438
+ _unused_attempt_counts,
1439
+ request_mappings,
1440
+ lines,
1441
+ ) = _build_wave_requests(
1442
+ state,
1443
+ jobs,
1444
+ district,
1445
+ source_commit,
1446
+ reference_commit,
1447
+ generated_commit,
1448
+ maximum_attempts,
1449
+ )
1450
+ _validate_remote_inputs(
1451
+ api,
1452
+ jobs,
1453
+ district,
1454
+ source_commit,
1455
+ reference_commit,
1456
+ generated_commit,
1457
+ hf_token,
1458
+ )
1459
+
1460
+ artifact_root = Path(str(state["workflowRoot"])) / "jsonl_only"
1461
+ artifact_root.mkdir(parents=True, exist_ok=True)
1462
+ jsonl_path = artifact_root / (
1463
+ "city_responses_set_" + str(set_number).zfill(3) + ".jsonl"
1464
+ )
1465
+ request_map_path = artifact_root / (
1466
+ "city_responses_set_" + str(set_number).zfill(3) + "_request_map.json"
1467
+ )
1468
+ _write_jsonl(jsonl_path, lines)
1469
+ _write_json(
1470
+ request_map_path,
1471
+ {
1472
+ "formatVersion": 1,
1473
+ "nodeVersion": NODE_VERSION,
1474
+ "onlyCreateJsonl": True,
1475
+ "districtName": district,
1476
+ "manifestSha256": str(state["manifestSha256"]),
1477
+ "batchSetNumber": set_number,
1478
+ "requestCount": len(request_mappings),
1479
+ "responsesModel": OPENAI_MODEL,
1480
+ "reasoning": OPENAI_REASONING,
1481
+ "responseModeration": RESPONSE_MODERATION,
1482
+ "imageTool": OPENAI_IMAGE_TOOL,
1483
+ "requests": request_mappings,
1484
+ },
1485
+ )
1486
+ url, commit_oid, path_in_repo = _upload_jsonl_artifact(
1487
+ jsonl_path,
1488
+ district,
1489
+ set_number,
1490
+ str(state["manifestSha256"]),
1491
+ hf_token,
1492
+ )
1493
+ state["lastJsonlOnly"] = {
1494
+ "createdUtc": _utc_now(),
1495
+ "batchSetNumber": set_number,
1496
+ "requestCount": len(request_mappings),
1497
+ "localPath": str(jsonl_path.resolve()),
1498
+ "requestMapPath": str(request_map_path.resolve()),
1499
+ "huggingFaceRepoPath": path_in_repo,
1500
+ "huggingFaceCommit": commit_oid,
1501
+ "huggingFaceUrl": url,
1502
+ }
1503
+ _write_json(Path(str(state["statePath"])), state)
1504
+ status = (
1505
+ "Only_Create_Jsonl is ON. Created and uploaded logical Batch set "
1506
+ + str(set_number)
1507
+ + " with "
1508
+ + str(len(lines))
1509
+ + " /v1/responses request(s). No OpenAI file was uploaded and no "
1510
+ + "OpenAI Batch was started. Image_A=Unity screenshot; Image_B=Depth; "
1511
+ + "Image_C=Segmentation; Image_D=Lineart; Image_E and later=published "
1512
+ + "generated continuity dependencies. Hugging Face JSONL: "
1513
+ + url
1514
+ )
1515
+ return status, set_number, str(jsonl_path.resolve()), url
1516
+
1517
+
1518
+ def _save_png(raw: bytes, directory: Path, job_custom_id: str) -> Path:
1519
+ directory.mkdir(parents=True, exist_ok=True)
1520
+ name = hashlib.sha256(job_custom_id.encode("utf-8")).hexdigest()[:20] + ".png"
1521
+ path = directory / name
1522
+ temporary = path.with_suffix(".png.part")
1523
+ temporary.write_bytes(raw)
1524
+ os.replace(temporary, path)
1525
+ return path
1526
+
1527
+
1528
+ def _parse_error_file(
1529
+ path: Optional[Path],
1530
+ request_by_batch_id: Mapping[str, Mapping[str, Any]],
1531
+ ) -> Dict[str, Dict[str, Any]]:
1532
+ result: Dict[str, Dict[str, Any]] = {}
1533
+ if path is None or not path.is_file():
1534
+ return result
1535
+ with path.open("r", encoding="utf-8-sig") as handle:
1536
+ for line_number, line in enumerate(handle, start=1):
1537
+ if not line.strip():
1538
+ continue
1539
+ record = json.loads(line)
1540
+ if not isinstance(record, Mapping):
1541
+ continue
1542
+ batch_custom_id = str(record.get("custom_id") or "")
1543
+ if batch_custom_id not in request_by_batch_id:
1544
+ continue
1545
+ response = record.get("response")
1546
+ response_map = response if isinstance(response, Mapping) else {}
1547
+ error = record.get("error") or response_map.get("body") or response_map
1548
+ result[batch_custom_id] = {
1549
+ "batchErrorLine": line_number,
1550
+ "statusCode": int(response_map.get("status_code") or 0),
1551
+ "error": error,
1552
+ }
1553
+ return result
1554
+
1555
+
1556
+ def _publish_results(
1557
+ state: Dict[str, Any],
1558
+ results: Sequence[Mapping[str, Any]],
1559
+ hf_token: str,
1560
+ ) -> str:
1561
+ if not results:
1562
+ return ""
1563
+ CommitOperationAdd, HfApi, _ = _load_hf_tools()
1564
+ api = HfApi(token=hf_token)
1565
+ parent_commit = _repo_commit(api, GENERATED_REPO_ID, hf_token)
1566
+ operations = [
1567
+ CommitOperationAdd(
1568
+ path_in_repo=str(result["targetPath"]),
1569
+ path_or_fileobj=str(result["localPath"]),
1570
+ )
1571
+ for result in results
1572
+ ]
1573
+ commit = api.create_commit(
1574
+ repo_id=GENERATED_REPO_ID,
1575
+ repo_type=REPO_TYPE,
1576
+ revision=REPO_REVISION,
1577
+ operations=operations,
1578
+ commit_message=(
1579
+ "City Responses Batch set "
1580
+ + str(results[0]["batchSetNumber"])
1581
+ + ": publish "
1582
+ + str(len(results))
1583
+ + " generated views"
1584
+ ),
1585
+ parent_commit=parent_commit,
1586
+ token=hf_token,
1587
+ )
1588
+ oid = str(getattr(commit, "oid", "") or "").strip()
1589
+ if not oid:
1590
+ raise RuntimeError("Hugging Face commit returned no oid.")
1591
+ state["lastGeneratedCommit"] = oid
1592
+ return oid
1593
+
1594
+
1595
+ def _process_terminal_batch(
1596
+ state: Dict[str, Any],
1597
+ batch: Mapping[str, Any],
1598
+ hf_token: str,
1599
+ openai_key: str,
1600
+ ) -> Tuple[int, int, str]:
1601
+ active = _require_mapping(state.get("activeBatch"), "activeBatch")
1602
+ run_root = Path(str(active["runRoot"]))
1603
+ requests = _require_list(active.get("requests"), "activeBatch.requests")
1604
+ request_by_batch_id: Dict[str, Mapping[str, Any]] = {
1605
+ str(request["batchCustomId"]): _require_mapping(request, "request mapping")
1606
+ for request in requests
1607
+ }
1608
+ output_path: Optional[Path] = None
1609
+ output_file_id = str(batch.get("output_file_id") or "")
1610
+ if _SAFE_FILE_ID.fullmatch(output_file_id):
1611
+ output_path = _download_openai_file(
1612
+ output_file_id,
1613
+ run_root / "openai_batch_output.jsonl",
1614
+ openai_key,
1615
+ )
1616
+ error_path: Optional[Path] = None
1617
+ error_file_id = str(batch.get("error_file_id") or "")
1618
+ if _SAFE_FILE_ID.fullmatch(error_file_id):
1619
+ error_path = _download_openai_file(
1620
+ error_file_id,
1621
+ run_root / "openai_batch_errors.jsonl",
1622
+ openai_key,
1623
+ )
1624
+ error_records = _parse_error_file(error_path, request_by_batch_id)
1625
+ results: List[Dict[str, Any]] = []
1626
+ failures: List[Dict[str, Any]] = []
1627
+ diagnostics: List[Dict[str, Any]] = []
1628
+ seen = set()
1629
+ if output_path is not None:
1630
+ with output_path.open("r", encoding="utf-8-sig") as handle:
1631
+ for line_number, line in enumerate(handle, start=1):
1632
+ if not line.strip():
1633
+ continue
1634
+ record = json.loads(line)
1635
+ if not isinstance(record, Mapping):
1636
+ raise ValueError(f"Batch output line {line_number} is not an object.")
1637
+ batch_custom_id = str(record.get("custom_id") or "")
1638
+ mapping = request_by_batch_id.get(batch_custom_id)
1639
+ if mapping is None or batch_custom_id in seen:
1640
+ raise ValueError(
1641
+ f"Unexpected or duplicate Batch custom_id {batch_custom_id!r}."
1642
+ )
1643
+ seen.add(batch_custom_id)
1644
+ diagnostic = _response_diagnostic(record, mapping, line_number)
1645
+ diagnostic["diagnosticIndex"] = len(diagnostics)
1646
+ diagnostics.append(diagnostic)
1647
+ try:
1648
+ response = _require_mapping(record.get("response"), "Batch response")
1649
+ status_code = int(response.get("status_code") or 0)
1650
+ if status_code < 200 or status_code >= 300:
1651
+ raise ValueError(f"Responses HTTP status {status_code}.")
1652
+ body = _require_mapping(response.get("body"), "Responses body")
1653
+ raw = _extract_single_png(body)
1654
+ local_path = _save_png(
1655
+ raw,
1656
+ run_root / "generated_images",
1657
+ str(mapping["jobCustomId"]),
1658
+ )
1659
+ results.append(
1660
+ {
1661
+ "batchCustomId": batch_custom_id,
1662
+ "jobCustomId": str(mapping["jobCustomId"]),
1663
+ "batchSetNumber": int(mapping["batchSetNumber"]),
1664
+ "attempt": int(mapping["attempt"]),
1665
+ "targetPath": str(mapping["targetPath"]),
1666
+ "localPath": str(local_path.resolve()),
1667
+ "sha256": hashlib.sha256(raw).hexdigest(),
1668
+ "byteCount": len(raw),
1669
+ "diagnosticIndex": int(diagnostic["diagnosticIndex"]),
1670
+ }
1671
+ )
1672
+ except Exception as exception:
1673
+ failures.append(
1674
+ {
1675
+ "batchCustomId": batch_custom_id,
1676
+ "jobCustomId": str(mapping["jobCustomId"]),
1677
+ "batchSetNumber": int(mapping["batchSetNumber"]),
1678
+ "attempt": int(mapping["attempt"]),
1679
+ "failureKind": "response_or_image_failure",
1680
+ "reason": str(exception)[:1600],
1681
+ "diagnosticIndex": int(diagnostic["diagnosticIndex"]),
1682
+ }
1683
+ )
1684
+ for batch_custom_id, mapping in request_by_batch_id.items():
1685
+ if batch_custom_id in seen:
1686
+ continue
1687
+ error = error_records.get(batch_custom_id)
1688
+ failures.append(
1689
+ {
1690
+ "batchCustomId": batch_custom_id,
1691
+ "jobCustomId": str(mapping["jobCustomId"]),
1692
+ "batchSetNumber": int(mapping["batchSetNumber"]),
1693
+ "attempt": int(mapping["attempt"]),
1694
+ "failureKind": "batch_request_error",
1695
+ "reason": str(error.get("error") if error else batch.get("errors") or batch.get("status"))[:1600],
1696
+ "batchError": error,
1697
+ }
1698
+ )
1699
+
1700
+ commit_oid = _publish_results(state, results, hf_token)
1701
+ completed = set(str(value) for value in state.get("completedCustomIds") or [])
1702
+ result_history = list(state.get("resultHistory") or [])
1703
+ for result in results:
1704
+ completed.add(str(result["jobCustomId"]))
1705
+ result_history.append(dict(result, publishedCommit=commit_oid))
1706
+ failure_history = list(state.get("failureHistory") or [])
1707
+ failure_history.extend(failures)
1708
+ state["completedCustomIds"] = sorted(completed)
1709
+ state["resultHistory"] = result_history
1710
+ state["failureHistory"] = failure_history
1711
+ state["lastProcessedBatchId"] = str(active["batchId"])
1712
+ state["activeBatch"] = None
1713
+ _write_json(
1714
+ run_root / "openai_response_diagnostics.json",
1715
+ {
1716
+ "formatVersion": 1,
1717
+ "nodeVersion": NODE_VERSION,
1718
+ "batchId": str(active["batchId"]),
1719
+ "batchSetNumber": int(active["batchSetNumber"]),
1720
+ "rawOutputPath": str(output_path.resolve()) if output_path else "",
1721
+ "rawErrorPath": str(error_path.resolve()) if error_path else "",
1722
+ "returnedOutputCount": len(seen),
1723
+ "successfulImageCount": len(results),
1724
+ "failureCount": len(failures),
1725
+ "requestConfiguration": {
1726
+ "responsesModel": OPENAI_MODEL,
1727
+ "reasoning": OPENAI_REASONING,
1728
+ "responseModeration": RESPONSE_MODERATION,
1729
+ "imageTool": OPENAI_IMAGE_TOOL,
1730
+ "toolChoice": {"type": "image_generation"},
1731
+ "store": False,
1732
+ },
1733
+ "entries": diagnostics,
1734
+ "failures": failures,
1735
+ },
1736
+ )
1737
+ _write_json(Path(str(state["statePath"])), state)
1738
+ return len(results), len(failures), commit_oid
1739
+
1740
+
1741
+ def _initialize_state(
1742
+ manifest: Mapping[str, Any],
1743
+ district: str,
1744
+ source_commit: str,
1745
+ reference_commit: str,
1746
+ manifest_sha256: str,
1747
+ jobs: Sequence[Mapping[str, Any]],
1748
+ ) -> Dict[str, Any]:
1749
+ root = _workflow_root(district, manifest_sha256)
1750
+ root.mkdir(parents=True, exist_ok=True)
1751
+ state_path = root / "workflow_state.json"
1752
+ state: Dict[str, Any] = {
1753
+ "formatVersion": 1,
1754
+ "nodeVersion": NODE_VERSION,
1755
+ "createdUtc": _utc_now(),
1756
+ "districtName": district,
1757
+ "manifestSha256": manifest_sha256,
1758
+ "sourceCommit": source_commit,
1759
+ "referenceCommit": reference_commit,
1760
+ "sourceRepoId": SOURCE_REPO_ID,
1761
+ "generatedRepoId": GENERATED_REPO_ID,
1762
+ "referenceRepoId": REFERENCE_REPO_ID,
1763
+ "workflowRoot": str(root.resolve()),
1764
+ "statePath": str(state_path.resolve()),
1765
+ "logicalBatchSetCount": int(manifest.get("batchSetCount") or 0),
1766
+ "totalImageCount": len(jobs),
1767
+ "completedCustomIds": [],
1768
+ "attemptCounts": {},
1769
+ "openAiBatchSequence": 0,
1770
+ "activeBatch": None,
1771
+ "resultHistory": [],
1772
+ "failureHistory": [],
1773
+ }
1774
+ _write_json(root / "uploaded_navigation_manifest.json", manifest)
1775
+ _write_json(state_path, state)
1776
+ return state
1777
+
1778
+
1779
+ def _load_or_initialize_state(
1780
+ manifest: Mapping[str, Any],
1781
+ jobs: Sequence[Mapping[str, Any]],
1782
+ district: str,
1783
+ source_commit: str,
1784
+ reference_commit: str,
1785
+ preview_json: Path,
1786
+ ) -> Dict[str, Any]:
1787
+ root = preview_json.parent
1788
+ state_path = root / "workflow_state.json"
1789
+ preview_data = _read_json(preview_json)
1790
+ manifest_sha256 = str(preview_data["manifestSha256"])
1791
+ if state_path.is_file():
1792
+ state = _read_json(state_path)
1793
+ if str(state.get("manifestSha256") or "") != manifest_sha256:
1794
+ raise ValueError("Saved workflow identity does not match this manifest.")
1795
+ has_started = bool(
1796
+ state.get("activeBatch")
1797
+ or state.get("completedCustomIds")
1798
+ or state.get("attemptCounts")
1799
+ or int(state.get("openAiBatchSequence") or 0) > 0
1800
+ )
1801
+ if not has_started and not isinstance(state.get("lastJsonlOnly"), Mapping):
1802
+ # Refresh an untouched state, but keep commits pinned after a JSONL
1803
+ # export. Uploading the artifact advances the source repository; if
1804
+ # that new commit were substituted into Image_A URLs, an otherwise
1805
+ # identical second export would produce different JSONL and would no
1806
+ # longer match the file the user inspected.
1807
+ state["sourceCommit"] = source_commit
1808
+ state["referenceCommit"] = reference_commit
1809
+ state["nodeVersion"] = NODE_VERSION
1810
+ _write_json(state_path, state)
1811
+ return state
1812
+ return _initialize_state(
1813
+ manifest,
1814
+ district,
1815
+ source_commit,
1816
+ reference_commit,
1817
+ manifest_sha256,
1818
+ jobs,
1819
+ )
1820
+
1821
+
1822
+ def _next_incomplete_set(
1823
+ jobs: Sequence[Mapping[str, Any]],
1824
+ completed_ids: Sequence[str],
1825
+ ) -> Tuple[int, List[Mapping[str, Any]]]:
1826
+ completed = set(str(value) for value in completed_ids)
1827
+ incomplete = [job for job in jobs if str(job["customId"]) not in completed]
1828
+ if not incomplete:
1829
+ return 0, []
1830
+ set_number = min(int(job["_batchSetNumber"]) for job in incomplete)
1831
+ return (
1832
+ set_number,
1833
+ [job for job in incomplete if int(job["_batchSetNumber"]) == set_number],
1834
+ )
1835
+
1836
+
1837
+ def _prepare(
1838
+ district_name: str,
1839
+ hf_key: str,
1840
+ ) -> Tuple[
1841
+ str,
1842
+ Optional[str],
1843
+ Dict[str, Any],
1844
+ List[Dict[str, Any]],
1845
+ str,
1846
+ str,
1847
+ Path,
1848
+ Path,
1849
+ str,
1850
+ ]:
1851
+ district = _safe_district_name(district_name)
1852
+ hf_token = _optional_secret(hf_key)
1853
+ manifest, source_commit, manifest_sha256, _ = _download_manifest(
1854
+ district, hf_token
1855
+ )
1856
+ jobs = _validate_manifest(manifest, district)
1857
+ _, HfApi, _ = _load_hf_tools()
1858
+ reference_commit = _repo_commit(
1859
+ HfApi(token=hf_token), REFERENCE_REPO_ID, hf_token
1860
+ )
1861
+ preview_json, preview_text, all_prompts = _write_prompt_preview(
1862
+ manifest,
1863
+ jobs,
1864
+ district,
1865
+ source_commit,
1866
+ reference_commit,
1867
+ manifest_sha256,
1868
+ )
1869
+ return (
1870
+ district,
1871
+ hf_token,
1872
+ manifest,
1873
+ jobs,
1874
+ source_commit,
1875
+ reference_commit,
1876
+ preview_json,
1877
+ preview_text,
1878
+ all_prompts,
1879
+ )
1880
+
1881
+
1882
+ def run_city_responses_batch(
1883
+ action: str,
1884
+ district_name: str,
1885
+ hf_key: str,
1886
+ openai_api_key: str,
1887
+ maximum_attempts: int,
1888
+ only_create_jsonl: bool = False,
1889
+ ) -> Tuple[str, str, int, int, int, str, str]:
1890
+ (
1891
+ district,
1892
+ optional_hf_token,
1893
+ manifest,
1894
+ jobs,
1895
+ source_commit,
1896
+ reference_commit,
1897
+ preview_json,
1898
+ preview_text,
1899
+ all_prompts,
1900
+ ) = _prepare(district_name, hf_key)
1901
+ total = len(jobs)
1902
+ if action not in {ACTION_PREVIEW, ACTION_RUN}:
1903
+ raise ValueError(f"Unknown ACTION {action!r}.")
1904
+ maximum_attempts = int(maximum_attempts)
1905
+ if maximum_attempts < 1 or maximum_attempts > 50:
1906
+ raise ValueError("MAX_ATTEMPTS must be between 1 and 50.")
1907
+ if action == ACTION_PREVIEW and not only_create_jsonl:
1908
+ return (
1909
+ f"Preview only. Validated {total} prompts in "
1910
+ f"{manifest['batchSetCount']} logical Batch sets. No OpenAI file or "
1911
+ f"Batch was created. Read {preview_text.resolve()} before changing "
1912
+ "ACTION to RUN_OR_CONTINUE.",
1913
+ "",
1914
+ 0,
1915
+ 0,
1916
+ total,
1917
+ str(preview_text.resolve()),
1918
+ all_prompts,
1919
+ )
1920
+ hf_token = _required_secret(hf_key, "Hugging Face token")
1921
+ state = _load_or_initialize_state(
1922
+ manifest,
1923
+ jobs,
1924
+ district,
1925
+ source_commit,
1926
+ reference_commit,
1927
+ preview_json,
1928
+ )
1929
+
1930
+ if only_create_jsonl:
1931
+ completed_count = len(state.get("completedCustomIds") or [])
1932
+ active_value = state.get("activeBatch")
1933
+ if isinstance(active_value, Mapping):
1934
+ jsonl_path = Path(str(active_value.get("jsonlPath") or ""))
1935
+ if not jsonl_path.is_file():
1936
+ raise FileNotFoundError(
1937
+ "The active Batch JSONL is missing locally: " + str(jsonl_path)
1938
+ )
1939
+ set_number = int(active_value.get("batchSetNumber") or 0)
1940
+ url, commit_oid, path_in_repo = _upload_jsonl_artifact(
1941
+ jsonl_path,
1942
+ district,
1943
+ set_number,
1944
+ str(state["manifestSha256"]),
1945
+ hf_token,
1946
+ )
1947
+ state["lastJsonlOnly"] = {
1948
+ "createdUtc": _utc_now(),
1949
+ "batchSetNumber": set_number,
1950
+ "requestCount": int(active_value.get("requestCount") or 0),
1951
+ "localPath": str(jsonl_path.resolve()),
1952
+ "huggingFaceRepoPath": path_in_repo,
1953
+ "huggingFaceCommit": commit_oid,
1954
+ "huggingFaceUrl": url,
1955
+ "source": "existing_active_batch",
1956
+ }
1957
+ _write_json(Path(str(state["statePath"])), state)
1958
+ return (
1959
+ "Only_Create_Jsonl is ON. Uploaded the exact JSONL already used "
1960
+ "by active Batch "
1961
+ + str(active_value.get("batchId") or "")
1962
+ + ". No OpenAI API call was made. Image_A=Unity screenshot; "
1963
+ "Image_B=Depth; Image_C=Segmentation; Image_D=Lineart; Image_E "
1964
+ "and later=published generated continuity dependencies. "
1965
+ "Hugging Face JSONL: "
1966
+ + url,
1967
+ str(active_value.get("batchId") or ""),
1968
+ set_number,
1969
+ completed_count,
1970
+ total,
1971
+ str(preview_text.resolve()),
1972
+ all_prompts,
1973
+ )
1974
+ set_number, ready_jobs = _next_incomplete_set(
1975
+ jobs, list(state.get("completedCustomIds") or [])
1976
+ )
1977
+ if not ready_jobs:
1978
+ return (
1979
+ "Only_Create_Jsonl is ON, but all city images are already "
1980
+ "complete; there is no pending logical Batch set to export. "
1981
+ "No OpenAI API call was made.",
1982
+ "",
1983
+ int(manifest["batchSetCount"]),
1984
+ total,
1985
+ total,
1986
+ str(preview_text.resolve()),
1987
+ all_prompts,
1988
+ )
1989
+ status, set_number, local_jsonl, _url = _create_jsonl_only(
1990
+ state,
1991
+ ready_jobs,
1992
+ district,
1993
+ hf_token,
1994
+ maximum_attempts,
1995
+ )
1996
+ return (
1997
+ status + " Local JSONL: " + local_jsonl,
1998
+ "",
1999
+ set_number,
2000
+ completed_count,
2001
+ total,
2002
+ str(preview_text.resolve()),
2003
+ all_prompts,
2004
+ )
2005
+
2006
+ openai_key = _required_secret(openai_api_key, "OpenAI API key")
2007
+
2008
+ progress_note = ""
2009
+ active_value = state.get("activeBatch")
2010
+ if isinstance(active_value, Mapping):
2011
+ batch_id = str(active_value.get("batchId") or "")
2012
+ batch = _batch_status(batch_id, openai_key)
2013
+ if str(batch.get("id") or "") != batch_id:
2014
+ raise ValueError("OpenAI returned a different Batch identity.")
2015
+ if str(batch.get("endpoint") or "") != OPENAI_BATCH_ENDPOINT:
2016
+ raise ValueError("Saved OpenAI Batch does not target /v1/responses.")
2017
+ if str(batch.get("input_file_id") or "") != str(
2018
+ active_value.get("inputFileId") or ""
2019
+ ):
2020
+ raise ValueError("OpenAI returned a different Batch input file identity.")
2021
+ metadata = batch.get("metadata")
2022
+ if (
2023
+ not isinstance(metadata, Mapping)
2024
+ or str(metadata.get("purpose") or "") != PURPOSE
2025
+ or str(metadata.get("batch_set") or "")
2026
+ != str(active_value.get("batchSetNumber") or "")
2027
+ ):
2028
+ raise ValueError("OpenAI Batch metadata does not identify this city wave.")
2029
+ status = str(batch.get("status") or "unknown")
2030
+ total_requests, completed_requests, failed_requests = _request_counts(batch)
2031
+ if status in _WAITING_BATCH_STATUSES:
2032
+ return (
2033
+ f"Waiting for Responses Batch {batch_id}: status={status}, "
2034
+ f"total={total_requests}, completed={completed_requests}, "
2035
+ f"failed={failed_requests}. Logical Batch set "
2036
+ f"{active_value['batchSetNumber']} contains "
2037
+ f"{active_value['requestCount']} request(s).",
2038
+ batch_id,
2039
+ int(active_value["batchSetNumber"]),
2040
+ len(state.get("completedCustomIds") or []),
2041
+ total,
2042
+ str(preview_text.resolve()),
2043
+ all_prompts,
2044
+ )
2045
+ if status not in _TERMINAL_BATCH_STATUSES:
2046
+ raise ValueError(f"Unknown OpenAI Batch status {status!r}.")
2047
+ success_count, failure_count, commit_oid = _process_terminal_batch(
2048
+ state,
2049
+ batch,
2050
+ hf_token,
2051
+ openai_key,
2052
+ )
2053
+ progress_note = (
2054
+ f"Processed Batch {batch_id} ({status}): published {success_count}, "
2055
+ f"will retry {failure_count}."
2056
+ + (f" Hugging Face commit {commit_oid}." if commit_oid else "")
2057
+ + " "
2058
+ )
2059
+
2060
+ completed_ids = list(state.get("completedCustomIds") or [])
2061
+ next_set_number, ready_jobs = _next_incomplete_set(jobs, completed_ids)
2062
+ if not ready_jobs:
2063
+ state["completedUtc"] = str(state.get("completedUtc") or _utc_now())
2064
+ _write_json(Path(str(state["statePath"])), state)
2065
+ return (
2066
+ progress_note
2067
+ + f"All done. Published all {total} city images across "
2068
+ f"{manifest['batchSetCount']} logical Batch sets.",
2069
+ str(state.get("lastProcessedBatchId") or ""),
2070
+ int(manifest["batchSetCount"]),
2071
+ total,
2072
+ total,
2073
+ str(preview_text.resolve()),
2074
+ all_prompts,
2075
+ )
2076
+
2077
+ start_status, batch_id, started_set, artifact = _start_wave(
2078
+ state,
2079
+ ready_jobs,
2080
+ district,
2081
+ hf_token,
2082
+ openai_key,
2083
+ maximum_attempts,
2084
+ )
2085
+ return (
2086
+ progress_note
2087
+ + start_status
2088
+ + f" Progress: {len(completed_ids)}/{total}. JSONL: {artifact}",
2089
+ batch_id,
2090
+ started_set,
2091
+ len(completed_ids),
2092
+ total,
2093
+ str(preview_text.resolve()),
2094
+ all_prompts,
2095
+ )
2096
+
2097
+
2098
+ class GPT_TO_HF_CityResponsesBatch:
2099
+ @classmethod
2100
+ def INPUT_TYPES(cls):
2101
+ return {
2102
+ "required": {
2103
+ "ACTION": (
2104
+ [ACTION_PREVIEW, ACTION_RUN],
2105
+ {"default": ACTION_PREVIEW},
2106
+ ),
2107
+ "Only_Create_Jsonl": (
2108
+ "BOOLEAN",
2109
+ {"default": False},
2110
+ ),
2111
+ "DISTRICT_NAME": (
2112
+ "STRING",
2113
+ {"default": "District_19_I", "multiline": False},
2114
+ ),
2115
+ "hf_key": (
2116
+ "STRING",
2117
+ {"default": "hf_xxx", "multiline": False},
2118
+ ),
2119
+ "openAi_key": (
2120
+ "STRING",
2121
+ {"default": "sk-xxx", "multiline": False},
2122
+ ),
2123
+ "MAX_ATTEMPTS": (
2124
+ "INT",
2125
+ {"default": 8, "min": 1, "max": 50, "step": 1},
2126
+ ),
2127
+ }
2128
+ }
2129
+
2130
+ RETURN_TYPES = ("STRING", "STRING", "INT", "INT", "INT", "STRING", "STRING")
2131
+ RETURN_NAMES = (
2132
+ "status",
2133
+ "batch_id",
2134
+ "logical_batch_set",
2135
+ "completed_images",
2136
+ "total_images",
2137
+ "prompt_preview_path",
2138
+ "all_prompts",
2139
+ )
2140
+ FUNCTION = "run"
2141
+ CATEGORY = "Salia/OpenAI Batch"
2142
+ OUTPUT_NODE = True
2143
+
2144
+ @classmethod
2145
+ def IS_CHANGED(cls, **kwargs):
2146
+ return float("nan")
2147
+
2148
+ def run(
2149
+ self,
2150
+ ACTION: str,
2151
+ DISTRICT_NAME: str,
2152
+ hf_key: str,
2153
+ openAi_key: str,
2154
+ MAX_ATTEMPTS: int,
2155
+ Only_Create_Jsonl: bool,
2156
+ ):
2157
+ return run_city_responses_batch(
2158
+ ACTION,
2159
+ DISTRICT_NAME,
2160
+ hf_key,
2161
+ openAi_key,
2162
+ MAX_ATTEMPTS,
2163
+ Only_Create_Jsonl,
2164
+ )
2165
+
2166
+
2167
+ NODE_CLASS_MAPPINGS = {
2168
+ "GPT_TO_HF_CityResponsesBatch": GPT_TO_HF_CityResponsesBatch,
2169
+ }
2170
+
2171
+ NODE_DISPLAY_NAME_MAPPINGS = {
2172
+ "GPT_TO_HF_CityResponsesBatch": (
2173
+ "GPT TO HF — City Responses Batch Sets (Preview First)"
2174
+ ),
2175
+ }