Upload GPT_TO_HF_CityResponsesBatch.py
Browse files- GPT_TO_HF_CityResponsesBatch.py +2175 -0
GPT_TO_HF_CityResponsesBatch.py
ADDED
|
@@ -0,0 +1,2175 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import base64
|
| 4 |
+
import hashlib
|
| 5 |
+
import json
|
| 6 |
+
import os
|
| 7 |
+
import re
|
| 8 |
+
import secrets
|
| 9 |
+
import shutil
|
| 10 |
+
import urllib.error
|
| 11 |
+
import urllib.parse
|
| 12 |
+
import urllib.request
|
| 13 |
+
from datetime import datetime, timezone
|
| 14 |
+
from io import BytesIO
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple
|
| 17 |
+
|
| 18 |
+
try:
|
| 19 |
+
import folder_paths
|
| 20 |
+
except ImportError: # Allows offline tests outside ComfyUI.
|
| 21 |
+
folder_paths = None
|
| 22 |
+
|
| 23 |
+
try:
|
| 24 |
+
from PIL import Image
|
| 25 |
+
except ImportError: # Raised with a useful message only when an image is decoded.
|
| 26 |
+
Image = None
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
NODE_VERSION = "2.1.0"
|
| 30 |
+
PURPOSE = "salia-city-responses-batch-waves"
|
| 31 |
+
OPENAI_BATCH_ENDPOINT = "/v1/responses"
|
| 32 |
+
OPENAI_MODEL = "gpt-5.6-sol"
|
| 33 |
+
OPENAI_IMAGE_MODEL = "gpt-image-2.5-sunburst"
|
| 34 |
+
OPENAI_REASONING: Dict[str, str] = {
|
| 35 |
+
"effort": "medium",
|
| 36 |
+
"mode": "standard",
|
| 37 |
+
"summary": "auto",
|
| 38 |
+
}
|
| 39 |
+
RESPONSE_MODERATION: Dict[str, Any] = {
|
| 40 |
+
"model": "omni-moderation-latest",
|
| 41 |
+
"policy": {
|
| 42 |
+
"input": {"mode": "score"},
|
| 43 |
+
"output": {"mode": "score"},
|
| 44 |
+
},
|
| 45 |
+
}
|
| 46 |
+
OPENAI_IMAGE_TOOL: Dict[str, Any] = {
|
| 47 |
+
"type": "image_generation",
|
| 48 |
+
"model": OPENAI_IMAGE_MODEL,
|
| 49 |
+
"action": "generate",
|
| 50 |
+
"size": "1536x1024",
|
| 51 |
+
"quality": "auto",
|
| 52 |
+
"output_format": "png",
|
| 53 |
+
"background": "opaque",
|
| 54 |
+
"moderation": "low",
|
| 55 |
+
"partial_images": 0,
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
SOURCE_REPO_ID = "saliacoel/maps_imgs"
|
| 59 |
+
GENERATED_REPO_ID = "saliacoel/map_imgs_gpt"
|
| 60 |
+
REFERENCE_REPO_ID = "saliacoel/x"
|
| 61 |
+
REPO_TYPE = "model"
|
| 62 |
+
REPO_REVISION = "main"
|
| 63 |
+
MANIFEST_FILE_NAME = "gpt_navigation_manifest.json"
|
| 64 |
+
STATE_DIRECTORY_NAME = "Salia_City_Responses_Batch"
|
| 65 |
+
EXPECTED_MANIFEST_VERSION = 5
|
| 66 |
+
EXPECTED_BATCH_PLAN_VERSION = 3
|
| 67 |
+
EXPECTED_BATCH_STRATEGY = (
|
| 68 |
+
"entrance_fronts_first_scene_tree_with_deferred_cycles_v3"
|
| 69 |
+
)
|
| 70 |
+
EXPECTED_PROMPT_VERSION = "city_responses_continuity_prompt_v2"
|
| 71 |
+
EXPECTED_REFERENCE_TYPES = (
|
| 72 |
+
"4_way_Junction",
|
| 73 |
+
"Left_Forward",
|
| 74 |
+
"Forward_Right",
|
| 75 |
+
"Left_Right",
|
| 76 |
+
"Forward_Only",
|
| 77 |
+
"Left_Only",
|
| 78 |
+
"Right_Only",
|
| 79 |
+
"Dead_End",
|
| 80 |
+
"straight_door",
|
| 81 |
+
"door_Left",
|
| 82 |
+
"door_Right",
|
| 83 |
+
"station_gate_Front_Direct",
|
| 84 |
+
"station_gate_End_Left",
|
| 85 |
+
"station_gate_End_Right",
|
| 86 |
+
"station_gate_Forward_Left",
|
| 87 |
+
"station_gate_Forward_Right",
|
| 88 |
+
"station_gate_Distant_Front",
|
| 89 |
+
"station_gate_Distant_Left",
|
| 90 |
+
"station_gate_Distant_Right",
|
| 91 |
+
)
|
| 92 |
+
|
| 93 |
+
ACTION_PREVIEW = "PREVIEW_PROMPTS_ONLY"
|
| 94 |
+
ACTION_RUN = "RUN_OR_CONTINUE"
|
| 95 |
+
_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
| 96 |
+
_SAFE_BATCH_ID = re.compile(r"^batch_[A-Za-z0-9_-]{1,200}$")
|
| 97 |
+
_SAFE_FILE_ID = re.compile(r"^file-[A-Za-z0-9_-]{1,200}$")
|
| 98 |
+
_TERMINAL_BATCH_STATUSES = {"completed", "failed", "expired", "cancelled"}
|
| 99 |
+
_WAITING_BATCH_STATUSES = {
|
| 100 |
+
"validating",
|
| 101 |
+
"in_progress",
|
| 102 |
+
"finalizing",
|
| 103 |
+
"cancelling",
|
| 104 |
+
}
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def _utc_now() -> str:
|
| 108 |
+
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def _output_root() -> Path:
|
| 112 |
+
if folder_paths is not None:
|
| 113 |
+
return Path(folder_paths.get_output_directory())
|
| 114 |
+
return Path.cwd() / "output"
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def _optional_secret(value: str) -> Optional[str]:
|
| 118 |
+
cleaned = str(value or "").strip()
|
| 119 |
+
if cleaned.lower() in {"", "none", "xxx", "hf_xxx", "sk_xxx", "sk-xxx"}:
|
| 120 |
+
return None
|
| 121 |
+
return cleaned
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def _required_secret(value: str, label: str) -> str:
|
| 125 |
+
cleaned = _optional_secret(value)
|
| 126 |
+
if not cleaned:
|
| 127 |
+
raise ValueError(f"A real {label} is required for RUN_OR_CONTINUE.")
|
| 128 |
+
return cleaned
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def _safe_district_name(value: str) -> str:
|
| 132 |
+
name = str(value or "").strip().strip("\"'")
|
| 133 |
+
if (
|
| 134 |
+
not name
|
| 135 |
+
or name in {".", ".."}
|
| 136 |
+
or "/" in name
|
| 137 |
+
or "\\" in name
|
| 138 |
+
or len(name) > 180
|
| 139 |
+
or any(ord(character) < 32 or ord(character) == 127 for character in name)
|
| 140 |
+
):
|
| 141 |
+
raise ValueError("DISTRICT_NAME must be one safe Hugging Face folder name.")
|
| 142 |
+
return name
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def _safe_file_name(value: Any, label: str) -> str:
|
| 146 |
+
name = str(value or "").strip()
|
| 147 |
+
if (
|
| 148 |
+
not name
|
| 149 |
+
or name in {".", ".."}
|
| 150 |
+
or "/" in name
|
| 151 |
+
or "\\" in name
|
| 152 |
+
or len(name) > 240
|
| 153 |
+
or any(ord(character) < 32 or ord(character) == 127 for character in name)
|
| 154 |
+
):
|
| 155 |
+
raise ValueError(f"Unsafe {label}: {name!r}.")
|
| 156 |
+
return name
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def _write_json(path: Path, value: Any) -> None:
|
| 160 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 161 |
+
temporary = path.with_suffix(path.suffix + ".part")
|
| 162 |
+
temporary.write_text(
|
| 163 |
+
json.dumps(value, ensure_ascii=False, indent=2) + "\n",
|
| 164 |
+
encoding="utf-8",
|
| 165 |
+
)
|
| 166 |
+
os.replace(temporary, path)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def _read_json(path: Path) -> Dict[str, Any]:
|
| 170 |
+
parsed = json.loads(path.read_text(encoding="utf-8-sig"))
|
| 171 |
+
if not isinstance(parsed, dict):
|
| 172 |
+
raise ValueError(f"Expected a JSON object in {path}.")
|
| 173 |
+
return parsed
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def _write_jsonl(path: Path, values: Sequence[Mapping[str, Any]]) -> None:
|
| 177 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 178 |
+
temporary = path.with_suffix(path.suffix + ".part")
|
| 179 |
+
with temporary.open("w", encoding="utf-8", newline="\n") as handle:
|
| 180 |
+
for value in values:
|
| 181 |
+
handle.write(
|
| 182 |
+
json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
| 183 |
+
+ "\n"
|
| 184 |
+
)
|
| 185 |
+
os.replace(temporary, path)
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def _load_hf_tools():
|
| 189 |
+
try:
|
| 190 |
+
from huggingface_hub import CommitOperationAdd, HfApi, hf_hub_download
|
| 191 |
+
except ImportError as exception:
|
| 192 |
+
raise RuntimeError(
|
| 193 |
+
"huggingface_hub is missing. Install it with ComfyUI's Python and "
|
| 194 |
+
"restart ComfyUI."
|
| 195 |
+
) from exception
|
| 196 |
+
return CommitOperationAdd, HfApi, hf_hub_download
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
def _repo_commit(api: Any, repo_id: str, token: Optional[str]) -> str:
|
| 200 |
+
info = api.repo_info(
|
| 201 |
+
repo_id=repo_id,
|
| 202 |
+
repo_type=REPO_TYPE,
|
| 203 |
+
revision=REPO_REVISION,
|
| 204 |
+
token=token,
|
| 205 |
+
)
|
| 206 |
+
commit = str(getattr(info, "sha", "") or "").strip()
|
| 207 |
+
if not commit:
|
| 208 |
+
raise ValueError(f"Could not resolve the current commit of {repo_id}.")
|
| 209 |
+
return commit
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
def _hf_url(repo_id: str, revision: str, path_in_repo: str) -> str:
|
| 213 |
+
return (
|
| 214 |
+
"https://huggingface.co/"
|
| 215 |
+
+ repo_id
|
| 216 |
+
+ "/resolve/"
|
| 217 |
+
+ urllib.parse.quote(str(revision), safe="")
|
| 218 |
+
+ "/"
|
| 219 |
+
+ urllib.parse.quote(str(path_in_repo), safe="/")
|
| 220 |
+
)
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def _hf_blob_url(repo_id: str, revision: str, path_in_repo: str) -> str:
|
| 224 |
+
return (
|
| 225 |
+
"https://huggingface.co/"
|
| 226 |
+
+ repo_id
|
| 227 |
+
+ "/blob/"
|
| 228 |
+
+ urllib.parse.quote(str(revision), safe="")
|
| 229 |
+
+ "/"
|
| 230 |
+
+ urllib.parse.quote(str(path_in_repo), safe="/")
|
| 231 |
+
)
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def _repo_path(district: str, file_name: str) -> str:
|
| 235 |
+
return district + "/" + _safe_file_name(file_name, "repository filename")
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
def _download_manifest(
|
| 239 |
+
district: str,
|
| 240 |
+
hf_token: Optional[str],
|
| 241 |
+
) -> Tuple[Dict[str, Any], str, str, Path]:
|
| 242 |
+
_, HfApi, hf_hub_download = _load_hf_tools()
|
| 243 |
+
api = HfApi(token=hf_token)
|
| 244 |
+
source_commit = _repo_commit(api, SOURCE_REPO_ID, hf_token)
|
| 245 |
+
path_in_repo = _repo_path(district, MANIFEST_FILE_NAME)
|
| 246 |
+
local = Path(
|
| 247 |
+
hf_hub_download(
|
| 248 |
+
repo_id=SOURCE_REPO_ID,
|
| 249 |
+
filename=path_in_repo,
|
| 250 |
+
repo_type=REPO_TYPE,
|
| 251 |
+
revision=source_commit,
|
| 252 |
+
token=hf_token,
|
| 253 |
+
force_download=True,
|
| 254 |
+
)
|
| 255 |
+
)
|
| 256 |
+
raw = local.read_bytes()
|
| 257 |
+
manifest_sha256 = hashlib.sha256(raw).hexdigest()
|
| 258 |
+
parsed = json.loads(raw.decode("utf-8-sig"))
|
| 259 |
+
if not isinstance(parsed, dict):
|
| 260 |
+
raise ValueError("The downloaded navigation manifest is not a JSON object.")
|
| 261 |
+
return parsed, source_commit, manifest_sha256, local
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
def _image_label(zero_based_index: int) -> str:
|
| 265 |
+
value = int(zero_based_index) + 1
|
| 266 |
+
if value <= 0:
|
| 267 |
+
raise ValueError("Image label index must be non-negative.")
|
| 268 |
+
letters = ""
|
| 269 |
+
while value:
|
| 270 |
+
value, remainder = divmod(value - 1, 26)
|
| 271 |
+
letters = chr(ord("A") + remainder) + letters
|
| 272 |
+
return "Image_" + letters
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
def _segmentation_file_name(reference_type: str) -> str:
|
| 276 |
+
return (
|
| 277 |
+
"4_way_Junction_Segmentation.png"
|
| 278 |
+
if reference_type == "4_way_Junction"
|
| 279 |
+
else reference_type + "_Segment.png"
|
| 280 |
+
)
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
def _require_mapping(value: Any, label: str) -> Mapping[str, Any]:
|
| 284 |
+
if not isinstance(value, Mapping):
|
| 285 |
+
raise ValueError(f"{label} must be a JSON object.")
|
| 286 |
+
return value
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
def _require_list(value: Any, label: str) -> List[Any]:
|
| 290 |
+
if not isinstance(value, list):
|
| 291 |
+
raise ValueError(f"{label} must be a JSON array.")
|
| 292 |
+
return value
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
def _validate_manifest(
|
| 296 |
+
manifest: Mapping[str, Any],
|
| 297 |
+
requested_district: str,
|
| 298 |
+
) -> List[Dict[str, Any]]:
|
| 299 |
+
if int(manifest.get("formatVersion") or 0) != EXPECTED_MANIFEST_VERSION:
|
| 300 |
+
raise ValueError(
|
| 301 |
+
f"Unsupported navigation manifest formatVersion; expected "
|
| 302 |
+
f"{EXPECTED_MANIFEST_VERSION}. Recapture with Unity V6.0.22's v5 exporter."
|
| 303 |
+
)
|
| 304 |
+
if int(manifest.get("batchPlanVersion") or 0) != EXPECTED_BATCH_PLAN_VERSION:
|
| 305 |
+
raise ValueError(
|
| 306 |
+
f"Unsupported batchPlanVersion; expected {EXPECTED_BATCH_PLAN_VERSION}."
|
| 307 |
+
)
|
| 308 |
+
if str(manifest.get("districtName") or "") != requested_district:
|
| 309 |
+
raise ValueError(
|
| 310 |
+
"The manifest districtName does not equal the requested Hugging Face "
|
| 311 |
+
"folder name."
|
| 312 |
+
)
|
| 313 |
+
expected_top_level = {
|
| 314 |
+
"sourceImageRepoId": SOURCE_REPO_ID,
|
| 315 |
+
"generatedImageRepoId": GENERATED_REPO_ID,
|
| 316 |
+
"referenceImageRepoId": REFERENCE_REPO_ID,
|
| 317 |
+
"repoRevision": REPO_REVISION,
|
| 318 |
+
"batchStrategy": EXPECTED_BATCH_STRATEGY,
|
| 319 |
+
"promptVersion": EXPECTED_PROMPT_VERSION,
|
| 320 |
+
}
|
| 321 |
+
for key, expected in expected_top_level.items():
|
| 322 |
+
actual = str(manifest.get(key) or "")
|
| 323 |
+
if actual != expected:
|
| 324 |
+
raise ValueError(f"Manifest {key} is {actual!r}; expected {expected!r}.")
|
| 325 |
+
|
| 326 |
+
catalog = _require_list(manifest.get("referenceCatalog"), "referenceCatalog")
|
| 327 |
+
catalog_by_type: Dict[str, Mapping[str, Any]] = {}
|
| 328 |
+
for index, raw_entry in enumerate(catalog):
|
| 329 |
+
entry = _require_mapping(raw_entry, f"referenceCatalog[{index}]")
|
| 330 |
+
reference_type = str(entry.get("referenceType") or "")
|
| 331 |
+
if reference_type in catalog_by_type:
|
| 332 |
+
raise ValueError(f"Duplicate reference type {reference_type!r}.")
|
| 333 |
+
catalog_by_type[reference_type] = entry
|
| 334 |
+
if tuple(catalog_by_type.keys()) != EXPECTED_REFERENCE_TYPES:
|
| 335 |
+
raise ValueError(
|
| 336 |
+
f"referenceCatalog must contain the {len(EXPECTED_REFERENCE_TYPES)} "
|
| 337 |
+
"canonical types in canonical order."
|
| 338 |
+
)
|
| 339 |
+
for reference_type in EXPECTED_REFERENCE_TYPES:
|
| 340 |
+
entry = catalog_by_type[reference_type]
|
| 341 |
+
expected_names = {
|
| 342 |
+
"depthFileName": reference_type + "_Depth.png",
|
| 343 |
+
"segmentationFileName": _segmentation_file_name(reference_type),
|
| 344 |
+
"lineartFileName": reference_type + "_Lineart.png",
|
| 345 |
+
}
|
| 346 |
+
for key, expected in expected_names.items():
|
| 347 |
+
if str(entry.get(key) or "") != expected:
|
| 348 |
+
raise ValueError(
|
| 349 |
+
f"Reference {reference_type!r} has invalid {key}; expected "
|
| 350 |
+
f"{expected!r}."
|
| 351 |
+
)
|
| 352 |
+
|
| 353 |
+
raw_sets = _require_list(manifest.get("batchSets"), "batchSets")
|
| 354 |
+
if int(manifest.get("batchSetCount") or -1) != len(raw_sets) or not raw_sets:
|
| 355 |
+
raise ValueError("batchSetCount does not match the non-empty batchSets array.")
|
| 356 |
+
|
| 357 |
+
jobs: List[Dict[str, Any]] = []
|
| 358 |
+
jobs_by_id: Dict[str, Dict[str, Any]] = {}
|
| 359 |
+
output_names = set()
|
| 360 |
+
set_ids = set()
|
| 361 |
+
declared_dependencies_by_set: Dict[int, List[int]] = {}
|
| 362 |
+
for set_index, raw_set in enumerate(raw_sets):
|
| 363 |
+
batch_set = _require_mapping(raw_set, f"batchSets[{set_index}]")
|
| 364 |
+
set_number = int(batch_set.get("batchSetNumber") or 0)
|
| 365 |
+
if set_number != set_index + 1:
|
| 366 |
+
raise ValueError("Batch-set numbers must be consecutive and one-based.")
|
| 367 |
+
set_id = str(batch_set.get("id") or "").strip()
|
| 368 |
+
if not set_id or set_id in set_ids:
|
| 369 |
+
raise ValueError(f"Missing or duplicate Batch-set id {set_id!r}.")
|
| 370 |
+
set_ids.add(set_id)
|
| 371 |
+
raw_depends_on = _require_list(
|
| 372 |
+
batch_set.get("dependsOnBatchSets"),
|
| 373 |
+
f"batchSets[{set_index}].dependsOnBatchSets",
|
| 374 |
+
)
|
| 375 |
+
depends_on = [int(value) for value in raw_depends_on]
|
| 376 |
+
if (
|
| 377 |
+
depends_on != sorted(set(depends_on))
|
| 378 |
+
or any(value < 1 or value >= set_number for value in depends_on)
|
| 379 |
+
):
|
| 380 |
+
raise ValueError(
|
| 381 |
+
f"Batch set {set_number} has invalid dependsOnBatchSets."
|
| 382 |
+
)
|
| 383 |
+
declared_dependencies_by_set[set_number] = depends_on
|
| 384 |
+
raw_images = _require_list(
|
| 385 |
+
batch_set.get("images"), f"batchSets[{set_index}].images"
|
| 386 |
+
)
|
| 387 |
+
if int(batch_set.get("imageCount") or -1) != len(raw_images) or not raw_images:
|
| 388 |
+
raise ValueError(f"Batch set {set_number} has an invalid imageCount.")
|
| 389 |
+
for image_index, raw_image in enumerate(raw_images):
|
| 390 |
+
image = dict(
|
| 391 |
+
_require_mapping(
|
| 392 |
+
raw_image,
|
| 393 |
+
f"batchSets[{set_index}].images[{image_index}]",
|
| 394 |
+
)
|
| 395 |
+
)
|
| 396 |
+
custom_id = str(image.get("customId") or "").strip()
|
| 397 |
+
if not custom_id or custom_id in jobs_by_id:
|
| 398 |
+
raise ValueError(f"Missing or duplicate image customId {custom_id!r}.")
|
| 399 |
+
if str(image.get("kind") or "").lower() == "bench" or "bench" in str(
|
| 400 |
+
image.get("topology") or ""
|
| 401 |
+
).lower():
|
| 402 |
+
raise ValueError("Bench images are forbidden in the city Batch plan.")
|
| 403 |
+
source_name = _safe_file_name(
|
| 404 |
+
image.get("sourceScreenshotFileName"), "Unity screenshot filename"
|
| 405 |
+
)
|
| 406 |
+
target_name = _safe_file_name(
|
| 407 |
+
image.get("targetGeneratedFileName"), "generated filename"
|
| 408 |
+
)
|
| 409 |
+
if not target_name.lower().endswith(".png") or target_name in output_names:
|
| 410 |
+
raise ValueError(f"Invalid or duplicate generated filename {target_name!r}.")
|
| 411 |
+
output_names.add(target_name)
|
| 412 |
+
reference_type = str(image.get("referenceType") or "")
|
| 413 |
+
if reference_type not in catalog_by_type:
|
| 414 |
+
raise ValueError(
|
| 415 |
+
f"Image {custom_id!r} uses unknown referenceType {reference_type!r}."
|
| 416 |
+
)
|
| 417 |
+
reference_images = _require_mapping(
|
| 418 |
+
image.get("referenceImages"), f"referenceImages for {custom_id}"
|
| 419 |
+
)
|
| 420 |
+
if str(reference_images.get("referenceType") or "") != reference_type:
|
| 421 |
+
raise ValueError(
|
| 422 |
+
f"Image {custom_id!r} referenceImages.referenceType does not "
|
| 423 |
+
"match referenceType."
|
| 424 |
+
)
|
| 425 |
+
catalog_entry = catalog_by_type[reference_type]
|
| 426 |
+
for key in ("depthFileName", "segmentationFileName", "lineartFileName"):
|
| 427 |
+
if str(reference_images.get(key) or "") != str(
|
| 428 |
+
catalog_entry.get(key) or ""
|
| 429 |
+
):
|
| 430 |
+
raise ValueError(
|
| 431 |
+
f"Image {custom_id!r} does not use the catalog's {key}."
|
| 432 |
+
)
|
| 433 |
+
dependencies = _require_list(
|
| 434 |
+
image.get("dependencies"), f"dependencies for {custom_id}"
|
| 435 |
+
)
|
| 436 |
+
if int(image.get("inputImageCount") or -1) != 4 + len(dependencies):
|
| 437 |
+
raise ValueError(f"Image {custom_id!r} has an invalid inputImageCount.")
|
| 438 |
+
if str(image.get("promptVersion") or "") != EXPECTED_PROMPT_VERSION:
|
| 439 |
+
raise ValueError(f"Image {custom_id!r} has the wrong promptVersion.")
|
| 440 |
+
# Do not normalize this string: Unity authored the complete request prompt,
|
| 441 |
+
# and the Batch request must receive it byte-for-byte as decoded JSON text.
|
| 442 |
+
prompt = str(image.get("prompt") or "")
|
| 443 |
+
if not prompt.strip():
|
| 444 |
+
raise ValueError(f"Image {custom_id!r} has an empty prompt.")
|
| 445 |
+
for required_text in (
|
| 446 |
+
"Image_A is the authoritative raw Unity render",
|
| 447 |
+
"Image_B is the",
|
| 448 |
+
"Image_C is the",
|
| 449 |
+
"Image_D is the",
|
| 450 |
+
"Do not swap left and right",
|
| 451 |
+
):
|
| 452 |
+
if required_text not in prompt:
|
| 453 |
+
raise ValueError(
|
| 454 |
+
f"Image {custom_id!r} prompt is missing {required_text!r}."
|
| 455 |
+
)
|
| 456 |
+
image["_batchSetNumber"] = set_number
|
| 457 |
+
image["_batchSetId"] = set_id
|
| 458 |
+
image["_sourceName"] = source_name
|
| 459 |
+
image["_targetName"] = target_name
|
| 460 |
+
image["_referenceImages"] = dict(reference_images)
|
| 461 |
+
image["_dependencies"] = [
|
| 462 |
+
dict(_require_mapping(dep, f"dependency for {custom_id}"))
|
| 463 |
+
for dep in dependencies
|
| 464 |
+
]
|
| 465 |
+
image["_prompt"] = prompt
|
| 466 |
+
jobs.append(image)
|
| 467 |
+
jobs_by_id[custom_id] = image
|
| 468 |
+
|
| 469 |
+
if int(manifest.get("batchImageCount") or -1) != len(jobs):
|
| 470 |
+
raise ValueError("batchImageCount does not match the Batch image total.")
|
| 471 |
+
|
| 472 |
+
actual_dependencies_by_set: Dict[int, set] = {
|
| 473 |
+
number: set() for number in declared_dependencies_by_set
|
| 474 |
+
}
|
| 475 |
+
|
| 476 |
+
for job in jobs:
|
| 477 |
+
custom_id = str(job["customId"])
|
| 478 |
+
dependency_ids = set()
|
| 479 |
+
for dependency_index, dependency in enumerate(job["_dependencies"]):
|
| 480 |
+
dependency_id = str(dependency.get("customId") or "")
|
| 481 |
+
if not dependency_id or dependency_id in dependency_ids:
|
| 482 |
+
raise ValueError(
|
| 483 |
+
f"Image {custom_id!r} has a missing or duplicate dependency."
|
| 484 |
+
)
|
| 485 |
+
dependency_ids.add(dependency_id)
|
| 486 |
+
source = jobs_by_id.get(dependency_id)
|
| 487 |
+
if source is None:
|
| 488 |
+
raise ValueError(
|
| 489 |
+
f"Image {custom_id!r} depends on unknown image {dependency_id!r}."
|
| 490 |
+
)
|
| 491 |
+
if int(source["_batchSetNumber"]) >= int(job["_batchSetNumber"]):
|
| 492 |
+
raise ValueError(
|
| 493 |
+
f"Image {custom_id!r} has a same-wave or forward dependency."
|
| 494 |
+
)
|
| 495 |
+
actual_dependencies_by_set[int(job["_batchSetNumber"])].add(
|
| 496 |
+
int(source["_batchSetNumber"])
|
| 497 |
+
)
|
| 498 |
+
expected_label = _image_label(4 + dependency_index)
|
| 499 |
+
if str(dependency.get("imageLabel") or "") != expected_label:
|
| 500 |
+
raise ValueError(
|
| 501 |
+
f"Image {custom_id!r} dependency {dependency_index} must be "
|
| 502 |
+
f"labeled {expected_label}."
|
| 503 |
+
)
|
| 504 |
+
if str(dependency.get("generatedFileName") or "") != str(
|
| 505 |
+
source["_targetName"]
|
| 506 |
+
):
|
| 507 |
+
raise ValueError(
|
| 508 |
+
f"Image {custom_id!r} has a mismatched dependency filename."
|
| 509 |
+
)
|
| 510 |
+
if str(dependency.get("locationId") or "") != str(
|
| 511 |
+
source.get("locationId") or ""
|
| 512 |
+
):
|
| 513 |
+
raise ValueError(
|
| 514 |
+
f"Image {custom_id!r} has a mismatched dependency locationId."
|
| 515 |
+
)
|
| 516 |
+
if str(dependency.get("sourceReferenceType") or "") != str(
|
| 517 |
+
source.get("referenceType") or ""
|
| 518 |
+
):
|
| 519 |
+
raise ValueError(
|
| 520 |
+
f"Image {custom_id!r} has a mismatched dependency "
|
| 521 |
+
"sourceReferenceType."
|
| 522 |
+
)
|
| 523 |
+
if not str(dependency.get("relationship") or "").strip():
|
| 524 |
+
raise ValueError(
|
| 525 |
+
f"Image {custom_id!r} has a dependency without a relationship."
|
| 526 |
+
)
|
| 527 |
+
movement = str(dependency.get("movementInstruction") or "").strip()
|
| 528 |
+
if not movement or movement not in str(job["_prompt"]):
|
| 529 |
+
raise ValueError(
|
| 530 |
+
f"Image {custom_id!r} is missing its exact movement instruction."
|
| 531 |
+
)
|
| 532 |
+
expected_primary = (
|
| 533 |
+
str(job["_dependencies"][0].get("customId") or "")
|
| 534 |
+
if job["_dependencies"]
|
| 535 |
+
else ""
|
| 536 |
+
)
|
| 537 |
+
if str(job.get("primaryContinuityCustomId") or "") != expected_primary:
|
| 538 |
+
raise ValueError(
|
| 539 |
+
f"Image {custom_id!r} has an invalid primaryContinuityCustomId."
|
| 540 |
+
)
|
| 541 |
+
if int(job["_batchSetNumber"]) > 1 and not job["_dependencies"]:
|
| 542 |
+
raise ValueError(
|
| 543 |
+
f"Image {custom_id!r} is outside Batch set 1 but has no dependency."
|
| 544 |
+
)
|
| 545 |
+
|
| 546 |
+
for set_number, declared in declared_dependencies_by_set.items():
|
| 547 |
+
actual = sorted(actual_dependencies_by_set[set_number])
|
| 548 |
+
if declared != actual:
|
| 549 |
+
raise ValueError(
|
| 550 |
+
f"Batch set {set_number} declares dependencies {declared}, but its "
|
| 551 |
+
f"images actually require {actual}."
|
| 552 |
+
)
|
| 553 |
+
|
| 554 |
+
actual_reference_counts: Dict[str, int] = {
|
| 555 |
+
reference_type: 0 for reference_type in EXPECTED_REFERENCE_TYPES
|
| 556 |
+
}
|
| 557 |
+
for job in jobs:
|
| 558 |
+
actual_reference_counts[str(job["referenceType"])] += 1
|
| 559 |
+
for reference_type, entry in catalog_by_type.items():
|
| 560 |
+
if int(entry.get("usedByImageCount") or 0) != actual_reference_counts[
|
| 561 |
+
reference_type
|
| 562 |
+
]:
|
| 563 |
+
raise ValueError(
|
| 564 |
+
f"Reference {reference_type!r} has an invalid usedByImageCount."
|
| 565 |
+
)
|
| 566 |
+
required_reference_types = sum(
|
| 567 |
+
1 for count in actual_reference_counts.values() if count > 0
|
| 568 |
+
)
|
| 569 |
+
expected_reference_counts = {
|
| 570 |
+
"referenceTypeCount": len(EXPECTED_REFERENCE_TYPES),
|
| 571 |
+
"referenceFileCount": len(EXPECTED_REFERENCE_TYPES) * 3,
|
| 572 |
+
"requiredReferenceTypeCount": required_reference_types,
|
| 573 |
+
"requiredReferenceFileCount": required_reference_types * 3,
|
| 574 |
+
}
|
| 575 |
+
for key, expected in expected_reference_counts.items():
|
| 576 |
+
if int(manifest.get(key) or -1) != expected:
|
| 577 |
+
raise ValueError(f"Manifest {key} is invalid; expected {expected}.")
|
| 578 |
+
|
| 579 |
+
raw_locations = _require_list(manifest.get("locations"), "locations")
|
| 580 |
+
if len(raw_locations) != len(jobs):
|
| 581 |
+
raise ValueError("locations must contain exactly one entry per Batch image.")
|
| 582 |
+
locations_by_custom_id: Dict[str, Mapping[str, Any]] = {}
|
| 583 |
+
for index, raw_location in enumerate(raw_locations):
|
| 584 |
+
location_entry = _require_mapping(raw_location, f"locations[{index}]")
|
| 585 |
+
custom_id = str(location_entry.get("customId") or "")
|
| 586 |
+
if not custom_id or custom_id in locations_by_custom_id:
|
| 587 |
+
raise ValueError(f"Missing or duplicate location customId {custom_id!r}.")
|
| 588 |
+
locations_by_custom_id[custom_id] = location_entry
|
| 589 |
+
if set(locations_by_custom_id) != set(jobs_by_id):
|
| 590 |
+
raise ValueError("locations and batchSets do not contain the same customIds.")
|
| 591 |
+
for custom_id, job in jobs_by_id.items():
|
| 592 |
+
location_entry = locations_by_custom_id[custom_id]
|
| 593 |
+
location = _require_mapping(
|
| 594 |
+
location_entry.get("location"), f"location object for {custom_id}"
|
| 595 |
+
)
|
| 596 |
+
if str(location.get("id") or "") != str(job.get("locationId") or ""):
|
| 597 |
+
raise ValueError(f"Location id mismatch for image {custom_id!r}.")
|
| 598 |
+
if str(location_entry.get("sourceScreenshotFileName") or "") != str(
|
| 599 |
+
job["_sourceName"]
|
| 600 |
+
):
|
| 601 |
+
raise ValueError(f"Screenshot filename mismatch for image {custom_id!r}.")
|
| 602 |
+
|
| 603 |
+
first_set = [job for job in jobs if int(job["_batchSetNumber"]) == 1]
|
| 604 |
+
if not first_set or any(job["_dependencies"] for job in first_set):
|
| 605 |
+
raise ValueError(
|
| 606 |
+
"Batch set 1 must contain dependency-free entrance identity seeds."
|
| 607 |
+
)
|
| 608 |
+
allowed_seed_types = {"straight_door", "station_gate_Front_Direct"}
|
| 609 |
+
if any(str(job.get("referenceType") or "") not in allowed_seed_types for job in first_set):
|
| 610 |
+
raise ValueError(
|
| 611 |
+
"Batch set 1 may contain only straight-door and station-front seeds."
|
| 612 |
+
)
|
| 613 |
+
if int(manifest.get("entranceSeedCount") or -1) != len(first_set):
|
| 614 |
+
raise ValueError("entranceSeedCount does not match Batch set 1.")
|
| 615 |
+
raw_seed_ids = _require_list(
|
| 616 |
+
manifest.get("entranceSeedCustomIds"), "entranceSeedCustomIds"
|
| 617 |
+
)
|
| 618 |
+
seed_ids = [str(value or "") for value in raw_seed_ids]
|
| 619 |
+
if len(seed_ids) != len(set(seed_ids)) or set(seed_ids) != {
|
| 620 |
+
str(job["customId"]) for job in first_set
|
| 621 |
+
}:
|
| 622 |
+
raise ValueError("entranceSeedCustomIds does not identify all Batch-set-1 seeds.")
|
| 623 |
+
root_id = str(manifest.get("batchRootCustomId") or "")
|
| 624 |
+
root = jobs_by_id.get(root_id)
|
| 625 |
+
if root is None or int(root["_batchSetNumber"]) != 1:
|
| 626 |
+
raise ValueError("batchRootCustomId must identify a Batch-set-1 entrance seed.")
|
| 627 |
+
if str(manifest.get("batchRootLocationId") or "") != str(
|
| 628 |
+
root.get("locationId") or ""
|
| 629 |
+
):
|
| 630 |
+
raise ValueError("batchRootLocationId does not match batchRootCustomId.")
|
| 631 |
+
if int(manifest.get("excludedBenchCount", -1)) != 0:
|
| 632 |
+
raise ValueError("The v5 city Batch manifest must exclude all benches.")
|
| 633 |
+
return jobs
|
| 634 |
+
|
| 635 |
+
|
| 636 |
+
def _workflow_root(district: str, manifest_sha256: str) -> Path:
|
| 637 |
+
district_key = re.sub(r"[^A-Za-z0-9._-]+", "_", district).strip("._")
|
| 638 |
+
return (
|
| 639 |
+
_output_root()
|
| 640 |
+
/ STATE_DIRECTORY_NAME
|
| 641 |
+
/ (district_key or "district")
|
| 642 |
+
/ manifest_sha256[:20]
|
| 643 |
+
)
|
| 644 |
+
|
| 645 |
+
|
| 646 |
+
def _preview_inputs(
|
| 647 |
+
manifest: Mapping[str, Any],
|
| 648 |
+
job: Mapping[str, Any],
|
| 649 |
+
district: str,
|
| 650 |
+
source_commit: str,
|
| 651 |
+
reference_commit: str,
|
| 652 |
+
generated_revision: str,
|
| 653 |
+
) -> List[Dict[str, str]]:
|
| 654 |
+
references = _require_mapping(job.get("_referenceImages"), "referenceImages")
|
| 655 |
+
inputs: List[Dict[str, str]] = [
|
| 656 |
+
{
|
| 657 |
+
"label": "Image_A",
|
| 658 |
+
"role": "authoritative Unity target view",
|
| 659 |
+
"repoId": SOURCE_REPO_ID,
|
| 660 |
+
"revision": source_commit,
|
| 661 |
+
"path": _repo_path(district, str(job["_sourceName"])),
|
| 662 |
+
},
|
| 663 |
+
{
|
| 664 |
+
"label": "Image_B",
|
| 665 |
+
"role": "Depth structural reference",
|
| 666 |
+
"repoId": REFERENCE_REPO_ID,
|
| 667 |
+
"revision": reference_commit,
|
| 668 |
+
"path": str(references["depthFileName"]),
|
| 669 |
+
},
|
| 670 |
+
{
|
| 671 |
+
"label": "Image_C",
|
| 672 |
+
"role": "Segmentation structural reference",
|
| 673 |
+
"repoId": REFERENCE_REPO_ID,
|
| 674 |
+
"revision": reference_commit,
|
| 675 |
+
"path": str(references["segmentationFileName"]),
|
| 676 |
+
},
|
| 677 |
+
{
|
| 678 |
+
"label": "Image_D",
|
| 679 |
+
"role": "Lineart structural reference",
|
| 680 |
+
"repoId": REFERENCE_REPO_ID,
|
| 681 |
+
"revision": reference_commit,
|
| 682 |
+
"path": str(references["lineartFileName"]),
|
| 683 |
+
},
|
| 684 |
+
]
|
| 685 |
+
for dependency in job["_dependencies"]:
|
| 686 |
+
inputs.append(
|
| 687 |
+
{
|
| 688 |
+
"label": str(dependency["imageLabel"]),
|
| 689 |
+
"role": "previously generated continuity view",
|
| 690 |
+
"repoId": GENERATED_REPO_ID,
|
| 691 |
+
"revision": generated_revision,
|
| 692 |
+
"path": _repo_path(district, str(dependency["generatedFileName"])),
|
| 693 |
+
}
|
| 694 |
+
)
|
| 695 |
+
for item in inputs:
|
| 696 |
+
item["url"] = _hf_url(item["repoId"], item["revision"], item["path"])
|
| 697 |
+
return inputs
|
| 698 |
+
|
| 699 |
+
|
| 700 |
+
def _write_prompt_preview(
|
| 701 |
+
manifest: Mapping[str, Any],
|
| 702 |
+
jobs: Sequence[Mapping[str, Any]],
|
| 703 |
+
district: str,
|
| 704 |
+
source_commit: str,
|
| 705 |
+
reference_commit: str,
|
| 706 |
+
manifest_sha256: str,
|
| 707 |
+
) -> Tuple[Path, Path, str]:
|
| 708 |
+
root = _workflow_root(district, manifest_sha256)
|
| 709 |
+
entries: List[Dict[str, Any]] = []
|
| 710 |
+
text_parts: List[str] = []
|
| 711 |
+
for job in jobs:
|
| 712 |
+
inputs = _preview_inputs(
|
| 713 |
+
manifest,
|
| 714 |
+
job,
|
| 715 |
+
district,
|
| 716 |
+
source_commit,
|
| 717 |
+
reference_commit,
|
| 718 |
+
REPO_REVISION,
|
| 719 |
+
)
|
| 720 |
+
entry = {
|
| 721 |
+
"batchSetNumber": int(job["_batchSetNumber"]),
|
| 722 |
+
"batchSetId": str(job["_batchSetId"]),
|
| 723 |
+
"customId": str(job["customId"]),
|
| 724 |
+
"locationId": str(job.get("locationId") or ""),
|
| 725 |
+
"referenceType": str(job.get("referenceType") or ""),
|
| 726 |
+
"targetGeneratedFileName": str(job["_targetName"]),
|
| 727 |
+
"inputs": inputs,
|
| 728 |
+
"prompt": str(job["_prompt"]),
|
| 729 |
+
}
|
| 730 |
+
entries.append(entry)
|
| 731 |
+
text_parts.append(
|
| 732 |
+
"=" * 80
|
| 733 |
+
+ f"\nBATCH SET {entry['batchSetNumber']:03d} | {entry['customId']}"
|
| 734 |
+
+ f"\nTARGET: {entry['targetGeneratedFileName']}"
|
| 735 |
+
+ "\n\nINPUTS\n"
|
| 736 |
+
+ "\n".join(
|
| 737 |
+
f"{item['label']} = {item['url']} ({item['role']})"
|
| 738 |
+
for item in inputs
|
| 739 |
+
)
|
| 740 |
+
+ "\n\nPROMPT\n"
|
| 741 |
+
+ entry["prompt"]
|
| 742 |
+
+ "\n"
|
| 743 |
+
)
|
| 744 |
+
preview = {
|
| 745 |
+
"formatVersion": 1,
|
| 746 |
+
"nodeVersion": NODE_VERSION,
|
| 747 |
+
"createdUtc": _utc_now(),
|
| 748 |
+
"districtName": district,
|
| 749 |
+
"manifestSha256": manifest_sha256,
|
| 750 |
+
"sourceCommit": source_commit,
|
| 751 |
+
"referenceCommit": reference_commit,
|
| 752 |
+
"generatedPreviewRevision": REPO_REVISION,
|
| 753 |
+
"batchSetCount": int(manifest.get("batchSetCount") or 0),
|
| 754 |
+
"imageCount": len(jobs),
|
| 755 |
+
"entries": entries,
|
| 756 |
+
}
|
| 757 |
+
json_path = root / "all_prompt_previews.json"
|
| 758 |
+
text_path = root / "all_prompt_previews.txt"
|
| 759 |
+
_write_json(json_path, preview)
|
| 760 |
+
full_text = "\n".join(text_parts)
|
| 761 |
+
text_path.parent.mkdir(parents=True, exist_ok=True)
|
| 762 |
+
temporary = text_path.with_suffix(".txt.part")
|
| 763 |
+
temporary.write_text(full_text, encoding="utf-8")
|
| 764 |
+
os.replace(temporary, text_path)
|
| 765 |
+
return json_path, text_path, full_text
|
| 766 |
+
|
| 767 |
+
|
| 768 |
+
def _openai_error(exception: urllib.error.HTTPError) -> str:
|
| 769 |
+
try:
|
| 770 |
+
body = exception.read().decode("utf-8", errors="replace")
|
| 771 |
+
parsed = json.loads(body)
|
| 772 |
+
error = parsed.get("error") if isinstance(parsed, dict) else None
|
| 773 |
+
if isinstance(error, Mapping):
|
| 774 |
+
return str(error.get("message") or body)[:1600]
|
| 775 |
+
return body[:1600]
|
| 776 |
+
except Exception:
|
| 777 |
+
return str(exception)
|
| 778 |
+
|
| 779 |
+
|
| 780 |
+
def _openai_json(
|
| 781 |
+
method: str,
|
| 782 |
+
path: str,
|
| 783 |
+
openai_key: str,
|
| 784 |
+
payload: Optional[Mapping[str, Any]] = None,
|
| 785 |
+
timeout: int = 300,
|
| 786 |
+
) -> Dict[str, Any]:
|
| 787 |
+
data = None
|
| 788 |
+
headers = {
|
| 789 |
+
"Authorization": "Bearer " + openai_key,
|
| 790 |
+
"User-Agent": f"Salia-City-Responses-Batch/{NODE_VERSION}",
|
| 791 |
+
}
|
| 792 |
+
if payload is not None:
|
| 793 |
+
data = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
| 794 |
+
headers["Content-Type"] = "application/json"
|
| 795 |
+
request = urllib.request.Request(
|
| 796 |
+
"https://api.openai.com" + path,
|
| 797 |
+
data=data,
|
| 798 |
+
headers=headers,
|
| 799 |
+
method=method,
|
| 800 |
+
)
|
| 801 |
+
try:
|
| 802 |
+
with urllib.request.urlopen(request, timeout=timeout) as response:
|
| 803 |
+
parsed = json.loads(response.read().decode("utf-8"))
|
| 804 |
+
except urllib.error.HTTPError as exception:
|
| 805 |
+
raise RuntimeError(
|
| 806 |
+
f"OpenAI API {method} {path} failed: {_openai_error(exception)}"
|
| 807 |
+
) from exception
|
| 808 |
+
if not isinstance(parsed, dict):
|
| 809 |
+
raise RuntimeError(f"OpenAI API {path} returned a non-object response.")
|
| 810 |
+
return parsed
|
| 811 |
+
|
| 812 |
+
|
| 813 |
+
def _upload_openai_batch_file(path: Path, openai_key: str) -> Dict[str, Any]:
|
| 814 |
+
boundary = "----SaliaCityResponses" + secrets.token_hex(16)
|
| 815 |
+
body = bytearray()
|
| 816 |
+
body.extend(f"--{boundary}\r\n".encode("ascii"))
|
| 817 |
+
body.extend(b'Content-Disposition: form-data; name="purpose"\r\n\r\n')
|
| 818 |
+
body.extend(b"batch\r\n")
|
| 819 |
+
body.extend(f"--{boundary}\r\n".encode("ascii"))
|
| 820 |
+
body.extend(
|
| 821 |
+
(
|
| 822 |
+
'Content-Disposition: form-data; name="file"; filename="'
|
| 823 |
+
+ path.name
|
| 824 |
+
+ '"\r\n'
|
| 825 |
+
).encode("utf-8")
|
| 826 |
+
)
|
| 827 |
+
body.extend(b"Content-Type: application/jsonl\r\n\r\n")
|
| 828 |
+
body.extend(path.read_bytes())
|
| 829 |
+
body.extend(b"\r\n")
|
| 830 |
+
body.extend(f"--{boundary}--\r\n".encode("ascii"))
|
| 831 |
+
request = urllib.request.Request(
|
| 832 |
+
"https://api.openai.com/v1/files",
|
| 833 |
+
data=bytes(body),
|
| 834 |
+
headers={
|
| 835 |
+
"Authorization": "Bearer " + openai_key,
|
| 836 |
+
"Content-Type": "multipart/form-data; boundary=" + boundary,
|
| 837 |
+
"User-Agent": f"Salia-City-Responses-Batch/{NODE_VERSION}",
|
| 838 |
+
},
|
| 839 |
+
method="POST",
|
| 840 |
+
)
|
| 841 |
+
try:
|
| 842 |
+
with urllib.request.urlopen(request, timeout=300) as response:
|
| 843 |
+
parsed = json.loads(response.read().decode("utf-8"))
|
| 844 |
+
except urllib.error.HTTPError as exception:
|
| 845 |
+
raise RuntimeError(
|
| 846 |
+
"OpenAI Batch file upload failed: " + _openai_error(exception)
|
| 847 |
+
) from exception
|
| 848 |
+
file_id = str(parsed.get("id") or "") if isinstance(parsed, dict) else ""
|
| 849 |
+
if not _SAFE_FILE_ID.fullmatch(file_id):
|
| 850 |
+
raise RuntimeError("OpenAI did not return a valid Batch input file id.")
|
| 851 |
+
return parsed
|
| 852 |
+
|
| 853 |
+
|
| 854 |
+
def _download_openai_file(file_id: str, destination: Path, openai_key: str) -> Path:
|
| 855 |
+
if not _SAFE_FILE_ID.fullmatch(file_id):
|
| 856 |
+
raise ValueError(f"Invalid OpenAI file id {file_id!r}.")
|
| 857 |
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
| 858 |
+
temporary = destination.with_suffix(destination.suffix + ".part")
|
| 859 |
+
request = urllib.request.Request(
|
| 860 |
+
"https://api.openai.com/v1/files/"
|
| 861 |
+
+ urllib.parse.quote(file_id, safe="")
|
| 862 |
+
+ "/content",
|
| 863 |
+
headers={
|
| 864 |
+
"Authorization": "Bearer " + openai_key,
|
| 865 |
+
"User-Agent": f"Salia-City-Responses-Batch/{NODE_VERSION}",
|
| 866 |
+
},
|
| 867 |
+
method="GET",
|
| 868 |
+
)
|
| 869 |
+
try:
|
| 870 |
+
with urllib.request.urlopen(request, timeout=3600) as response, temporary.open(
|
| 871 |
+
"wb"
|
| 872 |
+
) as output:
|
| 873 |
+
shutil.copyfileobj(response, output, length=1024 * 1024)
|
| 874 |
+
os.replace(temporary, destination)
|
| 875 |
+
except Exception:
|
| 876 |
+
if temporary.exists():
|
| 877 |
+
temporary.unlink()
|
| 878 |
+
raise
|
| 879 |
+
return destination
|
| 880 |
+
|
| 881 |
+
|
| 882 |
+
def _batch_status(batch_id: str, openai_key: str) -> Dict[str, Any]:
|
| 883 |
+
if not _SAFE_BATCH_ID.fullmatch(batch_id):
|
| 884 |
+
raise ValueError("Saved OpenAI Batch id is invalid.")
|
| 885 |
+
return _openai_json(
|
| 886 |
+
"GET",
|
| 887 |
+
"/v1/batches/" + urllib.parse.quote(batch_id, safe=""),
|
| 888 |
+
openai_key,
|
| 889 |
+
)
|
| 890 |
+
|
| 891 |
+
|
| 892 |
+
def _request_counts(batch: Mapping[str, Any]) -> Tuple[int, int, int]:
|
| 893 |
+
counts = batch.get("request_counts")
|
| 894 |
+
if not isinstance(counts, Mapping):
|
| 895 |
+
return 0, 0, 0
|
| 896 |
+
return (
|
| 897 |
+
int(counts.get("total") or 0),
|
| 898 |
+
int(counts.get("completed") or 0),
|
| 899 |
+
int(counts.get("failed") or 0),
|
| 900 |
+
)
|
| 901 |
+
|
| 902 |
+
|
| 903 |
+
def _assistant_messages(body: Mapping[str, Any]) -> List[str]:
|
| 904 |
+
result: List[str] = []
|
| 905 |
+
output = body.get("output")
|
| 906 |
+
if not isinstance(output, list):
|
| 907 |
+
return result
|
| 908 |
+
for item in output:
|
| 909 |
+
if not isinstance(item, Mapping) or str(item.get("type") or "") != "message":
|
| 910 |
+
continue
|
| 911 |
+
content = item.get("content")
|
| 912 |
+
if not isinstance(content, list):
|
| 913 |
+
continue
|
| 914 |
+
for part in content:
|
| 915 |
+
if isinstance(part, Mapping):
|
| 916 |
+
text = str(part.get("text") or "").strip()
|
| 917 |
+
if text:
|
| 918 |
+
result.append(text)
|
| 919 |
+
return result
|
| 920 |
+
|
| 921 |
+
|
| 922 |
+
def _response_diagnostic(
|
| 923 |
+
record: Mapping[str, Any],
|
| 924 |
+
request_mapping: Mapping[str, Any],
|
| 925 |
+
line_number: int,
|
| 926 |
+
) -> Dict[str, Any]:
|
| 927 |
+
response = record.get("response")
|
| 928 |
+
response_map = response if isinstance(response, Mapping) else {}
|
| 929 |
+
body_value = response_map.get("body")
|
| 930 |
+
body = body_value if isinstance(body_value, Mapping) else {}
|
| 931 |
+
output = body.get("output") if isinstance(body.get("output"), list) else []
|
| 932 |
+
calls: List[Dict[str, Any]] = []
|
| 933 |
+
reasoning_items: List[Dict[str, Any]] = []
|
| 934 |
+
for output_index, item in enumerate(output):
|
| 935 |
+
if not isinstance(item, Mapping):
|
| 936 |
+
continue
|
| 937 |
+
item_type = str(item.get("type") or "")
|
| 938 |
+
if item_type == "image_generation_call":
|
| 939 |
+
result_value = item.get("result")
|
| 940 |
+
calls.append(
|
| 941 |
+
{
|
| 942 |
+
"outputIndex": output_index,
|
| 943 |
+
"id": str(item.get("id") or ""),
|
| 944 |
+
"status": str(item.get("status") or "unknown"),
|
| 945 |
+
"revised_prompt": item.get("revised_prompt"),
|
| 946 |
+
"revisedPromptPresent": "revised_prompt" in item,
|
| 947 |
+
"resultPresent": isinstance(result_value, str)
|
| 948 |
+
and bool(result_value),
|
| 949 |
+
"resultBase64CharacterCount": len(result_value)
|
| 950 |
+
if isinstance(result_value, str)
|
| 951 |
+
else 0,
|
| 952 |
+
"availableFields": sorted(str(key) for key in item.keys()),
|
| 953 |
+
"allFieldsExceptResult": {
|
| 954 |
+
str(key): value
|
| 955 |
+
for key, value in item.items()
|
| 956 |
+
if str(key) != "result"
|
| 957 |
+
},
|
| 958 |
+
}
|
| 959 |
+
)
|
| 960 |
+
elif item_type == "reasoning":
|
| 961 |
+
reasoning_items.append(
|
| 962 |
+
{
|
| 963 |
+
"outputIndex": output_index,
|
| 964 |
+
"id": str(item.get("id") or ""),
|
| 965 |
+
"status": str(item.get("status") or ""),
|
| 966 |
+
"summary": item.get("summary"),
|
| 967 |
+
"content": item.get("content"),
|
| 968 |
+
"encryptedContentPresent": bool(item.get("encrypted_content")),
|
| 969 |
+
"availableFields": sorted(str(key) for key in item.keys()),
|
| 970 |
+
}
|
| 971 |
+
)
|
| 972 |
+
reasoning = body.get("reasoning") if isinstance(body.get("reasoning"), Mapping) else {}
|
| 973 |
+
usage = body.get("usage") if isinstance(body.get("usage"), Mapping) else {}
|
| 974 |
+
output_details = (
|
| 975 |
+
usage.get("output_tokens_details")
|
| 976 |
+
if isinstance(usage.get("output_tokens_details"), Mapping)
|
| 977 |
+
else {}
|
| 978 |
+
)
|
| 979 |
+
tool_usage = (
|
| 980 |
+
body.get("tool_usage") if isinstance(body.get("tool_usage"), Mapping) else {}
|
| 981 |
+
)
|
| 982 |
+
image_usage = (
|
| 983 |
+
tool_usage.get("image_gen")
|
| 984 |
+
if isinstance(tool_usage.get("image_gen"), Mapping)
|
| 985 |
+
else {}
|
| 986 |
+
)
|
| 987 |
+
messages = _assistant_messages(body)
|
| 988 |
+
completed_image_present = any(
|
| 989 |
+
call.get("status") == "completed" and call.get("resultPresent")
|
| 990 |
+
for call in calls
|
| 991 |
+
)
|
| 992 |
+
return {
|
| 993 |
+
"batchOutputLine": line_number,
|
| 994 |
+
"batchCustomId": str(record.get("custom_id") or ""),
|
| 995 |
+
"jobCustomId": str(request_mapping.get("jobCustomId") or ""),
|
| 996 |
+
"batchSetNumber": int(request_mapping.get("batchSetNumber") or 0),
|
| 997 |
+
"attempt": int(request_mapping.get("attempt") or 0),
|
| 998 |
+
"targetPath": str(request_mapping.get("targetPath") or ""),
|
| 999 |
+
"responseStatusCode": int(response_map.get("status_code") or 0),
|
| 1000 |
+
"batchRequestId": str(response_map.get("request_id") or ""),
|
| 1001 |
+
"responseId": str(body.get("id") or ""),
|
| 1002 |
+
"responseStatus": str(body.get("status") or ""),
|
| 1003 |
+
"responseModel": str(body.get("model") or ""),
|
| 1004 |
+
"responseError": body.get("error"),
|
| 1005 |
+
"responseIncompleteDetails": body.get("incomplete_details"),
|
| 1006 |
+
"responseModeration": body.get("moderation"),
|
| 1007 |
+
"responseReasoning": dict(reasoning),
|
| 1008 |
+
"reasoningEffort": reasoning.get("effort"),
|
| 1009 |
+
"reasoningMode": reasoning.get("mode"),
|
| 1010 |
+
"reasoningSummarySetting": reasoning.get("summary"),
|
| 1011 |
+
"reasoningTokenCount": int(output_details.get("reasoning_tokens") or 0),
|
| 1012 |
+
"responseUsage": dict(usage),
|
| 1013 |
+
"imageGenUsage": dict(image_usage),
|
| 1014 |
+
"imageGenerationCalls": calls,
|
| 1015 |
+
"reasoningOutputItems": reasoning_items,
|
| 1016 |
+
"assistantMessages": messages,
|
| 1017 |
+
"assistantFailureMessage": "" if completed_image_present else "\n".join(messages),
|
| 1018 |
+
"topLevelBatchError": record.get("error"),
|
| 1019 |
+
}
|
| 1020 |
+
|
| 1021 |
+
|
| 1022 |
+
def _extract_single_png(body: Mapping[str, Any]) -> bytes:
|
| 1023 |
+
if str(body.get("status") or "") != "completed" or body.get("error"):
|
| 1024 |
+
raise ValueError(
|
| 1025 |
+
"Responses request did not complete: "
|
| 1026 |
+
+ str(body.get("error") or body.get("status") or "unknown")
|
| 1027 |
+
)
|
| 1028 |
+
output = body.get("output")
|
| 1029 |
+
if not isinstance(output, list):
|
| 1030 |
+
raise ValueError("Responses result has no output array.")
|
| 1031 |
+
encoded_results: List[str] = []
|
| 1032 |
+
statuses: List[str] = []
|
| 1033 |
+
for item in output:
|
| 1034 |
+
if not isinstance(item, Mapping) or str(item.get("type") or "") != "image_generation_call":
|
| 1035 |
+
continue
|
| 1036 |
+
statuses.append(str(item.get("status") or "unknown"))
|
| 1037 |
+
encoded = item.get("result")
|
| 1038 |
+
if str(item.get("status") or "") == "completed" and isinstance(encoded, str) and encoded:
|
| 1039 |
+
encoded_results.append(encoded)
|
| 1040 |
+
if len(encoded_results) != 1:
|
| 1041 |
+
raise ValueError(
|
| 1042 |
+
f"Expected exactly one completed image result; received "
|
| 1043 |
+
f"{len(encoded_results)}. Call statuses: {', '.join(statuses) or 'none'}."
|
| 1044 |
+
)
|
| 1045 |
+
raw = base64.b64decode(encoded_results[0], validate=True)
|
| 1046 |
+
if not raw.startswith(_PNG_SIGNATURE):
|
| 1047 |
+
raise ValueError("Generated result is not a PNG.")
|
| 1048 |
+
if Image is None:
|
| 1049 |
+
raise RuntimeError("Pillow is required to validate generated image dimensions.")
|
| 1050 |
+
with Image.open(BytesIO(raw)) as image:
|
| 1051 |
+
if image.size != (1536, 1024):
|
| 1052 |
+
raise ValueError(
|
| 1053 |
+
f"Generated PNG dimensions are {image.size}; expected (1536, 1024)."
|
| 1054 |
+
)
|
| 1055 |
+
image.verify()
|
| 1056 |
+
return raw
|
| 1057 |
+
|
| 1058 |
+
|
| 1059 |
+
def _validate_remote_inputs(
|
| 1060 |
+
api: Any,
|
| 1061 |
+
jobs: Sequence[Mapping[str, Any]],
|
| 1062 |
+
district: str,
|
| 1063 |
+
source_commit: str,
|
| 1064 |
+
reference_commit: str,
|
| 1065 |
+
generated_commit: Optional[str],
|
| 1066 |
+
hf_token: str,
|
| 1067 |
+
) -> None:
|
| 1068 |
+
missing: List[str] = []
|
| 1069 |
+
checked = set()
|
| 1070 |
+
for job in jobs:
|
| 1071 |
+
candidates = [
|
| 1072 |
+
(SOURCE_REPO_ID, source_commit, _repo_path(district, str(job["_sourceName"]))),
|
| 1073 |
+
(
|
| 1074 |
+
REFERENCE_REPO_ID,
|
| 1075 |
+
reference_commit,
|
| 1076 |
+
str(job["_referenceImages"]["depthFileName"]),
|
| 1077 |
+
),
|
| 1078 |
+
(
|
| 1079 |
+
REFERENCE_REPO_ID,
|
| 1080 |
+
reference_commit,
|
| 1081 |
+
str(job["_referenceImages"]["segmentationFileName"]),
|
| 1082 |
+
),
|
| 1083 |
+
(
|
| 1084 |
+
REFERENCE_REPO_ID,
|
| 1085 |
+
reference_commit,
|
| 1086 |
+
str(job["_referenceImages"]["lineartFileName"]),
|
| 1087 |
+
),
|
| 1088 |
+
]
|
| 1089 |
+
for dependency in job["_dependencies"]:
|
| 1090 |
+
if not generated_commit:
|
| 1091 |
+
missing.append(
|
| 1092 |
+
GENERATED_REPO_ID
|
| 1093 |
+
+ ":"
|
| 1094 |
+
+ _repo_path(district, str(dependency["generatedFileName"]))
|
| 1095 |
+
+ " (no generated repository commit)"
|
| 1096 |
+
)
|
| 1097 |
+
else:
|
| 1098 |
+
candidates.append(
|
| 1099 |
+
(
|
| 1100 |
+
GENERATED_REPO_ID,
|
| 1101 |
+
generated_commit,
|
| 1102 |
+
_repo_path(district, str(dependency["generatedFileName"])),
|
| 1103 |
+
)
|
| 1104 |
+
)
|
| 1105 |
+
for repo_id, revision, path_in_repo in candidates:
|
| 1106 |
+
key = (repo_id, revision, path_in_repo)
|
| 1107 |
+
if key in checked:
|
| 1108 |
+
continue
|
| 1109 |
+
checked.add(key)
|
| 1110 |
+
if not api.file_exists(
|
| 1111 |
+
repo_id=repo_id,
|
| 1112 |
+
filename=path_in_repo,
|
| 1113 |
+
repo_type=REPO_TYPE,
|
| 1114 |
+
revision=revision,
|
| 1115 |
+
token=hf_token,
|
| 1116 |
+
):
|
| 1117 |
+
missing.append(f"{repo_id}@{revision}:{path_in_repo}")
|
| 1118 |
+
if missing:
|
| 1119 |
+
raise FileNotFoundError(
|
| 1120 |
+
"Cannot start this wave because required input files are missing: "
|
| 1121 |
+
+ "; ".join(missing[:30])
|
| 1122 |
+
)
|
| 1123 |
+
|
| 1124 |
+
|
| 1125 |
+
def _batch_line(
|
| 1126 |
+
request_mapping: Mapping[str, Any],
|
| 1127 |
+
job: Mapping[str, Any],
|
| 1128 |
+
inputs: Sequence[Mapping[str, str]],
|
| 1129 |
+
) -> Dict[str, Any]:
|
| 1130 |
+
content: List[Dict[str, Any]] = [
|
| 1131 |
+
{"type": "input_text", "text": str(job["_prompt"])}
|
| 1132 |
+
]
|
| 1133 |
+
for item in inputs:
|
| 1134 |
+
content.append(
|
| 1135 |
+
{
|
| 1136 |
+
"type": "input_text",
|
| 1137 |
+
"text": str(item["label"])
|
| 1138 |
+
+ " follows: "
|
| 1139 |
+
+ str(item["role"])
|
| 1140 |
+
+ ".",
|
| 1141 |
+
}
|
| 1142 |
+
)
|
| 1143 |
+
content.append({"type": "input_image", "image_url": str(item["url"])})
|
| 1144 |
+
return {
|
| 1145 |
+
"custom_id": str(request_mapping["batchCustomId"]),
|
| 1146 |
+
"method": "POST",
|
| 1147 |
+
"url": OPENAI_BATCH_ENDPOINT,
|
| 1148 |
+
"body": {
|
| 1149 |
+
"model": OPENAI_MODEL,
|
| 1150 |
+
"input": [{"role": "user", "content": content}],
|
| 1151 |
+
"reasoning": dict(OPENAI_REASONING),
|
| 1152 |
+
"moderation": json.loads(json.dumps(RESPONSE_MODERATION)),
|
| 1153 |
+
"tools": [dict(OPENAI_IMAGE_TOOL)],
|
| 1154 |
+
"tool_choice": {"type": "image_generation"},
|
| 1155 |
+
"store": False,
|
| 1156 |
+
},
|
| 1157 |
+
}
|
| 1158 |
+
|
| 1159 |
+
|
| 1160 |
+
def _build_wave_requests(
|
| 1161 |
+
state: Mapping[str, Any],
|
| 1162 |
+
jobs: Sequence[Mapping[str, Any]],
|
| 1163 |
+
district: str,
|
| 1164 |
+
source_commit: str,
|
| 1165 |
+
reference_commit: str,
|
| 1166 |
+
generated_commit: str,
|
| 1167 |
+
maximum_attempts: int,
|
| 1168 |
+
) -> Tuple[int, Dict[str, int], List[Dict[str, Any]], List[Dict[str, Any]]]:
|
| 1169 |
+
if not jobs:
|
| 1170 |
+
raise ValueError("Cannot build an empty city Batch wave.")
|
| 1171 |
+
set_numbers = {int(job["_batchSetNumber"]) for job in jobs}
|
| 1172 |
+
if len(set_numbers) != 1:
|
| 1173 |
+
raise ValueError("One OpenAI Batch may contain only one logical Batch set.")
|
| 1174 |
+
set_number = next(iter(set_numbers))
|
| 1175 |
+
completed = set(str(value) for value in state.get("completedCustomIds") or [])
|
| 1176 |
+
for job in jobs:
|
| 1177 |
+
for dependency in job["_dependencies"]:
|
| 1178 |
+
if str(dependency["customId"]) not in completed:
|
| 1179 |
+
raise ValueError(
|
| 1180 |
+
f"Image {job['customId']!r} is not ready; dependency "
|
| 1181 |
+
f"{dependency['customId']!r} is unpublished."
|
| 1182 |
+
)
|
| 1183 |
+
|
| 1184 |
+
attempt_counts = dict(state.get("attemptCounts") or {})
|
| 1185 |
+
request_mappings: List[Dict[str, Any]] = []
|
| 1186 |
+
lines: List[Dict[str, Any]] = []
|
| 1187 |
+
for job_index, job in enumerate(jobs):
|
| 1188 |
+
job_id = str(job["customId"])
|
| 1189 |
+
attempt = int(attempt_counts.get(job_id) or 0) + 1
|
| 1190 |
+
if attempt > maximum_attempts:
|
| 1191 |
+
raise RuntimeError(
|
| 1192 |
+
f"Image {job_id!r} reached MAX_ATTEMPTS={maximum_attempts}. "
|
| 1193 |
+
"Inspect its saved diagnostics before trying again."
|
| 1194 |
+
)
|
| 1195 |
+
fingerprint = (
|
| 1196 |
+
str(state["manifestSha256"])
|
| 1197 |
+
+ "\0"
|
| 1198 |
+
+ job_id
|
| 1199 |
+
+ "\0"
|
| 1200 |
+
+ str(attempt)
|
| 1201 |
+
)
|
| 1202 |
+
batch_custom_id = (
|
| 1203 |
+
"city_"
|
| 1204 |
+
+ str(set_number).zfill(3)
|
| 1205 |
+
+ "_"
|
| 1206 |
+
+ str(job_index).zfill(4)
|
| 1207 |
+
+ "_a"
|
| 1208 |
+
+ str(attempt).zfill(2)
|
| 1209 |
+
+ "_"
|
| 1210 |
+
+ hashlib.sha256(fingerprint.encode("utf-8")).hexdigest()[:12]
|
| 1211 |
+
)
|
| 1212 |
+
mapping = {
|
| 1213 |
+
"batchCustomId": batch_custom_id,
|
| 1214 |
+
"jobCustomId": job_id,
|
| 1215 |
+
"batchSetNumber": set_number,
|
| 1216 |
+
"attempt": attempt,
|
| 1217 |
+
"targetPath": _repo_path(district, str(job["_targetName"])),
|
| 1218 |
+
"prompt": str(job["_prompt"]),
|
| 1219 |
+
}
|
| 1220 |
+
inputs = _preview_inputs(
|
| 1221 |
+
state,
|
| 1222 |
+
job,
|
| 1223 |
+
district,
|
| 1224 |
+
source_commit,
|
| 1225 |
+
reference_commit,
|
| 1226 |
+
generated_commit or REPO_REVISION,
|
| 1227 |
+
)
|
| 1228 |
+
mapping["inputs"] = inputs
|
| 1229 |
+
request_mappings.append(mapping)
|
| 1230 |
+
lines.append(_batch_line(mapping, job, inputs))
|
| 1231 |
+
attempt_counts[job_id] = attempt
|
| 1232 |
+
return set_number, attempt_counts, request_mappings, lines
|
| 1233 |
+
|
| 1234 |
+
|
| 1235 |
+
def _start_wave(
|
| 1236 |
+
state: Dict[str, Any],
|
| 1237 |
+
jobs: Sequence[Mapping[str, Any]],
|
| 1238 |
+
district: str,
|
| 1239 |
+
hf_token: str,
|
| 1240 |
+
openai_key: str,
|
| 1241 |
+
maximum_attempts: int,
|
| 1242 |
+
) -> Tuple[str, str, int, str]:
|
| 1243 |
+
_, HfApi, _ = _load_hf_tools()
|
| 1244 |
+
api = HfApi(token=hf_token)
|
| 1245 |
+
generated_commit: Optional[str] = _repo_commit(
|
| 1246 |
+
api, GENERATED_REPO_ID, hf_token
|
| 1247 |
+
)
|
| 1248 |
+
source_commit = str(state["sourceCommit"])
|
| 1249 |
+
reference_commit = str(state["referenceCommit"])
|
| 1250 |
+
(
|
| 1251 |
+
set_number,
|
| 1252 |
+
attempt_counts,
|
| 1253 |
+
request_mappings,
|
| 1254 |
+
lines,
|
| 1255 |
+
) = _build_wave_requests(
|
| 1256 |
+
state,
|
| 1257 |
+
jobs,
|
| 1258 |
+
district,
|
| 1259 |
+
source_commit,
|
| 1260 |
+
reference_commit,
|
| 1261 |
+
generated_commit or REPO_REVISION,
|
| 1262 |
+
maximum_attempts,
|
| 1263 |
+
)
|
| 1264 |
+
_validate_remote_inputs(
|
| 1265 |
+
api,
|
| 1266 |
+
jobs,
|
| 1267 |
+
district,
|
| 1268 |
+
source_commit,
|
| 1269 |
+
reference_commit,
|
| 1270 |
+
generated_commit,
|
| 1271 |
+
hf_token,
|
| 1272 |
+
)
|
| 1273 |
+
|
| 1274 |
+
wave_sequence = int(state.get("openAiBatchSequence") or 0) + 1
|
| 1275 |
+
batches_root = Path(str(state["workflowRoot"])) / "openai_batches"
|
| 1276 |
+
while True:
|
| 1277 |
+
run_root = batches_root / (
|
| 1278 |
+
"batch_"
|
| 1279 |
+
+ str(wave_sequence).zfill(3)
|
| 1280 |
+
+ "_set_"
|
| 1281 |
+
+ str(set_number).zfill(3)
|
| 1282 |
+
)
|
| 1283 |
+
if not run_root.exists():
|
| 1284 |
+
break
|
| 1285 |
+
# A prior upload/create attempt may have failed before its Batch id could
|
| 1286 |
+
# be persisted. Keep that evidence and use a fresh local run directory.
|
| 1287 |
+
wave_sequence += 1
|
| 1288 |
+
run_root.mkdir(parents=True, exist_ok=False)
|
| 1289 |
+
jsonl_path = run_root / "city_responses_batch.jsonl"
|
| 1290 |
+
request_map_path = run_root / "request_map.json"
|
| 1291 |
+
_write_jsonl(jsonl_path, lines)
|
| 1292 |
+
_write_json(
|
| 1293 |
+
request_map_path,
|
| 1294 |
+
{
|
| 1295 |
+
"formatVersion": 1,
|
| 1296 |
+
"nodeVersion": NODE_VERSION,
|
| 1297 |
+
"districtName": district,
|
| 1298 |
+
"manifestSha256": str(state["manifestSha256"]),
|
| 1299 |
+
"batchSetNumber": set_number,
|
| 1300 |
+
"requestCount": len(request_mappings),
|
| 1301 |
+
"responsesModel": OPENAI_MODEL,
|
| 1302 |
+
"reasoning": OPENAI_REASONING,
|
| 1303 |
+
"responseModeration": RESPONSE_MODERATION,
|
| 1304 |
+
"imageTool": OPENAI_IMAGE_TOOL,
|
| 1305 |
+
"requests": request_mappings,
|
| 1306 |
+
},
|
| 1307 |
+
)
|
| 1308 |
+
uploaded = _upload_openai_batch_file(jsonl_path, openai_key)
|
| 1309 |
+
input_file_id = str(uploaded["id"])
|
| 1310 |
+
batch = _openai_json(
|
| 1311 |
+
"POST",
|
| 1312 |
+
"/v1/batches",
|
| 1313 |
+
openai_key,
|
| 1314 |
+
{
|
| 1315 |
+
"input_file_id": input_file_id,
|
| 1316 |
+
"endpoint": OPENAI_BATCH_ENDPOINT,
|
| 1317 |
+
"completion_window": "24h",
|
| 1318 |
+
"metadata": {
|
| 1319 |
+
"purpose": PURPOSE,
|
| 1320 |
+
"node_version": NODE_VERSION,
|
| 1321 |
+
"batch_set": str(set_number),
|
| 1322 |
+
"manifest": str(state["manifestSha256"])[:32],
|
| 1323 |
+
},
|
| 1324 |
+
},
|
| 1325 |
+
)
|
| 1326 |
+
batch_id = str(batch.get("id") or "")
|
| 1327 |
+
if not _SAFE_BATCH_ID.fullmatch(batch_id):
|
| 1328 |
+
raise RuntimeError("OpenAI did not return a valid Batch id.")
|
| 1329 |
+
state["attemptCounts"] = attempt_counts
|
| 1330 |
+
state["openAiBatchSequence"] = wave_sequence
|
| 1331 |
+
state["activeBatch"] = {
|
| 1332 |
+
"batchId": batch_id,
|
| 1333 |
+
"inputFileId": input_file_id,
|
| 1334 |
+
"batchSetNumber": set_number,
|
| 1335 |
+
"requestCount": len(request_mappings),
|
| 1336 |
+
"requests": request_mappings,
|
| 1337 |
+
"runRoot": str(run_root.resolve()),
|
| 1338 |
+
"jsonlPath": str(jsonl_path.resolve()),
|
| 1339 |
+
"requestMapPath": str(request_map_path.resolve()),
|
| 1340 |
+
"createdUtc": _utc_now(),
|
| 1341 |
+
"generatedInputCommit": generated_commit or "",
|
| 1342 |
+
}
|
| 1343 |
+
_write_json(Path(str(state["statePath"])), state)
|
| 1344 |
+
return (
|
| 1345 |
+
f"Started OpenAI Responses Batch {batch_id}: logical Batch set "
|
| 1346 |
+
f"{set_number}, {len(jobs)} image request(s), attempts "
|
| 1347 |
+
+ ", ".join(
|
| 1348 |
+
f"{mapping['jobCustomId']}={mapping['attempt']}"
|
| 1349 |
+
for mapping in request_mappings[:8]
|
| 1350 |
+
)
|
| 1351 |
+
+ (" ..." if len(request_mappings) > 8 else "")
|
| 1352 |
+
+ ".",
|
| 1353 |
+
batch_id,
|
| 1354 |
+
set_number,
|
| 1355 |
+
str(jsonl_path.resolve()),
|
| 1356 |
+
)
|
| 1357 |
+
|
| 1358 |
+
|
| 1359 |
+
def _upload_jsonl_artifact(
|
| 1360 |
+
jsonl_path: Path,
|
| 1361 |
+
district: str,
|
| 1362 |
+
set_number: int,
|
| 1363 |
+
manifest_sha256: str,
|
| 1364 |
+
hf_token: str,
|
| 1365 |
+
) -> Tuple[str, str, str]:
|
| 1366 |
+
raw = jsonl_path.read_bytes()
|
| 1367 |
+
jsonl_sha256 = hashlib.sha256(raw).hexdigest()
|
| 1368 |
+
file_name = (
|
| 1369 |
+
"city_responses_"
|
| 1370 |
+
+ manifest_sha256[:12]
|
| 1371 |
+
+ "_set_"
|
| 1372 |
+
+ str(set_number).zfill(3)
|
| 1373 |
+
+ "_"
|
| 1374 |
+
+ jsonl_sha256[:12]
|
| 1375 |
+
+ ".jsonl"
|
| 1376 |
+
)
|
| 1377 |
+
path_in_repo = (
|
| 1378 |
+
district
|
| 1379 |
+
+ "/openai_batch_jsonl/"
|
| 1380 |
+
+ _safe_file_name(file_name, "JSONL artifact filename")
|
| 1381 |
+
)
|
| 1382 |
+
CommitOperationAdd, HfApi, _ = _load_hf_tools()
|
| 1383 |
+
api = HfApi(token=hf_token)
|
| 1384 |
+
parent_commit = _repo_commit(api, SOURCE_REPO_ID, hf_token)
|
| 1385 |
+
if api.file_exists(
|
| 1386 |
+
repo_id=SOURCE_REPO_ID,
|
| 1387 |
+
filename=path_in_repo,
|
| 1388 |
+
repo_type=REPO_TYPE,
|
| 1389 |
+
revision=parent_commit,
|
| 1390 |
+
token=hf_token,
|
| 1391 |
+
):
|
| 1392 |
+
commit_oid = parent_commit
|
| 1393 |
+
else:
|
| 1394 |
+
commit = api.create_commit(
|
| 1395 |
+
repo_id=SOURCE_REPO_ID,
|
| 1396 |
+
repo_type=REPO_TYPE,
|
| 1397 |
+
revision=REPO_REVISION,
|
| 1398 |
+
operations=[
|
| 1399 |
+
CommitOperationAdd(
|
| 1400 |
+
path_in_repo=path_in_repo,
|
| 1401 |
+
path_or_fileobj=str(jsonl_path),
|
| 1402 |
+
)
|
| 1403 |
+
],
|
| 1404 |
+
commit_message=(
|
| 1405 |
+
"City Responses JSONL only: Batch set "
|
| 1406 |
+
+ str(set_number)
|
| 1407 |
+
+ " ("
|
| 1408 |
+
+ jsonl_sha256[:12]
|
| 1409 |
+
+ ")"
|
| 1410 |
+
),
|
| 1411 |
+
parent_commit=parent_commit,
|
| 1412 |
+
token=hf_token,
|
| 1413 |
+
)
|
| 1414 |
+
commit_oid = str(getattr(commit, "oid", "") or "").strip()
|
| 1415 |
+
if not commit_oid:
|
| 1416 |
+
raise RuntimeError("Hugging Face JSONL commit returned no oid.")
|
| 1417 |
+
return (
|
| 1418 |
+
_hf_blob_url(SOURCE_REPO_ID, commit_oid, path_in_repo),
|
| 1419 |
+
commit_oid,
|
| 1420 |
+
path_in_repo,
|
| 1421 |
+
)
|
| 1422 |
+
|
| 1423 |
+
|
| 1424 |
+
def _create_jsonl_only(
|
| 1425 |
+
state: Dict[str, Any],
|
| 1426 |
+
jobs: Sequence[Mapping[str, Any]],
|
| 1427 |
+
district: str,
|
| 1428 |
+
hf_token: str,
|
| 1429 |
+
maximum_attempts: int,
|
| 1430 |
+
) -> Tuple[str, int, str, str]:
|
| 1431 |
+
_, HfApi, _ = _load_hf_tools()
|
| 1432 |
+
api = HfApi(token=hf_token)
|
| 1433 |
+
generated_commit = _repo_commit(api, GENERATED_REPO_ID, hf_token)
|
| 1434 |
+
source_commit = str(state["sourceCommit"])
|
| 1435 |
+
reference_commit = str(state["referenceCommit"])
|
| 1436 |
+
(
|
| 1437 |
+
set_number,
|
| 1438 |
+
_unused_attempt_counts,
|
| 1439 |
+
request_mappings,
|
| 1440 |
+
lines,
|
| 1441 |
+
) = _build_wave_requests(
|
| 1442 |
+
state,
|
| 1443 |
+
jobs,
|
| 1444 |
+
district,
|
| 1445 |
+
source_commit,
|
| 1446 |
+
reference_commit,
|
| 1447 |
+
generated_commit,
|
| 1448 |
+
maximum_attempts,
|
| 1449 |
+
)
|
| 1450 |
+
_validate_remote_inputs(
|
| 1451 |
+
api,
|
| 1452 |
+
jobs,
|
| 1453 |
+
district,
|
| 1454 |
+
source_commit,
|
| 1455 |
+
reference_commit,
|
| 1456 |
+
generated_commit,
|
| 1457 |
+
hf_token,
|
| 1458 |
+
)
|
| 1459 |
+
|
| 1460 |
+
artifact_root = Path(str(state["workflowRoot"])) / "jsonl_only"
|
| 1461 |
+
artifact_root.mkdir(parents=True, exist_ok=True)
|
| 1462 |
+
jsonl_path = artifact_root / (
|
| 1463 |
+
"city_responses_set_" + str(set_number).zfill(3) + ".jsonl"
|
| 1464 |
+
)
|
| 1465 |
+
request_map_path = artifact_root / (
|
| 1466 |
+
"city_responses_set_" + str(set_number).zfill(3) + "_request_map.json"
|
| 1467 |
+
)
|
| 1468 |
+
_write_jsonl(jsonl_path, lines)
|
| 1469 |
+
_write_json(
|
| 1470 |
+
request_map_path,
|
| 1471 |
+
{
|
| 1472 |
+
"formatVersion": 1,
|
| 1473 |
+
"nodeVersion": NODE_VERSION,
|
| 1474 |
+
"onlyCreateJsonl": True,
|
| 1475 |
+
"districtName": district,
|
| 1476 |
+
"manifestSha256": str(state["manifestSha256"]),
|
| 1477 |
+
"batchSetNumber": set_number,
|
| 1478 |
+
"requestCount": len(request_mappings),
|
| 1479 |
+
"responsesModel": OPENAI_MODEL,
|
| 1480 |
+
"reasoning": OPENAI_REASONING,
|
| 1481 |
+
"responseModeration": RESPONSE_MODERATION,
|
| 1482 |
+
"imageTool": OPENAI_IMAGE_TOOL,
|
| 1483 |
+
"requests": request_mappings,
|
| 1484 |
+
},
|
| 1485 |
+
)
|
| 1486 |
+
url, commit_oid, path_in_repo = _upload_jsonl_artifact(
|
| 1487 |
+
jsonl_path,
|
| 1488 |
+
district,
|
| 1489 |
+
set_number,
|
| 1490 |
+
str(state["manifestSha256"]),
|
| 1491 |
+
hf_token,
|
| 1492 |
+
)
|
| 1493 |
+
state["lastJsonlOnly"] = {
|
| 1494 |
+
"createdUtc": _utc_now(),
|
| 1495 |
+
"batchSetNumber": set_number,
|
| 1496 |
+
"requestCount": len(request_mappings),
|
| 1497 |
+
"localPath": str(jsonl_path.resolve()),
|
| 1498 |
+
"requestMapPath": str(request_map_path.resolve()),
|
| 1499 |
+
"huggingFaceRepoPath": path_in_repo,
|
| 1500 |
+
"huggingFaceCommit": commit_oid,
|
| 1501 |
+
"huggingFaceUrl": url,
|
| 1502 |
+
}
|
| 1503 |
+
_write_json(Path(str(state["statePath"])), state)
|
| 1504 |
+
status = (
|
| 1505 |
+
"Only_Create_Jsonl is ON. Created and uploaded logical Batch set "
|
| 1506 |
+
+ str(set_number)
|
| 1507 |
+
+ " with "
|
| 1508 |
+
+ str(len(lines))
|
| 1509 |
+
+ " /v1/responses request(s). No OpenAI file was uploaded and no "
|
| 1510 |
+
+ "OpenAI Batch was started. Image_A=Unity screenshot; Image_B=Depth; "
|
| 1511 |
+
+ "Image_C=Segmentation; Image_D=Lineart; Image_E and later=published "
|
| 1512 |
+
+ "generated continuity dependencies. Hugging Face JSONL: "
|
| 1513 |
+
+ url
|
| 1514 |
+
)
|
| 1515 |
+
return status, set_number, str(jsonl_path.resolve()), url
|
| 1516 |
+
|
| 1517 |
+
|
| 1518 |
+
def _save_png(raw: bytes, directory: Path, job_custom_id: str) -> Path:
|
| 1519 |
+
directory.mkdir(parents=True, exist_ok=True)
|
| 1520 |
+
name = hashlib.sha256(job_custom_id.encode("utf-8")).hexdigest()[:20] + ".png"
|
| 1521 |
+
path = directory / name
|
| 1522 |
+
temporary = path.with_suffix(".png.part")
|
| 1523 |
+
temporary.write_bytes(raw)
|
| 1524 |
+
os.replace(temporary, path)
|
| 1525 |
+
return path
|
| 1526 |
+
|
| 1527 |
+
|
| 1528 |
+
def _parse_error_file(
|
| 1529 |
+
path: Optional[Path],
|
| 1530 |
+
request_by_batch_id: Mapping[str, Mapping[str, Any]],
|
| 1531 |
+
) -> Dict[str, Dict[str, Any]]:
|
| 1532 |
+
result: Dict[str, Dict[str, Any]] = {}
|
| 1533 |
+
if path is None or not path.is_file():
|
| 1534 |
+
return result
|
| 1535 |
+
with path.open("r", encoding="utf-8-sig") as handle:
|
| 1536 |
+
for line_number, line in enumerate(handle, start=1):
|
| 1537 |
+
if not line.strip():
|
| 1538 |
+
continue
|
| 1539 |
+
record = json.loads(line)
|
| 1540 |
+
if not isinstance(record, Mapping):
|
| 1541 |
+
continue
|
| 1542 |
+
batch_custom_id = str(record.get("custom_id") or "")
|
| 1543 |
+
if batch_custom_id not in request_by_batch_id:
|
| 1544 |
+
continue
|
| 1545 |
+
response = record.get("response")
|
| 1546 |
+
response_map = response if isinstance(response, Mapping) else {}
|
| 1547 |
+
error = record.get("error") or response_map.get("body") or response_map
|
| 1548 |
+
result[batch_custom_id] = {
|
| 1549 |
+
"batchErrorLine": line_number,
|
| 1550 |
+
"statusCode": int(response_map.get("status_code") or 0),
|
| 1551 |
+
"error": error,
|
| 1552 |
+
}
|
| 1553 |
+
return result
|
| 1554 |
+
|
| 1555 |
+
|
| 1556 |
+
def _publish_results(
|
| 1557 |
+
state: Dict[str, Any],
|
| 1558 |
+
results: Sequence[Mapping[str, Any]],
|
| 1559 |
+
hf_token: str,
|
| 1560 |
+
) -> str:
|
| 1561 |
+
if not results:
|
| 1562 |
+
return ""
|
| 1563 |
+
CommitOperationAdd, HfApi, _ = _load_hf_tools()
|
| 1564 |
+
api = HfApi(token=hf_token)
|
| 1565 |
+
parent_commit = _repo_commit(api, GENERATED_REPO_ID, hf_token)
|
| 1566 |
+
operations = [
|
| 1567 |
+
CommitOperationAdd(
|
| 1568 |
+
path_in_repo=str(result["targetPath"]),
|
| 1569 |
+
path_or_fileobj=str(result["localPath"]),
|
| 1570 |
+
)
|
| 1571 |
+
for result in results
|
| 1572 |
+
]
|
| 1573 |
+
commit = api.create_commit(
|
| 1574 |
+
repo_id=GENERATED_REPO_ID,
|
| 1575 |
+
repo_type=REPO_TYPE,
|
| 1576 |
+
revision=REPO_REVISION,
|
| 1577 |
+
operations=operations,
|
| 1578 |
+
commit_message=(
|
| 1579 |
+
"City Responses Batch set "
|
| 1580 |
+
+ str(results[0]["batchSetNumber"])
|
| 1581 |
+
+ ": publish "
|
| 1582 |
+
+ str(len(results))
|
| 1583 |
+
+ " generated views"
|
| 1584 |
+
),
|
| 1585 |
+
parent_commit=parent_commit,
|
| 1586 |
+
token=hf_token,
|
| 1587 |
+
)
|
| 1588 |
+
oid = str(getattr(commit, "oid", "") or "").strip()
|
| 1589 |
+
if not oid:
|
| 1590 |
+
raise RuntimeError("Hugging Face commit returned no oid.")
|
| 1591 |
+
state["lastGeneratedCommit"] = oid
|
| 1592 |
+
return oid
|
| 1593 |
+
|
| 1594 |
+
|
| 1595 |
+
def _process_terminal_batch(
|
| 1596 |
+
state: Dict[str, Any],
|
| 1597 |
+
batch: Mapping[str, Any],
|
| 1598 |
+
hf_token: str,
|
| 1599 |
+
openai_key: str,
|
| 1600 |
+
) -> Tuple[int, int, str]:
|
| 1601 |
+
active = _require_mapping(state.get("activeBatch"), "activeBatch")
|
| 1602 |
+
run_root = Path(str(active["runRoot"]))
|
| 1603 |
+
requests = _require_list(active.get("requests"), "activeBatch.requests")
|
| 1604 |
+
request_by_batch_id: Dict[str, Mapping[str, Any]] = {
|
| 1605 |
+
str(request["batchCustomId"]): _require_mapping(request, "request mapping")
|
| 1606 |
+
for request in requests
|
| 1607 |
+
}
|
| 1608 |
+
output_path: Optional[Path] = None
|
| 1609 |
+
output_file_id = str(batch.get("output_file_id") or "")
|
| 1610 |
+
if _SAFE_FILE_ID.fullmatch(output_file_id):
|
| 1611 |
+
output_path = _download_openai_file(
|
| 1612 |
+
output_file_id,
|
| 1613 |
+
run_root / "openai_batch_output.jsonl",
|
| 1614 |
+
openai_key,
|
| 1615 |
+
)
|
| 1616 |
+
error_path: Optional[Path] = None
|
| 1617 |
+
error_file_id = str(batch.get("error_file_id") or "")
|
| 1618 |
+
if _SAFE_FILE_ID.fullmatch(error_file_id):
|
| 1619 |
+
error_path = _download_openai_file(
|
| 1620 |
+
error_file_id,
|
| 1621 |
+
run_root / "openai_batch_errors.jsonl",
|
| 1622 |
+
openai_key,
|
| 1623 |
+
)
|
| 1624 |
+
error_records = _parse_error_file(error_path, request_by_batch_id)
|
| 1625 |
+
results: List[Dict[str, Any]] = []
|
| 1626 |
+
failures: List[Dict[str, Any]] = []
|
| 1627 |
+
diagnostics: List[Dict[str, Any]] = []
|
| 1628 |
+
seen = set()
|
| 1629 |
+
if output_path is not None:
|
| 1630 |
+
with output_path.open("r", encoding="utf-8-sig") as handle:
|
| 1631 |
+
for line_number, line in enumerate(handle, start=1):
|
| 1632 |
+
if not line.strip():
|
| 1633 |
+
continue
|
| 1634 |
+
record = json.loads(line)
|
| 1635 |
+
if not isinstance(record, Mapping):
|
| 1636 |
+
raise ValueError(f"Batch output line {line_number} is not an object.")
|
| 1637 |
+
batch_custom_id = str(record.get("custom_id") or "")
|
| 1638 |
+
mapping = request_by_batch_id.get(batch_custom_id)
|
| 1639 |
+
if mapping is None or batch_custom_id in seen:
|
| 1640 |
+
raise ValueError(
|
| 1641 |
+
f"Unexpected or duplicate Batch custom_id {batch_custom_id!r}."
|
| 1642 |
+
)
|
| 1643 |
+
seen.add(batch_custom_id)
|
| 1644 |
+
diagnostic = _response_diagnostic(record, mapping, line_number)
|
| 1645 |
+
diagnostic["diagnosticIndex"] = len(diagnostics)
|
| 1646 |
+
diagnostics.append(diagnostic)
|
| 1647 |
+
try:
|
| 1648 |
+
response = _require_mapping(record.get("response"), "Batch response")
|
| 1649 |
+
status_code = int(response.get("status_code") or 0)
|
| 1650 |
+
if status_code < 200 or status_code >= 300:
|
| 1651 |
+
raise ValueError(f"Responses HTTP status {status_code}.")
|
| 1652 |
+
body = _require_mapping(response.get("body"), "Responses body")
|
| 1653 |
+
raw = _extract_single_png(body)
|
| 1654 |
+
local_path = _save_png(
|
| 1655 |
+
raw,
|
| 1656 |
+
run_root / "generated_images",
|
| 1657 |
+
str(mapping["jobCustomId"]),
|
| 1658 |
+
)
|
| 1659 |
+
results.append(
|
| 1660 |
+
{
|
| 1661 |
+
"batchCustomId": batch_custom_id,
|
| 1662 |
+
"jobCustomId": str(mapping["jobCustomId"]),
|
| 1663 |
+
"batchSetNumber": int(mapping["batchSetNumber"]),
|
| 1664 |
+
"attempt": int(mapping["attempt"]),
|
| 1665 |
+
"targetPath": str(mapping["targetPath"]),
|
| 1666 |
+
"localPath": str(local_path.resolve()),
|
| 1667 |
+
"sha256": hashlib.sha256(raw).hexdigest(),
|
| 1668 |
+
"byteCount": len(raw),
|
| 1669 |
+
"diagnosticIndex": int(diagnostic["diagnosticIndex"]),
|
| 1670 |
+
}
|
| 1671 |
+
)
|
| 1672 |
+
except Exception as exception:
|
| 1673 |
+
failures.append(
|
| 1674 |
+
{
|
| 1675 |
+
"batchCustomId": batch_custom_id,
|
| 1676 |
+
"jobCustomId": str(mapping["jobCustomId"]),
|
| 1677 |
+
"batchSetNumber": int(mapping["batchSetNumber"]),
|
| 1678 |
+
"attempt": int(mapping["attempt"]),
|
| 1679 |
+
"failureKind": "response_or_image_failure",
|
| 1680 |
+
"reason": str(exception)[:1600],
|
| 1681 |
+
"diagnosticIndex": int(diagnostic["diagnosticIndex"]),
|
| 1682 |
+
}
|
| 1683 |
+
)
|
| 1684 |
+
for batch_custom_id, mapping in request_by_batch_id.items():
|
| 1685 |
+
if batch_custom_id in seen:
|
| 1686 |
+
continue
|
| 1687 |
+
error = error_records.get(batch_custom_id)
|
| 1688 |
+
failures.append(
|
| 1689 |
+
{
|
| 1690 |
+
"batchCustomId": batch_custom_id,
|
| 1691 |
+
"jobCustomId": str(mapping["jobCustomId"]),
|
| 1692 |
+
"batchSetNumber": int(mapping["batchSetNumber"]),
|
| 1693 |
+
"attempt": int(mapping["attempt"]),
|
| 1694 |
+
"failureKind": "batch_request_error",
|
| 1695 |
+
"reason": str(error.get("error") if error else batch.get("errors") or batch.get("status"))[:1600],
|
| 1696 |
+
"batchError": error,
|
| 1697 |
+
}
|
| 1698 |
+
)
|
| 1699 |
+
|
| 1700 |
+
commit_oid = _publish_results(state, results, hf_token)
|
| 1701 |
+
completed = set(str(value) for value in state.get("completedCustomIds") or [])
|
| 1702 |
+
result_history = list(state.get("resultHistory") or [])
|
| 1703 |
+
for result in results:
|
| 1704 |
+
completed.add(str(result["jobCustomId"]))
|
| 1705 |
+
result_history.append(dict(result, publishedCommit=commit_oid))
|
| 1706 |
+
failure_history = list(state.get("failureHistory") or [])
|
| 1707 |
+
failure_history.extend(failures)
|
| 1708 |
+
state["completedCustomIds"] = sorted(completed)
|
| 1709 |
+
state["resultHistory"] = result_history
|
| 1710 |
+
state["failureHistory"] = failure_history
|
| 1711 |
+
state["lastProcessedBatchId"] = str(active["batchId"])
|
| 1712 |
+
state["activeBatch"] = None
|
| 1713 |
+
_write_json(
|
| 1714 |
+
run_root / "openai_response_diagnostics.json",
|
| 1715 |
+
{
|
| 1716 |
+
"formatVersion": 1,
|
| 1717 |
+
"nodeVersion": NODE_VERSION,
|
| 1718 |
+
"batchId": str(active["batchId"]),
|
| 1719 |
+
"batchSetNumber": int(active["batchSetNumber"]),
|
| 1720 |
+
"rawOutputPath": str(output_path.resolve()) if output_path else "",
|
| 1721 |
+
"rawErrorPath": str(error_path.resolve()) if error_path else "",
|
| 1722 |
+
"returnedOutputCount": len(seen),
|
| 1723 |
+
"successfulImageCount": len(results),
|
| 1724 |
+
"failureCount": len(failures),
|
| 1725 |
+
"requestConfiguration": {
|
| 1726 |
+
"responsesModel": OPENAI_MODEL,
|
| 1727 |
+
"reasoning": OPENAI_REASONING,
|
| 1728 |
+
"responseModeration": RESPONSE_MODERATION,
|
| 1729 |
+
"imageTool": OPENAI_IMAGE_TOOL,
|
| 1730 |
+
"toolChoice": {"type": "image_generation"},
|
| 1731 |
+
"store": False,
|
| 1732 |
+
},
|
| 1733 |
+
"entries": diagnostics,
|
| 1734 |
+
"failures": failures,
|
| 1735 |
+
},
|
| 1736 |
+
)
|
| 1737 |
+
_write_json(Path(str(state["statePath"])), state)
|
| 1738 |
+
return len(results), len(failures), commit_oid
|
| 1739 |
+
|
| 1740 |
+
|
| 1741 |
+
def _initialize_state(
|
| 1742 |
+
manifest: Mapping[str, Any],
|
| 1743 |
+
district: str,
|
| 1744 |
+
source_commit: str,
|
| 1745 |
+
reference_commit: str,
|
| 1746 |
+
manifest_sha256: str,
|
| 1747 |
+
jobs: Sequence[Mapping[str, Any]],
|
| 1748 |
+
) -> Dict[str, Any]:
|
| 1749 |
+
root = _workflow_root(district, manifest_sha256)
|
| 1750 |
+
root.mkdir(parents=True, exist_ok=True)
|
| 1751 |
+
state_path = root / "workflow_state.json"
|
| 1752 |
+
state: Dict[str, Any] = {
|
| 1753 |
+
"formatVersion": 1,
|
| 1754 |
+
"nodeVersion": NODE_VERSION,
|
| 1755 |
+
"createdUtc": _utc_now(),
|
| 1756 |
+
"districtName": district,
|
| 1757 |
+
"manifestSha256": manifest_sha256,
|
| 1758 |
+
"sourceCommit": source_commit,
|
| 1759 |
+
"referenceCommit": reference_commit,
|
| 1760 |
+
"sourceRepoId": SOURCE_REPO_ID,
|
| 1761 |
+
"generatedRepoId": GENERATED_REPO_ID,
|
| 1762 |
+
"referenceRepoId": REFERENCE_REPO_ID,
|
| 1763 |
+
"workflowRoot": str(root.resolve()),
|
| 1764 |
+
"statePath": str(state_path.resolve()),
|
| 1765 |
+
"logicalBatchSetCount": int(manifest.get("batchSetCount") or 0),
|
| 1766 |
+
"totalImageCount": len(jobs),
|
| 1767 |
+
"completedCustomIds": [],
|
| 1768 |
+
"attemptCounts": {},
|
| 1769 |
+
"openAiBatchSequence": 0,
|
| 1770 |
+
"activeBatch": None,
|
| 1771 |
+
"resultHistory": [],
|
| 1772 |
+
"failureHistory": [],
|
| 1773 |
+
}
|
| 1774 |
+
_write_json(root / "uploaded_navigation_manifest.json", manifest)
|
| 1775 |
+
_write_json(state_path, state)
|
| 1776 |
+
return state
|
| 1777 |
+
|
| 1778 |
+
|
| 1779 |
+
def _load_or_initialize_state(
|
| 1780 |
+
manifest: Mapping[str, Any],
|
| 1781 |
+
jobs: Sequence[Mapping[str, Any]],
|
| 1782 |
+
district: str,
|
| 1783 |
+
source_commit: str,
|
| 1784 |
+
reference_commit: str,
|
| 1785 |
+
preview_json: Path,
|
| 1786 |
+
) -> Dict[str, Any]:
|
| 1787 |
+
root = preview_json.parent
|
| 1788 |
+
state_path = root / "workflow_state.json"
|
| 1789 |
+
preview_data = _read_json(preview_json)
|
| 1790 |
+
manifest_sha256 = str(preview_data["manifestSha256"])
|
| 1791 |
+
if state_path.is_file():
|
| 1792 |
+
state = _read_json(state_path)
|
| 1793 |
+
if str(state.get("manifestSha256") or "") != manifest_sha256:
|
| 1794 |
+
raise ValueError("Saved workflow identity does not match this manifest.")
|
| 1795 |
+
has_started = bool(
|
| 1796 |
+
state.get("activeBatch")
|
| 1797 |
+
or state.get("completedCustomIds")
|
| 1798 |
+
or state.get("attemptCounts")
|
| 1799 |
+
or int(state.get("openAiBatchSequence") or 0) > 0
|
| 1800 |
+
)
|
| 1801 |
+
if not has_started and not isinstance(state.get("lastJsonlOnly"), Mapping):
|
| 1802 |
+
# Refresh an untouched state, but keep commits pinned after a JSONL
|
| 1803 |
+
# export. Uploading the artifact advances the source repository; if
|
| 1804 |
+
# that new commit were substituted into Image_A URLs, an otherwise
|
| 1805 |
+
# identical second export would produce different JSONL and would no
|
| 1806 |
+
# longer match the file the user inspected.
|
| 1807 |
+
state["sourceCommit"] = source_commit
|
| 1808 |
+
state["referenceCommit"] = reference_commit
|
| 1809 |
+
state["nodeVersion"] = NODE_VERSION
|
| 1810 |
+
_write_json(state_path, state)
|
| 1811 |
+
return state
|
| 1812 |
+
return _initialize_state(
|
| 1813 |
+
manifest,
|
| 1814 |
+
district,
|
| 1815 |
+
source_commit,
|
| 1816 |
+
reference_commit,
|
| 1817 |
+
manifest_sha256,
|
| 1818 |
+
jobs,
|
| 1819 |
+
)
|
| 1820 |
+
|
| 1821 |
+
|
| 1822 |
+
def _next_incomplete_set(
|
| 1823 |
+
jobs: Sequence[Mapping[str, Any]],
|
| 1824 |
+
completed_ids: Sequence[str],
|
| 1825 |
+
) -> Tuple[int, List[Mapping[str, Any]]]:
|
| 1826 |
+
completed = set(str(value) for value in completed_ids)
|
| 1827 |
+
incomplete = [job for job in jobs if str(job["customId"]) not in completed]
|
| 1828 |
+
if not incomplete:
|
| 1829 |
+
return 0, []
|
| 1830 |
+
set_number = min(int(job["_batchSetNumber"]) for job in incomplete)
|
| 1831 |
+
return (
|
| 1832 |
+
set_number,
|
| 1833 |
+
[job for job in incomplete if int(job["_batchSetNumber"]) == set_number],
|
| 1834 |
+
)
|
| 1835 |
+
|
| 1836 |
+
|
| 1837 |
+
def _prepare(
|
| 1838 |
+
district_name: str,
|
| 1839 |
+
hf_key: str,
|
| 1840 |
+
) -> Tuple[
|
| 1841 |
+
str,
|
| 1842 |
+
Optional[str],
|
| 1843 |
+
Dict[str, Any],
|
| 1844 |
+
List[Dict[str, Any]],
|
| 1845 |
+
str,
|
| 1846 |
+
str,
|
| 1847 |
+
Path,
|
| 1848 |
+
Path,
|
| 1849 |
+
str,
|
| 1850 |
+
]:
|
| 1851 |
+
district = _safe_district_name(district_name)
|
| 1852 |
+
hf_token = _optional_secret(hf_key)
|
| 1853 |
+
manifest, source_commit, manifest_sha256, _ = _download_manifest(
|
| 1854 |
+
district, hf_token
|
| 1855 |
+
)
|
| 1856 |
+
jobs = _validate_manifest(manifest, district)
|
| 1857 |
+
_, HfApi, _ = _load_hf_tools()
|
| 1858 |
+
reference_commit = _repo_commit(
|
| 1859 |
+
HfApi(token=hf_token), REFERENCE_REPO_ID, hf_token
|
| 1860 |
+
)
|
| 1861 |
+
preview_json, preview_text, all_prompts = _write_prompt_preview(
|
| 1862 |
+
manifest,
|
| 1863 |
+
jobs,
|
| 1864 |
+
district,
|
| 1865 |
+
source_commit,
|
| 1866 |
+
reference_commit,
|
| 1867 |
+
manifest_sha256,
|
| 1868 |
+
)
|
| 1869 |
+
return (
|
| 1870 |
+
district,
|
| 1871 |
+
hf_token,
|
| 1872 |
+
manifest,
|
| 1873 |
+
jobs,
|
| 1874 |
+
source_commit,
|
| 1875 |
+
reference_commit,
|
| 1876 |
+
preview_json,
|
| 1877 |
+
preview_text,
|
| 1878 |
+
all_prompts,
|
| 1879 |
+
)
|
| 1880 |
+
|
| 1881 |
+
|
| 1882 |
+
def run_city_responses_batch(
|
| 1883 |
+
action: str,
|
| 1884 |
+
district_name: str,
|
| 1885 |
+
hf_key: str,
|
| 1886 |
+
openai_api_key: str,
|
| 1887 |
+
maximum_attempts: int,
|
| 1888 |
+
only_create_jsonl: bool = False,
|
| 1889 |
+
) -> Tuple[str, str, int, int, int, str, str]:
|
| 1890 |
+
(
|
| 1891 |
+
district,
|
| 1892 |
+
optional_hf_token,
|
| 1893 |
+
manifest,
|
| 1894 |
+
jobs,
|
| 1895 |
+
source_commit,
|
| 1896 |
+
reference_commit,
|
| 1897 |
+
preview_json,
|
| 1898 |
+
preview_text,
|
| 1899 |
+
all_prompts,
|
| 1900 |
+
) = _prepare(district_name, hf_key)
|
| 1901 |
+
total = len(jobs)
|
| 1902 |
+
if action not in {ACTION_PREVIEW, ACTION_RUN}:
|
| 1903 |
+
raise ValueError(f"Unknown ACTION {action!r}.")
|
| 1904 |
+
maximum_attempts = int(maximum_attempts)
|
| 1905 |
+
if maximum_attempts < 1 or maximum_attempts > 50:
|
| 1906 |
+
raise ValueError("MAX_ATTEMPTS must be between 1 and 50.")
|
| 1907 |
+
if action == ACTION_PREVIEW and not only_create_jsonl:
|
| 1908 |
+
return (
|
| 1909 |
+
f"Preview only. Validated {total} prompts in "
|
| 1910 |
+
f"{manifest['batchSetCount']} logical Batch sets. No OpenAI file or "
|
| 1911 |
+
f"Batch was created. Read {preview_text.resolve()} before changing "
|
| 1912 |
+
"ACTION to RUN_OR_CONTINUE.",
|
| 1913 |
+
"",
|
| 1914 |
+
0,
|
| 1915 |
+
0,
|
| 1916 |
+
total,
|
| 1917 |
+
str(preview_text.resolve()),
|
| 1918 |
+
all_prompts,
|
| 1919 |
+
)
|
| 1920 |
+
hf_token = _required_secret(hf_key, "Hugging Face token")
|
| 1921 |
+
state = _load_or_initialize_state(
|
| 1922 |
+
manifest,
|
| 1923 |
+
jobs,
|
| 1924 |
+
district,
|
| 1925 |
+
source_commit,
|
| 1926 |
+
reference_commit,
|
| 1927 |
+
preview_json,
|
| 1928 |
+
)
|
| 1929 |
+
|
| 1930 |
+
if only_create_jsonl:
|
| 1931 |
+
completed_count = len(state.get("completedCustomIds") or [])
|
| 1932 |
+
active_value = state.get("activeBatch")
|
| 1933 |
+
if isinstance(active_value, Mapping):
|
| 1934 |
+
jsonl_path = Path(str(active_value.get("jsonlPath") or ""))
|
| 1935 |
+
if not jsonl_path.is_file():
|
| 1936 |
+
raise FileNotFoundError(
|
| 1937 |
+
"The active Batch JSONL is missing locally: " + str(jsonl_path)
|
| 1938 |
+
)
|
| 1939 |
+
set_number = int(active_value.get("batchSetNumber") or 0)
|
| 1940 |
+
url, commit_oid, path_in_repo = _upload_jsonl_artifact(
|
| 1941 |
+
jsonl_path,
|
| 1942 |
+
district,
|
| 1943 |
+
set_number,
|
| 1944 |
+
str(state["manifestSha256"]),
|
| 1945 |
+
hf_token,
|
| 1946 |
+
)
|
| 1947 |
+
state["lastJsonlOnly"] = {
|
| 1948 |
+
"createdUtc": _utc_now(),
|
| 1949 |
+
"batchSetNumber": set_number,
|
| 1950 |
+
"requestCount": int(active_value.get("requestCount") or 0),
|
| 1951 |
+
"localPath": str(jsonl_path.resolve()),
|
| 1952 |
+
"huggingFaceRepoPath": path_in_repo,
|
| 1953 |
+
"huggingFaceCommit": commit_oid,
|
| 1954 |
+
"huggingFaceUrl": url,
|
| 1955 |
+
"source": "existing_active_batch",
|
| 1956 |
+
}
|
| 1957 |
+
_write_json(Path(str(state["statePath"])), state)
|
| 1958 |
+
return (
|
| 1959 |
+
"Only_Create_Jsonl is ON. Uploaded the exact JSONL already used "
|
| 1960 |
+
"by active Batch "
|
| 1961 |
+
+ str(active_value.get("batchId") or "")
|
| 1962 |
+
+ ". No OpenAI API call was made. Image_A=Unity screenshot; "
|
| 1963 |
+
"Image_B=Depth; Image_C=Segmentation; Image_D=Lineart; Image_E "
|
| 1964 |
+
"and later=published generated continuity dependencies. "
|
| 1965 |
+
"Hugging Face JSONL: "
|
| 1966 |
+
+ url,
|
| 1967 |
+
str(active_value.get("batchId") or ""),
|
| 1968 |
+
set_number,
|
| 1969 |
+
completed_count,
|
| 1970 |
+
total,
|
| 1971 |
+
str(preview_text.resolve()),
|
| 1972 |
+
all_prompts,
|
| 1973 |
+
)
|
| 1974 |
+
set_number, ready_jobs = _next_incomplete_set(
|
| 1975 |
+
jobs, list(state.get("completedCustomIds") or [])
|
| 1976 |
+
)
|
| 1977 |
+
if not ready_jobs:
|
| 1978 |
+
return (
|
| 1979 |
+
"Only_Create_Jsonl is ON, but all city images are already "
|
| 1980 |
+
"complete; there is no pending logical Batch set to export. "
|
| 1981 |
+
"No OpenAI API call was made.",
|
| 1982 |
+
"",
|
| 1983 |
+
int(manifest["batchSetCount"]),
|
| 1984 |
+
total,
|
| 1985 |
+
total,
|
| 1986 |
+
str(preview_text.resolve()),
|
| 1987 |
+
all_prompts,
|
| 1988 |
+
)
|
| 1989 |
+
status, set_number, local_jsonl, _url = _create_jsonl_only(
|
| 1990 |
+
state,
|
| 1991 |
+
ready_jobs,
|
| 1992 |
+
district,
|
| 1993 |
+
hf_token,
|
| 1994 |
+
maximum_attempts,
|
| 1995 |
+
)
|
| 1996 |
+
return (
|
| 1997 |
+
status + " Local JSONL: " + local_jsonl,
|
| 1998 |
+
"",
|
| 1999 |
+
set_number,
|
| 2000 |
+
completed_count,
|
| 2001 |
+
total,
|
| 2002 |
+
str(preview_text.resolve()),
|
| 2003 |
+
all_prompts,
|
| 2004 |
+
)
|
| 2005 |
+
|
| 2006 |
+
openai_key = _required_secret(openai_api_key, "OpenAI API key")
|
| 2007 |
+
|
| 2008 |
+
progress_note = ""
|
| 2009 |
+
active_value = state.get("activeBatch")
|
| 2010 |
+
if isinstance(active_value, Mapping):
|
| 2011 |
+
batch_id = str(active_value.get("batchId") or "")
|
| 2012 |
+
batch = _batch_status(batch_id, openai_key)
|
| 2013 |
+
if str(batch.get("id") or "") != batch_id:
|
| 2014 |
+
raise ValueError("OpenAI returned a different Batch identity.")
|
| 2015 |
+
if str(batch.get("endpoint") or "") != OPENAI_BATCH_ENDPOINT:
|
| 2016 |
+
raise ValueError("Saved OpenAI Batch does not target /v1/responses.")
|
| 2017 |
+
if str(batch.get("input_file_id") or "") != str(
|
| 2018 |
+
active_value.get("inputFileId") or ""
|
| 2019 |
+
):
|
| 2020 |
+
raise ValueError("OpenAI returned a different Batch input file identity.")
|
| 2021 |
+
metadata = batch.get("metadata")
|
| 2022 |
+
if (
|
| 2023 |
+
not isinstance(metadata, Mapping)
|
| 2024 |
+
or str(metadata.get("purpose") or "") != PURPOSE
|
| 2025 |
+
or str(metadata.get("batch_set") or "")
|
| 2026 |
+
!= str(active_value.get("batchSetNumber") or "")
|
| 2027 |
+
):
|
| 2028 |
+
raise ValueError("OpenAI Batch metadata does not identify this city wave.")
|
| 2029 |
+
status = str(batch.get("status") or "unknown")
|
| 2030 |
+
total_requests, completed_requests, failed_requests = _request_counts(batch)
|
| 2031 |
+
if status in _WAITING_BATCH_STATUSES:
|
| 2032 |
+
return (
|
| 2033 |
+
f"Waiting for Responses Batch {batch_id}: status={status}, "
|
| 2034 |
+
f"total={total_requests}, completed={completed_requests}, "
|
| 2035 |
+
f"failed={failed_requests}. Logical Batch set "
|
| 2036 |
+
f"{active_value['batchSetNumber']} contains "
|
| 2037 |
+
f"{active_value['requestCount']} request(s).",
|
| 2038 |
+
batch_id,
|
| 2039 |
+
int(active_value["batchSetNumber"]),
|
| 2040 |
+
len(state.get("completedCustomIds") or []),
|
| 2041 |
+
total,
|
| 2042 |
+
str(preview_text.resolve()),
|
| 2043 |
+
all_prompts,
|
| 2044 |
+
)
|
| 2045 |
+
if status not in _TERMINAL_BATCH_STATUSES:
|
| 2046 |
+
raise ValueError(f"Unknown OpenAI Batch status {status!r}.")
|
| 2047 |
+
success_count, failure_count, commit_oid = _process_terminal_batch(
|
| 2048 |
+
state,
|
| 2049 |
+
batch,
|
| 2050 |
+
hf_token,
|
| 2051 |
+
openai_key,
|
| 2052 |
+
)
|
| 2053 |
+
progress_note = (
|
| 2054 |
+
f"Processed Batch {batch_id} ({status}): published {success_count}, "
|
| 2055 |
+
f"will retry {failure_count}."
|
| 2056 |
+
+ (f" Hugging Face commit {commit_oid}." if commit_oid else "")
|
| 2057 |
+
+ " "
|
| 2058 |
+
)
|
| 2059 |
+
|
| 2060 |
+
completed_ids = list(state.get("completedCustomIds") or [])
|
| 2061 |
+
next_set_number, ready_jobs = _next_incomplete_set(jobs, completed_ids)
|
| 2062 |
+
if not ready_jobs:
|
| 2063 |
+
state["completedUtc"] = str(state.get("completedUtc") or _utc_now())
|
| 2064 |
+
_write_json(Path(str(state["statePath"])), state)
|
| 2065 |
+
return (
|
| 2066 |
+
progress_note
|
| 2067 |
+
+ f"All done. Published all {total} city images across "
|
| 2068 |
+
f"{manifest['batchSetCount']} logical Batch sets.",
|
| 2069 |
+
str(state.get("lastProcessedBatchId") or ""),
|
| 2070 |
+
int(manifest["batchSetCount"]),
|
| 2071 |
+
total,
|
| 2072 |
+
total,
|
| 2073 |
+
str(preview_text.resolve()),
|
| 2074 |
+
all_prompts,
|
| 2075 |
+
)
|
| 2076 |
+
|
| 2077 |
+
start_status, batch_id, started_set, artifact = _start_wave(
|
| 2078 |
+
state,
|
| 2079 |
+
ready_jobs,
|
| 2080 |
+
district,
|
| 2081 |
+
hf_token,
|
| 2082 |
+
openai_key,
|
| 2083 |
+
maximum_attempts,
|
| 2084 |
+
)
|
| 2085 |
+
return (
|
| 2086 |
+
progress_note
|
| 2087 |
+
+ start_status
|
| 2088 |
+
+ f" Progress: {len(completed_ids)}/{total}. JSONL: {artifact}",
|
| 2089 |
+
batch_id,
|
| 2090 |
+
started_set,
|
| 2091 |
+
len(completed_ids),
|
| 2092 |
+
total,
|
| 2093 |
+
str(preview_text.resolve()),
|
| 2094 |
+
all_prompts,
|
| 2095 |
+
)
|
| 2096 |
+
|
| 2097 |
+
|
| 2098 |
+
class GPT_TO_HF_CityResponsesBatch:
|
| 2099 |
+
@classmethod
|
| 2100 |
+
def INPUT_TYPES(cls):
|
| 2101 |
+
return {
|
| 2102 |
+
"required": {
|
| 2103 |
+
"ACTION": (
|
| 2104 |
+
[ACTION_PREVIEW, ACTION_RUN],
|
| 2105 |
+
{"default": ACTION_PREVIEW},
|
| 2106 |
+
),
|
| 2107 |
+
"Only_Create_Jsonl": (
|
| 2108 |
+
"BOOLEAN",
|
| 2109 |
+
{"default": False},
|
| 2110 |
+
),
|
| 2111 |
+
"DISTRICT_NAME": (
|
| 2112 |
+
"STRING",
|
| 2113 |
+
{"default": "District_19_I", "multiline": False},
|
| 2114 |
+
),
|
| 2115 |
+
"hf_key": (
|
| 2116 |
+
"STRING",
|
| 2117 |
+
{"default": "hf_xxx", "multiline": False},
|
| 2118 |
+
),
|
| 2119 |
+
"openAi_key": (
|
| 2120 |
+
"STRING",
|
| 2121 |
+
{"default": "sk-xxx", "multiline": False},
|
| 2122 |
+
),
|
| 2123 |
+
"MAX_ATTEMPTS": (
|
| 2124 |
+
"INT",
|
| 2125 |
+
{"default": 8, "min": 1, "max": 50, "step": 1},
|
| 2126 |
+
),
|
| 2127 |
+
}
|
| 2128 |
+
}
|
| 2129 |
+
|
| 2130 |
+
RETURN_TYPES = ("STRING", "STRING", "INT", "INT", "INT", "STRING", "STRING")
|
| 2131 |
+
RETURN_NAMES = (
|
| 2132 |
+
"status",
|
| 2133 |
+
"batch_id",
|
| 2134 |
+
"logical_batch_set",
|
| 2135 |
+
"completed_images",
|
| 2136 |
+
"total_images",
|
| 2137 |
+
"prompt_preview_path",
|
| 2138 |
+
"all_prompts",
|
| 2139 |
+
)
|
| 2140 |
+
FUNCTION = "run"
|
| 2141 |
+
CATEGORY = "Salia/OpenAI Batch"
|
| 2142 |
+
OUTPUT_NODE = True
|
| 2143 |
+
|
| 2144 |
+
@classmethod
|
| 2145 |
+
def IS_CHANGED(cls, **kwargs):
|
| 2146 |
+
return float("nan")
|
| 2147 |
+
|
| 2148 |
+
def run(
|
| 2149 |
+
self,
|
| 2150 |
+
ACTION: str,
|
| 2151 |
+
DISTRICT_NAME: str,
|
| 2152 |
+
hf_key: str,
|
| 2153 |
+
openAi_key: str,
|
| 2154 |
+
MAX_ATTEMPTS: int,
|
| 2155 |
+
Only_Create_Jsonl: bool,
|
| 2156 |
+
):
|
| 2157 |
+
return run_city_responses_batch(
|
| 2158 |
+
ACTION,
|
| 2159 |
+
DISTRICT_NAME,
|
| 2160 |
+
hf_key,
|
| 2161 |
+
openAi_key,
|
| 2162 |
+
MAX_ATTEMPTS,
|
| 2163 |
+
Only_Create_Jsonl,
|
| 2164 |
+
)
|
| 2165 |
+
|
| 2166 |
+
|
| 2167 |
+
NODE_CLASS_MAPPINGS = {
|
| 2168 |
+
"GPT_TO_HF_CityResponsesBatch": GPT_TO_HF_CityResponsesBatch,
|
| 2169 |
+
}
|
| 2170 |
+
|
| 2171 |
+
NODE_DISPLAY_NAME_MAPPINGS = {
|
| 2172 |
+
"GPT_TO_HF_CityResponsesBatch": (
|
| 2173 |
+
"GPT TO HF — City Responses Batch Sets (Preview First)"
|
| 2174 |
+
),
|
| 2175 |
+
}
|