Upload Backblaze_PNG_Upload.py
Browse files- Backblaze_PNG_Upload.py +951 -106
Backblaze_PNG_Upload.py
CHANGED
|
@@ -1,45 +1,63 @@
|
|
| 1 |
-
"""Single-file ComfyUI node:
|
| 2 |
-
|
| 3 |
-
INSTALL:
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
https://www.backblaze.com/apidocs/b2-authorize-account
|
| 37 |
-
https://www.backblaze.com/apidocs/b2-create-bucket
|
| 38 |
https://www.backblaze.com/apidocs/b2-upload-file
|
| 39 |
https://www.backblaze.com/apidocs/b2-get-download-authorization
|
|
|
|
|
|
|
| 40 |
"""
|
| 41 |
|
| 42 |
import base64
|
|
|
|
| 43 |
import hashlib
|
| 44 |
import json
|
| 45 |
import os
|
|
@@ -48,7 +66,8 @@ import socket
|
|
| 48 |
import tempfile
|
| 49 |
import threading
|
| 50 |
import time
|
| 51 |
-
|
|
|
|
| 52 |
from email.utils import parsedate_to_datetime
|
| 53 |
from http.client import HTTPException
|
| 54 |
from pathlib import Path
|
|
@@ -66,7 +85,7 @@ _DEFAULT_PNG = (
|
|
| 66 |
_MAX_PNG_BYTES = 128 * 1024 * 1024
|
| 67 |
_MAX_JSON_BYTES = 4 * 1024 * 1024
|
| 68 |
_LINK_SECONDS = 7 * 24 * 60 * 60
|
| 69 |
-
_ATTEMPTS =
|
| 70 |
_TIMEOUT_SECONDS = 120
|
| 71 |
_TOKEN_ERRORS = {"bad_auth_token", "expired_auth_token"}
|
| 72 |
_KEY_ID_CACHE = {}
|
|
@@ -140,6 +159,7 @@ class _RequestError(RuntimeError):
|
|
| 140 |
self.status = status
|
| 141 |
self.code = code if re.fullmatch(r"[A-Za-z0-9_]{1,80}", code) else "http_error"
|
| 142 |
self.retry_after = retry_after
|
|
|
|
| 143 |
super().__init__("HTTP {} ({})".format(status, self.code))
|
| 144 |
|
| 145 |
@property
|
|
@@ -161,7 +181,7 @@ class _HTTPSRedirect(HTTPRedirectHandler):
|
|
| 161 |
|
| 162 |
|
| 163 |
def _delay(attempt, error):
|
| 164 |
-
seconds =
|
| 165 |
value = error.retry_after
|
| 166 |
if value:
|
| 167 |
try:
|
|
@@ -174,13 +194,12 @@ def _delay(attempt, error):
|
|
| 174 |
seconds = max(seconds, date.timestamp() - time.time())
|
| 175 |
except (TypeError, ValueError, OverflowError):
|
| 176 |
pass
|
| 177 |
-
|
| 178 |
-
raise RuntimeError("The server requested a longer retry delay. Run again later.")
|
| 179 |
-
time.sleep(seconds)
|
| 180 |
|
| 181 |
|
| 182 |
def _read_url(url, *, data=None, headers=None, limit=_MAX_JSON_BYTES,
|
| 183 |
source=False, timeout=_TIMEOUT_SECONDS):
|
|
|
|
| 184 |
request_headers = {"User-Agent": "ComfyUI-Backblaze-PNG/1.0"}
|
| 185 |
request_headers.update(headers or {})
|
| 186 |
opener = build_opener(_HTTPSRedirect() if source else _NoRedirect())
|
|
@@ -192,6 +211,7 @@ def _read_url(url, *, data=None, headers=None, limit=_MAX_JSON_BYTES,
|
|
| 192 |
raise RuntimeError("The response exceeds the allowed size (PNG limit: 128 MiB).")
|
| 193 |
chunks, total = [], 0
|
| 194 |
while True:
|
|
|
|
| 195 |
chunk = response.read(min(1024 * 1024, limit + 1 - total))
|
| 196 |
if not chunk:
|
| 197 |
break
|
|
@@ -234,6 +254,7 @@ def _json_request(url, *, payload=None, headers=None, retries=True):
|
|
| 234 |
raise _RequestError(502, "invalid_json_response")
|
| 235 |
return result
|
| 236 |
except _RequestError as exc:
|
|
|
|
| 237 |
if not retries or not exc.temporary or attempt == _ATTEMPTS - 1:
|
| 238 |
raise
|
| 239 |
_delay(attempt, exc)
|
|
@@ -372,27 +393,30 @@ class _B2Client:
|
|
| 372 |
return owned[name]
|
| 373 |
raise RuntimeError("Could not choose an available private bucket name. Run again later.")
|
| 374 |
|
| 375 |
-
def
|
| 376 |
result = self.call("b2_list_file_names", {
|
| 377 |
"bucketId": bucket_id, "prefix": name, "startFileName": name, "maxFileCount": 1})
|
| 378 |
-
return
|
| 379 |
-
|
| 380 |
-
|
| 381 |
|
| 382 |
-
def
|
|
|
|
|
|
|
|
|
|
| 383 |
sha1 = hashlib.sha1(data).hexdigest()
|
| 384 |
for attempt in range(_ATTEMPTS):
|
| 385 |
# Also covers a timed-out upload which actually succeeded on the server.
|
| 386 |
-
|
| 387 |
-
|
| 388 |
-
return
|
| 389 |
upload = self.call("b2_get_upload_url", {"bucketId": bucket["bucketId"]})
|
| 390 |
upload_url = _b2_base_url(upload["uploadUrl"], upload=True)
|
| 391 |
try:
|
| 392 |
raw = _read_url(upload_url, data=data, headers={
|
| 393 |
"Authorization": upload["authorizationToken"],
|
| 394 |
"X-Bz-File-Name": quote(name, safe=""),
|
| 395 |
-
"Content-Type":
|
| 396 |
"Content-Length": str(len(data)),
|
| 397 |
"X-Bz-Content-Sha1": sha1,
|
| 398 |
})
|
|
@@ -404,9 +428,9 @@ class _B2Client:
|
|
| 404 |
or result.get("contentSha1") != sha1
|
| 405 |
or result.get("contentLength") != len(data)):
|
| 406 |
raise _RequestError(502, "invalid_upload_response")
|
| 407 |
-
|
| 408 |
-
return
|
| 409 |
except _RequestError as exc:
|
|
|
|
| 410 |
retryable = exc.temporary or exc.code in _TOKEN_ERRORS or exc.code == "auth_token_limit"
|
| 411 |
if not retryable or attempt == _ATTEMPTS - 1:
|
| 412 |
raise
|
|
@@ -422,82 +446,903 @@ class _B2Client:
|
|
| 422 |
+ urlencode({"Authorization": result["authorizationToken"]}))
|
| 423 |
|
| 424 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 425 |
class BackblazePNGUpload:
|
| 426 |
@classmethod
|
| 427 |
def INPUT_TYPES(cls):
|
| 428 |
return {"required": {
|
| 429 |
"master_key": ("STRING", {"default": "", "multiline": False,
|
| 430 |
-
"tooltip": "
|
| 431 |
-
|
| 432 |
-
|
| 433 |
-
|
|
|
|
| 434 |
}, "optional": {
|
| 435 |
"key_id": ("STRING", {"default": "", "multiline": False,
|
| 436 |
-
"tooltip": "
|
| 437 |
-
"verified saved ID, or the ID from KEY_ID:SECRET. First use of a bare "
|
| 438 |
-
"secret requires this ID; Backblaze cannot retrieve it from the secret."}),
|
| 439 |
}}
|
| 440 |
|
| 441 |
-
RETURN_TYPES = ("STRING", "STRING")
|
| 442 |
-
RETURN_NAMES = ("
|
| 443 |
FUNCTION = "upload"
|
| 444 |
CATEGORY = "Salia/Backblaze"
|
| 445 |
OUTPUT_NODE = True
|
| 446 |
-
DESCRIPTION =
|
| 447 |
-
"Returns a seven-day download URL and STATUS with the key ID. "
|
| 448 |
-
"Blank key_id uses a saved ID; first-time bare secrets still require an ID.")
|
| 449 |
|
| 450 |
@classmethod
|
| 451 |
-
def IS_CHANGED(cls, master_key,
|
| 452 |
-
# A source URL can change; private download URLs must never stay cached forever.
|
| 453 |
return float("nan")
|
| 454 |
|
| 455 |
-
def upload(self, master_key,
|
| 456 |
-
|
| 457 |
-
|
|
|
|
|
|
|
| 458 |
secret = ""
|
| 459 |
-
cache_note = ""
|
| 460 |
-
link = ""
|
| 461 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 462 |
resolved_id, secret, source = _credential_parts(master_key, key_id)
|
| 463 |
-
|
| 464 |
-
|
| 465 |
-
data = _download_png(png_url)
|
| 466 |
-
name = "images/" + hashlib.sha256(data).hexdigest() + ".png"
|
| 467 |
-
client = _B2Client(basic_auth)
|
| 468 |
cache_note = _remember_key_id(secret, resolved_id)
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
|
| 472 |
-
|
| 473 |
-
|
| 474 |
-
|
| 475 |
-
|
| 476 |
-
|
| 477 |
-
|
| 478 |
-
|
| 479 |
-
|
| 480 |
-
|
| 481 |
-
|
| 482 |
-
|
| 483 |
-
|
|
|
|
|
|
|
|
|
|
| 484 |
except Exception as exc:
|
| 485 |
-
|
| 486 |
-
status = "ERROR.
|
| 487 |
-
|
| 488 |
-
|
| 489 |
-
|
| 490 |
-
|
| 491 |
-
|
| 492 |
-
|
| 493 |
-
if isinstance(master_key, str):
|
| 494 |
-
sensitive += [master_key.strip(), master_key.strip().split(":", 1)[-1]]
|
| 495 |
-
for value in sorted(set(sensitive), key=len, reverse=True):
|
| 496 |
if value:
|
| 497 |
status = status.replace(value, "[REDACTED]")
|
| 498 |
-
print("[Backblaze
|
| 499 |
-
return {"ui": {"text": [status] + ([
|
| 500 |
|
| 501 |
|
| 502 |
NODE_CLASS_MAPPINGS = {"BackblazePNGUpload": BackblazePNGUpload}
|
| 503 |
-
NODE_DISPLAY_NAME_MAPPINGS = {"BackblazePNGUpload": "Backblaze
|
|
|
|
| 1 |
+
"""Single-file ComfyUI node: resumable Hugging Face district -> Backblaze mirror.
|
| 2 |
+
|
| 3 |
+
INSTALL: Replace this file in ComfyUI/custom_nodes/, restart ComfyUI, then re-add
|
| 4 |
+
"Backblaze District Mirror (Auto / Resume)". No __init__.py or packages needed.
|
| 5 |
+
Node class ID remains BackblazePNGUpload. The old png_url input is replaced.
|
| 6 |
+
|
| 7 |
+
REQUIRED INPUTS:
|
| 8 |
+
master_key: Backblaze master secret, or legacy KEY_ID:SECRET.
|
| 9 |
+
District_Name: e.g. District_19_K. Recursively scans that folder in the model
|
| 10 |
+
repository saliacoel/map_imgs_gpt, branch main.
|
| 11 |
+
hf_key: Hugging Face token with WRITE access to that repository.
|
| 12 |
+
OPTIONAL INPUT:
|
| 13 |
+
key_id: Backblaze key ID/account ID. Blank uses the ID in KEY_ID:SECRET
|
| 14 |
+
or the ID remembered after a previous successful authentication.
|
| 15 |
+
Backblaze cannot discover a first-time ID from the secret alone.
|
| 16 |
+
OUTPUTS: dictionary_url, STATUS, dictionary_json (all STRING).
|
| 17 |
+
|
| 18 |
+
The node publishes DICT_BLAZE_<District_Name>.json at the repository root.
|
| 19 |
+
Its "dictionary" maps complete Hugging Face main-branch image URLs to Backblaze
|
| 20 |
+
URLs. "files" contains source hashes, pinned source URLs/commits, B2 object IDs,
|
| 21 |
+
checksums, URL expiration timestamps, statuses, attempts and errors. Other fields
|
| 22 |
+
record source inventory/commit history, added/changed/removed paths, counts and
|
| 23 |
+
run/checkpoint history. Source deletions never delete Backblaze objects.
|
| 24 |
+
|
| 25 |
+
Every scan/download is pinned to one source commit. Unchanged source hashes reuse
|
| 26 |
+
existing Backblaze objects. Re-running also refreshes private links without
|
| 27 |
+
redownloading unchanged images. Links are valid for at most seven days; stored
|
| 28 |
+
objects do not expire. Anyone who can read the dictionary can use its bearer URLs
|
| 29 |
+
until expiration. API secrets and account-wide authorization tokens are NOT put
|
| 30 |
+
in the dictionary. One scoped download token serves only this district's prefix.
|
| 31 |
+
|
| 32 |
+
Transient requests get 8 attempts total, waiting 3,5,7,9,11,13,15 seconds. A longer
|
| 33 |
+
Retry-After is respected. Exhausted transient failures stop the run and trigger
|
| 34 |
+
a final partial dictionary save. Permanent bad source files are recorded and
|
| 35 |
+
skipped so later images can still upload. Run again to resume failed/pending work.
|
| 36 |
+
|
| 37 |
+
Progress is written atomically locally after each image, before requesting its
|
| 38 |
+
link, and after completion. Hugging Face checkpoints occur before image transfer,
|
| 39 |
+
every 25 files or 60 seconds, and on completion/failure/cancellation. Local state
|
| 40 |
+
survives a failed remote publication and retains source variants if scanning
|
| 41 |
+
fails. Local process/OS locks and remote parent-commit checks prevent conflicting
|
| 42 |
+
writers from silently overwriting progress. If HF rejects the final save, STATUS
|
| 43 |
+
says so, and dictionary_json plus the local checkpoint retain the result.
|
| 44 |
+
|
| 45 |
+
Runtime data lives under the ComfyUI user directory/backblaze_png_upload (fallback
|
| 46 |
+
~/.cache/comfyui-backblaze-png). Key cache stores only key ID + SHA-256(secret).
|
| 47 |
+
No manual configuration required. Limits: 128 MiB per image, 32 MiB per dictionary;
|
| 48 |
+
last 100 run records and up to 100 recent source commit summaries are retained.
|
| 49 |
+
Credentials in ComfyUI widgets may be saved in workflows; remove before sharing.
|
| 50 |
+
|
| 51 |
+
Verified API contracts:
|
| 52 |
https://www.backblaze.com/apidocs/b2-authorize-account
|
|
|
|
| 53 |
https://www.backblaze.com/apidocs/b2-upload-file
|
| 54 |
https://www.backblaze.com/apidocs/b2-get-download-authorization
|
| 55 |
+
https://huggingface.co/docs/huggingface_hub/package_reference/hf_api
|
| 56 |
+
https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/_commit_api.py
|
| 57 |
"""
|
| 58 |
|
| 59 |
import base64
|
| 60 |
+
import copy
|
| 61 |
import hashlib
|
| 62 |
import json
|
| 63 |
import os
|
|
|
|
| 66 |
import tempfile
|
| 67 |
import threading
|
| 68 |
import time
|
| 69 |
+
import uuid
|
| 70 |
+
from datetime import datetime, timezone
|
| 71 |
from email.utils import parsedate_to_datetime
|
| 72 |
from http.client import HTTPException
|
| 73 |
from pathlib import Path
|
|
|
|
| 85 |
_MAX_PNG_BYTES = 128 * 1024 * 1024
|
| 86 |
_MAX_JSON_BYTES = 4 * 1024 * 1024
|
| 87 |
_LINK_SECONDS = 7 * 24 * 60 * 60
|
| 88 |
+
_ATTEMPTS = 8
|
| 89 |
_TIMEOUT_SECONDS = 120
|
| 90 |
_TOKEN_ERRORS = {"bad_auth_token", "expired_auth_token"}
|
| 91 |
_KEY_ID_CACHE = {}
|
|
|
|
| 159 |
self.status = status
|
| 160 |
self.code = code if re.fullmatch(r"[A-Za-z0-9_]{1,80}", code) else "http_error"
|
| 161 |
self.retry_after = retry_after
|
| 162 |
+
self.attempts = 1
|
| 163 |
super().__init__("HTTP {} ({})".format(status, self.code))
|
| 164 |
|
| 165 |
@property
|
|
|
|
| 181 |
|
| 182 |
|
| 183 |
def _delay(attempt, error):
|
| 184 |
+
seconds = 3 + 2 * attempt
|
| 185 |
value = error.retry_after
|
| 186 |
if value:
|
| 187 |
try:
|
|
|
|
| 194 |
seconds = max(seconds, date.timestamp() - time.time())
|
| 195 |
except (TypeError, ValueError, OverflowError):
|
| 196 |
pass
|
| 197 |
+
_interruptible_sleep(seconds)
|
|
|
|
|
|
|
| 198 |
|
| 199 |
|
| 200 |
def _read_url(url, *, data=None, headers=None, limit=_MAX_JSON_BYTES,
|
| 201 |
source=False, timeout=_TIMEOUT_SECONDS):
|
| 202 |
+
_check_cancel()
|
| 203 |
request_headers = {"User-Agent": "ComfyUI-Backblaze-PNG/1.0"}
|
| 204 |
request_headers.update(headers or {})
|
| 205 |
opener = build_opener(_HTTPSRedirect() if source else _NoRedirect())
|
|
|
|
| 211 |
raise RuntimeError("The response exceeds the allowed size (PNG limit: 128 MiB).")
|
| 212 |
chunks, total = [], 0
|
| 213 |
while True:
|
| 214 |
+
_check_cancel()
|
| 215 |
chunk = response.read(min(1024 * 1024, limit + 1 - total))
|
| 216 |
if not chunk:
|
| 217 |
break
|
|
|
|
| 254 |
raise _RequestError(502, "invalid_json_response")
|
| 255 |
return result
|
| 256 |
except _RequestError as exc:
|
| 257 |
+
exc.attempts = attempt + 1
|
| 258 |
if not retries or not exc.temporary or attempt == _ATTEMPTS - 1:
|
| 259 |
raise
|
| 260 |
_delay(attempt, exc)
|
|
|
|
| 393 |
return owned[name]
|
| 394 |
raise RuntimeError("Could not choose an available private bucket name. Run again later.")
|
| 395 |
|
| 396 |
+
def find_file(self, bucket_id, name, sha1, length):
|
| 397 |
result = self.call("b2_list_file_names", {
|
| 398 |
"bucketId": bucket_id, "prefix": name, "startFileName": name, "maxFileCount": 1})
|
| 399 |
+
return next((item for item in result["files"]
|
| 400 |
+
if item.get("fileName") == name and item.get("action") == "upload"
|
| 401 |
+
and item.get("contentSha1") == sha1 and item.get("contentLength") == length), None)
|
| 402 |
|
| 403 |
+
def file_exists(self, bucket_id, name, sha1, length):
|
| 404 |
+
return self.find_file(bucket_id, name, sha1, length) is not None
|
| 405 |
+
|
| 406 |
+
def ensure_file(self, bucket, name, data, content_type="image/png"):
|
| 407 |
sha1 = hashlib.sha1(data).hexdigest()
|
| 408 |
for attempt in range(_ATTEMPTS):
|
| 409 |
# Also covers a timed-out upload which actually succeeded on the server.
|
| 410 |
+
existing = self.find_file(bucket["bucketId"], name, sha1, len(data))
|
| 411 |
+
if existing:
|
| 412 |
+
return existing
|
| 413 |
upload = self.call("b2_get_upload_url", {"bucketId": bucket["bucketId"]})
|
| 414 |
upload_url = _b2_base_url(upload["uploadUrl"], upload=True)
|
| 415 |
try:
|
| 416 |
raw = _read_url(upload_url, data=data, headers={
|
| 417 |
"Authorization": upload["authorizationToken"],
|
| 418 |
"X-Bz-File-Name": quote(name, safe=""),
|
| 419 |
+
"Content-Type": content_type,
|
| 420 |
"Content-Length": str(len(data)),
|
| 421 |
"X-Bz-Content-Sha1": sha1,
|
| 422 |
})
|
|
|
|
| 428 |
or result.get("contentSha1") != sha1
|
| 429 |
or result.get("contentLength") != len(data)):
|
| 430 |
raise _RequestError(502, "invalid_upload_response")
|
| 431 |
+
return result
|
|
|
|
| 432 |
except _RequestError as exc:
|
| 433 |
+
exc.attempts = attempt + 1
|
| 434 |
retryable = exc.temporary or exc.code in _TOKEN_ERRORS or exc.code == "auth_token_limit"
|
| 435 |
if not retryable or attempt == _ATTEMPTS - 1:
|
| 436 |
raise
|
|
|
|
| 446 |
+ urlencode({"Authorization": result["authorizationToken"]}))
|
| 447 |
|
| 448 |
|
| 449 |
+
|
| 450 |
+
|
| 451 |
+
import base64
|
| 452 |
+
import json
|
| 453 |
+
import math
|
| 454 |
+
import re
|
| 455 |
+
import socket
|
| 456 |
+
import time
|
| 457 |
+
from datetime import timezone
|
| 458 |
+
from email.utils import parsedate_to_datetime
|
| 459 |
+
from http.client import HTTPException
|
| 460 |
+
from urllib.error import HTTPError, URLError
|
| 461 |
+
from urllib.parse import quote, urljoin, urlsplit
|
| 462 |
+
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
| 463 |
+
|
| 464 |
+
|
| 465 |
+
_HF_REPO = "saliacoel/map_imgs_gpt"
|
| 466 |
+
_HF_ROOT = "https://huggingface.co"
|
| 467 |
+
_HF_API = _HF_ROOT + "/api/models/" + _HF_REPO
|
| 468 |
+
_HF_METADATA_LIMIT = 32 * 1024 * 1024
|
| 469 |
+
|
| 470 |
+
|
| 471 |
+
class _HFError(RuntimeError):
|
| 472 |
+
"""Safe for STATUS: never retain server bodies, URLs or credentials."""
|
| 473 |
+
|
| 474 |
+
def __init__(self, status, code="http_error", retry_after=None):
|
| 475 |
+
self.status = int(status)
|
| 476 |
+
self.code = code if re.fullmatch(r"[a-z_]{1,64}", str(code)) else "http_error"
|
| 477 |
+
self.retry_after = retry_after
|
| 478 |
+
self.attempts = 1
|
| 479 |
+
super().__init__("Hugging Face HTTP {} ({})".format(self.status, self.code))
|
| 480 |
+
|
| 481 |
+
@property
|
| 482 |
+
def temporary(self):
|
| 483 |
+
return (self.status in (0, 408, 429) or 500 <= self.status <= 599
|
| 484 |
+
or self.code == "rate_limited")
|
| 485 |
+
|
| 486 |
+
@property
|
| 487 |
+
def conflict(self):
|
| 488 |
+
return self.status in (409, 412) or self.code == "parent_commit_conflict"
|
| 489 |
+
|
| 490 |
+
|
| 491 |
+
class _HFSafeRedirect(HTTPRedirectHandler):
|
| 492 |
+
"""HF resolves large files to a signed CDN URL; do not send HF tokens there."""
|
| 493 |
+
|
| 494 |
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
| 495 |
+
old = urlsplit(req.full_url)
|
| 496 |
+
new = urlsplit(newurl)
|
| 497 |
+
if new.scheme.lower() != "https" or new.username or new.password:
|
| 498 |
+
raise _HFError(400, "unsafe_redirect")
|
| 499 |
+
# Commits must never be redirected/replayed to another endpoint.
|
| 500 |
+
if req.get_method() not in ("GET", "HEAD"):
|
| 501 |
+
raise _HFError(400, "unexpected_commit_redirect")
|
| 502 |
+
redirected = super().redirect_request(req, fp, code, msg, headers, newurl)
|
| 503 |
+
if redirected is not None and (old.scheme, old.netloc) != (new.scheme, new.netloc):
|
| 504 |
+
for key in list(redirected.headers):
|
| 505 |
+
if key.lower() in ("authorization", "cookie", "proxy-authorization"):
|
| 506 |
+
del redirected.headers[key]
|
| 507 |
+
for key in list(redirected.unredirected_hdrs):
|
| 508 |
+
if key.lower() in ("authorization", "cookie", "proxy-authorization"):
|
| 509 |
+
del redirected.unredirected_hdrs[key]
|
| 510 |
+
return redirected
|
| 511 |
+
|
| 512 |
+
|
| 513 |
+
def _hf_delay_seconds(attempt, retry_after):
|
| 514 |
+
"""attempt=0 after the first failure => 3,5,7,9,11,13,15 seconds."""
|
| 515 |
+
seconds = float(3 + 2 * attempt)
|
| 516 |
+
if retry_after:
|
| 517 |
+
try:
|
| 518 |
+
supplied = float(retry_after)
|
| 519 |
+
except (TypeError, ValueError):
|
| 520 |
+
try:
|
| 521 |
+
date = parsedate_to_datetime(str(retry_after))
|
| 522 |
+
if date.tzinfo is None:
|
| 523 |
+
date = date.replace(tzinfo=timezone.utc)
|
| 524 |
+
supplied = date.timestamp() - time.time()
|
| 525 |
+
except (TypeError, ValueError, OverflowError):
|
| 526 |
+
supplied = 0
|
| 527 |
+
if math.isfinite(supplied):
|
| 528 |
+
seconds = max(seconds, supplied)
|
| 529 |
+
return seconds
|
| 530 |
+
|
| 531 |
+
|
| 532 |
+
class _HFClient:
|
| 533 |
+
def __init__(self, token, check_cancel=None, sleep=None, on_retry=None):
|
| 534 |
+
token = str(token or "").strip()
|
| 535 |
+
if not token or any(c.isspace() for c in token):
|
| 536 |
+
raise ValueError("hf_key must be a Hugging Face token with write access to the repository.")
|
| 537 |
+
self._token = token
|
| 538 |
+
self._check_cancel = check_cancel or (lambda: None)
|
| 539 |
+
self._sleep = sleep or time.sleep
|
| 540 |
+
self._on_retry = on_retry or (lambda attempt, seconds, error: None)
|
| 541 |
+
self._opener = build_opener(_HFSafeRedirect())
|
| 542 |
+
|
| 543 |
+
def _wait(self, seconds):
|
| 544 |
+
# Long Retry-After values remain cancellable, with no busy waiting.
|
| 545 |
+
remaining = seconds
|
| 546 |
+
while remaining > 0:
|
| 547 |
+
self._check_cancel()
|
| 548 |
+
interval = min(1.0, remaining)
|
| 549 |
+
self._sleep(interval)
|
| 550 |
+
remaining -= interval
|
| 551 |
+
self._check_cancel()
|
| 552 |
+
|
| 553 |
+
@staticmethod
|
| 554 |
+
def _safe_url(url):
|
| 555 |
+
parts = urlsplit(url)
|
| 556 |
+
if (parts.scheme != "https" or parts.hostname != "huggingface.co"
|
| 557 |
+
or parts.port not in (None, 443) or parts.username or parts.password):
|
| 558 |
+
raise _HFError(400, "unsafe_api_url")
|
| 559 |
+
|
| 560 |
+
def _once(self, url, data=None, content_type=None, limit=_HF_METADATA_LIMIT):
|
| 561 |
+
self._safe_url(url)
|
| 562 |
+
self._check_cancel()
|
| 563 |
+
headers = {"User-Agent": "ComfyUI-Backblaze-District/3.0",
|
| 564 |
+
"Authorization": "Bearer " + self._token,
|
| 565 |
+
"Accept-Encoding": "identity"}
|
| 566 |
+
if content_type:
|
| 567 |
+
headers["Content-Type"] = content_type
|
| 568 |
+
request = Request(url, data=data, headers=headers)
|
| 569 |
+
try:
|
| 570 |
+
with self._opener.open(request, timeout=120) as response:
|
| 571 |
+
length = response.headers.get("Content-Length", "")
|
| 572 |
+
if length.isdigit() and int(length) > limit:
|
| 573 |
+
raise _HFError(413, "response_too_large")
|
| 574 |
+
chunks, total = [], 0
|
| 575 |
+
while True:
|
| 576 |
+
self._check_cancel()
|
| 577 |
+
chunk = response.read(min(1024 * 1024, limit + 1 - total))
|
| 578 |
+
if not chunk:
|
| 579 |
+
break
|
| 580 |
+
total += len(chunk)
|
| 581 |
+
if total > limit:
|
| 582 |
+
raise _HFError(413, "response_too_large")
|
| 583 |
+
chunks.append(chunk)
|
| 584 |
+
if length.isdigit() and total != int(length):
|
| 585 |
+
raise _HFError(0, "incomplete_response")
|
| 586 |
+
return b"".join(chunks), dict(response.headers.items())
|
| 587 |
+
except HTTPError as error:
|
| 588 |
+
status = error.code
|
| 589 |
+
retry_after = error.headers.get("Retry-After")
|
| 590 |
+
code = "http_error"
|
| 591 |
+
try:
|
| 592 |
+
# Only classify recognized server phrases. Never expose the body.
|
| 593 |
+
body = error.read(65536).decode("utf-8", "replace").lower()
|
| 594 |
+
if status in (401, 403, 429) and any(phrase in body for phrase in
|
| 595 |
+
("rate limit", "rate-limit", "rate_limit", "too many requests")):
|
| 596 |
+
code = "rate_limited"
|
| 597 |
+
elif status in (400, 409, 412) and any(phrase in body for phrase in
|
| 598 |
+
("parentcommit", "parent commit", "commit has changed", "concurrent")):
|
| 599 |
+
code = "parent_commit_conflict"
|
| 600 |
+
elif status in (401, 403):
|
| 601 |
+
code = "authentication_or_permission_denied"
|
| 602 |
+
elif status == 404:
|
| 603 |
+
code = "not_found"
|
| 604 |
+
except (OSError, HTTPException):
|
| 605 |
+
pass
|
| 606 |
+
finally:
|
| 607 |
+
error.close()
|
| 608 |
+
raise _HFError(status, code, retry_after) from None
|
| 609 |
+
except (URLError, TimeoutError, socket.timeout, ConnectionError, OSError, HTTPException):
|
| 610 |
+
raise _HFError(0, "connection_or_tls_error") from None
|
| 611 |
+
|
| 612 |
+
def _request(self, url, data=None, content_type=None, limit=_HF_METADATA_LIMIT, decode_json=False):
|
| 613 |
+
for attempt in range(8):
|
| 614 |
+
try:
|
| 615 |
+
raw, headers = self._once(url, data, content_type, limit)
|
| 616 |
+
if decode_json:
|
| 617 |
+
try:
|
| 618 |
+
return json.loads(raw), headers
|
| 619 |
+
except (ValueError, UnicodeError):
|
| 620 |
+
raise _HFError(502, "invalid_json_response") from None
|
| 621 |
+
return raw, headers
|
| 622 |
+
except _HFError as error:
|
| 623 |
+
error.attempts = attempt + 1
|
| 624 |
+
if not error.temporary or attempt == 7:
|
| 625 |
+
raise
|
| 626 |
+
delay = _hf_delay_seconds(attempt, error.retry_after)
|
| 627 |
+
self._on_retry(attempt + 1, delay, error)
|
| 628 |
+
self._wait(delay)
|
| 629 |
+
|
| 630 |
+
def _json(self, url, data=None, content_type=None):
|
| 631 |
+
return self._request(url, data, content_type, decode_json=True)
|
| 632 |
+
|
| 633 |
+
@staticmethod
|
| 634 |
+
def _next_link(headers, current_url):
|
| 635 |
+
# RFC Link URLs can include commas; match bracketed targets first.
|
| 636 |
+
value = next((v for k, v in headers.items() if k.lower() == "link"), "")
|
| 637 |
+
for match in re.finditer(r"<([^<>]+)>\s*;([^<>]*)(?=,\s*<|$)", value):
|
| 638 |
+
relation = re.search(r'\brel\s*=\s*(?:"([^"]*)"|([^;,\s]+))', match.group(2))
|
| 639 |
+
if relation and "next" in (relation.group(1) or relation.group(2)).split():
|
| 640 |
+
return urljoin(current_url, match.group(1))
|
| 641 |
+
return None
|
| 642 |
+
|
| 643 |
+
def info(self):
|
| 644 |
+
data, _ = self._json(_HF_API + "/revision/main")
|
| 645 |
+
if not isinstance(data, dict) or not re.fullmatch(r"[0-9a-fA-F]{40,64}", str(data.get("sha", ""))):
|
| 646 |
+
raise _HFError(502, "invalid_repository_info")
|
| 647 |
+
return data
|
| 648 |
+
|
| 649 |
+
def tree(self, revision, district):
|
| 650 |
+
"""Return every file (all extensions) within the recursively listed district."""
|
| 651 |
+
prefix = district.rstrip("/") + "/"
|
| 652 |
+
url = (_HF_API + "/tree/" + quote(revision, safe="") + "/" +
|
| 653 |
+
quote(district, safe="/") + "?recursive=true&expand=false&limit=1000")
|
| 654 |
+
files, seen_urls, seen_paths = [], set(), set()
|
| 655 |
+
expected_path = urlsplit(url).path
|
| 656 |
+
while url:
|
| 657 |
+
if url in seen_urls:
|
| 658 |
+
raise _HFError(502, "pagination_loop")
|
| 659 |
+
self._safe_url(url)
|
| 660 |
+
if urlsplit(url).path != expected_path:
|
| 661 |
+
raise _HFError(502, "unexpected_pagination_path")
|
| 662 |
+
seen_urls.add(url)
|
| 663 |
+
data, headers = self._json(url)
|
| 664 |
+
if not isinstance(data, list):
|
| 665 |
+
raise _HFError(502, "invalid_tree_response")
|
| 666 |
+
for item in data:
|
| 667 |
+
if not isinstance(item, dict):
|
| 668 |
+
raise _HFError(502, "invalid_tree_entry")
|
| 669 |
+
if item.get("type") != "file":
|
| 670 |
+
continue
|
| 671 |
+
path = item.get("path")
|
| 672 |
+
if (not isinstance(path, str) or not path.startswith(prefix)
|
| 673 |
+
or any(part in (".", "..", "") for part in path.split("/"))):
|
| 674 |
+
raise _HFError(502, "invalid_tree_path")
|
| 675 |
+
if path not in seen_paths:
|
| 676 |
+
seen_paths.add(path)
|
| 677 |
+
files.append(item)
|
| 678 |
+
url = self._next_link(headers, url)
|
| 679 |
+
return sorted(files, key=lambda item: item["path"])
|
| 680 |
+
|
| 681 |
+
def history(self, revision):
|
| 682 |
+
# Recent history is diagnostic; file blob/LFS identities determine changes.
|
| 683 |
+
data, _ = self._json(_HF_API + "/commits/" + quote(revision, safe="") + "?limit=100")
|
| 684 |
+
if not isinstance(data, list):
|
| 685 |
+
raise _HFError(502, "invalid_commit_history")
|
| 686 |
+
return [item for item in data[:100] if isinstance(item, dict)]
|
| 687 |
+
|
| 688 |
+
def read_file(self, path, revision="main", max_bytes=_HF_METADATA_LIMIT):
|
| 689 |
+
url = (_HF_ROOT + "/" + _HF_REPO + "/resolve/" +
|
| 690 |
+
quote(revision, safe="") + "/" + quote(path, safe="/"))
|
| 691 |
+
return self._request(url, limit=max_bytes)[0]
|
| 692 |
+
|
| 693 |
+
def commit_json(self, path, content_bytes, parent_sha, message):
|
| 694 |
+
"""POST one regular JSON blob; optimistic concurrency is mandatory.
|
| 695 |
+
|
| 696 |
+
The caller owns 409/412 reconciliation. In particular, after a response
|
| 697 |
+
was lost, reread the remote dictionary and compare before recommitting.
|
| 698 |
+
"""
|
| 699 |
+
if not re.fullmatch(r"[0-9a-fA-F]{40,64}", str(parent_sha)):
|
| 700 |
+
raise ValueError("A valid parent commit SHA is required for dictionary upload.")
|
| 701 |
+
if not isinstance(content_bytes, bytes):
|
| 702 |
+
raise TypeError("Dictionary content must be UTF-8 JSON bytes.")
|
| 703 |
+
# Explicit local bound, well below the documented regular-commit payload limit.
|
| 704 |
+
if len(content_bytes) > _HF_METADATA_LIMIT:
|
| 705 |
+
raise _HFError(413, "dictionary_exceeds_thirty_two_mib")
|
| 706 |
+
lines = [
|
| 707 |
+
{"key": "header", "value": {"summary": message, "description": "",
|
| 708 |
+
"parentCommit": parent_sha}},
|
| 709 |
+
{"key": "file", "value": {"path": path, "encoding": "base64",
|
| 710 |
+
"content": base64.b64encode(content_bytes).decode("ascii")}},
|
| 711 |
+
]
|
| 712 |
+
payload = ("\n".join(json.dumps(line, separators=(",", ":"), ensure_ascii=True)
|
| 713 |
+
for line in lines) + "\n").encode("utf-8")
|
| 714 |
+
result, _ = self._json(_HF_API + "/commit/main", payload, "application/x-ndjson")
|
| 715 |
+
if isinstance(result, dict) and result.get("success") is False:
|
| 716 |
+
raise _HFError(400, "commit_rejected")
|
| 717 |
+
if (not isinstance(result, dict) or not
|
| 718 |
+
re.fullmatch(r"[0-9a-fA-F]{40,64}", str(result.get("commitOid", "")))):
|
| 719 |
+
raise _HFError(502, "invalid_commit_response")
|
| 720 |
+
return result
|
| 721 |
+
|
| 722 |
+
|
| 723 |
+
# District mirroring and durable dictionary checkpoints.
|
| 724 |
+
_REPO_ID = "saliacoel/map_imgs_gpt"
|
| 725 |
+
_MANIFEST_LIMIT = 32 * 1024 * 1024
|
| 726 |
+
_CHECKPOINT_FILES = 25
|
| 727 |
+
_CHECKPOINT_SECONDS = 60
|
| 728 |
+
_IMAGE_TYPES = {
|
| 729 |
+
".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg",
|
| 730 |
+
".webp": "image/webp", ".gif": "image/gif", ".bmp": "image/bmp",
|
| 731 |
+
".tif": "image/tiff", ".tiff": "image/tiff", ".avif": "image/avif",
|
| 732 |
+
".heic": "image/heic", ".heif": "image/heif", ".ico": "image/x-icon",
|
| 733 |
+
".svg": "image/svg+xml", ".exr": "image/x-exr",
|
| 734 |
+
}
|
| 735 |
+
_DISTRICT_LOCKS = {}
|
| 736 |
+
_DISTRICT_LOCKS_GUARD = threading.Lock()
|
| 737 |
+
|
| 738 |
+
|
| 739 |
+
class _MirrorInterrupted(Exception):
|
| 740 |
+
pass
|
| 741 |
+
|
| 742 |
+
|
| 743 |
+
class _ConcurrentMirror(RuntimeError):
|
| 744 |
+
pass
|
| 745 |
+
|
| 746 |
+
|
| 747 |
+
def _check_cancel():
|
| 748 |
+
try:
|
| 749 |
+
import comfy.model_management as management
|
| 750 |
+
except ImportError:
|
| 751 |
+
return
|
| 752 |
+
if getattr(management, "processing_interrupted", lambda: False)():
|
| 753 |
+
raise _MirrorInterrupted("Cancelled by the user; completed uploads are saved.")
|
| 754 |
+
|
| 755 |
+
|
| 756 |
+
def _interruptible_sleep(seconds):
|
| 757 |
+
remaining = max(0.0, float(seconds))
|
| 758 |
+
while remaining > 0:
|
| 759 |
+
_check_cancel()
|
| 760 |
+
part = min(1.0, remaining)
|
| 761 |
+
time.sleep(part)
|
| 762 |
+
remaining -= part
|
| 763 |
+
|
| 764 |
+
|
| 765 |
+
def _utc_now():
|
| 766 |
+
return datetime.now(timezone.utc).isoformat(timespec="milliseconds")
|
| 767 |
+
|
| 768 |
+
|
| 769 |
+
def _resolve_url(path, revision="main"):
|
| 770 |
+
return "https://huggingface.co/" + _REPO_ID + "/resolve/" + quote(revision, safe="") + "/" + quote(path, safe="/")
|
| 771 |
+
|
| 772 |
+
|
| 773 |
+
def _json_bytes(document):
|
| 774 |
+
return (json.dumps(document, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode("utf-8")
|
| 775 |
+
|
| 776 |
+
|
| 777 |
+
def _atomic_json(path, document):
|
| 778 |
+
path = Path(path)
|
| 779 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 780 |
+
temporary = None
|
| 781 |
+
try:
|
| 782 |
+
with tempfile.NamedTemporaryFile(mode="wb", dir=str(path.parent), prefix=path.name + ".", delete=False) as stream:
|
| 783 |
+
temporary = stream.name
|
| 784 |
+
stream.write(_json_bytes(document))
|
| 785 |
+
stream.flush()
|
| 786 |
+
os.fsync(stream.fileno())
|
| 787 |
+
os.replace(temporary, path)
|
| 788 |
+
finally:
|
| 789 |
+
if temporary:
|
| 790 |
+
try:
|
| 791 |
+
os.unlink(temporary)
|
| 792 |
+
except OSError:
|
| 793 |
+
pass
|
| 794 |
+
|
| 795 |
+
|
| 796 |
+
def _source_identity(item):
|
| 797 |
+
lfs = item.get("lfs") or {}
|
| 798 |
+
sha256 = str(lfs.get("oid") or lfs.get("sha256") or "").removeprefix("sha256:")
|
| 799 |
+
oid = item.get("oid") or item.get("blob_id") or ""
|
| 800 |
+
size = lfs.get("size", item.get("size"))
|
| 801 |
+
if not isinstance(size, int) or size < 0:
|
| 802 |
+
raise ValueError("Hugging Face returned an invalid image size.")
|
| 803 |
+
if sha256 and re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
|
| 804 |
+
identity = "sha256:" + sha256.lower()
|
| 805 |
+
elif re.fullmatch(r"[0-9a-fA-F]{40}", str(oid)):
|
| 806 |
+
identity = "git:" + oid.lower()
|
| 807 |
+
else:
|
| 808 |
+
raise ValueError("Hugging Face did not return a usable source hash for an image.")
|
| 809 |
+
return identity, size
|
| 810 |
+
|
| 811 |
+
|
| 812 |
+
def _verify_source_bytes(data, record):
|
| 813 |
+
if len(data) != record["size"]:
|
| 814 |
+
raise ValueError("Downloaded image size does not match the pinned Hugging Face file.")
|
| 815 |
+
fingerprint = record["source_fingerprint"]
|
| 816 |
+
if fingerprint.startswith("sha256:"):
|
| 817 |
+
actual = "sha256:" + hashlib.sha256(data).hexdigest()
|
| 818 |
+
else:
|
| 819 |
+
actual = "git:" + hashlib.sha1(b"blob " + str(len(data)).encode("ascii") + b"\0" + data).hexdigest()
|
| 820 |
+
if actual != fingerprint:
|
| 821 |
+
raise ValueError("Downloaded image checksum does not match the pinned Hugging Face file.")
|
| 822 |
+
|
| 823 |
+
|
| 824 |
+
def _validate_manifest(doc, district):
|
| 825 |
+
if (not isinstance(doc, dict) or doc.get("schema_version") != 1
|
| 826 |
+
or doc.get("repo_id") != _REPO_ID or doc.get("district") != district
|
| 827 |
+
or not isinstance(doc.get("files"), dict) or not isinstance(doc.get("dictionary"), dict)):
|
| 828 |
+
raise ValueError("The existing district dictionary has an incompatible format; it was not overwritten.")
|
| 829 |
+
for path, record in doc["files"].items():
|
| 830 |
+
if not isinstance(path, str) or not path.startswith(district + "/") or not isinstance(record, dict):
|
| 831 |
+
raise ValueError("The existing district dictionary contains invalid file records; it was not overwritten.")
|
| 832 |
+
return doc
|
| 833 |
+
|
| 834 |
+
|
| 835 |
+
def _inventory_hash(files):
|
| 836 |
+
pairs = [(path, entry["source_fingerprint"]) for path, entry in sorted(files.items()) if entry.get("status") != "removed"]
|
| 837 |
+
return hashlib.sha256(json.dumps(pairs, separators=(",", ":")).encode()).hexdigest()
|
| 838 |
+
|
| 839 |
+
|
| 840 |
+
def _merge_successes(target, other):
|
| 841 |
+
"""Resume only verified upload records for the SAME source content/destination."""
|
| 842 |
+
if target.get("backblaze") != other.get("backblaze"):
|
| 843 |
+
return
|
| 844 |
+
for path, candidate in other.get("files", {}).items():
|
| 845 |
+
entry = target["files"].get(path)
|
| 846 |
+
if (not entry or entry.get("status") == "removed" or candidate.get("status") == "removed"
|
| 847 |
+
or entry.get("source_fingerprint") != candidate.get("source_fingerprint")
|
| 848 |
+
or not candidate.get("b2", {}).get("file_name")):
|
| 849 |
+
continue
|
| 850 |
+
mine = entry.get("b2", {})
|
| 851 |
+
theirs = candidate["b2"]
|
| 852 |
+
if (not mine or (candidate.get("status") == "uploaded" and entry.get("status") != "uploaded")
|
| 853 |
+
or theirs.get("url_expires_at_unix", 0) > mine.get("url_expires_at_unix", 0)):
|
| 854 |
+
entry["b2"] = copy.deepcopy(theirs)
|
| 855 |
+
entry["status"] = candidate.get("status", "stored")
|
| 856 |
+
entry["last_error"] = candidate.get("last_error")
|
| 857 |
+
entry["updated_at"] = candidate.get("updated_at")
|
| 858 |
+
entry["processing_attempts"] = max(entry.get("processing_attempts", 0), candidate.get("processing_attempts", 0))
|
| 859 |
+
|
| 860 |
+
|
| 861 |
+
def _recovery_records(document):
|
| 862 |
+
if not document:
|
| 863 |
+
return []
|
| 864 |
+
records = list(document.get("recovery_records", []))
|
| 865 |
+
for path, record in document.get("files", {}).items():
|
| 866 |
+
if record.get("b2", {}).get("file_name"):
|
| 867 |
+
records.append({"path": path, "record": copy.deepcopy(record),
|
| 868 |
+
"backblaze": copy.deepcopy(document.get("backblaze", {}))})
|
| 869 |
+
unique = {}
|
| 870 |
+
for item in records:
|
| 871 |
+
if not isinstance(item, dict) or not isinstance(item.get("record"), dict):
|
| 872 |
+
continue
|
| 873 |
+
key = (item.get("path"), item["record"].get("source_fingerprint"),
|
| 874 |
+
item.get("backblaze", {}).get("bucket_id"))
|
| 875 |
+
unique[key] = item
|
| 876 |
+
return list(unique.values())
|
| 877 |
+
|
| 878 |
+
|
| 879 |
+
class _CheckpointLock:
|
| 880 |
+
"""OS locks survive no crashes: the OS releases them when the process exits."""
|
| 881 |
+
def __init__(self, checkpoint_path):
|
| 882 |
+
self.path = Path(str(checkpoint_path) + ".lock")
|
| 883 |
+
self.stream = None
|
| 884 |
+
|
| 885 |
+
def acquire(self):
|
| 886 |
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
| 887 |
+
self.stream = self.path.open("a+b")
|
| 888 |
+
try:
|
| 889 |
+
if os.name == "nt":
|
| 890 |
+
import msvcrt
|
| 891 |
+
if self.path.stat().st_size == 0:
|
| 892 |
+
self.stream.write(b"0")
|
| 893 |
+
self.stream.flush()
|
| 894 |
+
self.stream.seek(0)
|
| 895 |
+
msvcrt.locking(self.stream.fileno(), msvcrt.LK_NBLCK, 1)
|
| 896 |
+
else:
|
| 897 |
+
import fcntl
|
| 898 |
+
fcntl.flock(self.stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
| 899 |
+
except OSError:
|
| 900 |
+
self.stream.close()
|
| 901 |
+
self.stream = None
|
| 902 |
+
raise RuntimeError("Another ComfyUI process is using this district checkpoint. Let it finish first.") from None
|
| 903 |
+
|
| 904 |
+
def release(self):
|
| 905 |
+
if self.stream is not None:
|
| 906 |
+
self.stream.close()
|
| 907 |
+
self.stream = None
|
| 908 |
+
|
| 909 |
+
|
| 910 |
+
class _DistrictMirror:
|
| 911 |
+
def __init__(self, hf, b2, district, secrets=()):
|
| 912 |
+
self.hf, self.b2, self.district = hf, b2, district
|
| 913 |
+
self.secrets = tuple(value for value in secrets if value)
|
| 914 |
+
self.filename = "DICT_BLAZE_" + district + ".json"
|
| 915 |
+
self.url = _resolve_url(self.filename)
|
| 916 |
+
self.run_id = uuid.uuid4().hex
|
| 917 |
+
self.doc = {
|
| 918 |
+
"schema_version": 1, "repo_id": _REPO_ID, "district": district,
|
| 919 |
+
"dictionary": {}, "files": {}, "history": [], "source": {},
|
| 920 |
+
"backblaze": {}, "status": "starting", "created_at": _utc_now(),
|
| 921 |
+
"retry_policy": {"attempts": 8, "wait_seconds": [3, 5, 7, 9, 11, 13, 15], "honor_retry_after": True},
|
| 922 |
+
"links": {"private_bucket": True, "valid_seconds": _LINK_SECONDS,
|
| 923 |
+
"scope": "district", "note": "Bearer URLs expire. Re-run this node to refresh. Stored objects do not expire."},
|
| 924 |
+
}
|
| 925 |
+
self.path = _key_id_cache_path().parent / "districts" / (
|
| 926 |
+
hashlib.sha256((_REPO_ID + ":" + b2.account_id).encode()).hexdigest()[:24] + "_" + self.filename)
|
| 927 |
+
self.remote_loaded = False
|
| 928 |
+
self.remote_digest = None
|
| 929 |
+
self.published = False
|
| 930 |
+
self.remote_existed = False
|
| 931 |
+
self.last_commit = None
|
| 932 |
+
self.last_checkpoint = 0.0
|
| 933 |
+
self.completed_since_checkpoint = 0
|
| 934 |
+
self.bucket = None
|
| 935 |
+
self.link_token = None
|
| 936 |
+
self.link_issued = 0.0
|
| 937 |
+
self.error = None
|
| 938 |
+
self.publish_error = None
|
| 939 |
+
self.local_error = None
|
| 940 |
+
self.run = {"run_id": self.run_id, "started_at": _utc_now(), "status": "starting",
|
| 941 |
+
"added": [], "changed": [], "removed": [], "reused": 0, "processed": 0,
|
| 942 |
+
"dictionary_commits": [], "inventory_complete": False}
|
| 943 |
+
|
| 944 |
+
def safe_error(self, exc):
|
| 945 |
+
if isinstance(exc, (_HFError, _RequestError, ValueError, RuntimeError, _MirrorInterrupted)):
|
| 946 |
+
text = str(exc)
|
| 947 |
+
else:
|
| 948 |
+
text = "Unexpected " + type(exc).__name__
|
| 949 |
+
for secret in sorted(self.secrets, key=len, reverse=True):
|
| 950 |
+
text = text.replace(secret, "[REDACTED]")
|
| 951 |
+
return text[:1500]
|
| 952 |
+
|
| 953 |
+
def read_remote(self, revision):
|
| 954 |
+
try:
|
| 955 |
+
data = self.hf.read_file(self.filename, revision, _MANIFEST_LIMIT)
|
| 956 |
+
except _HFError as exc:
|
| 957 |
+
if exc.status == 404:
|
| 958 |
+
return None, None
|
| 959 |
+
raise
|
| 960 |
+
try:
|
| 961 |
+
doc = json.loads(data)
|
| 962 |
+
except (ValueError, UnicodeError):
|
| 963 |
+
raise ValueError("The remote district dictionary is not valid JSON; it was not overwritten.") from None
|
| 964 |
+
return _validate_manifest(doc, self.district), hashlib.sha256(data).hexdigest()
|
| 965 |
+
|
| 966 |
+
def rebuild(self):
|
| 967 |
+
mapping = {}
|
| 968 |
+
counts = {"total": 0, "uploaded": 0, "stored": 0, "failed": 0, "pending": 0, "removed": 0}
|
| 969 |
+
now = time.time()
|
| 970 |
+
for path, entry in sorted(self.doc["files"].items()):
|
| 971 |
+
state = entry.get("status", "pending")
|
| 972 |
+
if state == "removed":
|
| 973 |
+
counts["removed"] += 1
|
| 974 |
+
continue
|
| 975 |
+
counts["total"] += 1
|
| 976 |
+
counts[state if state in counts else "pending"] += 1
|
| 977 |
+
b2 = entry.get("b2", {})
|
| 978 |
+
if (state == "uploaded" and b2.get("download_url")
|
| 979 |
+
and b2.get("url_expires_at_unix", 0) > now):
|
| 980 |
+
mapping[entry["source_url"]] = b2["download_url"]
|
| 981 |
+
counts["mapped"] = len(mapping)
|
| 982 |
+
self.doc["dictionary"] = mapping
|
| 983 |
+
self.doc["counts"] = counts
|
| 984 |
+
self.doc["updated_at"] = _utc_now()
|
| 985 |
+
self.doc["run_id"] = self.run_id
|
| 986 |
+
history = [item for item in self.doc.get("history", []) if item.get("run_id") != self.run_id]
|
| 987 |
+
self.doc["history"] = (history + [copy.deepcopy(self.run)])[-100:]
|
| 988 |
+
|
| 989 |
+
def save_local(self, strict=False):
|
| 990 |
+
self.rebuild()
|
| 991 |
+
try:
|
| 992 |
+
_atomic_json(self.path, self.doc)
|
| 993 |
+
self.local_error = None
|
| 994 |
+
except OSError as exc:
|
| 995 |
+
self.local_error = "Local checkpoint could not be saved: " + type(exc).__name__
|
| 996 |
+
if strict:
|
| 997 |
+
raise RuntimeError(self.local_error) from None
|
| 998 |
+
|
| 999 |
+
def publish(self):
|
| 1000 |
+
"""Compare-and-swap; merge concurrent same-inventory successes, never overwrite a newer inventory."""
|
| 1001 |
+
if not self.remote_loaded:
|
| 1002 |
+
raise RuntimeError("Remote state was not read successfully; refusing to overwrite it.")
|
| 1003 |
+
self.doc.pop("last_publication_error", None)
|
| 1004 |
+
checkpoint_id = uuid.uuid4().hex
|
| 1005 |
+
for conflict_attempt in range(8):
|
| 1006 |
+
info = self.hf.info()
|
| 1007 |
+
head = info["sha"]
|
| 1008 |
+
remote, digest = self.read_remote(head)
|
| 1009 |
+
if remote and remote.get("checkpoint_id") == checkpoint_id:
|
| 1010 |
+
# The previous POST committed but its response was lost.
|
| 1011 |
+
self.remote_digest = digest
|
| 1012 |
+
self._published(head)
|
| 1013 |
+
return
|
| 1014 |
+
if digest != self.remote_digest:
|
| 1015 |
+
if (not remote or remote.get("inventory_hash") != self.doc.get("inventory_hash")
|
| 1016 |
+
or remote.get("backblaze") != self.doc.get("backblaze")):
|
| 1017 |
+
raise _ConcurrentMirror("Another run changed this dictionary's source inventory or destination. Local progress is saved; run again to merge safely.")
|
| 1018 |
+
_merge_successes(self.doc, remote)
|
| 1019 |
+
known = {item.get("run_id") for item in self.doc.get("history", [])}
|
| 1020 |
+
self.doc.setdefault("history", []).extend(item for item in remote.get("history", []) if item.get("run_id") not in known)
|
| 1021 |
+
self.doc["checkpoint_id"] = checkpoint_id
|
| 1022 |
+
self.doc["hf_head_before_checkpoint"] = head
|
| 1023 |
+
self.rebuild()
|
| 1024 |
+
payload = _json_bytes(self.doc)
|
| 1025 |
+
try:
|
| 1026 |
+
result = self.hf.commit_json(self.filename, payload, head,
|
| 1027 |
+
"Backblaze mirror " + self.district + ": " + self.doc["status"])
|
| 1028 |
+
except _HFError as exc:
|
| 1029 |
+
if exc.conflict and conflict_attempt < 7:
|
| 1030 |
+
continue
|
| 1031 |
+
raise
|
| 1032 |
+
commit = result.get("commitOid") or result.get("oid") or result.get("commit_id")
|
| 1033 |
+
if not commit:
|
| 1034 |
+
# Verify content using the same checkpoint ID rather than claiming success.
|
| 1035 |
+
head = self.hf.info()["sha"]
|
| 1036 |
+
check, _ = self.read_remote(head)
|
| 1037 |
+
if not check or check.get("checkpoint_id") != checkpoint_id:
|
| 1038 |
+
raise RuntimeError("Hugging Face did not confirm the dictionary commit.")
|
| 1039 |
+
commit = head
|
| 1040 |
+
self.remote_digest = hashlib.sha256(payload).hexdigest()
|
| 1041 |
+
self._published(commit)
|
| 1042 |
+
return
|
| 1043 |
+
raise RuntimeError("The dictionary changed repeatedly during publication. Local progress is saved.")
|
| 1044 |
+
|
| 1045 |
+
def _published(self, commit):
|
| 1046 |
+
self.published = True
|
| 1047 |
+
self.remote_existed = True
|
| 1048 |
+
self.last_commit = commit
|
| 1049 |
+
self.publish_error = None
|
| 1050 |
+
self.run["dictionary_commits"].append(commit)
|
| 1051 |
+
self.doc["last_confirmed_dictionary_commit"] = commit
|
| 1052 |
+
self.last_checkpoint = time.monotonic()
|
| 1053 |
+
self.completed_since_checkpoint = 0
|
| 1054 |
+
self.save_local()
|
| 1055 |
+
|
| 1056 |
+
def reconcile(self, items, revision):
|
| 1057 |
+
previous = self.doc["files"]
|
| 1058 |
+
current = {}
|
| 1059 |
+
for item in items:
|
| 1060 |
+
path = item.get("path", "")
|
| 1061 |
+
if not path.startswith(self.district + "/") or item.get("type") != "file":
|
| 1062 |
+
continue
|
| 1063 |
+
suffix = Path(path).suffix.lower()
|
| 1064 |
+
if suffix not in _IMAGE_TYPES:
|
| 1065 |
+
continue
|
| 1066 |
+
fingerprint, size = _source_identity(item)
|
| 1067 |
+
old = previous.get(path)
|
| 1068 |
+
if old and old.get("source_fingerprint") == fingerprint and old.get("status") != "removed":
|
| 1069 |
+
entry = copy.deepcopy(old)
|
| 1070 |
+
else:
|
| 1071 |
+
entry = {"status": "pending", "processing_attempts": 0, "last_error": None}
|
| 1072 |
+
self.run["changed" if old else "added"].append(path)
|
| 1073 |
+
entry.update({"source_fingerprint": fingerprint, "size": size,
|
| 1074 |
+
"source_url": _resolve_url(path), "source_pinned_url": _resolve_url(path, revision),
|
| 1075 |
+
"seen_in_commit": revision, "content_type": _IMAGE_TYPES[suffix]})
|
| 1076 |
+
current[path] = entry
|
| 1077 |
+
for path, entry in previous.items():
|
| 1078 |
+
if path not in current:
|
| 1079 |
+
removed = copy.deepcopy(entry)
|
| 1080 |
+
if removed.get("status") != "removed":
|
| 1081 |
+
self.run["removed"].append(path)
|
| 1082 |
+
removed.update({"status": "removed", "removed_in_commit": revision})
|
| 1083 |
+
current[path] = removed
|
| 1084 |
+
self.doc["files"] = current
|
| 1085 |
+
self.doc["inventory_hash"] = _inventory_hash(current)
|
| 1086 |
+
self.doc["source"].update({"snapshot_commit": revision, "inventory_complete": True,
|
| 1087 |
+
"scanned_at": _utc_now(), "branch": "main"})
|
| 1088 |
+
self.run["source_commit"] = revision
|
| 1089 |
+
self.run["inventory_complete"] = True
|
| 1090 |
+
|
| 1091 |
+
def ensure_link(self, record):
|
| 1092 |
+
if not self.link_token or time.time() >= self.link_issued + _LINK_SECONDS - 3600:
|
| 1093 |
+
issued_before_request = time.time()
|
| 1094 |
+
grant = self.b2.call("b2_get_download_authorization", {
|
| 1095 |
+
"bucketId": self.bucket["bucketId"], "fileNamePrefix": "districts/" + self.district + "/",
|
| 1096 |
+
"validDurationInSeconds": _LINK_SECONDS})
|
| 1097 |
+
self.link_token = grant["authorizationToken"]
|
| 1098 |
+
self.link_issued = issued_before_request
|
| 1099 |
+
b2 = record["b2"]
|
| 1100 |
+
base = self.b2.download_url + "/file/" + quote(self.bucket["bucketName"], safe="") + "/" + quote(b2["file_name"], safe="/")
|
| 1101 |
+
b2.update({"download_url_base": base, "download_url": base + "?" + urlencode({"Authorization": self.link_token}),
|
| 1102 |
+
"url_issued_at_unix": self.link_issued, "url_expires_at_unix": self.link_issued + _LINK_SECONDS})
|
| 1103 |
+
|
| 1104 |
+
def process(self, path, entry, revision):
|
| 1105 |
+
_check_cancel()
|
| 1106 |
+
entry["processing_attempts"] = entry.get("processing_attempts", 0) + 1
|
| 1107 |
+
entry["last_error"] = None
|
| 1108 |
+
old = entry.get("b2", {})
|
| 1109 |
+
valid = (old.get("bucket_id") == self.bucket["bucketId"] and
|
| 1110 |
+
str(old.get("file_name", "")).startswith("districts/" + self.district + "/") and
|
| 1111 |
+
old.get("sha1") and old.get("size") == entry["size"])
|
| 1112 |
+
if valid and self.b2.file_exists(self.bucket["bucketId"], old["file_name"], old["sha1"], old["size"]):
|
| 1113 |
+
self.run["reused"] += 1
|
| 1114 |
+
else:
|
| 1115 |
+
if entry["size"] > _MAX_PNG_BYTES:
|
| 1116 |
+
raise ValueError("Image exceeds the supported 128 MiB per-file limit.")
|
| 1117 |
+
data = self.hf.read_file(path, revision, _MAX_PNG_BYTES)
|
| 1118 |
+
_verify_source_bytes(data, entry)
|
| 1119 |
+
sha256 = hashlib.sha256(data).hexdigest()
|
| 1120 |
+
name = "districts/" + self.district + "/" + sha256 + Path(path).suffix.lower()
|
| 1121 |
+
uploaded = self.b2.ensure_file(self.bucket, name, data, entry["content_type"])
|
| 1122 |
+
entry["b2"] = {"bucket_id": self.bucket["bucketId"], "bucket_name": self.bucket["bucketName"],
|
| 1123 |
+
"file_name": name, "file_id": (uploaded or {}).get("fileId"),
|
| 1124 |
+
"sha1": hashlib.sha1(data).hexdigest(), "sha256": sha256, "size": len(data)}
|
| 1125 |
+
entry["uploaded_from_commit"] = revision
|
| 1126 |
+
# Record successful storage BEFORE any link/token request can fail.
|
| 1127 |
+
entry.update({"status": "stored", "updated_at": _utc_now()})
|
| 1128 |
+
self.save_local(strict=True)
|
| 1129 |
+
self.ensure_link(entry)
|
| 1130 |
+
entry.update({"status": "uploaded", "last_error": None, "updated_at": _utc_now()})
|
| 1131 |
+
self.run["processed"] += 1
|
| 1132 |
+
self.completed_since_checkpoint += 1
|
| 1133 |
+
self.save_local(strict=True)
|
| 1134 |
+
|
| 1135 |
+
def execute(self):
|
| 1136 |
+
local = getattr(self, "local_resume", None)
|
| 1137 |
+
recovery = _recovery_records(local)
|
| 1138 |
+
if local and local.get("backblaze", {}).get("account_id") == self.b2.account_id:
|
| 1139 |
+
self.doc = copy.deepcopy(local)
|
| 1140 |
+
try:
|
| 1141 |
+
info = self.hf.info()
|
| 1142 |
+
revision = info["sha"]
|
| 1143 |
+
remote, self.remote_digest = self.read_remote(revision)
|
| 1144 |
+
self.remote_loaded = True
|
| 1145 |
+
self.remote_existed = remote is not None
|
| 1146 |
+
if remote:
|
| 1147 |
+
self.doc = copy.deepcopy(remote)
|
| 1148 |
+
if local:
|
| 1149 |
+
_merge_successes(self.doc, local)
|
| 1150 |
+
known_runs = {run.get("run_id") for run in self.doc.get("history", [])}
|
| 1151 |
+
self.doc.setdefault("history", []).extend(run for run in local.get("history", [])
|
| 1152 |
+
if run.get("run_id") not in known_runs)
|
| 1153 |
+
# Retain local-only/newer-content uploads even if listing now fails.
|
| 1154 |
+
# These are recovery candidates, never active mappings until inventory validation.
|
| 1155 |
+
self.doc["recovery_records"] = recovery + self.doc.get("recovery_records", [])
|
| 1156 |
+
self.run["attempted_source_commit"] = revision
|
| 1157 |
+
self.doc["status"] = "scanning"
|
| 1158 |
+
self.save_local()
|
| 1159 |
+
# Listing must finish before files are added/changed/marked removed.
|
| 1160 |
+
items = self.hf.tree(revision, self.district)
|
| 1161 |
+
self.reconcile(items, revision)
|
| 1162 |
+
try:
|
| 1163 |
+
history = self.hf.history(revision)
|
| 1164 |
+
self.doc["source"]["recent_commits"] = [
|
| 1165 |
+
{"id": item.get("id") or item.get("commit_id"), "title": item.get("title", ""),
|
| 1166 |
+
"date": item.get("date") or item.get("created_at")} for item in history[:100]]
|
| 1167 |
+
except _HFError as exc:
|
| 1168 |
+
self.run["history_warning"] = self.safe_error(exc)
|
| 1169 |
+
self.bucket = self.b2.ensure_bucket()
|
| 1170 |
+
destination = {"account_id": self.b2.account_id, "bucket_id": self.bucket["bucketId"],
|
| 1171 |
+
"bucket_name": self.bucket["bucketName"], "private": True}
|
| 1172 |
+
if self.doc.get("backblaze") != destination:
|
| 1173 |
+
for entry in self.doc["files"].values():
|
| 1174 |
+
entry.pop("b2", None)
|
| 1175 |
+
if entry.get("status") != "removed":
|
| 1176 |
+
entry["status"] = "pending"
|
| 1177 |
+
self.doc["backblaze"] = destination
|
| 1178 |
+
# Prefer additional successful local progress after a failed HF commit.
|
| 1179 |
+
# The pre-run local state is retained separately, never read after overwriting.
|
| 1180 |
+
if getattr(self, "local_resume", None):
|
| 1181 |
+
_merge_successes(self.doc, self.local_resume)
|
| 1182 |
+
for candidate in self.doc.get("recovery_records", []):
|
| 1183 |
+
_merge_successes(self.doc, {"backblaze": candidate.get("backblaze"),
|
| 1184 |
+
"files": {candidate["path"]: candidate["record"]}})
|
| 1185 |
+
self.doc.pop("recovery_records", None)
|
| 1186 |
+
self.doc["status"] = self.run["status"] = "running"
|
| 1187 |
+
self.save_local(strict=True)
|
| 1188 |
+
# Test actual HF write permission before transferring any images.
|
| 1189 |
+
self.publish()
|
| 1190 |
+
paths = [path for path in sorted(self.doc["files"]) if self.doc["files"][path].get("status") != "removed"]
|
| 1191 |
+
try:
|
| 1192 |
+
from comfy.utils import ProgressBar
|
| 1193 |
+
progress = ProgressBar(len(paths)) if paths else None
|
| 1194 |
+
except ImportError:
|
| 1195 |
+
progress = None
|
| 1196 |
+
print("[Backblaze District] {} images found in {} at commit {}.".format(len(paths), self.district, revision))
|
| 1197 |
+
for index, path in enumerate(paths, 1):
|
| 1198 |
+
entry = self.doc["files"][path]
|
| 1199 |
+
if progress is not None:
|
| 1200 |
+
progress.update_absolute(index - 1, len(paths))
|
| 1201 |
+
if index == 1 or index % 10 == 0:
|
| 1202 |
+
print("[Backblaze District] Processing image {}/{}.".format(index, len(paths)))
|
| 1203 |
+
try:
|
| 1204 |
+
self.process(path, entry, revision)
|
| 1205 |
+
except _MirrorInterrupted:
|
| 1206 |
+
raise
|
| 1207 |
+
except Exception as exc:
|
| 1208 |
+
entry.update({"status": "failed", "last_error": {"message": self.safe_error(exc),
|
| 1209 |
+
"attempts": getattr(exc, "attempts", None), "at": _utc_now()}, "updated_at": _utc_now()})
|
| 1210 |
+
self.save_local()
|
| 1211 |
+
if isinstance(exc, ValueError) or (isinstance(exc, _HFError) and exc.status in (404, 413)):
|
| 1212 |
+
# One permanently bad/oversized source must not block the rest.
|
| 1213 |
+
self.run.setdefault("permanent_file_errors", []).append(path)
|
| 1214 |
+
self.completed_since_checkpoint += 1
|
| 1215 |
+
if (self.completed_since_checkpoint >= _CHECKPOINT_FILES or
|
| 1216 |
+
time.monotonic() - self.last_checkpoint >= _CHECKPOINT_SECONDS):
|
| 1217 |
+
self.publish()
|
| 1218 |
+
continue
|
| 1219 |
+
raise
|
| 1220 |
+
if (self.completed_since_checkpoint >= _CHECKPOINT_FILES or
|
| 1221 |
+
time.monotonic() - self.last_checkpoint >= _CHECKPOINT_SECONDS):
|
| 1222 |
+
self.publish()
|
| 1223 |
+
if progress is not None:
|
| 1224 |
+
progress.update_absolute(len(paths), len(paths))
|
| 1225 |
+
self.rebuild()
|
| 1226 |
+
self.doc["status"] = self.run["status"] = (
|
| 1227 |
+
"complete" if self.doc["counts"]["mapped"] == self.doc["counts"]["total"] else "partial")
|
| 1228 |
+
except _MirrorInterrupted as exc:
|
| 1229 |
+
self.error = self.safe_error(exc)
|
| 1230 |
+
self.doc["status"] = self.run["status"] = "interrupted"
|
| 1231 |
+
except _ConcurrentMirror as exc:
|
| 1232 |
+
self.error = self.safe_error(exc)
|
| 1233 |
+
self.doc["status"] = self.run["status"] = "conflict"
|
| 1234 |
+
except Exception as exc:
|
| 1235 |
+
self.error = self.safe_error(exc)
|
| 1236 |
+
self.doc["status"] = self.run["status"] = "partial"
|
| 1237 |
+
finally:
|
| 1238 |
+
self.run["finished_at"] = _utc_now()
|
| 1239 |
+
self.run["error"] = self.error
|
| 1240 |
+
self.save_local()
|
| 1241 |
+
if self.remote_loaded and self.doc["status"] != "conflict":
|
| 1242 |
+
# A final dictionary save is attempted even after transfer errors/cancellation.
|
| 1243 |
+
saved_check = self.hf._check_cancel
|
| 1244 |
+
self.hf._check_cancel = lambda: None
|
| 1245 |
+
try:
|
| 1246 |
+
self.publish()
|
| 1247 |
+
except Exception as exc:
|
| 1248 |
+
self.publish_error = self.safe_error(exc)
|
| 1249 |
+
self.doc["last_publication_error"] = self.publish_error
|
| 1250 |
+
self.save_local()
|
| 1251 |
+
finally:
|
| 1252 |
+
self.hf._check_cancel = saved_check
|
| 1253 |
+
counts = self.doc.get("counts", {})
|
| 1254 |
+
status = "{}: {}/{} image URLs ready; {} reused, {} failed, {} pending.".format(
|
| 1255 |
+
self.doc["status"].upper(), counts.get("mapped", 0), counts.get("total", 0),
|
| 1256 |
+
self.run["reused"], counts.get("failed", 0), counts.get("pending", 0))
|
| 1257 |
+
if self.error:
|
| 1258 |
+
status += "\n" + self.error
|
| 1259 |
+
if self.publish_error or not self.published or self.doc["status"] == "conflict":
|
| 1260 |
+
status += "\nLATEST DICTIONARY WAS NOT PUBLISHED. " + (self.publish_error or "Hugging Face state was unavailable.")
|
| 1261 |
+
else:
|
| 1262 |
+
status += "\nDictionary saved to Hugging Face. Commit: " + str(self.last_commit)
|
| 1263 |
+
if self.local_error:
|
| 1264 |
+
status += "\n" + self.local_error
|
| 1265 |
+
else:
|
| 1266 |
+
status += "\nLocal checkpoint: " + str(self.path)
|
| 1267 |
+
status += "\nPrivate links expire after seven days; run again to refresh them."
|
| 1268 |
+
self.rebuild()
|
| 1269 |
+
return (self.url if self.remote_existed else "", status, _json_bytes(self.doc).decode("utf-8"))
|
| 1270 |
+
|
| 1271 |
+
|
| 1272 |
class BackblazePNGUpload:
|
| 1273 |
@classmethod
|
| 1274 |
def INPUT_TYPES(cls):
|
| 1275 |
return {"required": {
|
| 1276 |
"master_key": ("STRING", {"default": "", "multiline": False,
|
| 1277 |
+
"tooltip": "Backblaze master secret. KEY_ID:SECRET is also accepted."}),
|
| 1278 |
+
"District_Name": ("STRING", {"default": "District_19_K", "multiline": False,
|
| 1279 |
+
"tooltip": "Folder inside saliacoel/map_imgs_gpt. All image subfolders are included."}),
|
| 1280 |
+
"hf_key": ("STRING", {"default": "", "multiline": False,
|
| 1281 |
+
"tooltip": "Hugging Face token with write access to saliacoel/map_imgs_gpt."}),
|
| 1282 |
}, "optional": {
|
| 1283 |
"key_id": ("STRING", {"default": "", "multiline": False,
|
| 1284 |
+
"tooltip": "Backblaze key ID. Blank uses ID:SECRET or a previously verified saved ID."}),
|
|
|
|
|
|
|
| 1285 |
}}
|
| 1286 |
|
| 1287 |
+
RETURN_TYPES = ("STRING", "STRING", "STRING")
|
| 1288 |
+
RETURN_NAMES = ("dictionary_url", "STATUS", "dictionary_json")
|
| 1289 |
FUNCTION = "upload"
|
| 1290 |
CATEGORY = "Salia/Backblaze"
|
| 1291 |
OUTPUT_NODE = True
|
| 1292 |
+
DESCRIPTION = "Mirror every district image to Backblaze and publish a resumable DICT_BLAZE JSON to Hugging Face."
|
|
|
|
|
|
|
| 1293 |
|
| 1294 |
@classmethod
|
| 1295 |
+
def IS_CHANGED(cls, master_key, District_Name, hf_key, key_id=""):
|
|
|
|
| 1296 |
return float("nan")
|
| 1297 |
|
| 1298 |
+
def upload(self, master_key, District_Name, hf_key, key_id=""):
|
| 1299 |
+
mirror = None
|
| 1300 |
+
acquired = False
|
| 1301 |
+
lock = None
|
| 1302 |
+
file_lock = None
|
| 1303 |
secret = ""
|
|
|
|
|
|
|
| 1304 |
try:
|
| 1305 |
+
district = District_Name.strip()
|
| 1306 |
+
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", district) or district in (".", ".."):
|
| 1307 |
+
raise ValueError("District_Name must be a single folder name, for example District_19_K.")
|
| 1308 |
+
if not isinstance(hf_key, str) or not hf_key.strip() or any(c.isspace() for c in hf_key.strip()):
|
| 1309 |
+
raise ValueError("hf_key must contain a Hugging Face token with write access to the repository.")
|
| 1310 |
resolved_id, secret, source = _credential_parts(master_key, key_id)
|
| 1311 |
+
basic = base64.b64encode((resolved_id + ":" + secret).encode()).decode("ascii")
|
| 1312 |
+
b2 = _B2Client(basic)
|
|
|
|
|
|
|
|
|
|
| 1313 |
cache_note = _remember_key_id(secret, resolved_id)
|
| 1314 |
+
with _DISTRICT_LOCKS_GUARD:
|
| 1315 |
+
lock = _DISTRICT_LOCKS.setdefault((_REPO_ID, district), threading.Lock())
|
| 1316 |
+
acquired = lock.acquire(blocking=False)
|
| 1317 |
+
if not acquired:
|
| 1318 |
+
raise RuntimeError("This district is already running in this ComfyUI process. Let it finish first.")
|
| 1319 |
+
hf = _HFClient(hf_key.strip(), check_cancel=_check_cancel,
|
| 1320 |
+
on_retry=lambda attempt, seconds, error: print(
|
| 1321 |
+
"[Backblaze District] HF retry {}/8 in {} seconds ({}).".format(attempt + 1, seconds, error.code)))
|
| 1322 |
+
mirror = _DistrictMirror(hf, b2, district, secrets=(master_key, secret, hf_key.strip()))
|
| 1323 |
+
file_lock = _CheckpointLock(mirror.path)
|
| 1324 |
+
file_lock.acquire()
|
| 1325 |
+
try:
|
| 1326 |
+
with mirror.path.open("r", encoding="utf-8") as stream:
|
| 1327 |
+
mirror.local_resume = _validate_manifest(json.load(stream), district)
|
| 1328 |
+
except (OSError, ValueError, TypeError):
|
| 1329 |
+
mirror.local_resume = None
|
| 1330 |
+
url, status, json_text = mirror.execute()
|
| 1331 |
+
status += "\nKEY_ID: {} ({})\n{}".format(resolved_id, source, cache_note)
|
| 1332 |
except Exception as exc:
|
| 1333 |
+
url, json_text = "", _json_bytes(mirror.doc).decode() if mirror else "{}"
|
| 1334 |
+
status = "ERROR. " + (str(exc) if isinstance(exc, (ValueError, RuntimeError, _HFError, _RequestError, _MirrorInterrupted)) else type(exc).__name__)
|
| 1335 |
+
finally:
|
| 1336 |
+
if file_lock is not None:
|
| 1337 |
+
file_lock.release()
|
| 1338 |
+
if acquired:
|
| 1339 |
+
lock.release()
|
| 1340 |
+
for value in sorted({str(master_key).strip(), secret, str(hf_key).strip()}, key=len, reverse=True):
|
|
|
|
|
|
|
|
|
|
| 1341 |
if value:
|
| 1342 |
status = status.replace(value, "[REDACTED]")
|
| 1343 |
+
print("[Backblaze District] " + status)
|
| 1344 |
+
return {"ui": {"text": [status] + ([url] if url else [])}, "result": (url, status, json_text)}
|
| 1345 |
|
| 1346 |
|
| 1347 |
NODE_CLASS_MAPPINGS = {"BackblazePNGUpload": BackblazePNGUpload}
|
| 1348 |
+
NODE_DISPLAY_NAME_MAPPINGS = {"BackblazePNGUpload": "Backblaze District Mirror (Auto / Resume)"}
|