saliacoel commited on
Commit
9ea77e3
·
verified ·
1 Parent(s): eef5315

Upload Backblaze_PNG_Upload.py

Browse files
Files changed (1) hide show
  1. Backblaze_PNG_Upload.py +951 -106
Backblaze_PNG_Upload.py CHANGED
@@ -1,45 +1,63 @@
1
- """Single-file ComfyUI node: public Hugging Face PNG -> private Backblaze B2.
2
-
3
- INSTALL: Put this file directly in ComfyUI/custom_nodes/, then restart ComfyUI.
4
- NODE: Backblaze PNG Upload (Auto), category Salia/Backblaze.
5
-
6
- INPUTS:
7
- master_key: The application key SECRET. Legacy KEY_ID:SECRET also works.
8
- png_url: Public Hugging Face /resolve/ URL. /blob/ URLs also work.
9
- key_id: Master key ID (or account ID). When blank, extract the ID from
10
- KEY_ID:SECRET or reuse an ID verified previously with this secret.
11
- Backblaze has no API to discover the ID from a bare secret alone.
12
- On the first run, copy it once from your Backblaze App Keys page.
13
- OUTPUTS: png_url (STRING) and STATUS (STRING). The URL is valid for seven days.
14
- STATUS shows the selected key ID, its source, and success/failure. It never shows
15
- the secret. Errors return an empty URL and an explanatory STATUS, not a traceback.
16
-
17
- Automatically creates/reuses its own private bucket. Existing public buckets
18
- are never made private or used. PNG bytes are preserved, including transparency
19
- and metadata. Content-hash filenames reuse identical uploads on subsequent runs.
20
- Running again refreshes the link; the stored image itself does not expire.
21
- Anyone holding the returned link can download that image until the link expires.
22
-
23
- No SDK, pip install, manual config file, or __init__.py is required.
24
- Verified key IDs are saved automatically under the ComfyUI user directory
25
- (fallback: ~/.cache/comfyui-backblaze-png). Only ID + SHA-256(secret) are saved;
26
- the master secret and authorization tokens are never written to this cache.
27
- Uses Python's standard library. Source PNG limit: 128 MiB. Public HF URLs only.
28
- B2 Cloud Storage must already be enabled on the account. The master key has the
29
- needed permissions; a normal unrestricted key also works if granted listBuckets,
30
- writeBuckets, listFiles, writeFiles, and shareFiles.
31
-
32
- Credentials entered in widgets can be saved inside ComfyUI workflows/prompts;
33
- remove them before sharing workflows. This node never prints credentials.
34
-
35
- API references:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
36
  https://www.backblaze.com/apidocs/b2-authorize-account
37
- https://www.backblaze.com/apidocs/b2-create-bucket
38
  https://www.backblaze.com/apidocs/b2-upload-file
39
  https://www.backblaze.com/apidocs/b2-get-download-authorization
 
 
40
  """
41
 
42
  import base64
 
43
  import hashlib
44
  import json
45
  import os
@@ -48,7 +66,8 @@ import socket
48
  import tempfile
49
  import threading
50
  import time
51
- from datetime import timezone
 
52
  from email.utils import parsedate_to_datetime
53
  from http.client import HTTPException
54
  from pathlib import Path
@@ -66,7 +85,7 @@ _DEFAULT_PNG = (
66
  _MAX_PNG_BYTES = 128 * 1024 * 1024
67
  _MAX_JSON_BYTES = 4 * 1024 * 1024
68
  _LINK_SECONDS = 7 * 24 * 60 * 60
69
- _ATTEMPTS = 5
70
  _TIMEOUT_SECONDS = 120
71
  _TOKEN_ERRORS = {"bad_auth_token", "expired_auth_token"}
72
  _KEY_ID_CACHE = {}
@@ -140,6 +159,7 @@ class _RequestError(RuntimeError):
140
  self.status = status
141
  self.code = code if re.fullmatch(r"[A-Za-z0-9_]{1,80}", code) else "http_error"
142
  self.retry_after = retry_after
 
143
  super().__init__("HTTP {} ({})".format(status, self.code))
144
 
145
  @property
@@ -161,7 +181,7 @@ class _HTTPSRedirect(HTTPRedirectHandler):
161
 
162
 
163
  def _delay(attempt, error):
164
- seconds = min(2 ** attempt, 30)
165
  value = error.retry_after
166
  if value:
167
  try:
@@ -174,13 +194,12 @@ def _delay(attempt, error):
174
  seconds = max(seconds, date.timestamp() - time.time())
175
  except (TypeError, ValueError, OverflowError):
176
  pass
177
- if seconds > 60:
178
- raise RuntimeError("The server requested a longer retry delay. Run again later.")
179
- time.sleep(seconds)
180
 
181
 
182
  def _read_url(url, *, data=None, headers=None, limit=_MAX_JSON_BYTES,
183
  source=False, timeout=_TIMEOUT_SECONDS):
 
184
  request_headers = {"User-Agent": "ComfyUI-Backblaze-PNG/1.0"}
185
  request_headers.update(headers or {})
186
  opener = build_opener(_HTTPSRedirect() if source else _NoRedirect())
@@ -192,6 +211,7 @@ def _read_url(url, *, data=None, headers=None, limit=_MAX_JSON_BYTES,
192
  raise RuntimeError("The response exceeds the allowed size (PNG limit: 128 MiB).")
193
  chunks, total = [], 0
194
  while True:
 
195
  chunk = response.read(min(1024 * 1024, limit + 1 - total))
196
  if not chunk:
197
  break
@@ -234,6 +254,7 @@ def _json_request(url, *, payload=None, headers=None, retries=True):
234
  raise _RequestError(502, "invalid_json_response")
235
  return result
236
  except _RequestError as exc:
 
237
  if not retries or not exc.temporary or attempt == _ATTEMPTS - 1:
238
  raise
239
  _delay(attempt, exc)
@@ -372,27 +393,30 @@ class _B2Client:
372
  return owned[name]
373
  raise RuntimeError("Could not choose an available private bucket name. Run again later.")
374
 
375
- def file_exists(self, bucket_id, name, sha1, length):
376
  result = self.call("b2_list_file_names", {
377
  "bucketId": bucket_id, "prefix": name, "startFileName": name, "maxFileCount": 1})
378
- return any(item.get("fileName") == name and item.get("action") == "upload"
379
- and item.get("contentSha1") == sha1 and item.get("contentLength") == length
380
- for item in result["files"])
381
 
382
- def ensure_file(self, bucket, name, data):
 
 
 
383
  sha1 = hashlib.sha1(data).hexdigest()
384
  for attempt in range(_ATTEMPTS):
385
  # Also covers a timed-out upload which actually succeeded on the server.
386
- if self.file_exists(bucket["bucketId"], name, sha1, len(data)):
387
- print("[Backblaze PNG] Reusing an identical uploaded PNG.")
388
- return
389
  upload = self.call("b2_get_upload_url", {"bucketId": bucket["bucketId"]})
390
  upload_url = _b2_base_url(upload["uploadUrl"], upload=True)
391
  try:
392
  raw = _read_url(upload_url, data=data, headers={
393
  "Authorization": upload["authorizationToken"],
394
  "X-Bz-File-Name": quote(name, safe=""),
395
- "Content-Type": "image/png",
396
  "Content-Length": str(len(data)),
397
  "X-Bz-Content-Sha1": sha1,
398
  })
@@ -404,9 +428,9 @@ class _B2Client:
404
  or result.get("contentSha1") != sha1
405
  or result.get("contentLength") != len(data)):
406
  raise _RequestError(502, "invalid_upload_response")
407
- print("[Backblaze PNG] PNG uploaded and checksum confirmed.")
408
- return
409
  except _RequestError as exc:
 
410
  retryable = exc.temporary or exc.code in _TOKEN_ERRORS or exc.code == "auth_token_limit"
411
  if not retryable or attempt == _ATTEMPTS - 1:
412
  raise
@@ -422,82 +446,903 @@ class _B2Client:
422
  + urlencode({"Authorization": result["authorizationToken"]}))
423
 
424
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
425
  class BackblazePNGUpload:
426
  @classmethod
427
  def INPUT_TYPES(cls):
428
  return {"required": {
429
  "master_key": ("STRING", {"default": "", "multiline": False,
430
- "tooltip": "Master application key SECRET. Legacy KEY_ID:SECRET also works. "
431
- "Remove this credential before sharing a saved workflow."}),
432
- "png_url": ("STRING", {"default": _DEFAULT_PNG, "multiline": False,
433
- "tooltip": "Public Hugging Face PNG URL. Bucket and upload setup are automatic."}),
 
434
  }, "optional": {
435
  "key_id": ("STRING", {"default": "", "multiline": False,
436
- "tooltip": "Key ID (or account ID for a master key). Blank = use a previously "
437
- "verified saved ID, or the ID from KEY_ID:SECRET. First use of a bare "
438
- "secret requires this ID; Backblaze cannot retrieve it from the secret."}),
439
  }}
440
 
441
- RETURN_TYPES = ("STRING", "STRING")
442
- RETURN_NAMES = ("png_url", "STATUS")
443
  FUNCTION = "upload"
444
  CATEGORY = "Salia/Backblaze"
445
  OUTPUT_NODE = True
446
- DESCRIPTION = ("Uploads a Hugging Face PNG to an automatically created private Backblaze bucket. "
447
- "Returns a seven-day download URL and STATUS with the key ID. "
448
- "Blank key_id uses a saved ID; first-time bare secrets still require an ID.")
449
 
450
  @classmethod
451
- def IS_CHANGED(cls, master_key, png_url, key_id=""):
452
- # A source URL can change; private download URLs must never stay cached forever.
453
  return float("nan")
454
 
455
- def upload(self, master_key, png_url, key_id=""):
456
- resolved_id = ""
457
- source = ""
 
 
458
  secret = ""
459
- cache_note = ""
460
- link = ""
461
  try:
 
 
 
 
 
462
  resolved_id, secret, source = _credential_parts(master_key, key_id)
463
- basic_auth = base64.b64encode((resolved_id + ":" + secret).encode("utf-8")).decode("ascii")
464
- print("[Backblaze PNG] Downloading source PNG.")
465
- data = _download_png(png_url)
466
- name = "images/" + hashlib.sha256(data).hexdigest() + ".png"
467
- client = _B2Client(basic_auth)
468
  cache_note = _remember_key_id(secret, resolved_id)
469
- bucket = client.ensure_bucket()
470
- client.ensure_file(bucket, name, data)
471
- link = client.download_link(bucket, name)
472
- status = "OK. Private download link is valid for seven days."
473
- except _RequestError as exc:
474
- hints = {
475
- 0: "Check your internet connection and TLS certificates.",
476
- 401: "Check the key ID, application key, and key permissions.",
477
- 403: "Check B2 account status, storage/transaction caps, and key permissions.",
478
- 429: "Backblaze is rate limiting requests. Run again later.",
479
- }
480
- status = "ERROR. Backblaze failed: {}. {}".format(
481
- exc, hints.get(exc.status, "Check that B2 Cloud Storage is enabled, then try again."))
482
- except (ValueError, RuntimeError) as exc:
483
- status = "ERROR. " + str(exc)
 
 
 
484
  except Exception as exc:
485
- # Do not expose unexpected exception payloads, which may contain tokens.
486
- status = "ERROR. Unexpected {}. Check the inputs and try again.".format(type(exc).__name__)
487
- if resolved_id:
488
- status += "\nKEY_ID: {}\nID source: {}".format(resolved_id, source)
489
- if cache_note:
490
- status += "\n" + cache_note
491
- # Defense in depth: status may include a provider error or an ID pasted by mistake.
492
- sensitive = [secret]
493
- if isinstance(master_key, str):
494
- sensitive += [master_key.strip(), master_key.strip().split(":", 1)[-1]]
495
- for value in sorted(set(sensitive), key=len, reverse=True):
496
  if value:
497
  status = status.replace(value, "[REDACTED]")
498
- print("[Backblaze PNG] " + status)
499
- return {"ui": {"text": [status] + ([link] if link else [])}, "result": (link, status)}
500
 
501
 
502
  NODE_CLASS_MAPPINGS = {"BackblazePNGUpload": BackblazePNGUpload}
503
- NODE_DISPLAY_NAME_MAPPINGS = {"BackblazePNGUpload": "Backblaze PNG Upload (Auto)"}
 
1
+ """Single-file ComfyUI node: resumable Hugging Face district -> Backblaze mirror.
2
+
3
+ INSTALL: Replace this file in ComfyUI/custom_nodes/, restart ComfyUI, then re-add
4
+ "Backblaze District Mirror (Auto / Resume)". No __init__.py or packages needed.
5
+ Node class ID remains BackblazePNGUpload. The old png_url input is replaced.
6
+
7
+ REQUIRED INPUTS:
8
+ master_key: Backblaze master secret, or legacy KEY_ID:SECRET.
9
+ District_Name: e.g. District_19_K. Recursively scans that folder in the model
10
+ repository saliacoel/map_imgs_gpt, branch main.
11
+ hf_key: Hugging Face token with WRITE access to that repository.
12
+ OPTIONAL INPUT:
13
+ key_id: Backblaze key ID/account ID. Blank uses the ID in KEY_ID:SECRET
14
+ or the ID remembered after a previous successful authentication.
15
+ Backblaze cannot discover a first-time ID from the secret alone.
16
+ OUTPUTS: dictionary_url, STATUS, dictionary_json (all STRING).
17
+
18
+ The node publishes DICT_BLAZE_<District_Name>.json at the repository root.
19
+ Its "dictionary" maps complete Hugging Face main-branch image URLs to Backblaze
20
+ URLs. "files" contains source hashes, pinned source URLs/commits, B2 object IDs,
21
+ checksums, URL expiration timestamps, statuses, attempts and errors. Other fields
22
+ record source inventory/commit history, added/changed/removed paths, counts and
23
+ run/checkpoint history. Source deletions never delete Backblaze objects.
24
+
25
+ Every scan/download is pinned to one source commit. Unchanged source hashes reuse
26
+ existing Backblaze objects. Re-running also refreshes private links without
27
+ redownloading unchanged images. Links are valid for at most seven days; stored
28
+ objects do not expire. Anyone who can read the dictionary can use its bearer URLs
29
+ until expiration. API secrets and account-wide authorization tokens are NOT put
30
+ in the dictionary. One scoped download token serves only this district's prefix.
31
+
32
+ Transient requests get 8 attempts total, waiting 3,5,7,9,11,13,15 seconds. A longer
33
+ Retry-After is respected. Exhausted transient failures stop the run and trigger
34
+ a final partial dictionary save. Permanent bad source files are recorded and
35
+ skipped so later images can still upload. Run again to resume failed/pending work.
36
+
37
+ Progress is written atomically locally after each image, before requesting its
38
+ link, and after completion. Hugging Face checkpoints occur before image transfer,
39
+ every 25 files or 60 seconds, and on completion/failure/cancellation. Local state
40
+ survives a failed remote publication and retains source variants if scanning
41
+ fails. Local process/OS locks and remote parent-commit checks prevent conflicting
42
+ writers from silently overwriting progress. If HF rejects the final save, STATUS
43
+ says so, and dictionary_json plus the local checkpoint retain the result.
44
+
45
+ Runtime data lives under the ComfyUI user directory/backblaze_png_upload (fallback
46
+ ~/.cache/comfyui-backblaze-png). Key cache stores only key ID + SHA-256(secret).
47
+ No manual configuration required. Limits: 128 MiB per image, 32 MiB per dictionary;
48
+ last 100 run records and up to 100 recent source commit summaries are retained.
49
+ Credentials in ComfyUI widgets may be saved in workflows; remove before sharing.
50
+
51
+ Verified API contracts:
52
  https://www.backblaze.com/apidocs/b2-authorize-account
 
53
  https://www.backblaze.com/apidocs/b2-upload-file
54
  https://www.backblaze.com/apidocs/b2-get-download-authorization
55
+ https://huggingface.co/docs/huggingface_hub/package_reference/hf_api
56
+ https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/_commit_api.py
57
  """
58
 
59
  import base64
60
+ import copy
61
  import hashlib
62
  import json
63
  import os
 
66
  import tempfile
67
  import threading
68
  import time
69
+ import uuid
70
+ from datetime import datetime, timezone
71
  from email.utils import parsedate_to_datetime
72
  from http.client import HTTPException
73
  from pathlib import Path
 
85
  _MAX_PNG_BYTES = 128 * 1024 * 1024
86
  _MAX_JSON_BYTES = 4 * 1024 * 1024
87
  _LINK_SECONDS = 7 * 24 * 60 * 60
88
+ _ATTEMPTS = 8
89
  _TIMEOUT_SECONDS = 120
90
  _TOKEN_ERRORS = {"bad_auth_token", "expired_auth_token"}
91
  _KEY_ID_CACHE = {}
 
159
  self.status = status
160
  self.code = code if re.fullmatch(r"[A-Za-z0-9_]{1,80}", code) else "http_error"
161
  self.retry_after = retry_after
162
+ self.attempts = 1
163
  super().__init__("HTTP {} ({})".format(status, self.code))
164
 
165
  @property
 
181
 
182
 
183
  def _delay(attempt, error):
184
+ seconds = 3 + 2 * attempt
185
  value = error.retry_after
186
  if value:
187
  try:
 
194
  seconds = max(seconds, date.timestamp() - time.time())
195
  except (TypeError, ValueError, OverflowError):
196
  pass
197
+ _interruptible_sleep(seconds)
 
 
198
 
199
 
200
  def _read_url(url, *, data=None, headers=None, limit=_MAX_JSON_BYTES,
201
  source=False, timeout=_TIMEOUT_SECONDS):
202
+ _check_cancel()
203
  request_headers = {"User-Agent": "ComfyUI-Backblaze-PNG/1.0"}
204
  request_headers.update(headers or {})
205
  opener = build_opener(_HTTPSRedirect() if source else _NoRedirect())
 
211
  raise RuntimeError("The response exceeds the allowed size (PNG limit: 128 MiB).")
212
  chunks, total = [], 0
213
  while True:
214
+ _check_cancel()
215
  chunk = response.read(min(1024 * 1024, limit + 1 - total))
216
  if not chunk:
217
  break
 
254
  raise _RequestError(502, "invalid_json_response")
255
  return result
256
  except _RequestError as exc:
257
+ exc.attempts = attempt + 1
258
  if not retries or not exc.temporary or attempt == _ATTEMPTS - 1:
259
  raise
260
  _delay(attempt, exc)
 
393
  return owned[name]
394
  raise RuntimeError("Could not choose an available private bucket name. Run again later.")
395
 
396
+ def find_file(self, bucket_id, name, sha1, length):
397
  result = self.call("b2_list_file_names", {
398
  "bucketId": bucket_id, "prefix": name, "startFileName": name, "maxFileCount": 1})
399
+ return next((item for item in result["files"]
400
+ if item.get("fileName") == name and item.get("action") == "upload"
401
+ and item.get("contentSha1") == sha1 and item.get("contentLength") == length), None)
402
 
403
+ def file_exists(self, bucket_id, name, sha1, length):
404
+ return self.find_file(bucket_id, name, sha1, length) is not None
405
+
406
+ def ensure_file(self, bucket, name, data, content_type="image/png"):
407
  sha1 = hashlib.sha1(data).hexdigest()
408
  for attempt in range(_ATTEMPTS):
409
  # Also covers a timed-out upload which actually succeeded on the server.
410
+ existing = self.find_file(bucket["bucketId"], name, sha1, len(data))
411
+ if existing:
412
+ return existing
413
  upload = self.call("b2_get_upload_url", {"bucketId": bucket["bucketId"]})
414
  upload_url = _b2_base_url(upload["uploadUrl"], upload=True)
415
  try:
416
  raw = _read_url(upload_url, data=data, headers={
417
  "Authorization": upload["authorizationToken"],
418
  "X-Bz-File-Name": quote(name, safe=""),
419
+ "Content-Type": content_type,
420
  "Content-Length": str(len(data)),
421
  "X-Bz-Content-Sha1": sha1,
422
  })
 
428
  or result.get("contentSha1") != sha1
429
  or result.get("contentLength") != len(data)):
430
  raise _RequestError(502, "invalid_upload_response")
431
+ return result
 
432
  except _RequestError as exc:
433
+ exc.attempts = attempt + 1
434
  retryable = exc.temporary or exc.code in _TOKEN_ERRORS or exc.code == "auth_token_limit"
435
  if not retryable or attempt == _ATTEMPTS - 1:
436
  raise
 
446
  + urlencode({"Authorization": result["authorizationToken"]}))
447
 
448
 
449
+
450
+
451
+ import base64
452
+ import json
453
+ import math
454
+ import re
455
+ import socket
456
+ import time
457
+ from datetime import timezone
458
+ from email.utils import parsedate_to_datetime
459
+ from http.client import HTTPException
460
+ from urllib.error import HTTPError, URLError
461
+ from urllib.parse import quote, urljoin, urlsplit
462
+ from urllib.request import HTTPRedirectHandler, Request, build_opener
463
+
464
+
465
+ _HF_REPO = "saliacoel/map_imgs_gpt"
466
+ _HF_ROOT = "https://huggingface.co"
467
+ _HF_API = _HF_ROOT + "/api/models/" + _HF_REPO
468
+ _HF_METADATA_LIMIT = 32 * 1024 * 1024
469
+
470
+
471
+ class _HFError(RuntimeError):
472
+ """Safe for STATUS: never retain server bodies, URLs or credentials."""
473
+
474
+ def __init__(self, status, code="http_error", retry_after=None):
475
+ self.status = int(status)
476
+ self.code = code if re.fullmatch(r"[a-z_]{1,64}", str(code)) else "http_error"
477
+ self.retry_after = retry_after
478
+ self.attempts = 1
479
+ super().__init__("Hugging Face HTTP {} ({})".format(self.status, self.code))
480
+
481
+ @property
482
+ def temporary(self):
483
+ return (self.status in (0, 408, 429) or 500 <= self.status <= 599
484
+ or self.code == "rate_limited")
485
+
486
+ @property
487
+ def conflict(self):
488
+ return self.status in (409, 412) or self.code == "parent_commit_conflict"
489
+
490
+
491
+ class _HFSafeRedirect(HTTPRedirectHandler):
492
+ """HF resolves large files to a signed CDN URL; do not send HF tokens there."""
493
+
494
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
495
+ old = urlsplit(req.full_url)
496
+ new = urlsplit(newurl)
497
+ if new.scheme.lower() != "https" or new.username or new.password:
498
+ raise _HFError(400, "unsafe_redirect")
499
+ # Commits must never be redirected/replayed to another endpoint.
500
+ if req.get_method() not in ("GET", "HEAD"):
501
+ raise _HFError(400, "unexpected_commit_redirect")
502
+ redirected = super().redirect_request(req, fp, code, msg, headers, newurl)
503
+ if redirected is not None and (old.scheme, old.netloc) != (new.scheme, new.netloc):
504
+ for key in list(redirected.headers):
505
+ if key.lower() in ("authorization", "cookie", "proxy-authorization"):
506
+ del redirected.headers[key]
507
+ for key in list(redirected.unredirected_hdrs):
508
+ if key.lower() in ("authorization", "cookie", "proxy-authorization"):
509
+ del redirected.unredirected_hdrs[key]
510
+ return redirected
511
+
512
+
513
+ def _hf_delay_seconds(attempt, retry_after):
514
+ """attempt=0 after the first failure => 3,5,7,9,11,13,15 seconds."""
515
+ seconds = float(3 + 2 * attempt)
516
+ if retry_after:
517
+ try:
518
+ supplied = float(retry_after)
519
+ except (TypeError, ValueError):
520
+ try:
521
+ date = parsedate_to_datetime(str(retry_after))
522
+ if date.tzinfo is None:
523
+ date = date.replace(tzinfo=timezone.utc)
524
+ supplied = date.timestamp() - time.time()
525
+ except (TypeError, ValueError, OverflowError):
526
+ supplied = 0
527
+ if math.isfinite(supplied):
528
+ seconds = max(seconds, supplied)
529
+ return seconds
530
+
531
+
532
+ class _HFClient:
533
+ def __init__(self, token, check_cancel=None, sleep=None, on_retry=None):
534
+ token = str(token or "").strip()
535
+ if not token or any(c.isspace() for c in token):
536
+ raise ValueError("hf_key must be a Hugging Face token with write access to the repository.")
537
+ self._token = token
538
+ self._check_cancel = check_cancel or (lambda: None)
539
+ self._sleep = sleep or time.sleep
540
+ self._on_retry = on_retry or (lambda attempt, seconds, error: None)
541
+ self._opener = build_opener(_HFSafeRedirect())
542
+
543
+ def _wait(self, seconds):
544
+ # Long Retry-After values remain cancellable, with no busy waiting.
545
+ remaining = seconds
546
+ while remaining > 0:
547
+ self._check_cancel()
548
+ interval = min(1.0, remaining)
549
+ self._sleep(interval)
550
+ remaining -= interval
551
+ self._check_cancel()
552
+
553
+ @staticmethod
554
+ def _safe_url(url):
555
+ parts = urlsplit(url)
556
+ if (parts.scheme != "https" or parts.hostname != "huggingface.co"
557
+ or parts.port not in (None, 443) or parts.username or parts.password):
558
+ raise _HFError(400, "unsafe_api_url")
559
+
560
+ def _once(self, url, data=None, content_type=None, limit=_HF_METADATA_LIMIT):
561
+ self._safe_url(url)
562
+ self._check_cancel()
563
+ headers = {"User-Agent": "ComfyUI-Backblaze-District/3.0",
564
+ "Authorization": "Bearer " + self._token,
565
+ "Accept-Encoding": "identity"}
566
+ if content_type:
567
+ headers["Content-Type"] = content_type
568
+ request = Request(url, data=data, headers=headers)
569
+ try:
570
+ with self._opener.open(request, timeout=120) as response:
571
+ length = response.headers.get("Content-Length", "")
572
+ if length.isdigit() and int(length) > limit:
573
+ raise _HFError(413, "response_too_large")
574
+ chunks, total = [], 0
575
+ while True:
576
+ self._check_cancel()
577
+ chunk = response.read(min(1024 * 1024, limit + 1 - total))
578
+ if not chunk:
579
+ break
580
+ total += len(chunk)
581
+ if total > limit:
582
+ raise _HFError(413, "response_too_large")
583
+ chunks.append(chunk)
584
+ if length.isdigit() and total != int(length):
585
+ raise _HFError(0, "incomplete_response")
586
+ return b"".join(chunks), dict(response.headers.items())
587
+ except HTTPError as error:
588
+ status = error.code
589
+ retry_after = error.headers.get("Retry-After")
590
+ code = "http_error"
591
+ try:
592
+ # Only classify recognized server phrases. Never expose the body.
593
+ body = error.read(65536).decode("utf-8", "replace").lower()
594
+ if status in (401, 403, 429) and any(phrase in body for phrase in
595
+ ("rate limit", "rate-limit", "rate_limit", "too many requests")):
596
+ code = "rate_limited"
597
+ elif status in (400, 409, 412) and any(phrase in body for phrase in
598
+ ("parentcommit", "parent commit", "commit has changed", "concurrent")):
599
+ code = "parent_commit_conflict"
600
+ elif status in (401, 403):
601
+ code = "authentication_or_permission_denied"
602
+ elif status == 404:
603
+ code = "not_found"
604
+ except (OSError, HTTPException):
605
+ pass
606
+ finally:
607
+ error.close()
608
+ raise _HFError(status, code, retry_after) from None
609
+ except (URLError, TimeoutError, socket.timeout, ConnectionError, OSError, HTTPException):
610
+ raise _HFError(0, "connection_or_tls_error") from None
611
+
612
+ def _request(self, url, data=None, content_type=None, limit=_HF_METADATA_LIMIT, decode_json=False):
613
+ for attempt in range(8):
614
+ try:
615
+ raw, headers = self._once(url, data, content_type, limit)
616
+ if decode_json:
617
+ try:
618
+ return json.loads(raw), headers
619
+ except (ValueError, UnicodeError):
620
+ raise _HFError(502, "invalid_json_response") from None
621
+ return raw, headers
622
+ except _HFError as error:
623
+ error.attempts = attempt + 1
624
+ if not error.temporary or attempt == 7:
625
+ raise
626
+ delay = _hf_delay_seconds(attempt, error.retry_after)
627
+ self._on_retry(attempt + 1, delay, error)
628
+ self._wait(delay)
629
+
630
+ def _json(self, url, data=None, content_type=None):
631
+ return self._request(url, data, content_type, decode_json=True)
632
+
633
+ @staticmethod
634
+ def _next_link(headers, current_url):
635
+ # RFC Link URLs can include commas; match bracketed targets first.
636
+ value = next((v for k, v in headers.items() if k.lower() == "link"), "")
637
+ for match in re.finditer(r"<([^<>]+)>\s*;([^<>]*)(?=,\s*<|$)", value):
638
+ relation = re.search(r'\brel\s*=\s*(?:"([^"]*)"|([^;,\s]+))', match.group(2))
639
+ if relation and "next" in (relation.group(1) or relation.group(2)).split():
640
+ return urljoin(current_url, match.group(1))
641
+ return None
642
+
643
+ def info(self):
644
+ data, _ = self._json(_HF_API + "/revision/main")
645
+ if not isinstance(data, dict) or not re.fullmatch(r"[0-9a-fA-F]{40,64}", str(data.get("sha", ""))):
646
+ raise _HFError(502, "invalid_repository_info")
647
+ return data
648
+
649
+ def tree(self, revision, district):
650
+ """Return every file (all extensions) within the recursively listed district."""
651
+ prefix = district.rstrip("/") + "/"
652
+ url = (_HF_API + "/tree/" + quote(revision, safe="") + "/" +
653
+ quote(district, safe="/") + "?recursive=true&expand=false&limit=1000")
654
+ files, seen_urls, seen_paths = [], set(), set()
655
+ expected_path = urlsplit(url).path
656
+ while url:
657
+ if url in seen_urls:
658
+ raise _HFError(502, "pagination_loop")
659
+ self._safe_url(url)
660
+ if urlsplit(url).path != expected_path:
661
+ raise _HFError(502, "unexpected_pagination_path")
662
+ seen_urls.add(url)
663
+ data, headers = self._json(url)
664
+ if not isinstance(data, list):
665
+ raise _HFError(502, "invalid_tree_response")
666
+ for item in data:
667
+ if not isinstance(item, dict):
668
+ raise _HFError(502, "invalid_tree_entry")
669
+ if item.get("type") != "file":
670
+ continue
671
+ path = item.get("path")
672
+ if (not isinstance(path, str) or not path.startswith(prefix)
673
+ or any(part in (".", "..", "") for part in path.split("/"))):
674
+ raise _HFError(502, "invalid_tree_path")
675
+ if path not in seen_paths:
676
+ seen_paths.add(path)
677
+ files.append(item)
678
+ url = self._next_link(headers, url)
679
+ return sorted(files, key=lambda item: item["path"])
680
+
681
+ def history(self, revision):
682
+ # Recent history is diagnostic; file blob/LFS identities determine changes.
683
+ data, _ = self._json(_HF_API + "/commits/" + quote(revision, safe="") + "?limit=100")
684
+ if not isinstance(data, list):
685
+ raise _HFError(502, "invalid_commit_history")
686
+ return [item for item in data[:100] if isinstance(item, dict)]
687
+
688
+ def read_file(self, path, revision="main", max_bytes=_HF_METADATA_LIMIT):
689
+ url = (_HF_ROOT + "/" + _HF_REPO + "/resolve/" +
690
+ quote(revision, safe="") + "/" + quote(path, safe="/"))
691
+ return self._request(url, limit=max_bytes)[0]
692
+
693
+ def commit_json(self, path, content_bytes, parent_sha, message):
694
+ """POST one regular JSON blob; optimistic concurrency is mandatory.
695
+
696
+ The caller owns 409/412 reconciliation. In particular, after a response
697
+ was lost, reread the remote dictionary and compare before recommitting.
698
+ """
699
+ if not re.fullmatch(r"[0-9a-fA-F]{40,64}", str(parent_sha)):
700
+ raise ValueError("A valid parent commit SHA is required for dictionary upload.")
701
+ if not isinstance(content_bytes, bytes):
702
+ raise TypeError("Dictionary content must be UTF-8 JSON bytes.")
703
+ # Explicit local bound, well below the documented regular-commit payload limit.
704
+ if len(content_bytes) > _HF_METADATA_LIMIT:
705
+ raise _HFError(413, "dictionary_exceeds_thirty_two_mib")
706
+ lines = [
707
+ {"key": "header", "value": {"summary": message, "description": "",
708
+ "parentCommit": parent_sha}},
709
+ {"key": "file", "value": {"path": path, "encoding": "base64",
710
+ "content": base64.b64encode(content_bytes).decode("ascii")}},
711
+ ]
712
+ payload = ("\n".join(json.dumps(line, separators=(",", ":"), ensure_ascii=True)
713
+ for line in lines) + "\n").encode("utf-8")
714
+ result, _ = self._json(_HF_API + "/commit/main", payload, "application/x-ndjson")
715
+ if isinstance(result, dict) and result.get("success") is False:
716
+ raise _HFError(400, "commit_rejected")
717
+ if (not isinstance(result, dict) or not
718
+ re.fullmatch(r"[0-9a-fA-F]{40,64}", str(result.get("commitOid", "")))):
719
+ raise _HFError(502, "invalid_commit_response")
720
+ return result
721
+
722
+
723
+ # District mirroring and durable dictionary checkpoints.
724
+ _REPO_ID = "saliacoel/map_imgs_gpt"
725
+ _MANIFEST_LIMIT = 32 * 1024 * 1024
726
+ _CHECKPOINT_FILES = 25
727
+ _CHECKPOINT_SECONDS = 60
728
+ _IMAGE_TYPES = {
729
+ ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg",
730
+ ".webp": "image/webp", ".gif": "image/gif", ".bmp": "image/bmp",
731
+ ".tif": "image/tiff", ".tiff": "image/tiff", ".avif": "image/avif",
732
+ ".heic": "image/heic", ".heif": "image/heif", ".ico": "image/x-icon",
733
+ ".svg": "image/svg+xml", ".exr": "image/x-exr",
734
+ }
735
+ _DISTRICT_LOCKS = {}
736
+ _DISTRICT_LOCKS_GUARD = threading.Lock()
737
+
738
+
739
+ class _MirrorInterrupted(Exception):
740
+ pass
741
+
742
+
743
+ class _ConcurrentMirror(RuntimeError):
744
+ pass
745
+
746
+
747
+ def _check_cancel():
748
+ try:
749
+ import comfy.model_management as management
750
+ except ImportError:
751
+ return
752
+ if getattr(management, "processing_interrupted", lambda: False)():
753
+ raise _MirrorInterrupted("Cancelled by the user; completed uploads are saved.")
754
+
755
+
756
+ def _interruptible_sleep(seconds):
757
+ remaining = max(0.0, float(seconds))
758
+ while remaining > 0:
759
+ _check_cancel()
760
+ part = min(1.0, remaining)
761
+ time.sleep(part)
762
+ remaining -= part
763
+
764
+
765
+ def _utc_now():
766
+ return datetime.now(timezone.utc).isoformat(timespec="milliseconds")
767
+
768
+
769
+ def _resolve_url(path, revision="main"):
770
+ return "https://huggingface.co/" + _REPO_ID + "/resolve/" + quote(revision, safe="") + "/" + quote(path, safe="/")
771
+
772
+
773
+ def _json_bytes(document):
774
+ return (json.dumps(document, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode("utf-8")
775
+
776
+
777
+ def _atomic_json(path, document):
778
+ path = Path(path)
779
+ path.parent.mkdir(parents=True, exist_ok=True)
780
+ temporary = None
781
+ try:
782
+ with tempfile.NamedTemporaryFile(mode="wb", dir=str(path.parent), prefix=path.name + ".", delete=False) as stream:
783
+ temporary = stream.name
784
+ stream.write(_json_bytes(document))
785
+ stream.flush()
786
+ os.fsync(stream.fileno())
787
+ os.replace(temporary, path)
788
+ finally:
789
+ if temporary:
790
+ try:
791
+ os.unlink(temporary)
792
+ except OSError:
793
+ pass
794
+
795
+
796
+ def _source_identity(item):
797
+ lfs = item.get("lfs") or {}
798
+ sha256 = str(lfs.get("oid") or lfs.get("sha256") or "").removeprefix("sha256:")
799
+ oid = item.get("oid") or item.get("blob_id") or ""
800
+ size = lfs.get("size", item.get("size"))
801
+ if not isinstance(size, int) or size < 0:
802
+ raise ValueError("Hugging Face returned an invalid image size.")
803
+ if sha256 and re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
804
+ identity = "sha256:" + sha256.lower()
805
+ elif re.fullmatch(r"[0-9a-fA-F]{40}", str(oid)):
806
+ identity = "git:" + oid.lower()
807
+ else:
808
+ raise ValueError("Hugging Face did not return a usable source hash for an image.")
809
+ return identity, size
810
+
811
+
812
+ def _verify_source_bytes(data, record):
813
+ if len(data) != record["size"]:
814
+ raise ValueError("Downloaded image size does not match the pinned Hugging Face file.")
815
+ fingerprint = record["source_fingerprint"]
816
+ if fingerprint.startswith("sha256:"):
817
+ actual = "sha256:" + hashlib.sha256(data).hexdigest()
818
+ else:
819
+ actual = "git:" + hashlib.sha1(b"blob " + str(len(data)).encode("ascii") + b"\0" + data).hexdigest()
820
+ if actual != fingerprint:
821
+ raise ValueError("Downloaded image checksum does not match the pinned Hugging Face file.")
822
+
823
+
824
+ def _validate_manifest(doc, district):
825
+ if (not isinstance(doc, dict) or doc.get("schema_version") != 1
826
+ or doc.get("repo_id") != _REPO_ID or doc.get("district") != district
827
+ or not isinstance(doc.get("files"), dict) or not isinstance(doc.get("dictionary"), dict)):
828
+ raise ValueError("The existing district dictionary has an incompatible format; it was not overwritten.")
829
+ for path, record in doc["files"].items():
830
+ if not isinstance(path, str) or not path.startswith(district + "/") or not isinstance(record, dict):
831
+ raise ValueError("The existing district dictionary contains invalid file records; it was not overwritten.")
832
+ return doc
833
+
834
+
835
+ def _inventory_hash(files):
836
+ pairs = [(path, entry["source_fingerprint"]) for path, entry in sorted(files.items()) if entry.get("status") != "removed"]
837
+ return hashlib.sha256(json.dumps(pairs, separators=(",", ":")).encode()).hexdigest()
838
+
839
+
840
+ def _merge_successes(target, other):
841
+ """Resume only verified upload records for the SAME source content/destination."""
842
+ if target.get("backblaze") != other.get("backblaze"):
843
+ return
844
+ for path, candidate in other.get("files", {}).items():
845
+ entry = target["files"].get(path)
846
+ if (not entry or entry.get("status") == "removed" or candidate.get("status") == "removed"
847
+ or entry.get("source_fingerprint") != candidate.get("source_fingerprint")
848
+ or not candidate.get("b2", {}).get("file_name")):
849
+ continue
850
+ mine = entry.get("b2", {})
851
+ theirs = candidate["b2"]
852
+ if (not mine or (candidate.get("status") == "uploaded" and entry.get("status") != "uploaded")
853
+ or theirs.get("url_expires_at_unix", 0) > mine.get("url_expires_at_unix", 0)):
854
+ entry["b2"] = copy.deepcopy(theirs)
855
+ entry["status"] = candidate.get("status", "stored")
856
+ entry["last_error"] = candidate.get("last_error")
857
+ entry["updated_at"] = candidate.get("updated_at")
858
+ entry["processing_attempts"] = max(entry.get("processing_attempts", 0), candidate.get("processing_attempts", 0))
859
+
860
+
861
+ def _recovery_records(document):
862
+ if not document:
863
+ return []
864
+ records = list(document.get("recovery_records", []))
865
+ for path, record in document.get("files", {}).items():
866
+ if record.get("b2", {}).get("file_name"):
867
+ records.append({"path": path, "record": copy.deepcopy(record),
868
+ "backblaze": copy.deepcopy(document.get("backblaze", {}))})
869
+ unique = {}
870
+ for item in records:
871
+ if not isinstance(item, dict) or not isinstance(item.get("record"), dict):
872
+ continue
873
+ key = (item.get("path"), item["record"].get("source_fingerprint"),
874
+ item.get("backblaze", {}).get("bucket_id"))
875
+ unique[key] = item
876
+ return list(unique.values())
877
+
878
+
879
+ class _CheckpointLock:
880
+ """OS locks survive no crashes: the OS releases them when the process exits."""
881
+ def __init__(self, checkpoint_path):
882
+ self.path = Path(str(checkpoint_path) + ".lock")
883
+ self.stream = None
884
+
885
+ def acquire(self):
886
+ self.path.parent.mkdir(parents=True, exist_ok=True)
887
+ self.stream = self.path.open("a+b")
888
+ try:
889
+ if os.name == "nt":
890
+ import msvcrt
891
+ if self.path.stat().st_size == 0:
892
+ self.stream.write(b"0")
893
+ self.stream.flush()
894
+ self.stream.seek(0)
895
+ msvcrt.locking(self.stream.fileno(), msvcrt.LK_NBLCK, 1)
896
+ else:
897
+ import fcntl
898
+ fcntl.flock(self.stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
899
+ except OSError:
900
+ self.stream.close()
901
+ self.stream = None
902
+ raise RuntimeError("Another ComfyUI process is using this district checkpoint. Let it finish first.") from None
903
+
904
+ def release(self):
905
+ if self.stream is not None:
906
+ self.stream.close()
907
+ self.stream = None
908
+
909
+
910
+ class _DistrictMirror:
911
+ def __init__(self, hf, b2, district, secrets=()):
912
+ self.hf, self.b2, self.district = hf, b2, district
913
+ self.secrets = tuple(value for value in secrets if value)
914
+ self.filename = "DICT_BLAZE_" + district + ".json"
915
+ self.url = _resolve_url(self.filename)
916
+ self.run_id = uuid.uuid4().hex
917
+ self.doc = {
918
+ "schema_version": 1, "repo_id": _REPO_ID, "district": district,
919
+ "dictionary": {}, "files": {}, "history": [], "source": {},
920
+ "backblaze": {}, "status": "starting", "created_at": _utc_now(),
921
+ "retry_policy": {"attempts": 8, "wait_seconds": [3, 5, 7, 9, 11, 13, 15], "honor_retry_after": True},
922
+ "links": {"private_bucket": True, "valid_seconds": _LINK_SECONDS,
923
+ "scope": "district", "note": "Bearer URLs expire. Re-run this node to refresh. Stored objects do not expire."},
924
+ }
925
+ self.path = _key_id_cache_path().parent / "districts" / (
926
+ hashlib.sha256((_REPO_ID + ":" + b2.account_id).encode()).hexdigest()[:24] + "_" + self.filename)
927
+ self.remote_loaded = False
928
+ self.remote_digest = None
929
+ self.published = False
930
+ self.remote_existed = False
931
+ self.last_commit = None
932
+ self.last_checkpoint = 0.0
933
+ self.completed_since_checkpoint = 0
934
+ self.bucket = None
935
+ self.link_token = None
936
+ self.link_issued = 0.0
937
+ self.error = None
938
+ self.publish_error = None
939
+ self.local_error = None
940
+ self.run = {"run_id": self.run_id, "started_at": _utc_now(), "status": "starting",
941
+ "added": [], "changed": [], "removed": [], "reused": 0, "processed": 0,
942
+ "dictionary_commits": [], "inventory_complete": False}
943
+
944
+ def safe_error(self, exc):
945
+ if isinstance(exc, (_HFError, _RequestError, ValueError, RuntimeError, _MirrorInterrupted)):
946
+ text = str(exc)
947
+ else:
948
+ text = "Unexpected " + type(exc).__name__
949
+ for secret in sorted(self.secrets, key=len, reverse=True):
950
+ text = text.replace(secret, "[REDACTED]")
951
+ return text[:1500]
952
+
953
+ def read_remote(self, revision):
954
+ try:
955
+ data = self.hf.read_file(self.filename, revision, _MANIFEST_LIMIT)
956
+ except _HFError as exc:
957
+ if exc.status == 404:
958
+ return None, None
959
+ raise
960
+ try:
961
+ doc = json.loads(data)
962
+ except (ValueError, UnicodeError):
963
+ raise ValueError("The remote district dictionary is not valid JSON; it was not overwritten.") from None
964
+ return _validate_manifest(doc, self.district), hashlib.sha256(data).hexdigest()
965
+
966
+ def rebuild(self):
967
+ mapping = {}
968
+ counts = {"total": 0, "uploaded": 0, "stored": 0, "failed": 0, "pending": 0, "removed": 0}
969
+ now = time.time()
970
+ for path, entry in sorted(self.doc["files"].items()):
971
+ state = entry.get("status", "pending")
972
+ if state == "removed":
973
+ counts["removed"] += 1
974
+ continue
975
+ counts["total"] += 1
976
+ counts[state if state in counts else "pending"] += 1
977
+ b2 = entry.get("b2", {})
978
+ if (state == "uploaded" and b2.get("download_url")
979
+ and b2.get("url_expires_at_unix", 0) > now):
980
+ mapping[entry["source_url"]] = b2["download_url"]
981
+ counts["mapped"] = len(mapping)
982
+ self.doc["dictionary"] = mapping
983
+ self.doc["counts"] = counts
984
+ self.doc["updated_at"] = _utc_now()
985
+ self.doc["run_id"] = self.run_id
986
+ history = [item for item in self.doc.get("history", []) if item.get("run_id") != self.run_id]
987
+ self.doc["history"] = (history + [copy.deepcopy(self.run)])[-100:]
988
+
989
+ def save_local(self, strict=False):
990
+ self.rebuild()
991
+ try:
992
+ _atomic_json(self.path, self.doc)
993
+ self.local_error = None
994
+ except OSError as exc:
995
+ self.local_error = "Local checkpoint could not be saved: " + type(exc).__name__
996
+ if strict:
997
+ raise RuntimeError(self.local_error) from None
998
+
999
+ def publish(self):
1000
+ """Compare-and-swap; merge concurrent same-inventory successes, never overwrite a newer inventory."""
1001
+ if not self.remote_loaded:
1002
+ raise RuntimeError("Remote state was not read successfully; refusing to overwrite it.")
1003
+ self.doc.pop("last_publication_error", None)
1004
+ checkpoint_id = uuid.uuid4().hex
1005
+ for conflict_attempt in range(8):
1006
+ info = self.hf.info()
1007
+ head = info["sha"]
1008
+ remote, digest = self.read_remote(head)
1009
+ if remote and remote.get("checkpoint_id") == checkpoint_id:
1010
+ # The previous POST committed but its response was lost.
1011
+ self.remote_digest = digest
1012
+ self._published(head)
1013
+ return
1014
+ if digest != self.remote_digest:
1015
+ if (not remote or remote.get("inventory_hash") != self.doc.get("inventory_hash")
1016
+ or remote.get("backblaze") != self.doc.get("backblaze")):
1017
+ raise _ConcurrentMirror("Another run changed this dictionary's source inventory or destination. Local progress is saved; run again to merge safely.")
1018
+ _merge_successes(self.doc, remote)
1019
+ known = {item.get("run_id") for item in self.doc.get("history", [])}
1020
+ self.doc.setdefault("history", []).extend(item for item in remote.get("history", []) if item.get("run_id") not in known)
1021
+ self.doc["checkpoint_id"] = checkpoint_id
1022
+ self.doc["hf_head_before_checkpoint"] = head
1023
+ self.rebuild()
1024
+ payload = _json_bytes(self.doc)
1025
+ try:
1026
+ result = self.hf.commit_json(self.filename, payload, head,
1027
+ "Backblaze mirror " + self.district + ": " + self.doc["status"])
1028
+ except _HFError as exc:
1029
+ if exc.conflict and conflict_attempt < 7:
1030
+ continue
1031
+ raise
1032
+ commit = result.get("commitOid") or result.get("oid") or result.get("commit_id")
1033
+ if not commit:
1034
+ # Verify content using the same checkpoint ID rather than claiming success.
1035
+ head = self.hf.info()["sha"]
1036
+ check, _ = self.read_remote(head)
1037
+ if not check or check.get("checkpoint_id") != checkpoint_id:
1038
+ raise RuntimeError("Hugging Face did not confirm the dictionary commit.")
1039
+ commit = head
1040
+ self.remote_digest = hashlib.sha256(payload).hexdigest()
1041
+ self._published(commit)
1042
+ return
1043
+ raise RuntimeError("The dictionary changed repeatedly during publication. Local progress is saved.")
1044
+
1045
+ def _published(self, commit):
1046
+ self.published = True
1047
+ self.remote_existed = True
1048
+ self.last_commit = commit
1049
+ self.publish_error = None
1050
+ self.run["dictionary_commits"].append(commit)
1051
+ self.doc["last_confirmed_dictionary_commit"] = commit
1052
+ self.last_checkpoint = time.monotonic()
1053
+ self.completed_since_checkpoint = 0
1054
+ self.save_local()
1055
+
1056
+ def reconcile(self, items, revision):
1057
+ previous = self.doc["files"]
1058
+ current = {}
1059
+ for item in items:
1060
+ path = item.get("path", "")
1061
+ if not path.startswith(self.district + "/") or item.get("type") != "file":
1062
+ continue
1063
+ suffix = Path(path).suffix.lower()
1064
+ if suffix not in _IMAGE_TYPES:
1065
+ continue
1066
+ fingerprint, size = _source_identity(item)
1067
+ old = previous.get(path)
1068
+ if old and old.get("source_fingerprint") == fingerprint and old.get("status") != "removed":
1069
+ entry = copy.deepcopy(old)
1070
+ else:
1071
+ entry = {"status": "pending", "processing_attempts": 0, "last_error": None}
1072
+ self.run["changed" if old else "added"].append(path)
1073
+ entry.update({"source_fingerprint": fingerprint, "size": size,
1074
+ "source_url": _resolve_url(path), "source_pinned_url": _resolve_url(path, revision),
1075
+ "seen_in_commit": revision, "content_type": _IMAGE_TYPES[suffix]})
1076
+ current[path] = entry
1077
+ for path, entry in previous.items():
1078
+ if path not in current:
1079
+ removed = copy.deepcopy(entry)
1080
+ if removed.get("status") != "removed":
1081
+ self.run["removed"].append(path)
1082
+ removed.update({"status": "removed", "removed_in_commit": revision})
1083
+ current[path] = removed
1084
+ self.doc["files"] = current
1085
+ self.doc["inventory_hash"] = _inventory_hash(current)
1086
+ self.doc["source"].update({"snapshot_commit": revision, "inventory_complete": True,
1087
+ "scanned_at": _utc_now(), "branch": "main"})
1088
+ self.run["source_commit"] = revision
1089
+ self.run["inventory_complete"] = True
1090
+
1091
+ def ensure_link(self, record):
1092
+ if not self.link_token or time.time() >= self.link_issued + _LINK_SECONDS - 3600:
1093
+ issued_before_request = time.time()
1094
+ grant = self.b2.call("b2_get_download_authorization", {
1095
+ "bucketId": self.bucket["bucketId"], "fileNamePrefix": "districts/" + self.district + "/",
1096
+ "validDurationInSeconds": _LINK_SECONDS})
1097
+ self.link_token = grant["authorizationToken"]
1098
+ self.link_issued = issued_before_request
1099
+ b2 = record["b2"]
1100
+ base = self.b2.download_url + "/file/" + quote(self.bucket["bucketName"], safe="") + "/" + quote(b2["file_name"], safe="/")
1101
+ b2.update({"download_url_base": base, "download_url": base + "?" + urlencode({"Authorization": self.link_token}),
1102
+ "url_issued_at_unix": self.link_issued, "url_expires_at_unix": self.link_issued + _LINK_SECONDS})
1103
+
1104
+ def process(self, path, entry, revision):
1105
+ _check_cancel()
1106
+ entry["processing_attempts"] = entry.get("processing_attempts", 0) + 1
1107
+ entry["last_error"] = None
1108
+ old = entry.get("b2", {})
1109
+ valid = (old.get("bucket_id") == self.bucket["bucketId"] and
1110
+ str(old.get("file_name", "")).startswith("districts/" + self.district + "/") and
1111
+ old.get("sha1") and old.get("size") == entry["size"])
1112
+ if valid and self.b2.file_exists(self.bucket["bucketId"], old["file_name"], old["sha1"], old["size"]):
1113
+ self.run["reused"] += 1
1114
+ else:
1115
+ if entry["size"] > _MAX_PNG_BYTES:
1116
+ raise ValueError("Image exceeds the supported 128 MiB per-file limit.")
1117
+ data = self.hf.read_file(path, revision, _MAX_PNG_BYTES)
1118
+ _verify_source_bytes(data, entry)
1119
+ sha256 = hashlib.sha256(data).hexdigest()
1120
+ name = "districts/" + self.district + "/" + sha256 + Path(path).suffix.lower()
1121
+ uploaded = self.b2.ensure_file(self.bucket, name, data, entry["content_type"])
1122
+ entry["b2"] = {"bucket_id": self.bucket["bucketId"], "bucket_name": self.bucket["bucketName"],
1123
+ "file_name": name, "file_id": (uploaded or {}).get("fileId"),
1124
+ "sha1": hashlib.sha1(data).hexdigest(), "sha256": sha256, "size": len(data)}
1125
+ entry["uploaded_from_commit"] = revision
1126
+ # Record successful storage BEFORE any link/token request can fail.
1127
+ entry.update({"status": "stored", "updated_at": _utc_now()})
1128
+ self.save_local(strict=True)
1129
+ self.ensure_link(entry)
1130
+ entry.update({"status": "uploaded", "last_error": None, "updated_at": _utc_now()})
1131
+ self.run["processed"] += 1
1132
+ self.completed_since_checkpoint += 1
1133
+ self.save_local(strict=True)
1134
+
1135
+ def execute(self):
1136
+ local = getattr(self, "local_resume", None)
1137
+ recovery = _recovery_records(local)
1138
+ if local and local.get("backblaze", {}).get("account_id") == self.b2.account_id:
1139
+ self.doc = copy.deepcopy(local)
1140
+ try:
1141
+ info = self.hf.info()
1142
+ revision = info["sha"]
1143
+ remote, self.remote_digest = self.read_remote(revision)
1144
+ self.remote_loaded = True
1145
+ self.remote_existed = remote is not None
1146
+ if remote:
1147
+ self.doc = copy.deepcopy(remote)
1148
+ if local:
1149
+ _merge_successes(self.doc, local)
1150
+ known_runs = {run.get("run_id") for run in self.doc.get("history", [])}
1151
+ self.doc.setdefault("history", []).extend(run for run in local.get("history", [])
1152
+ if run.get("run_id") not in known_runs)
1153
+ # Retain local-only/newer-content uploads even if listing now fails.
1154
+ # These are recovery candidates, never active mappings until inventory validation.
1155
+ self.doc["recovery_records"] = recovery + self.doc.get("recovery_records", [])
1156
+ self.run["attempted_source_commit"] = revision
1157
+ self.doc["status"] = "scanning"
1158
+ self.save_local()
1159
+ # Listing must finish before files are added/changed/marked removed.
1160
+ items = self.hf.tree(revision, self.district)
1161
+ self.reconcile(items, revision)
1162
+ try:
1163
+ history = self.hf.history(revision)
1164
+ self.doc["source"]["recent_commits"] = [
1165
+ {"id": item.get("id") or item.get("commit_id"), "title": item.get("title", ""),
1166
+ "date": item.get("date") or item.get("created_at")} for item in history[:100]]
1167
+ except _HFError as exc:
1168
+ self.run["history_warning"] = self.safe_error(exc)
1169
+ self.bucket = self.b2.ensure_bucket()
1170
+ destination = {"account_id": self.b2.account_id, "bucket_id": self.bucket["bucketId"],
1171
+ "bucket_name": self.bucket["bucketName"], "private": True}
1172
+ if self.doc.get("backblaze") != destination:
1173
+ for entry in self.doc["files"].values():
1174
+ entry.pop("b2", None)
1175
+ if entry.get("status") != "removed":
1176
+ entry["status"] = "pending"
1177
+ self.doc["backblaze"] = destination
1178
+ # Prefer additional successful local progress after a failed HF commit.
1179
+ # The pre-run local state is retained separately, never read after overwriting.
1180
+ if getattr(self, "local_resume", None):
1181
+ _merge_successes(self.doc, self.local_resume)
1182
+ for candidate in self.doc.get("recovery_records", []):
1183
+ _merge_successes(self.doc, {"backblaze": candidate.get("backblaze"),
1184
+ "files": {candidate["path"]: candidate["record"]}})
1185
+ self.doc.pop("recovery_records", None)
1186
+ self.doc["status"] = self.run["status"] = "running"
1187
+ self.save_local(strict=True)
1188
+ # Test actual HF write permission before transferring any images.
1189
+ self.publish()
1190
+ paths = [path for path in sorted(self.doc["files"]) if self.doc["files"][path].get("status") != "removed"]
1191
+ try:
1192
+ from comfy.utils import ProgressBar
1193
+ progress = ProgressBar(len(paths)) if paths else None
1194
+ except ImportError:
1195
+ progress = None
1196
+ print("[Backblaze District] {} images found in {} at commit {}.".format(len(paths), self.district, revision))
1197
+ for index, path in enumerate(paths, 1):
1198
+ entry = self.doc["files"][path]
1199
+ if progress is not None:
1200
+ progress.update_absolute(index - 1, len(paths))
1201
+ if index == 1 or index % 10 == 0:
1202
+ print("[Backblaze District] Processing image {}/{}.".format(index, len(paths)))
1203
+ try:
1204
+ self.process(path, entry, revision)
1205
+ except _MirrorInterrupted:
1206
+ raise
1207
+ except Exception as exc:
1208
+ entry.update({"status": "failed", "last_error": {"message": self.safe_error(exc),
1209
+ "attempts": getattr(exc, "attempts", None), "at": _utc_now()}, "updated_at": _utc_now()})
1210
+ self.save_local()
1211
+ if isinstance(exc, ValueError) or (isinstance(exc, _HFError) and exc.status in (404, 413)):
1212
+ # One permanently bad/oversized source must not block the rest.
1213
+ self.run.setdefault("permanent_file_errors", []).append(path)
1214
+ self.completed_since_checkpoint += 1
1215
+ if (self.completed_since_checkpoint >= _CHECKPOINT_FILES or
1216
+ time.monotonic() - self.last_checkpoint >= _CHECKPOINT_SECONDS):
1217
+ self.publish()
1218
+ continue
1219
+ raise
1220
+ if (self.completed_since_checkpoint >= _CHECKPOINT_FILES or
1221
+ time.monotonic() - self.last_checkpoint >= _CHECKPOINT_SECONDS):
1222
+ self.publish()
1223
+ if progress is not None:
1224
+ progress.update_absolute(len(paths), len(paths))
1225
+ self.rebuild()
1226
+ self.doc["status"] = self.run["status"] = (
1227
+ "complete" if self.doc["counts"]["mapped"] == self.doc["counts"]["total"] else "partial")
1228
+ except _MirrorInterrupted as exc:
1229
+ self.error = self.safe_error(exc)
1230
+ self.doc["status"] = self.run["status"] = "interrupted"
1231
+ except _ConcurrentMirror as exc:
1232
+ self.error = self.safe_error(exc)
1233
+ self.doc["status"] = self.run["status"] = "conflict"
1234
+ except Exception as exc:
1235
+ self.error = self.safe_error(exc)
1236
+ self.doc["status"] = self.run["status"] = "partial"
1237
+ finally:
1238
+ self.run["finished_at"] = _utc_now()
1239
+ self.run["error"] = self.error
1240
+ self.save_local()
1241
+ if self.remote_loaded and self.doc["status"] != "conflict":
1242
+ # A final dictionary save is attempted even after transfer errors/cancellation.
1243
+ saved_check = self.hf._check_cancel
1244
+ self.hf._check_cancel = lambda: None
1245
+ try:
1246
+ self.publish()
1247
+ except Exception as exc:
1248
+ self.publish_error = self.safe_error(exc)
1249
+ self.doc["last_publication_error"] = self.publish_error
1250
+ self.save_local()
1251
+ finally:
1252
+ self.hf._check_cancel = saved_check
1253
+ counts = self.doc.get("counts", {})
1254
+ status = "{}: {}/{} image URLs ready; {} reused, {} failed, {} pending.".format(
1255
+ self.doc["status"].upper(), counts.get("mapped", 0), counts.get("total", 0),
1256
+ self.run["reused"], counts.get("failed", 0), counts.get("pending", 0))
1257
+ if self.error:
1258
+ status += "\n" + self.error
1259
+ if self.publish_error or not self.published or self.doc["status"] == "conflict":
1260
+ status += "\nLATEST DICTIONARY WAS NOT PUBLISHED. " + (self.publish_error or "Hugging Face state was unavailable.")
1261
+ else:
1262
+ status += "\nDictionary saved to Hugging Face. Commit: " + str(self.last_commit)
1263
+ if self.local_error:
1264
+ status += "\n" + self.local_error
1265
+ else:
1266
+ status += "\nLocal checkpoint: " + str(self.path)
1267
+ status += "\nPrivate links expire after seven days; run again to refresh them."
1268
+ self.rebuild()
1269
+ return (self.url if self.remote_existed else "", status, _json_bytes(self.doc).decode("utf-8"))
1270
+
1271
+
1272
  class BackblazePNGUpload:
1273
  @classmethod
1274
  def INPUT_TYPES(cls):
1275
  return {"required": {
1276
  "master_key": ("STRING", {"default": "", "multiline": False,
1277
+ "tooltip": "Backblaze master secret. KEY_ID:SECRET is also accepted."}),
1278
+ "District_Name": ("STRING", {"default": "District_19_K", "multiline": False,
1279
+ "tooltip": "Folder inside saliacoel/map_imgs_gpt. All image subfolders are included."}),
1280
+ "hf_key": ("STRING", {"default": "", "multiline": False,
1281
+ "tooltip": "Hugging Face token with write access to saliacoel/map_imgs_gpt."}),
1282
  }, "optional": {
1283
  "key_id": ("STRING", {"default": "", "multiline": False,
1284
+ "tooltip": "Backblaze key ID. Blank uses ID:SECRET or a previously verified saved ID."}),
 
 
1285
  }}
1286
 
1287
+ RETURN_TYPES = ("STRING", "STRING", "STRING")
1288
+ RETURN_NAMES = ("dictionary_url", "STATUS", "dictionary_json")
1289
  FUNCTION = "upload"
1290
  CATEGORY = "Salia/Backblaze"
1291
  OUTPUT_NODE = True
1292
+ DESCRIPTION = "Mirror every district image to Backblaze and publish a resumable DICT_BLAZE JSON to Hugging Face."
 
 
1293
 
1294
  @classmethod
1295
+ def IS_CHANGED(cls, master_key, District_Name, hf_key, key_id=""):
 
1296
  return float("nan")
1297
 
1298
+ def upload(self, master_key, District_Name, hf_key, key_id=""):
1299
+ mirror = None
1300
+ acquired = False
1301
+ lock = None
1302
+ file_lock = None
1303
  secret = ""
 
 
1304
  try:
1305
+ district = District_Name.strip()
1306
+ if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", district) or district in (".", ".."):
1307
+ raise ValueError("District_Name must be a single folder name, for example District_19_K.")
1308
+ if not isinstance(hf_key, str) or not hf_key.strip() or any(c.isspace() for c in hf_key.strip()):
1309
+ raise ValueError("hf_key must contain a Hugging Face token with write access to the repository.")
1310
  resolved_id, secret, source = _credential_parts(master_key, key_id)
1311
+ basic = base64.b64encode((resolved_id + ":" + secret).encode()).decode("ascii")
1312
+ b2 = _B2Client(basic)
 
 
 
1313
  cache_note = _remember_key_id(secret, resolved_id)
1314
+ with _DISTRICT_LOCKS_GUARD:
1315
+ lock = _DISTRICT_LOCKS.setdefault((_REPO_ID, district), threading.Lock())
1316
+ acquired = lock.acquire(blocking=False)
1317
+ if not acquired:
1318
+ raise RuntimeError("This district is already running in this ComfyUI process. Let it finish first.")
1319
+ hf = _HFClient(hf_key.strip(), check_cancel=_check_cancel,
1320
+ on_retry=lambda attempt, seconds, error: print(
1321
+ "[Backblaze District] HF retry {}/8 in {} seconds ({}).".format(attempt + 1, seconds, error.code)))
1322
+ mirror = _DistrictMirror(hf, b2, district, secrets=(master_key, secret, hf_key.strip()))
1323
+ file_lock = _CheckpointLock(mirror.path)
1324
+ file_lock.acquire()
1325
+ try:
1326
+ with mirror.path.open("r", encoding="utf-8") as stream:
1327
+ mirror.local_resume = _validate_manifest(json.load(stream), district)
1328
+ except (OSError, ValueError, TypeError):
1329
+ mirror.local_resume = None
1330
+ url, status, json_text = mirror.execute()
1331
+ status += "\nKEY_ID: {} ({})\n{}".format(resolved_id, source, cache_note)
1332
  except Exception as exc:
1333
+ url, json_text = "", _json_bytes(mirror.doc).decode() if mirror else "{}"
1334
+ status = "ERROR. " + (str(exc) if isinstance(exc, (ValueError, RuntimeError, _HFError, _RequestError, _MirrorInterrupted)) else type(exc).__name__)
1335
+ finally:
1336
+ if file_lock is not None:
1337
+ file_lock.release()
1338
+ if acquired:
1339
+ lock.release()
1340
+ for value in sorted({str(master_key).strip(), secret, str(hf_key).strip()}, key=len, reverse=True):
 
 
 
1341
  if value:
1342
  status = status.replace(value, "[REDACTED]")
1343
+ print("[Backblaze District] " + status)
1344
+ return {"ui": {"text": [status] + ([url] if url else [])}, "result": (url, status, json_text)}
1345
 
1346
 
1347
  NODE_CLASS_MAPPINGS = {"BackblazePNGUpload": BackblazePNGUpload}
1348
+ NODE_DISPLAY_NAME_MAPPINGS = {"BackblazePNGUpload": "Backblaze District Mirror (Auto / Resume)"}