File size: 57,743 Bytes
e13472e
 
 
 
 
 
 
 
4b54047
 
aa1490a
dcd43e2
4b54047
 
81a8217
e13472e
 
 
 
 
 
 
 
 
 
81a8217
dcd43e2
81a8217
aa1490a
dcd43e2
 
aa1490a
81a8217
 
6032706
 
 
 
 
81a8217
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6032706
81a8217
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ffcfecb
81a8217
 
ffcfecb
81a8217
 
 
 
ffcfecb
 
 
 
 
 
6032706
 
 
 
 
81a8217
 
 
 
6032706
81a8217
 
 
 
 
 
 
 
 
 
 
 
 
dcd43e2
 
aa1490a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b2aa446
 
 
 
 
 
a5981db
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6032706
 
 
 
 
 
81a8217
b2aa446
 
 
 
 
6032706
b2aa446
 
6032706
 
 
 
 
 
 
b2aa446
dcd43e2
 
 
 
 
81a8217
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
dcd43e2
a5981db
 
 
dcd43e2
a5981db
 
 
dcd43e2
7f72224
 
 
 
dcd43e2
7f72224
 
a5981db
dcd43e2
 
 
 
 
b2aa446
 
dcd43e2
aa1490a
 
 
 
7f72224
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b2aa446
7f72224
 
 
dcd43e2
 
 
 
 
81a8217
 
 
 
 
7f72224
dcd43e2
 
7f72224
dcd43e2
 
ffcfecb
 
 
 
 
 
7f72224
 
 
 
 
 
 
 
 
dcd43e2
 
 
aa1490a
81a8217
7f72224
 
 
81a8217
7f72224
ffcfecb
7f72224
81a8217
 
 
 
 
 
 
 
 
 
 
 
 
7f72224
81a8217
 
 
 
 
 
 
 
 
 
aa1490a
7f72224
 
 
81a8217
 
ffcfecb
dcd43e2
 
 
81a8217
dcd43e2
 
81a8217
 
 
 
dcd43e2
81a8217
 
 
dcd43e2
81a8217
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7f72224
81a8217
aa1490a
 
 
 
 
 
 
 
 
 
 
 
 
dcd43e2
 
 
 
 
 
6032706
 
 
 
7f72224
6032706
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7f72224
 
 
6032706
 
7f72224
 
6032706
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7f72224
6032706
 
 
 
 
 
 
a5981db
 
 
6032706
7f72224
 
 
 
 
 
a5981db
7f72224
 
 
6032706
 
 
 
 
 
 
 
7f72224
6032706
 
 
 
7f72224
6032706
 
 
 
 
 
7f72224
 
 
 
 
 
 
 
 
 
 
 
6032706
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7f72224
 
6032706
 
7f72224
6032706
 
 
 
 
 
4b54047
 
 
 
 
 
 
 
 
 
 
 
 
 
bdac669
 
 
4b54047
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bdac669
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4b54047
 
 
 
 
 
 
 
 
 
 
 
 
bdac669
4b54047
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bdac669
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4b54047
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bdac669
 
4b54047
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bdac669
 
4b54047
 
 
 
e13472e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4b54047
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e13472e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b2aa446
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1fc14b3
 
 
b2aa446
 
 
 
1fc14b3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b2aa446
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e13472e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b2aa446
e13472e
dcd43e2
 
53bf512
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
"""Tenant-and-connection scoped connector sync lifecycle state.

This data-layer module records what a provider runner did. It deliberately does not import or
call providers: the API layer owns runners while every connector shares this durable vocabulary.
"""
from __future__ import annotations

import datetime as _dt
import hashlib
import json
import logging
import threading
import time
import uuid
from copy import deepcopy

import core.store as store


STATE_KEY = "connector_sync_state"
STATES = ("queued", "syncing", "materializing", "ready", "partial", "error")
_ERROR_LIMIT = 50
_DEFAULT_CONNECTION = "default"
_PROVIDERS = {}

# Ordinary requests coalesce per provider/tenant. Explicit events retain separate durable receipts.
_SYNC_REQUESTS = {}
_SYNC_REQUEST_RUNTIMES = {}
_SYNC_NOT_BEFORE = {}
_SYNC_REQUEST_LOCK = threading.Lock()
_SYNC_WAKE = threading.Event()
_SYNC_DUE_TIMER = None
SYNC_REQUESTS_KEY = "connector_sync_requests_v1"
NOTIFICATION_TARGETING_CONTRACT = "amazon-table-events-v1"
# Ordinary requests the web process hands to the sync worker live in this member of the desk.
REMOTE_MEMBER = "ordinary"
REMOTE_KEEP_SECONDS = 3600
_REQUEST_LISTENERS = []
_REMOTE_IDS = {}
_EVENT_DEDUPE_SECONDS = 600


def _target_tables(provider, tenant, rt, tables, event_id):
    from core.connector_spec import AMAZON_TABLES
    known = {row.source.removeprefix("amzn_") for row in AMAZON_TABLES
             if not row.native.startswith("derived:")}
    if (provider != "amazon_sp" or rt is None or rt.key != tenant
            or not isinstance(tables, list) or not tables
            or any(not isinstance(table, str) or table not in known for table in tables)):
        raise ValueError("amazon_notification_target_invalid")
    if not isinstance(event_id, str) or not event_id.strip() or len(tables) != 1:
        raise ValueError("Amazon notifications require one table and an event ID.")
    return list(tables)


def _sync_request_document(current):
    document = deepcopy(current) if current is not None else {}
    if not isinstance(document, dict):
        raise ValueError("The stored sync request desk is unavailable.")
    for member in ("requests", "events", REMOTE_MEMBER):
        document.setdefault(member, {})
        if not isinstance(document[member], dict):
            raise ValueError("The stored sync request desk is unavailable.")
    return document


def _cache_sync_requests(rt, document):
    """Refresh this tenant's durable receipts under the desk lock without touching ordinary work."""
    rows = {}
    for identity, row in document["requests"].items():
        if not isinstance(row, dict) or row.get("id") != identity or row.get("tenant") != rt.key:
            raise ValueError("The stored sync receipt does not match its tenant or identity.")
        _target_tables(row.get("provider"), rt.key, rt, row.get("tables"), row.get("eventId"))
        rows[(row["provider"], rt.key, identity)] = deepcopy(row)
    for key in list(_SYNC_REQUESTS):
        if len(key) == 3 and key[1] == rt.key:
            del _SYNC_REQUESTS[key]
    _SYNC_REQUESTS.update(rows)
    _SYNC_REQUEST_RUNTIMES[rt.key] = rt


def _wake_sync_requests(stamp, held=()):
    _arm_sync_wake(stamp)
    if any(row["state"] == "queued" and (row.get("notBefore") or 0) <= stamp
           and row.get("requestedBy") not in held
           for row in _SYNC_REQUESTS.values()):
        _SYNC_WAKE.set()


def resume_held_requests(now=None):
    """Wake the resident loop for queued work a hold left waiting."""
    with _SYNC_REQUEST_LOCK:
        _wake_sync_requests(time.time() if now is None else now)


def restore_sync_requests(rt, *, now=None, requeue=True):
    """Restore durable events and requeue claims abandoned by a previous process.

    requeue=False only reads: a process that never claims must not requeue another one's claim.
    """
    stamp = time.time() if now is None else now
    with _SYNC_REQUEST_LOCK:
        document = _sync_request_document(rt.get(SYNC_REQUESTS_KEY, fresh=True))
        abandoned = {identity for identity, row in document["requests"].items()
                     if requeue and row.get("state") == "running" and
                     _SYNC_REQUESTS.get((row.get("provider"), rt.key, identity), {}).get("state") != "running"}
        if abandoned:
            def restore(current):
                doc = _sync_request_document(current)
                for identity in abandoned:
                    row = doc["requests"].get(identity)
                    if row and row["state"] == "running":
                        row.update(state="queued", startedAt=None, claimToken="", restoredAt=stamp)
                return doc
            document = rt.update(SYNC_REQUESTS_KEY, restore, flush="sync")
        _cache_sync_requests(rt, document)
        _wake_sync_requests(stamp)
        return [deepcopy(row) for row in document["requests"].values()]


def _arm_sync_wake(now):
    """Called under the desk lock. A future claim wakes the resident loop once, at its floor.

    Bootstrap re-arms this disposable alarm from the provider's durable not-before marker.
    The alarm only wakes the single writer; it neither queues work nor calls a provider.
    """
    global _SYNC_DUE_TIMER
    if _SYNC_DUE_TIMER is not None:
        _SYNC_DUE_TIMER.cancel()
        _SYNC_DUE_TIMER = None
    due = [row["notBefore"] for row in _SYNC_REQUESTS.values()
           if row["state"] == "queued" and row.get("notBefore") and row["notBefore"] > now]
    if due:
        _SYNC_DUE_TIMER = threading.Timer(min(due) - now, _SYNC_WAKE.set)
        _SYNC_DUE_TIMER.daemon = True
        _SYNC_DUE_TIMER.start()


def defer_sync(provider, tenant, not_before):
    """Install a provider's durable checkpoint boundary, including on process bootstrap.

    The provider owns persistence and restores this boundary before re-queuing its work.
    This module only claims intents; it never reads a provider mirror.
    """
    with _SYNC_REQUEST_LOCK:
        key = (provider, tenant)
        if not_before is None:
            _SYNC_NOT_BEFORE.pop(key, None)
        else:
            _SYNC_NOT_BEFORE[key] = float(not_before)
        receipt = _SYNC_REQUESTS.get(key)
        if receipt and receipt["state"] == "queued":
            receipt["notBefore"] = _SYNC_NOT_BEFORE.get(key)
            _arm_sync_wake(time.time())
            if not not_before or float(not_before) <= time.time():
                _SYNC_WAKE.set()


def sync_not_before(provider, tenant):
    """The provider's installed not-before floor as an epoch, or None."""
    with _SYNC_REQUEST_LOCK:
        return _SYNC_NOT_BEFORE.get((provider, tenant))


def _clear_sync_defer(provider, tenant):
    """Remove one obsolete provider hold under the desk lock, retaining explicit continuation waits."""
    key = (provider, tenant)
    held = _SYNC_NOT_BEFORE.pop(key, None)
    receipt = _SYNC_REQUESTS.get(key)
    if (held is not None and receipt and receipt["state"] == "queued"
            and receipt.get("requestedBy") != "first load" and receipt.get("notBefore") == held):
        receipt["notBefore"] = None


def clear_sync_defer(provider, tenant):
    """Release one provider hold after its producer adopts partition-specific waits."""
    with _SYNC_REQUEST_LOCK:
        _clear_sync_defer(provider, tenant)
        _wake_sync_requests(time.time())


def register_request_listener(fn):
    """Call fn(receipt, connection_id) after each accepted ordinary request, outside the desk lock."""
    if fn not in _REQUEST_LISTENERS:
        _REQUEST_LISTENERS.append(fn)


def request_sync(provider, tenant, *, requested_by="", now=None, not_before=None,
                 rt=None, tables=None, event_id="", connection_id=None, wait_cause=""):
    """Queue ordinary work or durably acknowledge one tenant-bound Amazon event.

    `wait_cause` is the safe sentence a receipt held by its not-before floor shows instead of the bare stamp.
    """
    receipt = _request_sync(provider, tenant, requested_by=requested_by, now=now,
                            not_before=not_before, rt=rt, tables=tables, event_id=event_id,
                            wait_cause=wait_cause)
    if tables is None and receipt.get("accepted"):
        for listener in list(_REQUEST_LISTENERS):
            listener(dict(receipt), connection_id)
    return receipt


def _request_sync(provider, tenant, *, requested_by="", now=None, not_before=None,
                  rt=None, tables=None, event_id="", wait_cause=""):
    if provider not in _PROVIDERS:
        raise ValueError("This connector is not registered for sync.")
    tenant = str(tenant or "").strip()
    if not tenant:
        raise ValueError("A tenant is required for sync.")
    if tables is None and event_id:
        raise ValueError("Amazon notifications require an explicit table target.")
    if tables is not None:
        targets = _target_tables(provider, tenant, rt, tables, event_id)
        stamp, identity = time.time() if now is None else now, uuid.uuid4().hex
        event_key = hashlib.sha256(json.dumps([provider, event_id], separators=(",", ":")).encode()).hexdigest()
        receipt = {"id": identity, "provider": provider, "tenant": tenant, "tables": targets,
                   "eventId": event_id, "requestedBy": str(requested_by or ""),
                   "requestedAt": stamp, "notBefore": float(not_before) if not_before is not None else None,
                   "state": "queued", "startedAt": None, "finishedAt": None,
                   "detail": "", "accepted": False, "claimToken": ""}

        def enqueue_event(current):
            document = _sync_request_document(current)
            event = document["events"].get(event_key)
            if event and stamp < event["firstAcceptedAt"] + _EVENT_DEDUPE_SECONDS:
                if event["requestId"] not in document["requests"]:
                    raise ValueError("The stored Amazon event receipt is unavailable.")
                return document
            document["requests"][identity] = deepcopy(receipt)
            document["events"][event_key] = {"requestId": identity, "firstAcceptedAt": stamp}
            return document

        with _SYNC_REQUEST_LOCK:
            document = rt.update(SYNC_REQUESTS_KEY, enqueue_event, flush="sync")
            _cache_sync_requests(rt, document)
            accepted_id = document["events"][event_key]["requestId"]
            result = deepcopy(document["requests"][accepted_id])
            _wake_sync_requests(stamp)
            return {**result, "accepted": accepted_id == identity, "duplicate": accepted_id != identity}
    with _SYNC_REQUEST_LOCK:
        if provider == "amazon_sp" and requested_by != "first load":
            # Amazon's history floor belongs to owed tables, never a loaded-table refresh.
            _clear_sync_defer(provider, tenant)
        previous = _SYNC_REQUESTS.get((provider, tenant))
        replaces_amazon_history = (provider == "amazon_sp" and previous
                                   and previous.get("requestedBy") == "first load"
                                   and requested_by != "first load")
        if previous and previous["state"] in ("queued", "running"):
            if not (previous["state"] == "queued" and previous.get("requestedBy") == "first load"
                    and requested_by != "first load"):
                return {**_request_wait(previous), "accepted": False}
        receipt = {"id": uuid.uuid4().hex, "provider": provider, "tenant": tenant, "state": "queued",
                   "requestedAt": time.time() if now is None else now,
                   "notBefore": max(float(not_before or 0), _SYNC_NOT_BEFORE.get((provider, tenant), 0),
                                    float(previous.get("notBefore") or 0) if previous
                                    and previous["state"] == "queued" and not replaces_amazon_history else 0) or None,
                   "startedAt": None, "finishedAt": None, "detail": "",
                   "requestedBy": str(requested_by or ""), "accepted": True}
        if requested_by == "first load" and previous and previous.get("firstLoad"):
            # A continuation keeps the safe numeric progress visible while waiting for the writer.
            receipt.update(detail=previous["detail"], firstLoad=dict(previous["firstLoad"]))
        if wait_cause:
            receipt.update(detail=str(wait_cause)[:300], waitCause=str(wait_cause)[:300])
        _SYNC_REQUESTS[(provider, tenant)] = receipt
        stamp = time.time() if now is None else now
        _arm_sync_wake(stamp)
        if (receipt.get("notBefore") or 0) <= stamp:
            _SYNC_WAKE.set()
        return _request_wait(receipt, now=stamp)


def _request_order(receipt):
    """Manual work precedes first-load continuations, with FIFO within each class."""
    return receipt.get("requestedBy") == "first load", receipt.get("requestedAt") or 0


def _request_wait(receipt, *, now=None):
    """Describe a queued receipt's executing competitor or provider floor without claiming a start."""
    result = deepcopy(receipt)
    result.setdefault("waitingFor", None)
    if receipt["state"] != "queued":
        result["waitingFor"] = None
        return result
    running = [row for row in _SYNC_REQUESTS.values()
               if row["tenant"] == receipt["tenant"] and row["state"] == "running"
               and row.get("startedAt") is not None]
    if running:
        active = max(running, key=lambda row: row["startedAt"])
        started = active["startedAt"]
        identity = active.get("id") or f"{active['provider']}:{started}"
        result["waitingFor"] = {"id": identity, "provider": active["provider"], "startedAt": started}
        stamp = _dt.datetime.fromtimestamp(started, _dt.timezone.utc).isoformat()
        result["detail"] = f"Waiting for {active['provider']} receipt {identity}, started at {stamp}."
    elif result.get("waitingFor"):
        return result
    elif (receipt.get("notBefore") or 0) > (time.time() if now is None else now):
        stamp = _dt.datetime.fromtimestamp(receipt["notBefore"], _dt.timezone.utc).isoformat()
        result["detail"] = receipt.get("waitCause") or f"Waiting for the provider not-before floor at {stamp}."
    elif result.get("detail", "").startswith("Waiting for "):
        result["detail"] = ""
    return result


def sync_request(provider, tenant):
    """Return a detached current receipt, or None."""
    with _SYNC_REQUEST_LOCK:
        rows = [row for row in _SYNC_REQUESTS.values()
                if row["provider"] == provider and row["tenant"] == tenant]
        receipt = next((row for row in rows if row["state"] == "running"), None)
        if receipt is None:
            queued = [row for row in rows if row["state"] == "queued"]
            receipt = min(queued, key=_request_order) if queued else _SYNC_REQUESTS.get((provider, tenant))
        if receipt is None:
            return None
        return {**_request_wait(receipt), "accepted": False}


def take_sync_requests(*, now=None, limit=None, exclude_providers=(), refresh_remote=False,
                       exclude_requested_by=()):
    """Claim manual work first, preserving provider floors, tenant claims and event siblings.

    `exclude_requested_by` leaves those receipts queued and does not wake the loop for them.
    """
    if refresh_remote:
        with _SYNC_REQUEST_LOCK:
            runtimes = list(_SYNC_REQUEST_RUNTIMES.values())
        for rt in runtimes:
            try:
                poll_remote_requests(rt, now=now)
            except Exception as exc:
                logging.getLogger(__name__).warning("Sync desk refresh unavailable for %s (%s)",
                                                     rt.key, type(exc).__name__)
    with _SYNC_REQUEST_LOCK:
        _SYNC_WAKE.clear()
        claimed = []
        stamp = time.time() if now is None else now
        busy = {(row["provider"], row["tenant"]) for row in _SYNC_REQUESTS.values() if row["state"] == "running"}
        for receipt in sorted(list(_SYNC_REQUESTS.values()), key=_request_order):
            if limit is not None and len(claimed) >= limit:
                break
            pair = (receipt["provider"], receipt["tenant"])
            if (receipt["provider"] in exclude_providers or pair in busy or receipt["state"] != "queued"
                    or receipt.get("requestedBy") in exclude_requested_by
                    or max(receipt.get("notBefore") or 0, _SYNC_NOT_BEFORE.get(pair, 0)) > stamp):
                continue
            if receipt.get("tables") is not None:
                rt = _SYNC_REQUEST_RUNTIMES[receipt["tenant"]]
                identity, token = receipt["id"], uuid.uuid4().hex

                def claim_event(current):
                    document = _sync_request_document(current)
                    rows = [row for row in document["requests"].values()
                            if (row["provider"], row["tenant"]) == pair]
                    if any(row["state"] == "running" for row in rows):
                        return document
                    eligible = [row for row in rows if row["state"] == "queued"
                                and (row.get("notBefore") or 0) <= stamp]
                    row = min(eligible, key=_request_order) if eligible else None
                    if row is not None and row["id"] == identity:
                        row.update(state="running", startedAt=stamp, claimToken=token, accepted=False)
                    return document

                document = rt.update(SYNC_REQUESTS_KEY, claim_event, flush="sync")
                _cache_sync_requests(rt, document)
                receipt = document["requests"].get(identity, {})
                if receipt.get("claimToken") != token:
                    continue
            else:
                receipt.update(state="running", startedAt=stamp, accepted=False)
            receipt["waitingFor"] = None
            if receipt.get("detail", "").startswith("Waiting for "):
                receipt["detail"] = ""
            claimed.append(deepcopy(receipt))
            busy.add(pair)
        _wake_sync_requests(stamp, exclude_requested_by)
        return claimed


def finish_sync_request(provider, tenant, *, ok, detail="", now=None, first_load=None, request_id=None):
    """Finish a claimed intent. The API runner supplies a safe cause sentence, never raw errors."""
    with _SYNC_REQUEST_LOCK:
        receipt = next((row for row in _SYNC_REQUESTS.values() if row["provider"] == provider
                        and row["tenant"] == tenant and row["state"] == "running"
                        and (request_id is None or row.get("id") == request_id)), None)
        if receipt is None:
            return None
        stamp = time.time() if now is None else now
        completion = {"state": "done" if ok else "failed", "detail": str(detail or "")[:300],
                      "finishedAt": stamp, "accepted": False}
        if first_load is not None:
            completion["firstLoad"] = dict(first_load)
        if receipt.get("tables") is not None:
            rt = _SYNC_REQUEST_RUNTIMES[tenant]
            identity, token = receipt["id"], receipt["claimToken"]

            def finish_event(current):
                document = _sync_request_document(current)
                row = document["requests"].get(identity)
                if row and row["state"] == "running" and row.get("claimToken") == token:
                    row.update(completion)
                return document

            document = rt.update(SYNC_REQUESTS_KEY, finish_event, flush="sync")
            _cache_sync_requests(rt, document)
            receipt = document["requests"].get(identity, {})
            if receipt.get("claimToken") != token or receipt.get("finishedAt") != stamp:
                return None
        else:
            receipt.update(completion)
        result = deepcopy(receipt)
        result["waitingFor"] = None
        _wake_sync_requests(stamp)
    connector = {"amazon_sp": "amazon", "meta_ads": "meta"}.get(provider)
    if connector:
        try:
            from core.sync_events import emit
        except ImportError:
            pass  # C20: lane J supplies this module at combine.
        else:
            try:
                emit(tenant, connector, bool(ok), result["detail"],
                     _dt.datetime.fromtimestamp(result["finishedAt"], _dt.timezone.utc).isoformat())
            except Exception:
                logging.getLogger(__name__).exception("Sync completion listener failed")
    return result


def sync_wake_event():
    """The event the resident thread waits on between passes."""
    return _SYNC_WAKE


def queue_remote_request(rt, receipt, connection_id=None, *, now=None):
    """Web side: durably hand one accepted ordinary request to the sync worker process."""
    stamp = time.time() if now is None else now
    identity = receipt.get("id") or uuid.uuid4().hex
    entry = {"id": identity, "provider": receipt["provider"], "tenant": rt.key,
             "requestedBy": str(receipt.get("requestedBy") or ""),
             "requestedAt": float(receipt.get("requestedAt") or stamp),
             "notBefore": receipt.get("notBefore"),
             "connectionId": str(connection_id) if connection_id else None,
             "state": "queued", "startedAt": None, "finishedAt": None, "detail": ""}

    def add(current):
        document = _sync_request_document(current)
        document[REMOTE_MEMBER][identity] = entry
        return document

    rt.update(SYNC_REQUESTS_KEY, add, flush="sync")
    with _SYNC_REQUEST_LOCK:
        _REMOTE_IDS[(receipt["provider"], rt.key)] = identity
    return identity


def fail_remote_request(provider, tenant, detail, *, now=None):
    """Web side: a hand-off that could not be saved finishes its receipt instead of hanging."""
    with _SYNC_REQUEST_LOCK:
        receipt = _SYNC_REQUESTS.get((provider, tenant))
        if receipt and receipt["state"] == "queued":
            receipt.update(state="failed", detail=str(detail)[:300],
                           finishedAt=time.time() if now is None else now)


def _remote_progress(entry, receipt):
    """The worker receipt's progress for one entry, or None when that run does not cover it."""
    started = receipt.get("startedAt") if receipt else None
    if (started is None or entry["requestedAt"] > started
            or receipt["state"] not in ("running", "done", "failed")
            or (entry.get("requestedBy") != "first load" and receipt.get("requestedBy") == "first load")):
        return None
    progress = {"state": receipt["state"], "startedAt": started,
                "finishedAt": receipt.get("finishedAt"), "detail": receipt.get("detail") or "",
                "waitingFor": None}
    if receipt.get("firstLoad"):
        progress["firstLoad"] = dict(receipt["firstLoad"])
    return progress


def poll_remote_requests(rt, *, now=None):
    """Worker side: admit queued hand-offs, publish their progress, prune old ones.

    An entry leaves the queue only when a run that started after it was requested finishes.
    Returns the entries admitted as new local receipts.
    """
    stamp = time.time() if now is None else now
    admitted, changes, pruned = [], {}, []
    with _SYNC_REQUEST_LOCK:
        document = _sync_request_document(rt.get(SYNC_REQUESTS_KEY, fresh=True))
        for identity, entry in sorted(document[REMOTE_MEMBER].items(), key=lambda item: _request_order(item[1])):
            key = (entry.get("provider"), rt.key)
            if entry.get("tenant") != rt.key or key[0] not in _PROVIDERS:
                continue
            if entry["state"] in ("done", "failed"):
                if stamp - float(entry.get("finishedAt") or 0) > REMOTE_KEEP_SECONDS:
                    pruned.append(identity)
                continue
            manual_amazon = key[0] == "amazon_sp" and entry.get("requestedBy") != "first load"
            if manual_amazon:
                _clear_sync_defer(*key)
            receipt = _SYNC_REQUESTS.get(key)
            # Promote under the same lock as claims, so a continuation cannot win a manual hand-off race.
            if (receipt and receipt["state"] == "queued" and receipt.get("requestedBy") == "first load"
                    and entry.get("requestedBy") != "first load"):
                receipt.update(id=identity, requestedBy=entry.get("requestedBy") or "",
                               requestedAt=entry["requestedAt"],
                               connectionId=entry.get("connectionId"),
                               notBefore=max(0 if manual_amazon else float(receipt.get("notBefore") or 0),
                                             float(entry.get("notBefore") or 0),
                                             _SYNC_NOT_BEFORE.get(key, 0)) or None)
                admitted.append(deepcopy(entry))
            progress = _remote_progress(entry, receipt)
            if progress is not None:
                if any(entry.get(name) != value for name, value in progress.items()):
                    changes[identity] = progress
                continue
            # A "running" entry with no local run was abandoned by a worker restart: admit it again.
            if (entry["state"] in ("queued", "running")
                    and not (receipt and receipt["state"] in ("queued", "running"))):
                local = {"id": identity, "provider": key[0], "tenant": rt.key, "state": "queued",
                         "requestedAt": entry["requestedAt"],
                         "notBefore": max(float(entry.get("notBefore") or 0),
                                          _SYNC_NOT_BEFORE.get(key, 0)) or None,
                         "startedAt": None, "finishedAt": None, "detail": "",
                         "connectionId": entry.get("connectionId"),
                         "requestedBy": entry.get("requestedBy") or "", "accepted": True}
                if receipt and receipt.get("firstLoad") and local["requestedBy"] == "first load":
                    local.update(detail=receipt["detail"], firstLoad=dict(receipt["firstLoad"]))
                _SYNC_REQUESTS[key] = local
                _SYNC_REQUEST_RUNTIMES.setdefault(rt.key, rt)
                admitted.append(deepcopy(entry))
                receipt = local
            waiting = _request_wait({**entry, "state": "queued", "startedAt": None,
                                     "finishedAt": None, "waitingFor": None,
                                     "notBefore": max(float(entry.get("notBefore") or 0),
                                                      _SYNC_NOT_BEFORE.get(key, 0),
                                                      float(receipt.get("notBefore") or 0)
                                                      if receipt and receipt["state"] == "queued" else 0) or None},
                                    now=stamp)
            fields = {name: waiting.get(name) for name in
                      ("state", "startedAt", "finishedAt", "notBefore", "detail", "waitingFor")}
            if any(entry.get(name) != value for name, value in fields.items()):
                changes[identity] = fields
        if changes or pruned:
            def publish(current):
                doc = _sync_request_document(current)
                for identity, progress in changes.items():
                    if identity in doc[REMOTE_MEMBER]:
                        doc[REMOTE_MEMBER][identity].update(progress)
                for identity in pruned:
                    doc[REMOTE_MEMBER].pop(identity, None)
                return doc
            rt.update(SYNC_REQUESTS_KEY, publish, flush="sync")
        if admitted:
            _wake_sync_requests(stamp)
    return admitted


def apply_remote_progress(rt):
    """Web side: copy the worker's published progress onto this process's receipts."""
    document = _sync_request_document(rt.get(SYNC_REQUESTS_KEY, fresh=True))
    applied = 0
    with _SYNC_REQUEST_LOCK:
        for identity, entry in document[REMOTE_MEMBER].items():
            key = (entry.get("provider"), rt.key)
            receipt = _SYNC_REQUESTS.get(key)
            if _REMOTE_IDS.get(key) != identity or receipt is None:
                continue
            fields = {name: entry.get(name) for name in
                      ("state", "startedAt", "finishedAt", "detail", "notBefore", "waitingFor")}
            if entry.get("firstLoad"):
                fields["firstLoad"] = dict(entry["firstLoad"])
            if any(receipt.get(k) != v for k, v in fields.items()):
                receipt.update(fields)
                applied += 1
        # Event receipts the worker claimed or finished, refreshed without requeueing anything.
        _cache_sync_requests(rt, document)
    return applied

# One receipt per immutable database identity. No credentials, session, or source values.
JOBS_KEY = "connector_materialization_jobs"
MAX_ATTEMPTS = 3
LEASE_SECONDS = 300
LIFECYCLE_STATES = ("configured", "catalogue-ready", "queued", "syncing", "ready", "failed")


def job_key(provider, connection_id, database_key):
    return json.dumps([_key(provider), _slot(connection_id), _key(database_key)],
                      separators=(",", ":"))


def _job_public(row):
    answer = {k: v for k, v in row.items() if k != "leaseToken"}
    # Older status readers only know error. Keep one persisted cause and project it there.
    if row.get("state") == "queued" and row.get("stall"):
        answer["error"] = row["stall"]["cause"]
    answer["retryable"] = (row.get("state") == "failed"
                           and int(row.get("attempts", 0)) < MAX_ATTEMPTS)
    answer["maxAttempts"] = MAX_ATTEMPTS
    if row.get("state") == "failed" and not answer["retryable"]:
        answer["error"] = "Sync attempts exhausted. Check the connection and update its settings."
    return answer


def jobs(rt, provider=None, connection_id=None):
    """Persisted status, including receipts left by a previous process."""
    document = rt.get(JOBS_KEY, fresh=True) or {}
    return [_job_public(row) for row in document.values()
            if isinstance(row, dict)
            and (provider is None or row.get("provider") == provider)
            and (connection_id is None or row.get("connectionId") == _slot(connection_id))]


def job_summary(receipts):
    """Aggregate only durable facts; a retry of ready work must never claim it is queued."""
    if not receipts:
        return None
    states = {row.get("state") for row in receipts}
    state = next((value for value in ("syncing", "queued", "failed", "ready", "catalogue-ready", "configured")
                  if value in states), "configured")
    errors = [row["error"] for row in receipts if row.get("error")]
    stamp = max(row.get("at", 0) for row in receipts)
    return {"state": state, "at": _dt.datetime.fromtimestamp(stamp, _dt.timezone.utc).isoformat(),
            "retryable": any(row.get("retryable") for row in receipts),
            "databases": [row["databaseKey"] for row in receipts if row.get("state") == "ready"],
            "detail": f"Database sync {state}", "errors": errors, "errorsTotal": len(errors)}


def configuration_revision(provider, connection_id, database_key, model, field_choices):
    """Fingerprint this database's field choices, never rows or sibling configuration."""
    identity = database_identity(provider, connection_id, database_key, model)
    selected = []
    for key, value in (field_choices or {}).items():
        scoped = parse_field_key(key)
        if scoped and scoped[:4] == (identity["provider"], identity["connectionId"],
                                     identity["databaseKey"], identity["model"]):
            selected.append((scoped[4], value is True))
    return hashlib.sha256(json.dumps(sorted(selected), separators=(",", ":")).encode()).hexdigest()


def enqueue(rt, provider, connection_id, database_key, *, model="", retry=False,
            changed=False, revision=None, now=None):
    """A Save is idempotent. Failed work requires an explicit, bounded retry."""
    identity = database_identity(provider, connection_id, database_key, model)
    key = job_key(provider, connection_id, database_key)
    stamp, receipt = time.time() if now is None else now, uuid.uuid4().hex

    def apply(current):
        doc = dict(current or {})
        previous = doc.get(key) or {}
        updated = changed or (revision is not None and previous.get("revision") != revision)
        if not updated and previous.get("state") in ("queued", "syncing", "ready"):
            return doc
        attempts = int(previous.get("attempts", 0))
        if not updated and previous.get("state") == "failed" and (not retry or attempts >= MAX_ATTEMPTS):
            return doc
        doc[key] = {**identity, "id": receipt, "state": "queued", "at": stamp,
                    "attempts": attempts if retry and not updated else 0, "leaseUntil": 0,
                    "revision": revision,
                    "leaseToken": "", "error": "", "workspaceChanged": False}
        return doc

    landed = rt.update(JOBS_KEY, apply, flush="sync")
    return _job_public(landed[key])


def stall(rt, candidate, cause, *, now=None):
    """Record WHY queued work cannot start, without consuming an attempt."""
    key = job_key(candidate["provider"], candidate["connectionId"], candidate["databaseKey"])
    stamp = time.time() if now is None else now
    sentence = str(cause or "")[:300]

    def eligible(row):
        return (row.get("id") == candidate.get("id") and
                (row.get("state") == "queued" or
                 row.get("state") == "syncing" and row.get("leaseUntil", 0) <= stamp))

    previous = (rt.get(JOBS_KEY, fresh=True) or {}).get(key) or {}
    # Avoid even submitting a store update on the unchanged 15-second drain pass.
    if not eligible(previous) or (previous.get("stall") or {}).get("cause") == sentence:
        return _job_public(previous)

    def apply(current):
        doc = dict(current or {})
        row = dict(doc.get(key) or {})
        if not eligible(row) or (row.get("stall") or {}).get("cause") == sentence:
            return doc
        row.update(state="queued", leaseUntil=0, leaseToken="",
                   stall={"at": stamp, "cause": sentence})
        doc[key] = row
        return doc

    landed = rt.update(JOBS_KEY, apply, flush="sync")
    return _job_public(landed.get(key) or {})


def claim(rt, candidate, *, now=None):
    """Atomically claim queued or abandoned work; replay callbacks have no side effects."""
    key = job_key(candidate["provider"], candidate["connectionId"], candidate["databaseKey"])
    stamp, token = time.time() if now is None else now, uuid.uuid4().hex

    def apply(current):
        doc = dict(current or {})
        row = dict(doc.get(key) or {})
        if row.get("id") != candidate.get("id"):
            return doc
        if row.get("state") != "queued" and not (
                row.get("state") == "syncing" and row.get("leaseUntil", 0) <= stamp):
            return doc
        row.pop("stall", None)
        if int(row.get("attempts", 0)) >= MAX_ATTEMPTS:
            row.update(state="failed", at=stamp, leaseToken="", leaseUntil=0,
                       error="Sync attempts exhausted. Check the connection and update its settings.")
        else:
            row.update(state="syncing", at=stamp, attempts=int(row.get("attempts", 0)) + 1,
                       leaseToken=token, leaseUntil=stamp + LEASE_SECONDS)
        doc[key] = row
        return doc

    landed = rt.update(JOBS_KEY, apply, flush="sync")
    row = landed.get(key) or {}
    return dict(row) if row.get("leaseToken") == token else None


def owns(rt, job, *, now=None):
    row = (rt.get(JOBS_KEY, fresh=True) or {}).get(job_key(
        job["provider"], job["connectionId"], job["databaseKey"])) or {}
    return (row.get("id") == job.get("id") and row.get("state") == "syncing"
            and row.get("leaseToken") == job.get("leaseToken")
            and row.get("leaseUntil", 0) > (time.time() if now is None else now))


def progress(rt, job, *, rows, total):
    """Record published rows only while this worker still owns its unexpired lease."""
    key = job_key(job["provider"], job["connectionId"], job["databaseKey"])
    stamp = time.time()

    def apply(current):
        doc = dict(current or {})
        row = dict(doc.get(key) or {})
        if (row.get("id") != job.get("id") or row.get("state") != "syncing"
                or row.get("leaseToken") != job.get("leaseToken")
                or row.get("leaseUntil", 0) <= stamp):
            return doc
        row["progress"] = {"rows": rows, "total": total, "at": stamp}
        doc[key] = row
        return doc

    landed = rt.update(JOBS_KEY, apply, flush="sync")
    return _job_public(landed.get(key) or {})


def finish(rt, job, *, success=False, renew=False, interrupted=False, cause="", now=None):
    """A stale worker cannot acknowledge, renew, or overwrite a newer receipt."""
    key = job_key(job["provider"], job["connectionId"], job["databaseKey"])
    stamp = time.time() if now is None else now

    def apply(current):
        doc = dict(current or {})
        row = dict(doc.get(key) or {})
        if (row.get("id") != job.get("id") or row.get("state") != "syncing"
                or row.get("leaseToken") != job.get("leaseToken")
                or row.get("leaseUntil", 0) <= stamp):
            return doc
        if renew:
            row["leaseUntil"] = stamp + LEASE_SECONDS
        elif interrupted:
            # Disconnect suspends the current attempt, preserving earlier failures and identity.
            row.update(state="queued", at=stamp, attempts=max(0, int(row.get("attempts", 0)) - 1),
                       leaseUntil=0, leaseToken="", workspaceChanged=False, error="")
        else:
            row.update(state="ready" if success else "failed", at=stamp,
                       leaseUntil=0, leaseToken="", workspaceChanged=bool(success),
                       error="" if success else (str(cause or "")[:300] or
                             "Rows could not be synced. Check the connection and retry."))
        doc[key] = row
        return doc

    landed = rt.update(JOBS_KEY, apply, flush="sync")
    return _job_public(landed.get(key) or {})


def cancel(rt, provider, connection_id, database_key):
    key = job_key(provider, connection_id, database_key)
    stamp = time.time()

    def apply(current):
        doc = dict(current or {})
        if key in doc:
            doc[key] = {**doc[key], "state": "configured", "at": stamp,
                        "leaseToken": "", "leaseUntil": 0, "workspaceChanged": False,
                        "error": ""}
            doc[key].pop("stall", None)
            doc[key].pop("progress", None)
        return doc

    rt.update(JOBS_KEY, apply, flush="sync")


def _st(st):
    return st if st is not None else store


def _key(value):
    return str(value or "").strip()


def _slot(connection_id):
    return _key(connection_id) or _DEFAULT_CONNECTION


def _target(rt, st):
    return st if st is not None else rt


def database_identity(provider, connection_id, database_key, model=""):
    """Source-neutral identity; labels and sync state never participate in a database key."""
    if not _key(provider) or not _key(database_key):
        raise ValueError("provider and database key are required")
    return {"provider": _key(provider), "connectionId": _slot(connection_id),
            "databaseKey": _key(database_key), "model": _key(model)}


def field_key(provider, connection_id, database_key, model, raw_name):
    """An opaque, collision-free key for one source field in one database."""
    identity = database_identity(provider, connection_id, database_key, model)
    if not identity["model"] or not _key(raw_name):
        raise ValueError("model and raw field name are required")
    return json.dumps([identity["provider"], identity["connectionId"],
                       identity["databaseKey"], identity["model"], _key(raw_name)],
                      separators=(",", ":"), ensure_ascii=True)


def parse_field_key(value):
    """Read a scoped field key, or return None for a legacy model.field key."""
    try:
        parts = json.loads(value)
    except (TypeError, ValueError):
        return None
    if not isinstance(parts, list) or len(parts) != 5 or not all(
            isinstance(part, str) and part for part in parts):
        return None
    return (*parts,)


def _public(provider, connection_id, row):
    if not isinstance(row, dict):
        return None
    answer = dict(row)
    answer["provider"] = provider
    answer["connectionId"] = connection_id
    # RULE: retryability follows the state. Storing it would let the two answers drift.
    answer["retryable"] = answer.get("state") in ("error", "partial")
    return answer


def record(rt, provider, connection_id, state, *, detail="", errors=(), databases=(), st=None):
    """Record one lifecycle transition and return its defensive public representation."""
    provider_key = _key(provider)
    if not provider_key:
        raise ValueError("provider is required")
    if state not in STATES:
        # RULE: unknown states are refused. A free-form status eventually stores a typo that the
        # UI cannot render as one of C6's six observable states.
        raise ValueError(f"unknown connector sync state: {state}")
    connection_key = _slot(connection_id)
    all_errors = [str(value) for value in (errors or ())]
    visible_errors = all_errors[:_ERROR_LIMIT]
    now = _dt.datetime.now(_dt.timezone.utc).isoformat()
    written = {}

    def _apply(current):
        document = dict(current or {})
        providers_doc = dict(document.get(provider_key) or {})
        previous = providers_doc.get(connection_key)
        previous_state = previous.get("state") if isinstance(previous, dict) else None
        row = {
            "state": state,
            # RULE: stamp every transition and name its predecessor. A resting state alone cannot
            # tell the UI whether work advanced, retried, or regressed.
            "at": now,
            "from": previous_state,
            "detail": str(detail or ""),
            # RULE: partial and error stay distinct. Partial exposes databases usable now;
            # error means no usable set was produced.
            "errors": visible_errors,
            # RULE: retain up to 50 errors and report the full count. A storage limit must be
            # observable rather than silently turning additional failures into zero information.
            "errorsTotal": len(all_errors),
            # These are governed table keys, not copied provider rows. In Odoo, row_limit == 0
            # keeps 963,783 GL lines out of the editable user_tables document.
            "databases": [str(value) for value in (databases or ()) if str(value)],
        }
        providers_doc[connection_key] = row
        # RULE: key state by provider and connection. The measured legacy defect is that
        # odoo_sync_status is one tenant document, so a second Odoo connection overwrites the
        # first one's status. The default slot matches Odoo's existing catalog convention.
        document[provider_key] = providers_doc
        written.update(row)
        return document

    _st(_target(rt, st)).update(STATE_KEY, _apply, flush="sync")
    return _public(provider_key, connection_key, written)


def read(rt, provider, connection_id=None, st=None):
    """Read one connection defensively; store failures mean no recorded state, never a 500."""
    try:
        provider_key = _key(provider)
        connection_key = _slot(connection_id)
        document = _st(_target(rt, st)).get(STATE_KEY) or {}
        providers_doc = document.get(provider_key) if isinstance(document, dict) else None
        row = providers_doc.get(connection_key) if isinstance(providers_doc, dict) else None
        return _public(provider_key, connection_key, row)
    except Exception:  # noqa: BLE001 - status reads cannot take down their caller
        return None


def read_all(rt, st=None):
    """Read every provider and connection defensively."""
    try:
        document = _st(_target(rt, st)).get(STATE_KEY) or {}
        answer = {}
        for provider_key, connections in document.items():
            if not isinstance(connections, dict):
                continue
            public_connections = {}
            for connection_key, row in connections.items():
                public = _public(str(provider_key), str(connection_key), row)
                if public is not None:
                    public_connections[str(connection_key)] = public
            if public_connections:
                answer[str(provider_key)] = public_connections
        return answer
    except Exception:  # noqa: BLE001 - status reads cannot take down their caller
        return {}


def clear(rt, provider, connection_id, st=None):
    """Remove only the named provider/connection state."""
    provider_key = _key(provider)
    connection_key = _slot(connection_id)

    def _drop(current):
        document = dict(current or {})
        connections = dict(document.get(provider_key) or {})
        connections.pop(connection_key, None)
        if connections:
            document[provider_key] = connections
        else:
            document.pop(provider_key, None)
        return document

    _st(_target(rt, st)).update(STATE_KEY, _drop, flush="sync")


# C-status: one store document per tenant that every card reads instead of process-local state.
# Schema, the one place the key names live:
#   STATUS_KEY -> {provider: {connectionId: {
#       "lastGoodSyncAt": ISO-8601 UTC,  advanced only, by a publication or a mirror reading after one
#       "pass": {"startedAt": ISO, "endedAt": ISO | None, "outcome": one of PASS_OUTCOMES,
#                "detail": str, "durationSeconds": float | None, ...extra writer fields},
#       "resync": {...the resync loop's state, JSON-safe..., "at": ISO},
#       "<section>": {...}   any other named section, merged by update_status,
#       "updatedAt": ISO}}}
#   Reserved provider "sync_worker" (connection "default"), written by the sync worker, never a connector:
#       "heartbeat": {pid, host, tenant, scope, at (epoch s), bootRestore}  in the worker tenant's document
#       "catchUp": {connector: catch-up progress}, "mirrors": {connector: {snapshotId, at}}  per tenant
#       "job": {id, submittedAt (ISO), image, tenant}  the web's last HF sync job, per tenant
#       "schedule": {nextPassAt (ISO), at (ISO), reason, jobId}  written by a --once job at its exit
#       "heartbeat.exitedAt" (epoch s)  a --once job's last beat; that beat reads as no live worker
# RULE: every writer merges one provider and connection entry. It never clobbers another
# connector's entry, and a store that refuses is logged and answered False, never raised into a pass.
STATUS_KEY = "connector_status_v1"
PASS_OUTCOMES = ("running", "done", "continuing", "waiting", "held", "failed")
_STATUS_RESERVED = ("lastGoodSyncAt", "updatedAt")


def _utc_stamp(value=None):
    """A UTC ISO stamp from a datetime, epoch seconds or ISO text, or None when unreadable."""
    if value is None:
        return _dt.datetime.now(_dt.timezone.utc).isoformat()
    try:
        if isinstance(value, _dt.datetime):
            parsed = value
        elif isinstance(value, (int, float)) and not isinstance(value, bool):
            parsed = _dt.datetime.fromtimestamp(float(value), _dt.timezone.utc)
        else:
            parsed = _dt.datetime.fromisoformat(str(value).strip().replace("Z", "+00:00"))
        if parsed.tzinfo is None:
            parsed = parsed.replace(tzinfo=_dt.timezone.utc)
        return parsed.astimezone(_dt.timezone.utc).isoformat()
    except (TypeError, ValueError, OverflowError, OSError):
        return None


def _status_update(rt, provider, connection_id, change):
    """Apply `change(entry)` to one provider and connection entry; False when the store refuses."""
    provider_key = _key(provider)
    if not provider_key:
        raise ValueError("provider is required")
    connection_key = _slot(connection_id)

    def _apply(current):
        document = dict(current) if isinstance(current, dict) else {}
        connections = document.get(provider_key)
        connections = dict(connections) if isinstance(connections, dict) else {}
        entry = connections.get(connection_key)
        entry = deepcopy(entry) if isinstance(entry, dict) else {}
        change(entry)
        entry["updatedAt"] = _utc_stamp()
        connections[connection_key] = entry
        document[provider_key] = connections
        return document

    try:
        rt.update(STATUS_KEY, _apply, flush="sync")
        return True
    except Exception as exc:  # noqa: BLE001 - a status write never fails the pass that made it
        logging.getLogger(__name__).warning("Connector status write refused for %s/%s (%s)",
                                            provider_key, connection_key, type(exc).__name__)
        return False


def _advance_good_sync(entry, at):
    stamp = _utc_stamp(at) if at is not None else None
    if stamp is None:
        return
    held = _utc_stamp(entry.get("lastGoodSyncAt")) if entry.get("lastGoodSyncAt") else None
    if held is None or _dt.datetime.fromisoformat(stamp) > _dt.datetime.fromisoformat(held):
        entry["lastGoodSyncAt"] = stamp


def _status_document(rt, fresh):
    # `fresh` reads past this process's cache: the sync worker writes from another process.
    return (rt.get(STATUS_KEY, fresh=True) if fresh else rt.get(STATUS_KEY)) or {}


def status(rt, provider, connection_id=None, *, fresh=False):
    """One connector's C-status entry as a copy, or None. A store read failure answers None."""
    try:
        document = _status_document(rt, fresh)
        connections = document.get(_key(provider)) if isinstance(document, dict) else None
        entry = connections.get(_slot(connection_id)) if isinstance(connections, dict) else None
        return deepcopy(entry) if isinstance(entry, dict) else None
    except Exception:  # noqa: BLE001 - status reads cannot take down their caller
        return None


def status_all(rt, *, fresh=False):
    """Every provider's C-status entries as a copy; a store read failure answers {}."""
    try:
        document = _status_document(rt, fresh)
        return {str(provider): deepcopy(connections) for provider, connections in document.items()
                if isinstance(connections, dict)} if isinstance(document, dict) else {}
    except Exception:  # noqa: BLE001 - status reads cannot take down their caller
        return {}


def last_good_sync(rt, provider, connection_id=None):
    """The stored lastGoodSyncAt as a UTC ISO stamp, or None."""
    entry = status(rt, provider, connection_id)
    return _utc_stamp(entry.get("lastGoodSyncAt")) if entry and entry.get("lastGoodSyncAt") else None


def record_good_sync(rt, provider, at=None, *, connection_id=None):
    """Advance lastGoodSyncAt to `at` (default now). An older stamp never moves it back."""
    return _status_update(rt, provider, connection_id, lambda entry: _advance_good_sync(entry, at))


def record_pass_start(rt, provider, *, at=None, connection_id=None):
    """Stamp a pass as running. The previous pass's end is dropped with it."""
    started = _utc_stamp(at)

    def change(entry):
        previous = entry.get("pass") if isinstance(entry.get("pass"), dict) else {}
        publication = {name: deepcopy(previous[name]) for name in
                       ("tableSync", "held", "notOffered", "notOfferedTables") if name in previous}
        entry["pass"] = {**publication, "startedAt": started, "endedAt": None, "outcome": "running",
                         "detail": "", "durationSeconds": None, "budgetSeconds": 1800}
    return _status_update(rt, provider, connection_id, change)


def record_pass_end(rt, provider, outcome, *, at=None, detail="", last_good_sync_at=None,
                    connection_id=None, extra=None):
    """Close the last pass with its outcome; `last_good_sync_at` advances lastGoodSyncAt."""
    if outcome not in PASS_OUTCOMES or outcome == "running":
        raise ValueError(f"unknown pass outcome: {outcome}")
    ended = _utc_stamp(at)

    def change(entry):
        previous = entry.get("pass") if isinstance(entry.get("pass"), dict) else {}
        started = previous.get("startedAt")
        finished = previous.get("publishedAt") or ended
        duration = None
        if started and finished:
            duration = round((_dt.datetime.fromisoformat(finished)
                              - _dt.datetime.fromisoformat(started)).total_seconds(), 3)
        publication = {name: deepcopy(previous[name]) for name in
                       ("tableSync", "held", "notOffered", "notOfferedTables", "publishedAt") if name in previous}
        row = {**publication, "startedAt": started, "endedAt": finished, "outcome": outcome,
               "detail": str(detail or "")[:300], "durationSeconds": duration,
               "budgetSeconds": 1800}
        fixed = {"startedAt", "endedAt", "outcome", "detail", "durationSeconds", "budgetSeconds"}
        row.update({str(k): v for k, v in dict(extra or {}).items() if str(k) not in fixed})
        entry["pass"] = row
        slot_date = str(row.get("slotDate") or started or ended or "")[:10]
        if slot_date:
            days = dict(entry.get("dailyPasses") or {})
            days[slot_date] = {"date": slot_date,
                               "minutes": round(duration / 60, 3) if duration is not None else None,
                               "budgetMinutes": 30, "outcome": outcome}
            entry["dailyPasses"] = days
        if outcome != "failed":
            # Any pass that did not fail proves the credential works again.
            entry.pop("credentialHold", None)
        _advance_good_sync(entry, last_good_sync_at)
    return _status_update(rt, provider, connection_id, change)


def credential_fingerprint(material):
    """The first 16 hex of sha256 over the credential material, or None when there is none.

    A list or tuple is joined line by line. The secret itself is never stored or returned.
    """
    if isinstance(material, (list, tuple)):
        material = "\n".join(str(part) for part in material)
    if not isinstance(material, str) or not material:
        return None
    return hashlib.sha256(material.encode("utf-8")).hexdigest()[:16]


def hold_credential(rt, provider, fingerprint, *, connection_id=None):
    """Record that the provider refused the credential with this fingerprint."""
    if not fingerprint:
        return False
    at = _utc_stamp()

    def change(entry):
        entry["credentialHold"] = {"fingerprint": str(fingerprint), "at": at}
    return _status_update(rt, provider, connection_id, change)


def credential_held(rt, provider, fingerprint, *, connection_id=None):
    """True when a scheduled pass must skip: the last pass failed on this same refused credential.

    An unknown fingerprint or an unreadable record answers False, so the pass runs.
    """
    if not fingerprint:
        return False
    entry = status(rt, provider, connection_id, fresh=True) or {}
    hold = entry.get("credentialHold")
    last_pass = entry.get("pass") if isinstance(entry.get("pass"), dict) else {}
    return (isinstance(hold, dict) and hold.get("fingerprint") == fingerprint
            and last_pass.get("outcome") == "failed")


def record_resync_state(rt, provider, state, *, connection_id=None):
    """Replace the connector's resync state with a JSON-safe copy of `state`."""
    try:
        snapshot = json.loads(json.dumps(dict(state or {}), default=str))
    except (TypeError, ValueError):
        snapshot = {}
    snapshot["at"] = _utc_stamp()

    def change(entry):
        entry["resync"] = snapshot
    return _status_update(rt, provider, connection_id, change)


def update_status(rt, provider, section, fields, *, connection_id=None):
    """Merge `fields` into one named section of a connector's entry (first loads, heartbeats)."""
    name = _key(section)
    if not name or name in _STATUS_RESERVED:
        raise ValueError(f"section {section!r} is not writable through update_status")
    try:
        values = json.loads(json.dumps(dict(fields or {}), default=str))
    except (TypeError, ValueError):
        values = {}

    def change(entry):
        current = entry.get(name) if isinstance(entry.get(name), dict) else {}
        entry[name] = {**current, **values}
    return _status_update(rt, provider, connection_id, change)


def register_provider(provider, *, label, retry_hint):
    """Register provider metadata; runners remain in the API layer."""
    provider_key = _key(provider)
    if not provider_key:
        raise ValueError("provider is required")
    _PROVIDERS[provider_key] = {
        "key": provider_key,
        "label": str(label or provider_key),
        "retryHint": str(retry_hint or ""),
    }


def providers():
    """Return a copy of registered provider metadata."""
    return {key: dict(value) for key, value in _PROVIDERS.items()}


register_provider("odoo", label="Odoo", retry_hint="Retry the Odoo database sync.")
register_provider(
    "amazon_sp",
    label="Amazon",
    retry_hint="Use Sync now to retry the Amazon pull.",
)
register_provider("meta_ads", label="Meta Ads",
                  retry_hint="Use Sync now to retry the Meta Ads pull.")
register_provider("amazon_ads", label="Amazon Ads",
                  retry_hint="Use Sync now to retry the Amazon Ads pull.")