File size: 103,751 Bytes
dc81d25
8fb0b86
65d3c91
8fb0b86
dc81d25
65d3c91
8fb0b86
 
 
 
 
 
 
 
 
 
 
 
 
dc81d25
8fb0b86
 
 
 
 
 
 
 
 
 
 
 
 
 
dc81d25
 
 
8fb0b86
dc81d25
8fb0b86
 
 
 
 
 
 
 
 
dc81d25
8fb0b86
 
 
0501224
dc81d25
 
 
 
0501224
 
 
 
8fb0b86
0501224
 
 
8fb0b86
 
0501224
8fb0b86
 
 
dc81d25
8fb0b86
0501224
 
 
 
8fb0b86
 
 
 
dc81d25
8fb0b86
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0501224
8fb0b86
 
 
 
dc81d25
8fb0b86
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0501224
dc81d25
8fb0b86
 
a0de066
 
8fb0b86
 
 
 
 
 
 
 
0501224
 
 
 
 
 
 
 
 
 
8fb0b86
 
 
0501224
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8fb0b86
dc81d25
 
 
 
 
8fb0b86
 
c376385
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8fb0b86
c376385
dc81d25
c376385
 
 
 
 
 
 
 
8fb0b86
 
 
 
 
 
 
 
 
 
 
c376385
8fb0b86
 
 
 
 
c376385
 
8fb0b86
 
 
 
 
 
 
c376385
8fb0b86
 
 
 
 
 
 
 
dc81d25
8fb0b86
 
 
 
 
 
 
 
c376385
 
 
 
 
 
8fb0b86
0501224
 
8fb0b86
 
c376385
65d3c91
 
dc81d25
8fb0b86
 
6072e5e
c376385
 
 
 
dc81d25
6072e5e
c376385
0501224
c376385
e7bfe80
c376385
 
dc81d25
 
 
 
 
 
 
8fb0b86
 
c376385
 
 
8fb0b86
 
 
 
c376385
 
 
 
 
 
 
 
 
 
a0de066
 
8fb0b86
 
 
 
 
 
 
 
 
 
a0de066
 
8fb0b86
 
c376385
 
 
 
dc81d25
c376385
 
 
 
 
 
 
 
 
8fb0b86
 
c376385
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
65d3c91
c376385
 
 
 
65d3c91
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c376385
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8fb0b86
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
dc81d25
c376385
8fb0b86
 
 
 
 
 
 
c376385
 
 
 
8fb0b86
c376385
8fb0b86
 
 
 
c376385
8fb0b86
c376385
 
 
 
8fb0b86
c376385
8fb0b86
 
c376385
 
 
8fb0b86
c376385
 
dc81d25
8fb0b86
c376385
 
 
 
8fb0b86
 
c376385
 
8fb0b86
 
 
c376385
 
8fb0b86
c376385
 
8fb0b86
 
 
c376385
8fb0b86
 
c376385
 
 
 
 
 
 
dc81d25
c376385
dc81d25
 
c376385
 
 
 
dc81d25
 
 
 
 
 
 
8fb0b86
dc81d25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c376385
 
 
 
 
 
 
 
dc81d25
c376385
 
dc81d25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c376385
dc81d25
 
 
 
 
 
 
 
 
 
 
c376385
dc81d25
 
 
8fb0b86
dc81d25
 
8fb0b86
dc81d25
 
 
 
 
 
 
 
 
 
 
c376385
dc81d25
 
c376385
dc81d25
 
 
c376385
 
dc81d25
 
 
c376385
8fb0b86
 
 
 
 
c376385
8fb0b86
 
c376385
8fb0b86
 
 
 
 
 
dc81d25
 
 
 
 
 
 
 
8fb0b86
 
 
 
 
 
 
 
 
dc81d25
8fb0b86
 
 
6072e5e
c376385
 
6072e5e
8fb0b86
dc81d25
 
 
c376385
dc81d25
 
 
 
c376385
 
 
dc81d25
 
 
 
 
 
 
 
 
 
0501224
 
 
 
 
 
 
 
 
 
 
dc81d25
0501224
 
 
 
6072e5e
dc81d25
c376385
 
 
6072e5e
 
dc81d25
c376385
6072e5e
c376385
 
 
6072e5e
c376385
 
8fb0b86
c376385
dc81d25
6072e5e
c376385
6072e5e
c376385
 
 
 
 
 
 
 
8fb0b86
c376385
 
0501224
 
c376385
 
 
 
8fb0b86
 
 
 
 
6072e5e
c376385
 
 
6072e5e
 
65d3c91
c376385
 
6072e5e
c376385
 
 
6072e5e
c376385
 
 
 
 
 
 
 
 
6072e5e
8fb0b86
dc81d25
 
 
 
c376385
8fb0b86
 
c376385
8fb0b86
28b20c0
 
c376385
 
 
 
 
 
28b20c0
 
 
 
a0de066
28b20c0
 
8fb0b86
 
 
 
 
 
 
 
c376385
8fb0b86
 
6072e5e
8fb0b86
 
 
 
dc81d25
 
 
8fb0b86
 
 
 
c376385
8fb0b86
c376385
8fb0b86
c376385
 
8fb0b86
 
 
 
 
 
 
 
 
 
 
 
 
dc81d25
 
 
 
 
 
8fb0b86
 
 
 
 
 
dc81d25
8fb0b86
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
dc81d25
8fb0b86
 
 
 
 
 
 
 
 
 
 
 
 
 
 
dc81d25
 
8fb0b86
 
 
 
 
 
 
 
 
 
 
 
dc81d25
8fb0b86
 
 
 
 
6072e5e
dc81d25
 
6072e5e
 
 
dc81d25
 
 
 
 
a0de066
dc81d25
6072e5e
dc81d25
a0de066
 
6072e5e
 
 
a0de066
 
 
 
6072e5e
 
 
dc81d25
6072e5e
a0de066
6072e5e
 
c376385
8fb0b86
 
 
c376385
8fb0b86
c376385
8fb0b86
 
 
dc81d25
c376385
dc81d25
 
 
c376385
dc81d25
c376385
e7bfe80
 
dc81d25
e7bfe80
 
dc81d25
 
 
e7bfe80
dc81d25
 
 
 
e7bfe80
 
dc81d25
c376385
 
 
 
 
 
 
8fb0b86
dc81d25
 
8fb0b86
 
 
 
dc81d25
c376385
dc81d25
 
 
 
 
 
 
8fb0b86
dc81d25
 
 
 
 
 
 
 
 
 
 
6072e5e
8fb0b86
 
 
 
c5cc338
c376385
 
c5cc338
 
a0de066
 
8fb0b86
6072e5e
a0de066
 
c376385
 
8fb0b86
dc81d25
8fb0b86
 
c376385
 
 
e7bfe80
dc81d25
 
6072e5e
c376385
 
 
 
6072e5e
8fb0b86
 
dc81d25
c376385
8fb0b86
 
 
 
 
 
 
c376385
 
6072e5e
dc81d25
a0de066
 
 
 
 
 
c376385
 
a0de066
 
 
 
 
 
 
 
 
dc81d25
0501224
dc81d25
8fb0b86
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a0de066
dc81d25
8fb0b86
a0de066
 
 
 
 
 
 
 
8fb0b86
 
 
 
 
65d3c91
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
dc81d25
 
 
 
 
 
 
 
 
 
 
 
 
 
c5cc338
c376385
 
 
 
65d3c91
dc81d25
a0de066
 
8fb0b86
dc81d25
 
 
 
 
 
 
 
 
 
 
 
 
 
c5cc338
 
 
dc81d25
 
 
c376385
 
 
 
 
dc81d25
 
c376385
 
 
65d3c91
 
 
c376385
 
8fb0b86
 
dc81d25
8fb0b86
 
 
dc81d25
8fb0b86
 
c376385
 
 
dc81d25
 
 
8fb0b86
 
 
 
 
 
 
 
 
 
c376385
8fb0b86
 
c376385
65d3c91
8fb0b86
 
 
 
 
 
 
 
 
 
 
 
 
c376385
 
 
 
8fb0b86
 
 
dc81d25
 
 
 
 
 
 
c5cc338
c376385
 
 
dc81d25
 
 
 
 
 
 
c5cc338
dc81d25
 
 
 
 
 
 
 
 
 
e7bfe80
 
 
dc81d25
 
 
 
 
 
 
 
8fb0b86
 
 
 
dc81d25
8fb0b86
dc81d25
8fb0b86
 
 
 
 
 
 
dc81d25
8fb0b86
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
dc81d25
8fb0b86
 
 
 
 
 
 
 
 
 
 
 
65d3c91
8fb0b86
 
c5cc338
dc81d25
 
 
65d3c91
dc81d25
 
 
 
8fb0b86
 
 
dc81d25
 
8fb0b86
 
 
65d3c91
dc81d25
 
 
 
65d3c91
 
 
 
dc81d25
 
 
65d3c91
8fb0b86
 
 
 
 
 
 
65d3c91
 
 
 
 
 
 
 
 
 
8fb0b86
 
 
 
 
 
a0de066
 
 
8fb0b86
a0de066
 
8fb0b86
 
 
 
 
 
 
 
65d3c91
8fb0b86
 
 
 
 
 
 
65d3c91
dc81d25
 
 
 
 
 
 
 
65d3c91
8fb0b86
 
 
 
 
 
 
 
 
dc81d25
8fb0b86
65d3c91
8fb0b86
dc81d25
65d3c91
8fb0b86
 
 
 
 
 
 
 
 
 
 
 
 
 
65d3c91
c376385
8fb0b86
 
 
dc81d25
8fb0b86
 
dc81d25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c376385
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
dc81d25
 
 
 
 
 
8fb0b86
dc81d25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c376385
dc81d25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8fb0b86
dc81d25
 
c5cc338
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
dc81d25
 
 
8fb0b86
dc81d25
 
 
 
 
 
 
 
 
 
8fb0b86
dc81d25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8fb0b86
 
dc81d25
 
8fb0b86
 
 
dc81d25
8fb0b86
 
 
 
 
 
dc81d25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8fb0b86
dc81d25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8fb0b86
65d3c91
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c376385
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
dc81d25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8fb0b86
 
dc81d25
 
 
8fb0b86
dc81d25
8fb0b86
 
dc81d25
8fb0b86
 
 
 
 
dc81d25
8fb0b86
 
 
 
 
dc81d25
8fb0b86
 
dc81d25
c376385
8fb0b86
 
 
dc81d25
8fb0b86
dc81d25
8fb0b86
 
 
dc81d25
8fb0b86
 
dc81d25
8fb0b86
dc81d25
8fb0b86
dc81d25
8fb0b86
 
dc81d25
8fb0b86
 
 
 
c376385
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
diff --git a/src/transformers/integrations/experts_offload.py b/src/transformers/integrations/experts_offload.py
new file mode 100644
index 0000000000..7a7686bdc0
--- /dev/null
+++ b/src/transformers/integrations/experts_offload.py
@@ -0,0 +1,1341 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Stream MoE experts from disk through a shared device cache. Inference only, on one device."""
+
+import heapq
+import json
+import math
+import os
+import re
+import threading
+from collections import OrderedDict
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+import torch
+from safetensors import safe_open
+
+from ..utils import logging
+from .moe import ExpertsInterface
+
+
+try:
+    import fcntl
+except ImportError:  # Windows
+    fcntl = None
+
+logger = logging.get_logger(__name__)
+
+_DTYPES = {"F32": torch.float32, "F16": torch.float16, "BF16": torch.bfloat16}
+_PACKED_DTYPES = {**_DTYPES, "U32": torch.uint32, "U8": torch.uint8}
+_QUANTIZED_FORMAT = "mlx-affine-expert-blob"
+_MLX_QUANTIZATION_KERNEL = ("kernels-community/mlx-quantization-metal-kernels", 2)
+_COUNTER_LOCK = threading.Lock()
+
+
+def _read_exact(path, offset, size, bypass_page_cache, what, out=None, fd=None):
+    """Read a byte range into ``out`` (or a new buffer), optionally bypassing or dropping the OS page cache.
+
+    ``fd`` is an open descriptor of ``path``; positional reads are thread-safe."""
+    # TODO(portability): implement Windows cache bypass; report unsupported OS/filesystem combinations.
+    data = bytearray(size) if out is None else out
+    own = fd is None
+    if own:
+        fd = os.open(path, os.O_RDONLY)
+        if bypass_page_cache and hasattr(fcntl, "F_NOCACHE"):
+            fcntl.fcntl(fd, fcntl.F_NOCACHE, 1)
+    try:
+        view = memoryview(data).cast("B")
+        done = 0
+        while done < size:
+            count = os.preadv(fd, [view[done:size]], offset + done)
+            if not count:
+                raise OSError(f"Short checkpoint read: {path}, {what}")
+            done += count
+        # TODO(portability): distinguish best-effort cache eviction from uncached reads.
+        if bypass_page_cache and hasattr(os, "posix_fadvise"):
+            os.posix_fadvise(fd, offset, size, os.POSIX_FADV_DONTNEED)
+    finally:
+        if own:
+            os.close(fd)
+    return data
+
+
+class SafetensorsReader:
+    """Validate safetensors headers and read tensors or contiguous expert slices without persistent mmap."""
+
+    def __init__(self, directory, bypass_page_cache=False, packed=False):
+        directory = Path(directory)
+        self.bypass_page_cache = bypass_page_cache
+        self.dtypes = _PACKED_DTYPES if packed else _DTYPES
+        index = directory / "model.safetensors.index.json"
+        if index.exists():
+            weight_map = json.loads(index.read_text())["weight_map"]
+            filenames = sorted(set(weight_map.values()))
+        else:
+            weight_map = None
+            filenames = ["model.safetensors"]
+        self.tensors = {}
+        self.read_bytes = 0
+        self.read_calls = 0
+        self._fds = {}
+        for filename in filenames:
+            path = (directory / filename).resolve()
+            if not path.is_file():
+                raise FileNotFoundError(f"Missing checkpoint shard: {path}")
+            # Validate offsets, sizes and overlaps before reading the header.
+            with safe_open(path, framework="pt", device="cpu") as handle:
+                keys = set(handle.keys())
+            with path.open("rb") as handle:
+                header_size = int.from_bytes(handle.read(8), "little")
+                header = json.loads(handle.read(header_size))
+            for name in keys:
+                info = header[name]
+                if name in self.tensors:
+                    raise ValueError(f"Duplicate checkpoint tensor: {name}")
+                self.tensors[name] = (path, 8 + header_size + info["data_offsets"][0], info)
+                if weight_map is not None and weight_map.get(name) != filename:
+                    raise ValueError(f"Checkpoint index disagrees with shard for {name}")
+        if weight_map is not None and set(weight_map) != set(self.tensors):
+            raise ValueError("Checkpoint index contains missing tensors")
+
+    def shape(self, name):
+        return tuple(self.tensors[name][2]["shape"])
+
+    def read(self, name, expert=None, out=None):
+        """Read a tensor, or one expert slice of it, into ``out`` (a CPU uint8 tensor of its size) when given."""
+        path, offset, info = self.tensors[name]
+        shape = tuple(info["shape"])
+        if info["dtype"] not in self.dtypes:
+            raise ValueError(f"Only F32/F16/BF16 (and packed) tensors can be read: {name} is {info['dtype']}")
+        dtype = self.dtypes[info["dtype"]]
+        item_size = torch.empty((), dtype=dtype).element_size()
+        if expert is not None:
+            if len(shape) < 2 or not 0 <= expert < shape[0]:
+                raise ValueError(f"Invalid expert slice {expert} for {name}: {shape}")
+            shape = shape[1:]
+            offset += expert * math.prod(shape) * item_size
+        size = math.prod(shape) * item_size
+        if out is not None:
+            if out.dtype != torch.uint8 or out.numel() != size or out.device.type != "cpu":
+                raise ValueError(f"`out` must be a CPU uint8 tensor of {size} bytes")
+            _read_exact(path, offset, size, self.bypass_page_cache, f"tensor {name}", out.numpy(), self._fd(path))
+            data = out
+        else:
+            data = torch.frombuffer(
+                _read_exact(path, offset, size, self.bypass_page_cache, f"tensor {name}", fd=self._fd(path)),
+                dtype=torch.uint8,
+            )
+        with _COUNTER_LOCK:
+            self.read_bytes += size
+            self.read_calls += 1
+        return data.view(dtype).reshape(shape)
+
+    def _fd(self, path):
+        """One read-only descriptor per shard, opened on first use and shared by the I/O threads."""
+        fd = self._fds.get(path)
+        if fd is None:
+            with _COUNTER_LOCK:
+                fd = self._fds.get(path)
+                if fd is None:
+                    fd = os.open(path, os.O_RDONLY)
+                    if self.bypass_page_cache and hasattr(fcntl, "F_NOCACHE"):
+                        fcntl.fcntl(fd, fcntl.F_NOCACHE, 1)
+                    self._fds[path] = fd
+        return fd
+
+    def __del__(self):
+        for fd in getattr(self, "_fds", {}).values():
+            try:
+                os.close(fd)
+            except OSError:
+                pass
+
+    def read_expert(self, prefix, expert):
+        if f"{prefix}.gate_up_proj" in self.tensors:
+            return self.read(f"{prefix}.gate_up_proj", expert), self.read(f"{prefix}.down_proj", expert)
+        gate_up = torch.cat([self.read(f"{prefix}.{expert}.{p}_proj.weight") for p in ("gate", "up")])
+        return gate_up, self.read(f"{prefix}.{expert}.down_proj.weight")
+
+
+def _layout_bytes(layout):
+    return sum(math.prod(shape) * dtype.itemsize for shape, dtype in layout)
+
+
+class _Pool:
+    """Device slots shared by the MoE layers whose experts have one layout; LRU (or LFU) within the pool."""
+
+    def __init__(self, layout, num_slots, packed):
+        self.layout = layout  # [(shape, dtype)] of one expert's tensors, in storage order
+        self.slot_bytes = _layout_bytes(layout)
+        self.num_slots = num_slots
+        self.packed = packed
+        self.slot_buffer = self.banks = None  # see `materialize`
+        self.free_slots = list(range(num_slots - 1, -1, -1))
+        self.cache = OrderedDict()  # key -> slot, least recently used first
+        self.heap = []  # LFU: (count, sequence, key); stale entries are discarded lazily
+        self.num_experts = 0  # over the pool's layers
+
+    def materialize(self, device):
+        if self.banks is not None:
+            return
+        if self.packed:
+            # One row of bytes per slot, viewed as the expert's tensors: one copy per install.
+            self.slot_buffer = torch.empty((self.num_slots, self.slot_bytes), device=device, dtype=torch.uint8)
+            self.banks = list(_split_expert(self.slot_buffer, self.layout))
+        else:
+            self.banks = [
+                torch.empty((self.num_slots, *shape), device=device, dtype=dtype) for shape, dtype in self.layout
+            ]
+
+
+class ExpertStore:
+    """Device pools of expert slots with LRU or decaying LFU admission, for one inference request at a time.
+
+    Layers whose experts share a layout share a pool; with several layouts (mixed quantization, an external expert
+    source), the budget gives every layer the same number of slots. Staging and uncached experts can exceed the budget.
+
+    Experts come from the checkpoint (floating point), from `quantized_experts` (MLX affine bytes), or from an
+    ``expert_source``: an object with ``layout(prefix)`` (``[(shape, dtype)]`` of one expert's tensors, stored
+    contiguously in that order), ``read(prefix, expert, out)`` (fill ``out``, a CPU uint8 tensor of the expert's size)
+    and ``read_bytes`` / ``read_calls`` counters; optionally ``prefix(module_name)`` (its key for an experts module)
+    and ``pool_key(prefix)`` (layers with equal keys share a pool; defaults to the layout)."""
+
+    def __init__(
+        self,
+        reader,
+        cache_bytes,
+        device,
+        dtype,
+        policy="lru",
+        decay_every=4096,
+        io_threads=0,
+        quantized=None,
+        source=None,
+    ):
+        if cache_bytes < 0:
+            raise ValueError("cache_bytes must be nonnegative")
+        if policy not in ("lru", "lfu"):
+            raise ValueError(f"Unknown cache policy {policy!r}; use 'lru' or 'lfu'")
+        if source is not None and quantized is not None:
+            raise ValueError("Use either quantized_experts or expert_source, not both")
+        self.policy = policy
+        self.decay_every = decay_every
+        self.counts = {}
+        self.sequence = 0
+        self.lookups = 0
+        self.not_admitted = 0
+        self.reader = reader
+        self.source = source
+        self.cache_bytes = cache_bytes
+        self.device = torch.device(device)
+        self.dtype = dtype
+        self.resident_bytes = 0
+        self.peak_cache_bytes = 0
+        self.hits = self.misses = self.evictions = 0
+        self.quantized = None
+        if quantized is not None:
+            info = json.loads((Path(quantized) / "quantization.json").read_text(encoding="utf-8"))
+            if info.get("format") != _QUANTIZED_FORMAT:
+                raise ValueError(f"Unsupported quantized experts format {info.get('format')!r}")
+            self.quantized = (
+                SafetensorsReader(quantized, reader.bypass_page_cache, packed=True),
+                info["bits"],
+                info["group_size"],
+            )
+            self.gather_qmm = _load_gather_qmm(self.device)
+        # Layers register first (`register`), then `allocate` sizes the pools from the whole set of layouts.
+        self.layouts = {}  # prefix -> (pool key, layout, num_experts)
+        self.pools = []
+        self.pool_of = {}
+        # expert -> slot per layer (the pool's num_slots when absent), kept on the host (installs and evictions are host
+        # bookkeeping) and copied to the device only when a device-side lookup needs it.
+        self.slot_tables = {}
+        self.host_slot_tables = {}
+        self.stale_slot_tables = set()
+        self.total_experts = 0
+        self.compute = "grouped_mm"
+        self.expert_compute = None  # see `experts_offload_forward`
+        self.uses = {}  # (prefix, expert) -> lookups, see `routing_profile`
+        self.preload_profile = None  # see `preload_from_profile`
+        self.executor = ThreadPoolExecutor(io_threads, thread_name_prefix="experts-offload-io") if io_threads else None
+        self.pending = {}
+        self.window = 2 * io_threads
+        self.prefetched = self.prefetch_used = self.installed_ahead = 0
+        # Long prefill calls (`cold_prefill_routes`): hits do not refresh, and the experts installed for a layer move to
+        # the cold end of its pool once that layer's call is done, so one long prompt does not flush the decode set.
+        self.cold = False
+        self.cold_installed = {}
+        # Retain host buffers until the next routing copy synchronizes the device.
+        self.inflight = []
+        # Recycled host buffers for expert bytes, all of the largest expert size, see `_host_buffer`.
+        self.free_host_buffers = []
+        self.host_buffer_bytes = 0
+        self.commit = False  # see `_commit_mps`
+        # Optional host tier (expert bytes, one layout): a pool of expert rows in host memory between storage and the
+        # device cache, see `allocate_host_cache`. `host_busy` holds the rows handed out and not yet released.
+        self.host_pool = None
+        self.host_slots = 0
+        self.host_cache = OrderedDict()
+        self.host_free = []
+        self.host_busy = set()
+        self.host_lock = threading.Lock()
+        self.host_hits = self.host_misses = 0
+
+    def register(self, prefix, module):
+        """Validate one experts module's tensors and record its layout; returns the checkpoint keys it replaces."""
+        if self.pools:
+            raise ValueError("Register every experts module before `allocate`")
+        shapes = {
+            "gate_up_proj": (module.num_experts, 2 * module.intermediate_dim, module.hidden_dim),
+            "down_proj": (module.num_experts, module.hidden_dim, module.intermediate_dim),
+        }
+        per_expert = [
+            f"{prefix}.{expert}.{projection}.weight"
+            for expert in range(module.num_experts)
+            for projection in ("gate_proj", "up_proj", "down_proj")
+        ]
+        if self.source is not None or (
+            self.quantized is not None and f"{prefix}.experts" in self.quantized[0].tensors
+        ):
+            # Experts come from elsewhere: skip whatever the checkpoint holds for them.
+            candidates = [f"{prefix}.{suffix}" for suffix in shapes] + per_expert
+            names = {name: None for name in candidates if name in self.reader.tensors}
+        elif f"{prefix}.gate_up_proj" in self.reader.tensors:
+            names = {f"{prefix}.{suffix}": shape for suffix, shape in shapes.items()}
+        else:
+            names = {}
+            for expert in range(module.num_experts):
+                for projection in ("gate_proj", "up_proj", "down_proj"):
+                    shape = (module.intermediate_dim, module.hidden_dim)
+                    if projection == "down_proj":
+                        shape = shape[::-1]
+                    names[f"{prefix}.{expert}.{projection}.weight"] = shape
+        for name, shape in names.items():
+            if shape is None:
+                continue
+            if name not in self.reader.tensors or self.reader.shape(name) != shape:
+                raise ValueError(f"Missing or incompatible expert tensor {name}; expected {shape}")
+        if self.source is not None:
+            layout = [(tuple(shape), dtype) for shape, dtype in self.source.layout(prefix)]
+            key = self.source.pool_key(prefix) if hasattr(self.source, "pool_key") else None
+        elif self.quantized is not None:
+            self._check_quantized(prefix, module)
+            layout, key = self._quantized_shapes(module), None
+        else:
+            layout = [
+                ((2 * module.intermediate_dim, module.hidden_dim), self.dtype),
+                ((module.hidden_dim, module.intermediate_dim), self.dtype),
+            ]
+            key = None
+        self.layouts[prefix] = (key if key is not None else tuple(layout), layout, module.num_experts)
+        self.total_experts += module.num_experts
+        return set(names)
+
+    def allocate(self, materialize=True):
+        """Size the device pools: one per layout, each layer getting the same number of slots. Their memory is taken
+        now, or with ``materialize=False`` at the first lookup (after loading, so the loading peak excludes them)."""
+        groups = {}
+        for prefix, (key, layout, _) in self.layouts.items():
+            groups.setdefault(key, (layout, []))[1].append(prefix)
+        layer_bytes = [_layout_bytes(layout) for _, layout, _ in self.layouts.values()]
+        if not groups:
+            raise ValueError("No experts module registered")
+        if len(groups) == 1:
+            sizes = [self.cache_bytes // layer_bytes[0]]
+        else:
+            per_layer = self.cache_bytes // sum(layer_bytes)
+            sizes = [per_layer * len(prefixes) for _, prefixes in groups.values()]
+        packed = self.quantized is not None
+        for (layout, prefixes), num_slots in zip(groups.values(), sizes):
+            pool = _Pool(layout, num_slots, packed)
+            self.pools.append(pool)
+            for prefix in prefixes:
+                self.pool_of[prefix] = pool
+                experts = self.layouts[prefix][2]
+                pool.num_experts += experts
+                self.slot_tables[prefix] = torch.full((experts,), num_slots, device=self.device)
+                self.host_slot_tables[prefix] = torch.full((experts,), num_slots)
+        self.host_buffer_bytes = max(pool.slot_bytes for pool in self.pools)
+        self.materialized = False
+        if materialize:
+            self.materialize()
+
+    def materialize(self):
+        """Take the pools' device memory (once), then fill it from ``preload_profile`` when set."""
+        if not self.materialized:
+            for pool in self.pools:
+                pool.materialize(self.device)
+            self.materialized = True
+            if self.preload_profile is not None:
+                self.preload_from_profile(self.preload_profile)
+
+    def routing_profile(self):
+        """Lookups per expert since loading, ``{prefix: {expert: count}}``: what ``preload_profile`` takes."""
+        profile = {}
+        for (prefix, expert), count in self.uses.items():
+            profile.setdefault(prefix, {})[expert] = count
+        return profile
+
+    def preload_from_profile(self, profile):
+        """Fill every layer's share of its pool with its most used experts in ``profile`` (``{prefix: {expert:
+        count}}``, or a path to it as JSON), least used first so the most used end up hottest. Reads in parallel."""
+        if not isinstance(profile, dict):
+            profile = json.loads(Path(profile).read_text(encoding="utf-8"))
+        layers = {}
+        for prefix in self.layouts:
+            layers.setdefault(id(self.pool_of[prefix]), []).append(prefix)
+        cold, self.cold = self.cold, False  # the first call may be a cold prefill: the preload is not part of it
+        for prefix in self.layouts:
+            pool = self.pool_of[prefix]
+            counts = {int(expert): count for expert, count in profile.get(prefix, {}).items()}
+            share = pool.num_slots // len(layers[id(pool)])
+            ranked = sorted(counts, key=counts.get, reverse=True)[:share]
+            self.prefetch(prefix, ranked)
+            for expert in reversed(ranked):
+                self._install((prefix, expert))
+            self.synchronize()  # releases this layer's host buffers before the next layer's reads
+        self.cold = cold
+
+    # Single-pool views, kept for callers that predate size classes.
+    @property
+    def num_slots(self):
+        return sum(pool.num_slots for pool in self.pools)
+
+    @property
+    def slot_bytes(self):
+        if len(self.pools) != 1:
+            raise ValueError("slot_bytes is defined for a single pool; see `pools`")
+        return self.pools[0].slot_bytes
+
+    @property
+    def cache(self):
+        """Every resident key -> slot (read-only view over the pools)."""
+        return {key: slot for pool in self.pools for key, slot in pool.cache.items()}
+
+    @property
+    def gate_up_pool(self):
+        return self.pools[0].banks[0] if len(self.pools) == 1 and self.quantized is None else None
+
+    @property
+    def down_pool(self):
+        return self.pools[0].banks[1] if len(self.pools) == 1 and self.quantized is None else None
+
+    def _quantized_shapes(self, module):
+        """Per-expert shapes and dtypes of the quantized tensors, in pool order."""
+        _, bits, group_size = self.quantized
+        shapes = []
+        for rows, columns in (
+            (2 * module.intermediate_dim, module.hidden_dim),
+            (module.hidden_dim, module.intermediate_dim),
+        ):
+            if columns % group_size or (columns * bits) % 32:
+                raise ValueError(f"Input size {columns} does not fit {bits}-bit groups of {group_size}")
+            shapes += [
+                ((rows, columns * bits // 32), torch.uint32),
+                ((rows, columns // group_size), self.dtype),
+                ((rows, columns // group_size), self.dtype),
+            ]
+        return shapes
+
+    def _check_quantized(self, prefix, module):
+        reader, name = self.quantized[0], f"{prefix}.experts"
+        size = sum(math.prod(shape) * dtype.itemsize for shape, dtype in self._quantized_shapes(module))
+        if name not in reader.tensors or reader.shape(name) != (module.num_experts, size):
+            raise ValueError(
+                f"Missing or incompatible quantized experts {name}; expected {module.num_experts} x {size} bytes "
+                "(quantized from a checkpoint in the execution dtype)"
+            )
+
+    def all_resident(self):
+        """Whether every expert is cached, allowing routing to stay on the device."""
+        return sum(len(pool.cache) for pool in self.pools) == self.total_experts
+
+    def _count(self, key):
+        self.lookups += 1
+        if self.policy != "lfu":
+            return
+        if self.decay_every and self.lookups % self.decay_every == 0:
+            self.counts = {k: v / 2 for k, v in self.counts.items() if v >= 0.25}
+            for pool in self.pools:
+                pool.heap = []
+                for resident in pool.cache:
+                    self._push(resident)
+        self.counts[key] = self.counts.get(key, 0) + 1
+        if key in self.pool_of[key[0]].cache:
+            self._push(key)
+
+    def _push(self, key):
+        self.sequence += 1
+        heapq.heappush(self.pool_of[key[0]].heap, (self.counts.get(key, 0), self.sequence, key))
+
+    def _least_used(self, pool):
+        while pool.heap:
+            count, _, key = pool.heap[0]
+            if key in pool.cache and count == self.counts.get(key, 0):
+                return key, count
+            heapq.heappop(pool.heap)
+        return None, None
+
+    def _evict(self, pool, victim):
+        slot = pool.cache.pop(victim)
+        self.resident_bytes -= pool.slot_bytes
+        self.evictions += 1
+        pool.free_slots.append(slot)
+        self.host_slot_tables[victim[0]][victim[1]] = pool.num_slots
+        self.stale_slot_tables.add(victim[0])
+
+    def _make_room(self, key, pinned=()):
+        """Evict unpinned experts of the key's pool until a slot is free; return False if admission is declined."""
+        pool = self.pool_of[key[0]]
+        if not pool.num_slots:
+            return False
+        if self.policy == "lfu":
+            while not pool.free_slots:
+                victim, count = self._least_used(pool)
+                if victim is None or victim in pinned or self.counts.get(key, 0) <= count:
+                    self.not_admitted += 1
+                    return False
+                heapq.heappop(pool.heap)
+                self._evict(pool, victim)
+            return True
+        while not pool.free_slots:
+            victim = next((k for k in pool.cache if k not in pinned), None)
+            if victim is None:
+                self.not_admitted += 1
+                return False
+            self._evict(pool, victim)
+        return True
+
+    def demote(self, prefix):
+        """Move the experts installed for ``prefix`` during a cold call to the cold end of their pool."""
+        cache = self.pool_of[prefix].cache
+        for key in self.cold_installed.pop(prefix, ()):
+            if key in cache:
+                cache.move_to_end(key, last=False)
+
+    def allocate_host_cache(self, host_cache_bytes):
+        """Keep up to ``host_cache_bytes`` of expert bytes in (pinned) host memory, between storage and the device
+        cache: a device-cache miss then costs one host-to-device copy instead of a storage read. Worth it when host
+        memory is larger than device memory (discrete GPUs); on unified memory the device cache already is host RAM."""
+        if self.quantized is None and self.source is None:
+            raise ValueError("host_cache_bytes needs quantized_experts or an expert_source")
+        if len(self.pools) != 1:
+            raise ValueError("host_cache_bytes needs experts of a single layout")
+        self.host_slots = host_cache_bytes // self.slot_bytes
+        if not self.host_slots:
+            return
+        self.host_pool = torch.empty(
+            (self.host_slots, self.slot_bytes), dtype=torch.uint8, pin_memory=self.device.type != "cpu"
+        )
+        self.host_free = list(range(self.host_slots))
+
+    def _host_row(self, tensor):
+        """The host-pool row ``tensor`` is, or None."""
+        if self.host_pool is None:
+            return None
+        offset = tensor.data_ptr() - self.host_pool.data_ptr()
+        if 0 <= offset < self.host_pool.numel() and offset % self.slot_bytes == 0:
+            return offset // self.slot_bytes
+        return None
+
+    def _release_host_row(self, tensor):
+        row = self._host_row(tensor)
+        if row is not None:
+            with self.host_lock:
+                self.host_busy.discard(row)
+        return row is not None
+
+    def _read_bytes(self, prefix, expert, out):
+        """Read one expert's bytes into ``out`` (a CPU uint8 tensor of its pool's slot size)."""
+        if self.source is not None:
+            self.source.read(prefix, expert, out)
+        else:
+            self.quantized[0].read(f"{prefix}.experts", expert, out=out)
+        return out
+
+    def _read_host_cached(self, prefix, expert):
+        """An expert through the host tier: a hit hands out its row, a miss reads into a free (or the least recently
+        used, not busy) row. Returns None when every row is busy. Safe on I/O threads."""
+        key = (prefix, expert)
+        with self.host_lock:
+            row = self.host_cache.get(key)
+            if row is not None:
+                self.host_cache.move_to_end(key)
+                self.host_busy.add(row)
+                self.host_hits += 1
+                return (self.host_pool[row],)
+            if self.host_free:
+                row = self.host_free.pop()
+            else:
+                victim = next((k for k, r in self.host_cache.items() if r not in self.host_busy), None)
+                if victim is None:
+                    return None
+                row = self.host_cache.pop(victim)
+            self.host_busy.add(row)
+            self.host_misses += 1
+        try:
+            self._read_bytes(prefix, expert, self.host_pool[row])
+        except BaseException:
+            with self.host_lock:
+                self.host_busy.discard(row)
+                self.host_free.append(row)
+            raise
+        with self.host_lock:
+            self.host_cache[key] = row
+        return (self.host_pool[row],)
+
+    def preload_host(self):
+        """Read every expert into the host tier now (in parallel when there are I/O threads)."""
+        keys = [(prefix, expert) for prefix, table in self.host_slot_tables.items() for expert in range(table.numel())]
+        if self.host_slots < len(keys):
+            raise ValueError(
+                f"host preload needs room for all {len(keys)} experts; the host tier has {self.host_slots}"
+            )
+        if self.executor is not None:
+            results = [f.result() for f in [self.executor.submit(self._read_host_cached, *key) for key in keys]]
+        else:
+            results = [self._read_host_cached(*key) for key in keys]
+        for tensors in results:
+            self._release_host_row(tensors[0])
+        self.host_hits = self.host_misses = 0
+
+    def _read_host(self, prefix, expert):
+        """Read expert bytes or floating weights on the host; safe to run on I/O threads."""
+        if self.host_pool is not None:
+            tensors = self._read_host_cached(prefix, expert)
+            if tensors is not None:
+                return tensors
+        if self.quantized is not None or self.source is not None:
+            # Straight into a recycled pinned buffer: copying pinned memory to the device costs the host ~5x less
+            # than pageable memory (0.03 vs 0.14 ms for a 2 MB expert on an M5).
+            return (self._read_bytes(prefix, expert, self._host_buffer()[: self.pool_of[prefix].slot_bytes]),)
+        return tuple(t.to(self.dtype) for t in self.reader.read_expert(prefix, expert))
+
+    def _host_buffer(self):
+        """A host buffer for one expert (of the largest size, so one size is recycled), pinned when the device can
+        use it; called from I/O threads."""
+        try:
+            return self.free_host_buffers.pop()
+        except IndexError:
+            return torch.empty(self.host_buffer_bytes, dtype=torch.uint8, pin_memory=self.device.type != "cpu")
+
+    def prefetch(self, prefix, experts):
+        """Start reading the experts of ``prefix`` that are neither resident nor already being read."""
+        if self.executor is None:
+            return
+        cache = self.pool_of[prefix].cache
+        for expert in experts:
+            key = (prefix, expert)
+            if key not in cache and key not in self.pending:
+                self.pending[key] = self.executor.submit(self._read_host, prefix, expert)
+                self.prefetched += 1
+
+    def discard(self, prefix):
+        """Drop the reads started for ``prefix`` that its forward call did not use (wrong predictions)."""
+        for key in [key for key in self.pending if key[0] == prefix]:
+            future = self.pending.pop(key)
+            if not future.cancel() and self.host_pool is not None:
+                # Already read (or reading): its host-tier row was handed out and will not be copied.
+                future.add_done_callback(self._release_unused)
+
+    def _release_unused(self, future):
+        if not future.cancelled() and future.exception() is None:
+            self._release_host_row(future.result()[0])
+
+    def _take_host(self, prefix, expert):
+        """One expert in host memory: from a read already started by :meth:`prefetch`, else read now."""
+        future = self.pending.pop((prefix, expert), None)
+        if future is not None and not future.cancelled():
+            self.prefetch_used += 1
+            return future.result()
+        return self._read_host(prefix, expert)
+
+    # TODO(portability): share pinned host staging for asynchronous accelerator copies with _install.
+    def _read(self, prefix, expert):
+        """One expert on the device, for a single use."""
+        tensors = self._take_host(prefix, expert)
+        self.inflight.append(tensors)
+        if self.quantized is not None or self.source is not None:
+            return _split_expert(tensors[0].to(self.device, non_blocking=True), self.pool_of[prefix].layout)
+        return tuple(t.to(self.device, non_blocking=True) for t in tensors)
+
+    def release_host_buffers(self):
+        """Called right after a host synchronization: every queued copy has read its host buffer."""
+        # TODO(portability): validate copy completion and slot reuse on each backend before releasing buffers.
+        if self.quantized is not None or self.source is not None:
+            # list.append / pop are atomic, so I/O threads can take buffers back while this runs.
+            for tensors in self.inflight:
+                if self._release_host_row(tensors[0]):
+                    continue  # a host-tier row: it stays cached, now free to be evicted
+                buffer = tensors[0] if tensors[0]._base is None else tensors[0]._base
+                if buffer.dtype == torch.uint8 and buffer.numel() == self.host_buffer_bytes:
+                    self.free_host_buffers.append(buffer)
+        self.inflight.clear()
+
+    def synchronize(self):
+        """Wait for queued device work, then release the host buffers of its copies."""
+        # TODO(portability): synchronize other accelerator backends.
+        if self.device.type == "mps":
+            torch.mps.synchronize()
+        elif self.device.type == "cuda":
+            torch.cuda.synchronize(self.device)
+        self.release_host_buffers()
+
+    def device_slot_table(self, prefix):
+        """The layer's expert -> slot table on the device, refreshed from the host copy when it changed."""
+        if prefix in self.stale_slot_tables:
+            snapshot = self.host_slot_tables[prefix].clone()
+            self.inflight.append((snapshot,))  # read by a queued non-blocking copy
+            self.slot_tables[prefix].copy_(snapshot, non_blocking=True)
+            self.stale_slot_tables.discard(prefix)
+        return self.slot_tables[prefix]
+
+    def device_slots(self, prefix, routing):
+        """Slots of host-side ``routing`` as a device tensor: one small copy instead of a device lookup."""
+        slots = self.host_slot_tables[prefix][routing]
+        self.inflight.append((slots,))
+        return slots.to(self.device, non_blocking=True)
+
+    def install_ahead(self, prefix, experts):
+        """Install prefetched experts while queued device work runs, without counting lookups."""
+        if not self.materialized:
+            self.materialize()
+        cache = self.pool_of[prefix].cache
+        for expert in experts:
+            key = (prefix, expert)
+            # Only finished reads: waiting here would delay the next layer's launches for guesses that may go unused.
+            if key in cache or key not in self.pending or not self.pending[key].done():
+                continue
+            if self._install(key) is None:
+                break  # the pool is full of pinned experts
+            self.installed_ahead += 1
+
+    def _install(self, key, pinned=()):
+        if not self._make_room(key, pinned):
+            return None
+        pool = self.pool_of[key[0]]
+        # Keep host buffers alive until the next host synchronization.
+        tensors = self._take_host(*key)
+        slot = pool.free_slots.pop()
+        self.inflight.append(tensors)
+        if pool.slot_buffer is not None:
+            pool.slot_buffer[slot].copy_(tensors[0], non_blocking=True)
+        elif self.source is not None:
+            offset = 0
+            for bank in pool.banks:
+                size = bank[slot].numel() * bank.element_size()
+                bank[slot].view(-1).view(torch.uint8).copy_(tensors[0][offset : offset + size], non_blocking=True)
+                offset += size
+        else:
+            for bank, tensor in zip(pool.banks, tensors):
+                bank[slot].copy_(tensor, non_blocking=True)
+        self.host_slot_tables[key[0]][key[1]] = slot
+        self.stale_slot_tables.add(key[0])
+        pool.cache[key] = slot
+        if self.cold:
+            self.cold_installed.setdefault(key[0], []).append(key)
+        self.resident_bytes += pool.slot_bytes
+        self.peak_cache_bytes = max(self.peak_cache_bytes, self.resident_bytes)
+        if self.policy == "lfu":
+            self._push(key)
+        return slot
+
+    def acquire(self, prefix, expert, pinned=()):
+        """Make an expert resident in its pool; returns its slot, or None when it runs once without being cached."""
+        if not self.materialized:
+            self.materialize()
+        key = (prefix, expert)
+        self._count(key)
+        self.uses[key] = self.uses.get(key, 0) + 1
+        cache = self.pool_of[prefix].cache
+        if key in cache:
+            self.hits += 1
+            if not self.cold:
+                cache.move_to_end(key)
+            return cache[key]
+        self.misses += 1
+        return self._install(key, pinned)
+
+    def expert_tensors(self, prefix, expert):
+        """The device tensors of one expert without counting a lookup: pool views when resident, else read for this use."""
+        pool = self.pool_of[prefix]
+        slot = pool.cache.get((prefix, expert))
+        if slot is None:
+            return self._read(prefix, expert)
+        return tuple(bank[slot] for bank in pool.banks)
+
+    def get(self, prefix, expert):
+        """Return the device tensors of one expert (pool views when cached, else read for this use only)."""
+        slot = self.acquire(prefix, expert)
+        if slot is None:
+            return self._read(prefix, expert)
+        return tuple(bank[slot] for bank in self.pool_of[prefix].banks)
+
+    def _readers(self):
+        return [self.reader] + ([self.quantized[0]] if self.quantized else []) + ([self.source] if self.source else [])
+
+    def preload(self):
+        """Load every expert now (reads in parallel), when the cache holds them all: no storage reads afterwards."""
+        for pool in self.pools:
+            if pool.num_slots < pool.num_experts:
+                raise ValueError(
+                    f"preload needs room for all {pool.num_experts} experts of a pool; it has {pool.num_slots} slots"
+                )
+        for prefix, table in self.host_slot_tables.items():
+            experts = list(range(table.numel()))
+            self.prefetch(prefix, experts)
+            for expert in experts:
+                self.acquire(prefix, expert)
+            self.synchronize()
+        self.hits = self.misses = 0
+
+    def stats(self):
+        return {
+            "hits": self.hits,
+            "misses": self.misses,
+            "evictions": self.evictions,
+            "not_admitted": self.not_admitted,
+            "policy": self.policy,
+            "slots": self.num_slots,
+            "pools": len(self.pools),
+            "prefetched": self.prefetched,
+            "prefetch_used": self.prefetch_used,
+            "installed_ahead": self.installed_ahead,
+            "resident_bytes": self.resident_bytes,
+            "peak_cache_bytes": self.peak_cache_bytes,
+            "read_bytes": sum(reader.read_bytes for reader in self._readers()),
+            "read_calls": sum(reader.read_calls for reader in self._readers()),
+            "host_slots": self.host_slots,
+            "host_hits": self.host_hits,
+            "host_misses": self.host_misses,
+        }
+
+
+class _PoolExperts:
+    """Expose a pool's banks to MoE kernels; slot num_slots uses their expert-parallel sentinel handling."""
+
+    def __init__(self, module, pool):
+        self.module = module
+        self.num_experts = pool.num_slots
+        self.gate_up_proj, self.down_proj = pool.banks
+        self.act_fn = module.act_fn
+        self.has_gate = getattr(module, "has_gate", True)
+        self.has_bias = getattr(module, "has_bias", False)
+        self.is_transposed = getattr(module, "is_transposed", False)
+        self.has_post_expert_norm = getattr(module, "has_post_expert_norm", False)
+        self.post_expert_norm = getattr(module, "post_expert_norm", None)
+        self._is_expert_parallel = True
+
+    def _apply_gate(self, gate_up_out):
+        return self.module._apply_gate(gate_up_out)
+
+
+def _run_expert(module, result, hidden_states, top_k_weights, token, slot, gate_up, down):
+    # Bound temporary activations during prefill.
+    for start in range(0, token.numel(), module.offload_token_chunk_size):
+        indices = token[start : start + module.offload_token_chunk_size]
+        slots = slot[start : start + module.offload_token_chunk_size]
+        activated = module._apply_gate(torch.nn.functional.linear(hidden_states[indices], gate_up))
+        output = torch.nn.functional.linear(activated, down)
+        output = output * top_k_weights[indices, slots, None]
+        result.index_add_(0, indices, output.to(result.dtype))
+
+
+def _split_expert(blob, shapes):
+    """Views of one expert's bytes as its quantized tensors: gate_up (weight, scales, biases), then down."""
+    # TODO(portability): validate uint32 views and strided packed banks on each accelerator backend.
+    tensors, offset = [], 0
+    for shape, dtype in shapes:
+        size = math.prod(shape) * dtype.itemsize
+        if offset % dtype.itemsize:
+            raise ValueError("Misaligned quantized expert layout")
+        tensors.append(blob[..., offset : offset + size].view(dtype).view(*blob.shape[:-1], *shape))
+        offset += size
+    return tuple(tensors)
+
+
+def _load_gather_qmm(device):
+    """MLX's quantized MoE matmul from the Hub (Metal) on MPS; elsewhere the slow PyTorch reference below."""
+    if device.type == "mps":
+        try:
+            from .hub_kernels import get_kernel
+
+            return get_kernel(_MLX_QUANTIZATION_KERNEL[0], version=_MLX_QUANTIZATION_KERNEL[1]).gather_qmm
+        except Exception as error:  # the kernels package is optional
+            logger.warning(f"Using the PyTorch reference for quantized experts; the Metal kernel failed: {error}")
+    # TODO(portability): add optimized non-Metal kernels; retain the reference as a correctness fallback.
+    return _gather_qmm_reference
+
+
+def _gather_qmm_reference(x, w, scales, biases, rhs_indices, group_size, bits, **_):
+    """`gather_qmm(x [S, 1, K], w [E, N, K * bits / 32], ..., rhs_indices [S], transpose=True)` -> [S, 1, N]."""
+    from .metal_quantization import _affine_dequantize_tensor
+
+    output = x.new_empty(*x.shape[:-1], w.shape[1])
+    for expert in torch.unique(rhs_indices).tolist():
+        rows = rhs_indices == expert
+        weight = _affine_dequantize_tensor(w[expert], scales[expert], biases[expert], group_size, bits)
+        output[rows] = x[rows] @ weight.to(x.dtype).T
+    return output
+
+
+def _quantized_mlp(module, store, x, banks, rhs, *, sorted_indices=False):
+    """Apply the quantized expert selected by rhs for each input row."""
+    _, bits, group_size = store.quantized
+
+    def matmul(inputs, weight, scales, biases):
+        return store.gather_qmm(
+            inputs[:, None, :],
+            weight,
+            scales,
+            biases,
+            rhs_indices=rhs,
+            transpose=True,
+            group_size=group_size,
+            bits=bits,
+            sorted_indices=sorted_indices,
+        ).squeeze(1)
+
+    return matmul(module._apply_gate(matmul(x, *banks[:3])), *banks[3:])
+
+
+def _sorted_kernel_applies(rows, experts):
+    """Whether MLX selects its batched sorted kernel (at least 16 rows and 4 per expert)."""
+    # TODO(portability): select this threshold and the float32 workaround by kernel/backend.
+    return rows >= 16 and rows >= 4 * experts
+
+
+# Experts per sorted `gather_qmm` call in prefill: bounds the transient bank when a long prompt routes to nearly every
+# expert of a layer (64 x 2.15 MB for Qwen3.8's 3-bit experts instead of 512).
+PREFILL_EXPERT_GROUP = 64
+
+
+def _quantized_sorted_forward(module, store, hidden_states, top_k_index, top_k_weights, experts, banks, result):
+    """Add the routes to ``experts`` (sorted by expert, over ``banks`` holding only them) into ``result`` (float32).
+
+    Uses float32 to avoid incorrect fp16/bf16 results in the v2 M5 kernel."""
+    top_k = top_k_index.shape[1]
+    lookup = torch.full((module.num_experts,), -1, dtype=torch.long, device=hidden_states.device)
+    lookup[torch.tensor(experts, device=hidden_states.device)] = torch.arange(
+        len(experts), device=hidden_states.device
+    )
+    flat = lookup[top_k_index.reshape(-1)]
+    selected = torch.nonzero(flat >= 0).squeeze(1)
+    rhs, permutation = torch.sort(flat[selected], stable=True)
+    order = selected[permutation]
+    banks = [bank.float() if bank.is_floating_point() else bank for bank in banks]
+
+    x = hidden_states.float()[order // top_k]
+    output = _quantized_mlp(module, store, x, banks, rhs, sorted_indices=True)
+    output = output * top_k_weights.reshape(-1)[order, None].float()
+    result.index_add_(0, order // top_k, output)
+
+
+def _quantized_pool_forward(module, store, pool, hidden_states, slots, top_k_weights):
+    """Resident quantized experts through `gather_qmm`; non-resident routes (slot ``num_slots``) contribute zero."""
+    num_tokens, top_k = slots.shape
+    rhs = slots.reshape(-1)
+    valid = rhs < pool.num_slots
+    weights = (top_k_weights.reshape(-1) * valid)[:, None]
+    output = _quantized_mlp(module, store, hidden_states.repeat_interleave(top_k, dim=0), pool.banks, rhs * valid)
+    return (output * weights.to(output.dtype)).view(num_tokens, top_k, -1).sum(dim=1).to(hidden_states.dtype)
+
+
+def _pool_forward(module, store, hidden_states, top_k_index, top_k_weights, slots=None):
+    pool = store.pool_of[module.offload_prefix]
+    if slots is None:
+        slots = store.device_slot_table(module.offload_prefix)[top_k_index]
+    if store.quantized is not None:
+        return _quantized_pool_forward(module, store, pool, hidden_states, slots, top_k_weights)
+    kernel = ExpertsInterface._global_mapping[store.compute]
+    return kernel(_PoolExperts(module, pool), hidden_states, slots, top_k_weights)
+
+
+def _commit_mps(store):
+    """Hand the queued MPS work to the GPU without waiting for it.
+
+    PyTorch's MPS stream keeps small kernels in a command buffer that it commits at the next synchronization, so after
+    each layer's routing copy the GPU would idle while the host encodes the next layer. Recording an event commits the
+    buffer.
+    """
+    # TODO(portability): MPS-only workaround for PyTorch's lazy command-buffer commits; check whether other backends
+    # (CUDA streams submit eagerly) need an equivalent, and drop it once PyTorch's MPS stream commits on its own.
+    if store.device.type == "mps":
+        torch.mps.Event().record()
+
+
+def experts_offload_forward(module, hidden_states, top_k_index, top_k_weights):
+    """The ``experts_offload`` experts implementation: resident experts from the cache, missing ones from disk.
+
+    With an ``expert_compute`` callable, this forward does the routing copy, reads, read-ahead, next-layer guesses and
+    installs, and the callable computes the output: ``expert_compute(module, store, hidden_states, top_k_index,
+    top_k_weights, routing, experts, pinned)``, where ``routing`` is ``top_k_index`` on the host, ``experts`` the sorted
+    routed experts and ``pinned`` the keys made resident for this call (the others are read with
+    ``store.expert_tensors``)."""
+    if module.training or torch.is_grad_enabled():
+        raise RuntimeError("experts_offload is inference-only: use eval() and torch.no_grad() / inference_mode()")
+    store = module.offload_store
+    if hidden_states.device != store.device or hidden_states.dtype != store.dtype:
+        raise ValueError(
+            "Moving/casting a streamed model after loading is unsupported; reload on the target device/dtype"
+        )
+    prefix = module.offload_prefix
+    if store.expert_compute is None and store.all_resident():
+        store.hits += top_k_index.numel()
+        return _pool_forward(module, store, hidden_states, top_k_index, top_k_weights)
+    # Copy routing and next-layer predictions to the host together.
+    next_layer = getattr(module, "offload_next", None)
+    predict = next_layer is not None and store.executor is not None
+    predict = predict and top_k_index.numel() <= module.offload_predict_max_routes
+    after_next = getattr(module, "offload_after_next", None) if predict else None
+    if predict:
+        next_prefix, router = next_layer
+        guesses = [torch.nn.functional.linear(hidden_states, router.weight).topk(router.top_k, dim=-1).indices]
+        if after_next is not None:
+            guesses.append(
+                torch.nn.functional.linear(hidden_states, after_next[1].weight).topk(router.top_k, dim=-1).indices
+            )
+        routes = torch.cat([top_k_index.reshape(-1)] + [guess.reshape(-1) for guess in guesses]).cpu()
+        routing = routes[: top_k_index.numel()].view(top_k_index.shape)
+    else:
+        routing = top_k_index.cpu()
+    # The routing copy also waits for earlier host-to-device copies.
+    store.release_host_buffers()
+    experts = torch.unique(routing).tolist()
+    if experts and not (experts[0] >= 0 and experts[-1] < module.num_experts):
+        raise ValueError("Expert parallelism and sentinel expert IDs are unsupported")
+    store.prefetch(prefix, experts[: store.window])
+    readahead = module.offload_readahead_routes
+    readahead_next = readahead and module.offload_next_prefix is not None and top_k_index.numel() >= readahead
+    if readahead_next:
+        # A long prompt routes to nearly every expert of every layer: read the next layer's while this one computes.
+        store.prefetch(module.offload_next_prefix, range(module.num_experts))
+    width = top_k_index.shape[0] * (next_layer[1].top_k if predict else 0)
+    guessed = torch.unique(routes[top_k_index.numel() : top_k_index.numel() + width]).tolist() if predict else []
+    if predict:
+        store.prefetch(next_prefix, guessed)
+    if after_next is not None:
+        store.prefetch(after_next[0], torch.unique(routes[top_k_index.numel() + width :]).tolist())
+    cold_routes = module.offload_cold_prefill_routes
+    store.cold = bool(cold_routes) and store.policy == "lru" and top_k_index.numel() >= cold_routes
+    try:
+        result = _offloaded_experts(module, store, prefix, hidden_states, top_k_index, top_k_weights, routing, experts)
+    finally:
+        store.discard(prefix)
+        # Also demotes what a previous cold call installed ahead for this layer; `store.cold` stays set until the next
+        # call, so the next layer's installs below are demoted after that layer runs.
+        store.demote(prefix)
+    if store.commit:
+        _commit_mps(store)
+    if guessed:
+        store.install_ahead(next_prefix, guessed)
+    if readahead_next and module.offload_prefill_install_ahead:
+        # The next layer's reads started above: queue the finished ones' copies behind this layer's compute, so that
+        # layer finds them installed instead of copying while the device idles after its routing copy.
+        store.install_ahead(module.offload_next_prefix, range(module.num_experts))
+    return result
+
+
+def _offloaded_experts(module, store, prefix, hidden_states, top_k_index, top_k_weights, routing, experts):
+    pool = store.pool_of[prefix]
+    pinned, uncached = set(), []
+    for position, expert in enumerate(experts):
+        store.prefetch(prefix, experts[position + 1 : position + 1 + store.window])
+        if store.acquire(prefix, expert, pinned) is None:
+            uncached.append(expert)
+        else:
+            pinned.add((prefix, expert))
+    if store.expert_compute is not None:
+        return store.expert_compute(module, store, hidden_states, top_k_index, top_k_weights, routing, experts, pinned)
+    if store.quantized is not None and _sorted_kernel_applies(top_k_index.numel(), len(experts)):
+        # A group of experts at a time: a prefill that routes to every expert never holds a whole layer of them.
+        result = torch.zeros(hidden_states.shape, dtype=torch.float32, device=hidden_states.device)
+        for start in range(0, len(experts), PREFILL_EXPERT_GROUP):
+            group = experts[start : start + PREFILL_EXPERT_GROUP]
+            per_expert = []
+            for position, expert in enumerate(group):
+                if (prefix, expert) in pinned:
+                    slot = pool.cache[(prefix, expert)]
+                    per_expert.append([bank[slot] for bank in pool.banks])
+                else:
+                    store.prefetch(prefix, experts[start + position + 1 : start + position + 1 + store.window])
+                    per_expert.append(store._read(prefix, expert))
+            banks = [torch.stack(parts) for parts in zip(*per_expert)]
+            del per_expert
+            _quantized_sorted_forward(module, store, hidden_states, top_k_index, top_k_weights, group, banks, result)
+            if store.inflight:
+                store.synchronize()
+        return result.to(hidden_states.dtype)
+    if pinned:
+        slots = store.device_slots(prefix, routing)
+        result = _pool_forward(module, store, hidden_states, top_k_index, top_k_weights, slots)
+    else:
+        result = torch.zeros_like(hidden_states)
+    for position, expert in enumerate(uncached):
+        store.prefetch(prefix, uncached[position + 1 : position + 1 + store.window])
+        token, slot = (t.to(hidden_states.device) for t in torch.where(routing == expert))
+        tensors = store._read(prefix, expert)
+        if store.quantized is not None:
+            banks = [tensor[None] for tensor in tensors]
+            rhs = torch.zeros_like(token)
+            output = _quantized_mlp(module, store, hidden_states[token], banks, rhs) * top_k_weights[token, slot, None]
+            result.index_add_(0, token, output.to(result.dtype))
+        else:
+            _run_expert(module, result, hidden_states, top_k_weights, token, slot, *tensors)
+    return result
+
+
+def _checkpoint_prefix(reader, name, quantized_reader=None):
+    """Find a unique packed, per-expert, or quantized checkpoint prefix, allowing a different model root."""
+    suffix = name.split(".", 1)[1] if name.startswith("model.") else name
+    found = set()
+    for key in reader.tensors:
+        if key.endswith(f"{suffix}.gate_up_proj"):
+            found.add(key.removesuffix(".gate_up_proj"))
+        elif key.endswith(f"{suffix}.0.gate_proj.weight"):
+            found.add(key.removesuffix(".0.gate_proj.weight"))
+    if not found and quantized_reader is not None:
+        found = {key.removesuffix(".experts") for key in quantized_reader.tensors if key.endswith(f"{suffix}.experts")}
+    if len(found) != 1:
+        raise ValueError(f"Cannot find the checkpoint experts of {name} (candidates: {sorted(found)})")
+    return found.pop()
+
+
+def split_experts_device_map(device_map, experts_offload):
+    """Turn ``device_map`` entries that send the experts to ``"disk"`` into expert streaming.
+
+    An entry whose key names experts modules (``"experts.*"``, ``"model.layers.*.mlp.experts"``, ...) and whose value is
+    ``"disk"`` streams every experts module from disk through the device cache, instead of Accelerate's disk offload
+    (which would read every expert of a layer at each forward). The rest of the map must place the model on one device.
+    Returns the device map without those entries and the ``experts_offload`` options (``cache_bytes`` required)."""
+    if not isinstance(device_map, dict):
+        return device_map, experts_offload
+    keys = [key for key, value in device_map.items() if value == "disk" and "experts" in str(key)]
+    if not keys:
+        return device_map, experts_offload
+    rest = {key: value for key, value in device_map.items() if key not in keys}
+    if not rest:
+        raise ValueError('Map the rest of the model to a device too, e.g. {"": "mps", "experts.*": "disk"}')
+    if "disk" in rest.values():
+        raise ValueError("Streamed experts cannot be combined with disk offload of other modules")
+    options = dict(experts_offload or {})
+    if "cache_bytes" not in options:
+        raise ValueError(
+            'Experts on "disk" stream through a device cache: pass its size, e.g. experts_offload={"cache_bytes": 8 << 30}'
+        )
+    return rest, options
+
+
+_OPTIONS = {
+    "cache_bytes",
+    "quantized_experts",
+    "cache_policy",
+    "io_threads",
+    "prefetch_depth",
+    "preload",
+    "bypass_page_cache",
+    "experts_implementation",
+    "token_chunk_size",
+    "predict_max_routes",
+    "mps_commit",
+    "host_cache_bytes",
+    "host_preload",
+    "prefill_readahead_routes",
+    "prefill_install_ahead",
+    "cold_prefill_routes",
+    "expert_source",
+    "expert_compute",
+    "preload_profile",
+}
+
+
+def prepare_experts_offload(model, checkpoint_files, *, dtype, device, options):
+    """Replace meta expert banks with a shared disk-backed cache and return checkpoint keys to skip.
+
+    Options:
+        cache_bytes (`int`, required): device memory for cached experts.
+        quantized_experts (`str` or `os.PathLike`, *optional*): a directory written by :func:`quantize_experts`; the
+            experts are then read and cached quantized and run by `gather_qmm`. The checkpoint may omit its own experts.
+        cache_policy (`str`, defaults to `"lru"`): `"lru"` or `"lfu"`.
+        io_threads (`int`, defaults to 4): threads reading experts; 0 reads synchronously and disables prefetch.
+        prefetch_depth (`int`, defaults to 2): how many layers ahead to guess and read experts (0 disables).
+        preload (`bool`, defaults to `False`): read every expert at load time; the cache must hold them all.
+        bypass_page_cache (`bool`, defaults to `False`): keep streamed bytes out of the OS page cache.
+        experts_implementation (`str`, defaults to `"grouped_mm"`): kernel for cached BF16 experts, or `"batched_mm"`.
+        token_chunk_size (`int`, defaults to 256): row chunk for experts that run uncached.
+        predict_max_routes (`int`, defaults to 64): prefetch only for calls with at most this many routes (decode).
+        prefill_readahead_routes (`int`, defaults to 4096): calls with at least this many routes (long prompts) also
+            start reading every expert of the next MoE layer, which such a prompt nearly all uses, so storage keeps
+            reading while the device computes. 0 disables.
+        host_cache_bytes (`int`, defaults to 0): host memory (pinned on accelerators) kept as a tier of quantized
+            experts between storage and the device cache; for discrete GPUs with more host than device memory.
+        host_preload (`bool`, defaults to `False`): read every expert into the host tier at load time.
+        prefill_install_ahead (`bool`, defaults to `False`): with `prefill_readahead_routes`, also install the next
+            layer's finished reads behind this layer's compute.
+        cold_prefill_routes (`int`, defaults to 0): calls with at least this many routes (long prompts) do not refresh
+            the LRU order on hits, and the experts they install move to the cold end once the layer is done, so a long
+            prompt does not evict the experts decoding keeps using. 0 disables. LRU only.
+        mps_commit (`bool`, defaults to `True`): on MPS, commit the queued work after each decoder-layer sub-block and
+            after the experts, so the GPU runs it while the host encodes what follows.
+        expert_source (*optional*): read experts from an external store instead of the checkpoint (see
+            :class:`ExpertStore`); needs `expert_compute`. Layers may differ in layout: each layout gets its own pool and
+            every layer the same number of slots.
+        preload_profile (`dict` or `str`, *optional*): a routing profile (``{prefix: {expert: count}}``, or a JSON file of
+            one, as returned by `model.experts_offload_store.routing_profile()`); the first forward fills each layer's
+            slots with its most used experts, so decoding starts from a warm cache.
+        expert_compute (`Callable`, *optional*): computes the experts output over the pools (see
+            :func:`experts_offload_forward`); required with `expert_source`, where the experts' format is the source's.
+    """
+    options = dict(options)
+    unknown = set(options) - _OPTIONS
+    if unknown or "cache_bytes" not in options:
+        raise ValueError(f"experts_offload needs `cache_bytes`; unknown options: {sorted(unknown)}")
+    if not checkpoint_files or not all(str(f).endswith(".safetensors") for f in checkpoint_files):
+        raise ValueError("experts_offload reads experts from safetensors checkpoints only")
+    if dtype not in _DTYPES.values():
+        raise ValueError("experts_offload needs a floating dtype (float32/float16/bfloat16)")
+    source = options.get("expert_source")
+    if (source is None) != (options.get("expert_compute") is None):
+        raise ValueError("expert_source and expert_compute go together")
+    if options.get("experts_implementation", "grouped_mm") not in ("grouped_mm", "batched_mm"):
+        raise ValueError("experts_offload supports experts_implementation 'grouped_mm' or 'batched_mm'")
+    ExpertsInterface.register("experts_offload", experts_offload_forward)
+    device = torch.empty(0, device=device).device
+    reader = SafetensorsReader(Path(checkpoint_files[0]).parent, options.get("bypass_page_cache", False))
+    store = ExpertStore(
+        reader,
+        options["cache_bytes"],
+        device,
+        dtype,
+        policy=options.get("cache_policy", "lru"),
+        io_threads=options.get("io_threads", 4),
+        quantized=options.get("quantized_experts"),
+        source=source,
+    )
+    store.compute = options.get("experts_implementation", "grouped_mm")
+    store.expert_compute = options.get("expert_compute")
+    store.preload_profile = options.get("preload_profile")
+    experts = [
+        (name, module)
+        for name, module in model.named_modules()
+        if hasattr(module, "_apply_gate") and isinstance(getattr(module, "gate_up_proj", None), torch.nn.Parameter)
+    ]
+    if not experts:
+        raise ValueError(f"{type(model).__name__} has no experts module that can be offloaded")
+    consumed = set()
+    for name, module in experts:
+        if module.has_bias or module.is_transposed or not module.is_concatenated or module.has_post_expert_norm:
+            raise ValueError(
+                f"experts_offload supports gated, concatenated, untransposed experts without bias: {name}"
+            )
+        if source is not None:
+            prefix = source.prefix(name) if hasattr(source, "prefix") else name
+        else:
+            prefix = _checkpoint_prefix(reader, name, store.quantized[0] if store.quantized else None)
+        consumed.update(store.register(prefix, module))
+        module.register_parameter("gate_up_proj", None)
+        module.register_parameter("down_proj", None)
+        module._is_hf_initialized = True  # Banks remain in the checkpoint.
+        module.config._experts_implementation = "experts_offload"
+        module.offload_store = store
+        module.offload_prefix = prefix
+        module.offload_token_chunk_size = options.get("token_chunk_size", 256)
+        module.offload_predict_max_routes = options.get("predict_max_routes", 64)
+        module.offload_next = module.offload_after_next = None
+        module.offload_readahead_routes = options.get("prefill_readahead_routes", 4096)
+        module.offload_prefill_install_ahead = options.get("prefill_install_ahead", False)
+        module.offload_cold_prefill_routes = options.get("cold_prefill_routes", 0)
+    store.allocate(materialize=False)
+    depth = options.get("prefetch_depth", 2)
+    routers = []
+    for name, module in experts:
+        router = getattr(model.get_submodule(name.rpartition(".")[0]), "gate", None)
+        usable = router is not None and hasattr(router, "weight") and hasattr(router, "top_k")
+        routers.append((module.offload_prefix, router) if usable else None)
+    for index, (_, module) in enumerate(experts):
+        module.offload_next_prefix = experts[index + 1][1].offload_prefix if index + 1 < len(experts) else None
+        if depth >= 1 and index + 1 < len(routers):
+            module.offload_next = routers[index + 1]
+        if depth >= 2 and index + 2 < len(routers):
+            module.offload_after_next = routers[index + 2]
+    if options.get("mps_commit", True):
+        store.commit = True
+
+        def commit_hook(module, args, output):
+            _commit_mps(store)  # returns nothing: a forward hook's return value would replace the output
+
+        for name, _ in experts:
+            layer = model.get_submodule(name.rpartition(".")[0].rpartition(".")[0])
+            for child in layer.children():
+                child.register_forward_hook(commit_hook)
+    if options.get("host_cache_bytes", 0):
+        store.allocate_host_cache(options["host_cache_bytes"])
+    if options.get("host_preload", False):
+        store.preload_host()
+    if options.get("preload", False):
+        store.preload()
+    model.experts_offload_store = store
+    return consumed
+
+
+def quantize_experts(checkpoint, output, *, bits=4, group_size=64, device="cpu"):
+    """Write MLX affine experts as uint8 rows: gate_up then down, each with weights, scales and biases.
+
+    Uses the Metal kernel on MPS, otherwise PyTorch. Peak memory is one experts module."""
+    from safetensors.torch import save_file
+
+    from .metal_quantization import _affine_quantize_tensor
+
+    device = torch.device(device)
+    quantize = lambda weight: _affine_quantize_tensor(weight, group_size, bits)  # noqa: E731
+    quantizer = "pytorch-reference"
+    # TODO(portability): Metal quantizer on MPS only; other accelerators use the PyTorch reference.
+    if device.type == "mps":
+        from .hub_kernels import get_kernel
+
+        kernel = get_kernel(_MLX_QUANTIZATION_KERNEL[0], version=_MLX_QUANTIZATION_KERNEL[1])
+        quantize = lambda weight: kernel.quantize(weight, group_size=group_size, bits=bits)  # noqa: E731
+        quantizer = "mlx-quantization-metal-kernels"
+    reader = SafetensorsReader(checkpoint)
+    output = Path(output)
+    output.mkdir(parents=True, exist_ok=True)
+    packed = sorted(name.removesuffix(".gate_up_proj") for name in reader.tensors if name.endswith(".gate_up_proj"))
+    per_expert = sorted(
+        name.removesuffix(".0.gate_proj.weight") for name in reader.tensors if name.endswith(".0.gate_proj.weight")
+    )
+    weight_map = {}
+    for index, prefix in enumerate(packed + per_expert):
+        if prefix in packed:
+            experts = reader.shape(f"{prefix}.gate_up_proj")[0]
+        else:
+            experts = sum(
+                1 for name in reader.tensors if re.fullmatch(rf"{re.escape(prefix)}\.\d+\.gate_proj\.weight", name)
+            )
+
+        blobs = []
+        for expert in range(experts):
+            parts = []
+            for weight in reader.read_expert(prefix, expert):
+                weight_q, scales, biases = quantize(weight.to(device))
+                parts += [weight_q, scales.to(weight.dtype), biases.to(weight.dtype)]
+            blobs.append(torch.cat([part.cpu().contiguous().view(torch.uint8).reshape(-1) for part in parts]))
+        filename = f"experts-{index:05d}.safetensors"
+        save_file({f"{prefix}.experts": torch.stack(blobs)}, output / filename, {"format": "pt"})
+        weight_map[f"{prefix}.experts"] = filename
+        logger.info(f"quantized {prefix}: {experts} experts")
+    (output / "model.safetensors.index.json").write_text(json.dumps({"metadata": {}, "weight_map": weight_map}))
+    info = {"format": _QUANTIZED_FORMAT, "bits": bits, "group_size": group_size, "quantizer": quantizer}
+    (output / "quantization.json").write_text(json.dumps(info, indent=2))
+    return output
diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py
index 99aef4a573..0cbc5cef9e 100644
--- a/src/transformers/modeling_utils.py
+++ b/src/transformers/modeling_utils.py
@@ -74,6 +74,7 @@ from .integrations.accelerate import (
 )
 from .integrations.deepspeed import _load_state_dict_into_zero3_model, prevent_arange_dtype_change
 from .integrations.eager_paged import eager_paged_attention_forward
+from .integrations.experts_offload import prepare_experts_offload, split_experts_device_map
 from .integrations.finegrained_fp8 import ALL_FP8_EXPERTS_FUNCTIONS
 from .integrations.flash_attention import flash_attention_forward
 from .integrations.flex_attention import flex_attention_forward
@@ -187,6 +188,8 @@ class LoadStateDictConfig:
     weights_only: bool = True
     weight_mapping: list[WeightConverter | WeightRenaming] | None = None
     disable_mmap: bool | None = None
+    # Checkpoint keys loaded by another component.
+    skip_keys: set[str] = field(default_factory=set)
 
     @property
     def is_quantized(self) -> bool:
@@ -4099,6 +4102,11 @@ class PreTrainedModel(
                 This should only be used as an inference optimization, as it can slightly change outputs. If omitted,
                 `from_pretrained()` falls back to `config.fusion_config` when available. Refer to the fusion mapping
                 guide in `docs/source/en/fusion_mapping.md` for more details.
+            experts_offload (`dict`, *optional*):
+                Options for MoE experts streamed from disk through a device cache, for inference: either with a
+                `device_map` entry mapping the experts to `"disk"` (`{"": "mps", "experts.*": "disk"}`), or alone on a
+                single-device map. Requires `cache_bytes`; see
+                `transformers.integrations.experts_offload.prepare_experts_offload` for the options.
             key_mapping (`dict[str, str], *optional*):
                 A potential mapping of the weight names if using a model on the Hub which is compatible to a Transformers
                 architecture, but was not converted accordingly.
@@ -4166,6 +4174,7 @@ class PreTrainedModel(
         use_kernels = kwargs.pop("use_kernels", False)
         kernel_config = kwargs.pop("kernel_config", None)
         key_mapping = kwargs.pop("key_mapping", None)
+        experts_offload = kwargs.pop("experts_offload", None)
 
         # Not used anymore -- remove them from the kwargs
         for name in ["mirror", "_fast_init", "low_cpu_mem_usage", "from_tf", "from_flax", "offload_state_dict"]:
@@ -4262,6 +4271,8 @@ class PreTrainedModel(
                 cache_dir=cache_dir,
             )
             download_kwargs["revision"] = revision
+        # `{"experts.*": "disk"}` entries stream the experts (see `split_experts_device_map`) instead of disk offload.
+        device_map, experts_offload = split_experts_device_map(device_map, experts_offload)
         device_map = check_and_set_device_map(device_map)  # warn, error and fix the device map
 
         user_agent = {"file_type": "model", "framework": "pytorch", "from_auto_class": from_auto_class}
@@ -4394,6 +4405,20 @@ class PreTrainedModel(
         if gguf_file:
             state_dict = hf_quantizer.get_state_dict(checkpoint_files[0], model)
 
+        skip_keys = set()
+        if experts_offload is not None:
+            devices = set(device_map.values()) if isinstance(device_map, dict) else {device_map or "cpu"}
+            # Metal quantization only converts nn.Linear modules, never the expert banks this removes.
+            dense_quantizer = hf_quantizer is not None and hf_quantizer.quantization_config.quant_method == "metal"
+            if len(devices) != 1 or gguf_file or (hf_quantizer is not None and not dense_quantizer) or state_dict:
+                raise ValueError(
+                    "experts_offload needs a single-device device_map and a safetensors checkpoint, unquantized or "
+                    "with MetalConfig for its dense layers"
+                )
+            skip_keys = prepare_experts_offload(
+                model, checkpoint_files, dtype=dtype, device=devices.pop(), options=experts_offload
+            )
+
         # Create the dtype_plan to potentially use the `keep_in_fp32` flags (this needs to be called on the already
         # instantiated model, as the flags can be modified by instances sometimes)
         dtype_plan = model._get_dtype_plan(dtype)
@@ -4425,6 +4450,7 @@ class PreTrainedModel(
             use_safetensors=use_safetensors,
             download_kwargs=download_kwargs,
             disable_mmap=disable_mmap,
+            skip_keys=skip_keys,
         )
         if distributed_checkpoint_dir is not None:
             loading_info = model._load_and_finalize_distributed_pretrained_model(
@@ -4551,6 +4577,8 @@ class PreTrainedModel(
                 merged_state_dict = {}
                 for file in checkpoint_files:
                     if load_config.disable_mmap or _is_on_hf_mount(file):
+                        if load_config.skip_keys:
+                            raise ValueError("experts_offload does not support disable_mmap or HF mounts")
                         with open(file, "rb") as _fh:
                             merged_state_dict.update(_safe_load_bytes(_fh.read()))
                         continue
@@ -4570,7 +4598,8 @@ class PreTrainedModel(
                     file_pointer = safe_open(file, framework="pt", device=device, backend=backend)
                     all_pointer.add(file_pointer)
                     for k in file_pointer.keys():
-                        merged_state_dict[k] = file_pointer.get_slice(k)  # don't materialize yet
+                        if k not in load_config.skip_keys:
+                            merged_state_dict[k] = file_pointer.get_slice(k)  # don't materialize yet
             # Checkpoints are .bin
             elif checkpoint_files is not None:
                 merged_state_dict = {}
diff --git a/tests/integrations/test_experts_offload.py b/tests/integrations/test_experts_offload.py
new file mode 100644
index 0000000000..ec02004e43
--- /dev/null
+++ b/tests/integrations/test_experts_offload.py
@@ -0,0 +1,535 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import math
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest import mock
+
+from transformers import is_torch_available
+from transformers.testing_utils import require_torch
+
+
+if is_torch_available():
+    import torch
+    from safetensors.torch import load_file, save_file
+
+    from transformers import OlmoeConfig, OlmoeForCausalLM, Qwen3MoeConfig, Qwen3MoeForCausalLM
+    from transformers.integrations.experts_offload import (
+        ExpertStore,
+        SafetensorsReader,
+        _split_expert,
+        quantize_experts,
+    )
+    from transformers.integrations.metal_quantization import _affine_dequantize_tensor
+
+
+HIDDEN, INTERMEDIATE, EXPERTS = 16, 8, 4
+EXPERT_BYTES = 3 * HIDDEN * INTERMEDIATE * 4  # float32
+IDS = [[1, 7, 3, 9, 2, 11, 5]]
+
+
+def tiny_qwen3_moe(seed=0):
+    torch.manual_seed(seed)
+    config = Qwen3MoeConfig(
+        vocab_size=64,
+        hidden_size=HIDDEN,
+        intermediate_size=24,
+        moe_intermediate_size=INTERMEDIATE,
+        num_hidden_layers=2,
+        num_attention_heads=2,
+        num_key_value_heads=1,
+        head_dim=8,
+        num_experts=EXPERTS,
+        num_experts_per_tok=2,
+    )
+    config._attn_implementation = "eager"
+    return Qwen3MoeForCausalLM(config).eval()
+
+
+def greedy(model, ids, tokens=6):
+    with torch.no_grad():
+        return model.generate(torch.tensor(ids), max_new_tokens=tokens, do_sample=False, pad_token_id=0)
+
+
+class PaddedSource:
+    """An external expert store over a model's experts: float32 gate_up and down, plus (layer 1) a padding tensor,
+    so the two layers differ in size and get one pool each."""
+
+    def __init__(self, model):
+        self.experts = {}
+        for index, layer in enumerate(model.model.layers):
+            experts = layer.mlp.experts
+            self.experts[f"source.{index}"] = (experts.gate_up_proj.detach(), experts.down_proj.detach(), index)
+        self.prefixes = list(self.experts)
+        self.read_bytes = self.read_calls = 0
+
+    def prefix(self, module_name):
+        return f"source.{module_name.split('.')[2]}"
+
+    def layout(self, prefix):
+        gate_up, down, index = self.experts[prefix]
+        layout = [(tuple(gate_up.shape[1:]), torch.float32), (tuple(down.shape[1:]), torch.float32)]
+        return layout + ([((16,), torch.uint8)] if index else [])
+
+    def expert_bytes(self, prefix):
+        return sum(math.prod(shape) * dtype.itemsize for shape, dtype in self.layout(prefix))
+
+    def read(self, prefix, expert, out):
+        gate_up, down, _ = self.experts[prefix]
+        blob = torch.cat([gate_up[expert].reshape(-1).view(torch.uint8), down[expert].reshape(-1).view(torch.uint8)])
+        out[: blob.numel()] = blob
+        out[blob.numel() :] = 7
+        self.read_bytes += out.numel()
+        self.read_calls += 1
+
+
+def reference_compute(module, store, hidden_states, top_k_index, top_k_weights, routing, experts, pinned):
+    """`expert_compute` for :class:`PaddedSource`: every route through `store.expert_tensors`."""
+    result = torch.zeros_like(hidden_states)
+    prefix = module.offload_prefix
+    for expert in experts:
+        gate_up, down, *padding = store.expert_tensors(prefix, expert)
+        if padding:
+            assert bool((padding[0] == 7).all())
+        token, slot = (t.to(hidden_states.device) for t in torch.where(routing == expert))
+        activated = module._apply_gate(torch.nn.functional.linear(hidden_states[token], gate_up))
+        output = torch.nn.functional.linear(activated, down) * top_k_weights[token, slot, None]
+        result.index_add_(0, token, output.to(result.dtype))
+    return result
+
+
+# TODO(portability): run on Linux/macOS/Windows and parameterize available devices; current loads use CPU.
+@require_torch
+class ExpertsOffloadTest(unittest.TestCase):
+    def load(self, path, **options):
+        return Qwen3MoeForCausalLM.from_pretrained(
+            path, attn_implementation="eager", experts_offload={"cache_bytes": 3 * EXPERT_BYTES, **options}
+        )
+
+    def test_loads_without_expert_banks_and_matches_eager(self):
+        reference = tiny_qwen3_moe()
+        with tempfile.TemporaryDirectory() as directory:
+            reference.save_pretrained(directory)
+            model, info = Qwen3MoeForCausalLM.from_pretrained(
+                directory,
+                attn_implementation="eager",
+                experts_offload={"cache_bytes": 3 * EXPERT_BYTES},
+                output_loading_info=True,
+            )
+            self.assertEqual((info["missing_keys"], info["unexpected_keys"]), (set(), set()))
+            experts = model.model.layers[0].mlp.experts
+            self.assertIsNone(experts.gate_up_proj)
+            self.assertEqual(experts.config._experts_implementation, "experts_offload")
+            self.assertFalse(any(name.endswith(("gate_up_proj", "down_proj")) for name in model.state_dict()))
+            self.assertIsNone(model.experts_offload_store.pools[0].banks)  # device pools are taken at the first use
+            torch.testing.assert_close(greedy(model, IDS), greedy(reference, IDS))
+            self.assertGreater(model.experts_offload_store.misses, 0)
+
+    def test_every_cache_size_and_kernel_matches_eager(self):
+        reference = tiny_qwen3_moe(1)
+        expected = greedy(reference, IDS)
+        with tempfile.TemporaryDirectory() as directory:
+            reference.save_pretrained(directory)
+            # Cover uncached, partial-cache and fully resident execution.
+            for slots in (0, 1, 3, 8):
+                for implementation in ("grouped_mm", "batched_mm"):
+                    with self.subTest(slots=slots, implementation=implementation):
+                        model = self.load(
+                            directory, cache_bytes=slots * EXPERT_BYTES, experts_implementation=implementation
+                        )
+                        store = model.experts_offload_store
+                        self.assertEqual(store.num_slots, slots)
+                        torch.testing.assert_close(greedy(model, IDS), expected)
+                        self.assertLessEqual(store.resident_bytes, slots * EXPERT_BYTES)
+                        self.assertEqual(store.all_resident(), slots == 8)
+
+    def test_reads_hits_evictions_and_zero_budget(self):
+        reference = tiny_qwen3_moe(2)
+        with tempfile.TemporaryDirectory() as directory:
+            reference.save_pretrained(directory)
+            for budget in (0, EXPERT_BYTES):
+                with self.subTest(budget=budget):
+                    # No I/O threads: no reads ahead, so the counts below are exact.
+                    model = self.load(directory, cache_bytes=budget, io_threads=0)
+                    store = model.experts_offload_store
+                    module, eager = model.model.layers[0].mlp.experts, reference.model.layers[0].mlp.experts
+                    hidden, weights = torch.randn(2, HIDDEN), torch.ones(2, 1)
+                    ids = torch.zeros(2, 1, dtype=torch.long)
+                    with torch.inference_mode():
+                        torch.testing.assert_close(module(hidden, ids, weights), eager(hidden, ids, weights))
+                        self.assertEqual(store.stats()["read_bytes"], EXPERT_BYTES)
+                        module(hidden, ids, weights)
+                        self.assertEqual(store.hits, int(budget > 0))
+                        module(hidden, ids + 1, weights)
+                        torch.testing.assert_close(module(hidden, ids, weights), eager(hidden, ids, weights))
+                    self.assertEqual(store.misses, 3 if budget else 4)
+                    self.assertLessEqual(store.resident_bytes, budget)
+
+    def test_prefetch_threads_and_depths_match_eager(self):
+        reference = tiny_qwen3_moe(3)
+        expected = greedy(reference, IDS, tokens=8)
+        with tempfile.TemporaryDirectory() as directory:
+            reference.save_pretrained(directory)
+            for io_threads, depth in ((0, 0), (1, 1), (4, 1), (4, 2)):
+                with self.subTest(io_threads=io_threads, depth=depth):
+                    model = self.load(directory, io_threads=io_threads, prefetch_depth=depth)
+                    layers = model.model.layers
+                    if depth:
+                        self.assertIs(layers[0].mlp.experts.offload_next[1], layers[1].mlp.gate)
+                    self.assertIsNone(layers[1].mlp.experts.offload_next)
+                    torch.testing.assert_close(greedy(model, IDS, tokens=8), expected)
+                    store = model.experts_offload_store
+                    self.assertEqual(store.pending, {})
+                    if io_threads:
+                        self.assertGreater(store.prefetch_used, 0)
+                    else:
+                        self.assertEqual(store.prefetched, 0)
+
+    def test_prefill_readahead_matches_eager(self):
+        reference = tiny_qwen3_moe(3)
+        expected = greedy(reference, IDS, tokens=8)
+        with tempfile.TemporaryDirectory() as directory:
+            reference.save_pretrained(directory)
+            # The 7-token prompt has 14 routes per layer, decode steps 2: a threshold of 8 reads ahead in prefill only.
+            for routes in (0, 8):
+                with self.subTest(prefill_readahead_routes=routes):
+                    model = self.load(directory, io_threads=2, prefetch_depth=0, prefill_readahead_routes=routes)
+                    layers = model.model.layers
+                    next_prefix = layers[1].mlp.experts.offload_prefix
+                    self.assertEqual(layers[0].mlp.experts.offload_next_prefix, next_prefix)
+                    self.assertIsNone(layers[1].mlp.experts.offload_next_prefix)
+                    store = model.experts_offload_store
+                    calls = []
+                    prefetch = store.prefetch
+
+                    def spy(prefix, experts):
+                        calls.append((prefix, experts))
+                        return prefetch(prefix, experts)
+
+                    with mock.patch.object(store, "prefetch", side_effect=spy):
+                        torch.testing.assert_close(greedy(model, IDS, tokens=8), expected)
+                    self.assertEqual(store.pending, {})
+                    readahead = [call for call in calls if call == (next_prefix, range(EXPERTS))]
+                    # Once per forward of layer 0 with enough routes: the prompt, not the decode steps.
+                    self.assertEqual(len(readahead), 1 if routes else 0)
+
+    def test_mps_commit_hooks(self):
+        reference = tiny_qwen3_moe(3)
+        expected = greedy(reference, IDS, tokens=8)
+        with tempfile.TemporaryDirectory() as directory:
+            reference.save_pretrained(directory)
+            for enabled in (True, False):
+                with self.subTest(mps_commit=enabled):
+                    with mock.patch("transformers.integrations.experts_offload._commit_mps") as commit:
+                        model = self.load(directory, mps_commit=enabled)
+                        torch.testing.assert_close(greedy(model, IDS, tokens=8), expected)
+                    self.assertEqual(commit.called, enabled)
+
+    def test_preload_and_fully_resident_layers_skip_the_host_copy(self):
+        reference = tiny_qwen3_moe(4)
+        with tempfile.TemporaryDirectory() as directory:
+            reference.save_pretrained(directory)
+            with self.assertRaises(ValueError):
+                self.load(directory, preload=True)  # 3 slots cannot hold 8 experts
+            model = self.load(directory, cache_bytes=8 * EXPERT_BYTES, preload=True)
+            store = model.experts_offload_store
+            self.assertTrue(store.all_resident())
+            read_bytes = store.stats()["read_bytes"]
+            original = torch.Tensor.cpu
+            with mock.patch.object(torch.Tensor, "cpu", autospec=True, side_effect=original) as cpu:
+                torch.testing.assert_close(greedy(model, IDS), greedy(reference, IDS))
+            self.assertFalse(any(call.args[0].dtype == torch.long for call in cpu.call_args_list))
+            self.assertEqual((store.misses, store.stats()["read_bytes"]), (0, read_bytes))
+
+    def test_per_expert_checkpoint_layout(self):
+        torch.manual_seed(5)
+        config = OlmoeConfig(
+            vocab_size=64,
+            hidden_size=HIDDEN,
+            intermediate_size=INTERMEDIATE,
+            num_hidden_layers=2,
+            num_attention_heads=2,
+            num_key_value_heads=2,
+            num_experts=EXPERTS,
+            num_experts_per_tok=2,
+        )
+        config._attn_implementation = "eager"
+        reference = OlmoeForCausalLM(config).eval()
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory)
+            reference.save_pretrained(path)
+            # Match the Hub checkpoint: one tensor per expert and projection.
+            tensors = load_file(path / "model.safetensors")
+            for name in [name for name in tensors if name.endswith("experts.gate_up_proj")]:
+                prefix = name.removesuffix(".gate_up_proj")
+                gate_up, down = tensors.pop(name), tensors.pop(f"{prefix}.down_proj")
+                for expert in range(EXPERTS):
+                    gate, up = gate_up[expert].chunk(2)
+                    tensors[f"{prefix}.{expert}.gate_proj.weight"] = gate.contiguous()
+                    tensors[f"{prefix}.{expert}.up_proj.weight"] = up.contiguous()
+                    tensors[f"{prefix}.{expert}.down_proj.weight"] = down[expert].contiguous()
+            save_file(tensors, path / "model.safetensors", {"format": "pt"})
+            model = OlmoeForCausalLM.from_pretrained(
+                path, attn_implementation="eager", experts_offload={"cache_bytes": 3 * EXPERT_BYTES}
+            )
+            torch.testing.assert_close(greedy(model, IDS), greedy(reference, IDS))
+
+    def test_quantized_experts_match_the_dequantized_model(self):
+        reference = tiny_qwen3_moe(6)
+        # 12 tokens x top-2 over 4 experts: prefill takes the sorted path, decode the pool path.
+        ids = [[1, 7, 3, 9, 2, 11, 5, 13, 4, 8, 6, 10]]
+        group = 8
+        shapes = [
+            ((2 * INTERMEDIATE, HIDDEN * 8 // 32), torch.uint32),
+            ((2 * INTERMEDIATE, HIDDEN // group), torch.float32),
+            ((2 * INTERMEDIATE, HIDDEN // group), torch.float32),
+            ((HIDDEN, INTERMEDIATE * 8 // 32), torch.uint32),
+            ((HIDDEN, INTERMEDIATE // group), torch.float32),
+            ((HIDDEN, INTERMEDIATE // group), torch.float32),
+        ]
+        with tempfile.TemporaryDirectory() as directory:
+            path, quantized = Path(directory) / "model", Path(directory) / "experts-q8"
+            reference.save_pretrained(path)
+            quantize_experts(path, quantized, bits=8, group_size=group)
+            # The reference runs the dequantized experts.
+            with torch.no_grad():
+                for index, layer in enumerate(reference.model.layers):
+                    blobs = load_file(quantized / f"experts-{index:05d}.safetensors")
+                    blobs = blobs[f"model.layers.{index}.mlp.experts.experts"]
+                    parts = [_split_expert(blobs[expert], shapes) for expert in range(EXPERTS)]
+                    for offset, projection in ((0, "gate_up_proj"), (3, "down_proj")):
+                        dequantized = [_affine_dequantize_tensor(*p[offset : offset + 3], group, 8) for p in parts]
+                        getattr(layer.mlp.experts, projection).copy_(torch.stack(dequantized))
+            expected = greedy(reference, ids)
+            for cache_bytes in (1000, 3000, 8000):
+                with self.subTest(cache_bytes=cache_bytes):
+                    model = self.load(path, cache_bytes=cache_bytes, quantized_experts=quantized)
+                    torch.testing.assert_close(greedy(model, ids), expected)
+            # The quantized experts can be the only copy: drop the BF16 experts from the checkpoint.
+            tensors = {k: v for k, v in load_file(path / "model.safetensors").items() if ".mlp.experts." not in k}
+            save_file(tensors, path / "model.safetensors", {"format": "pt"})
+            model = self.load(path, cache_bytes=3000, quantized_experts=quantized)
+            torch.testing.assert_close(greedy(model, ids), expected)
+
+    def test_host_tier(self):
+        reference = tiny_qwen3_moe(6)
+        ids = [[1, 7, 3, 9, 2, 11, 5, 13, 4, 8, 6, 10]]
+        with tempfile.TemporaryDirectory() as directory:
+            path, quantized = Path(directory) / "model", Path(directory) / "experts-q8"
+            reference.save_pretrained(path)
+            quantize_experts(path, quantized, bits=8, group_size=8)
+            expected = greedy(self.load(path, cache_bytes=3000, quantized_experts=quantized), ids)
+            slot_bytes = self.load(
+                path, cache_bytes=3000, quantized_experts=quantized
+            ).experts_offload_store.slot_bytes
+            total = 2 * EXPERTS  # two layers
+            for host_experts, preload, io_threads in (
+                (total, False, 2),
+                (3, False, 2),
+                (total, True, 0),
+                (total, True, 4),
+            ):
+                with self.subTest(host_experts=host_experts, preload=preload, io_threads=io_threads):
+                    model = self.load(
+                        path, cache_bytes=3000, quantized_experts=quantized, io_threads=io_threads,
+                        host_cache_bytes=host_experts * slot_bytes, host_preload=preload,
+                    )  # fmt: skip
+                    store = model.experts_offload_store
+                    self.assertEqual(store.host_slots, host_experts)
+                    reads = store.stats()["read_calls"]
+                    self.assertEqual(reads, total if preload else 0)
+                    torch.testing.assert_close(greedy(model, ids), expected)
+                    torch.testing.assert_close(greedy(model, ids), expected)
+                    if host_experts == total:
+                        # Every expert was read at most once: device-cache misses are served from host memory.
+                        self.assertEqual(store.stats()["read_calls"], total)
+                        self.assertGreater(store.host_hits, 0)
+                    store.synchronize()  # rows of the last copies are released at the next synchronization
+                    self.assertEqual(store.host_busy, set())  # every handed-out row was released
+            with self.assertRaises(ValueError):
+                self.load(path, cache_bytes=3000, host_cache_bytes=1 << 20)  # needs quantized experts
+
+    def test_device_map_sends_experts_to_disk(self):
+        reference = tiny_qwen3_moe(10)
+        expected = greedy(reference, IDS)
+        with tempfile.TemporaryDirectory() as directory:
+            reference.save_pretrained(directory)
+            for key in ("experts.*", "model.layers.*.mlp.experts"):
+                with self.subTest(key=key):
+                    model = Qwen3MoeForCausalLM.from_pretrained(
+                        directory,
+                        attn_implementation="eager",
+                        device_map={"": "cpu", key: "disk"},
+                        experts_offload={"cache_bytes": 3 * EXPERT_BYTES},
+                    )
+                    experts = model.model.layers[0].mlp.experts
+                    self.assertIsNone(experts.gate_up_proj)
+                    self.assertEqual(experts.config._experts_implementation, "experts_offload")
+                    torch.testing.assert_close(greedy(model, IDS), expected)
+                    self.assertGreater(model.experts_offload_store.misses, 0)
+            for device_map, options in (
+                ({"experts.*": "disk"}, {"cache_bytes": EXPERT_BYTES}),  # nothing on a device
+                ({"": "cpu", "experts.*": "disk"}, None),  # no cache size
+            ):
+                with self.subTest(device_map=device_map), self.assertRaises(ValueError):
+                    Qwen3MoeForCausalLM.from_pretrained(directory, device_map=device_map, experts_offload=options)
+
+    def test_routing_profile_preload(self):
+        reference = tiny_qwen3_moe(11)
+        expected = greedy(reference, IDS)
+        with tempfile.TemporaryDirectory() as directory:
+            reference.save_pretrained(directory)
+            model = self.load(directory, cache_bytes=4 * EXPERT_BYTES, io_threads=0)
+            greedy(model, IDS)
+            profile = model.experts_offload_store.routing_profile()
+            self.assertEqual(
+                sum(map(sum, (p.values() for p in profile.values()))), model.experts_offload_store.lookups
+            )
+            path = Path(directory) / "profile.json"
+            path.write_text(json.dumps(profile))
+            for preload in (profile, str(path)):
+                model = self.load(directory, cache_bytes=4 * EXPERT_BYTES, io_threads=2, preload_profile=preload)
+                store = model.experts_offload_store
+                self.assertIsNone(store.pools[0].banks)  # nothing read before the first forward
+                torch.testing.assert_close(greedy(model, IDS), expected)
+                for prefix, counts in profile.items():
+                    top = sorted(counts, key=counts.get, reverse=True)[:2]  # 4 slots, 2 layers
+                    self.assertTrue(all((prefix, expert) in store.cache for expert in top[:1]))
+                self.assertLess(store.misses, model.experts_offload_store.lookups)
+
+    def test_expert_source_with_mixed_layouts(self):
+        reference = tiny_qwen3_moe(8)
+        expected = greedy(reference, IDS, tokens=8)
+        source = PaddedSource(reference)
+        with tempfile.TemporaryDirectory() as directory:
+            reference.save_pretrained(directory)
+            sizes = [source.expert_bytes(prefix) for prefix in source.prefixes]
+            self.assertNotEqual(*sizes)  # one pool per layer
+            for io_threads, depth, slots in ((0, 0, 0), (0, 0, 2), (2, 1, 2), (2, 2, EXPERTS)):
+                with self.subTest(io_threads=io_threads, depth=depth, slots=slots):
+                    model = self.load(
+                        directory, cache_bytes=slots * sum(sizes), io_threads=io_threads, prefetch_depth=depth,
+                        expert_source=source, expert_compute=reference_compute,
+                    )  # fmt: skip
+                    store = model.experts_offload_store
+                    self.assertEqual(len(store.pools), 2)
+                    self.assertEqual([pool.num_slots for pool in store.pools], [slots, slots])
+                    self.assertEqual(model.model.layers[1].mlp.experts.offload_prefix, source.prefixes[1])
+                    torch.testing.assert_close(greedy(model, IDS, tokens=8), expected)
+                    self.assertEqual(store.pending, {})
+                    self.assertGreater(store.stats()["read_calls"], 0)
+                    for pool in store.pools:
+                        self.assertLessEqual(len(pool.cache), pool.num_slots)
+                    if slots == EXPERTS:
+                        self.assertEqual(store.misses, 2 * EXPERTS - store.installed_ahead)
+            with self.assertRaises(ValueError):
+                self.load(directory, expert_source=source)  # needs expert_compute
+
+    def test_cold_prefill_routes_matches_eager(self):
+        reference = tiny_qwen3_moe(9)
+        expected = greedy(reference, IDS, tokens=8)
+        with tempfile.TemporaryDirectory() as directory:
+            reference.save_pretrained(directory)
+            # 14 routes per layer in the 7-token prompt, 2 per decode step.
+            model = self.load(
+                directory, io_threads=2, prefetch_depth=0, cold_prefill_routes=8, prefill_readahead_routes=8,
+                prefill_install_ahead=True,
+            )  # fmt: skip
+            torch.testing.assert_close(greedy(model, IDS, tokens=8), expected)
+            store = model.experts_offload_store
+            self.assertEqual(store.cold_installed, {})
+            self.assertFalse(store.cold)
+
+    def test_inference_only_and_option_validation(self):
+        reference = tiny_qwen3_moe(7)
+        with tempfile.TemporaryDirectory() as directory:
+            reference.save_pretrained(directory)
+            with self.assertRaises(ValueError):
+                self.load(directory, cache_size=1)
+            with self.assertRaises(ValueError):
+                Qwen3MoeForCausalLM.from_pretrained(directory, experts_offload={"io_threads": 2})
+            with self.assertRaises(ValueError):
+                self.load(directory, experts_implementation="eager")
+            model = self.load(directory)
+            with self.assertRaises(RuntimeError):
+                model(torch.tensor(IDS))  # grad enabled
+            model.train()
+            with torch.no_grad(), self.assertRaises(RuntimeError):
+                model(torch.tensor(IDS))
+
+
+@require_torch
+class ExpertStorePolicyTest(unittest.TestCase):
+    def make_store(self, policy, cache_experts, experts=5, hidden=4, intermediate=2):
+        banks = {
+            "layer.gate_up_proj": torch.arange(experts * 2 * intermediate * hidden, dtype=torch.float32).view(
+                experts, 2 * intermediate, hidden
+            ),
+            "layer.down_proj": torch.arange(experts * hidden * intermediate, dtype=torch.float32).view(
+                experts, hidden, intermediate
+            ),
+        }
+        reader = SimpleNamespace(
+            tensors=banks,
+            bypass_page_cache=False,
+            shape=lambda name: tuple(banks[name].shape),
+            read=lambda name, expert=None: banks[name][expert].clone(),
+            read_bytes=0,
+            read_calls=0,
+        )
+        reader.read_expert = lambda prefix, expert: SafetensorsReader.read_expert(reader, prefix, expert)
+        expert_bytes = 3 * intermediate * hidden * 4
+        store = ExpertStore(reader, cache_experts * expert_bytes, "cpu", torch.float32, policy=policy)
+        store.register("layer", SimpleNamespace(num_experts=experts, intermediate_dim=intermediate, hidden_dim=hidden))
+        store.allocate()
+        return store, banks
+
+    def test_lfu_keeps_a_hot_set_where_lru_thrashes(self):
+        # Repeat a decoding cycle longer than the cache.
+        pattern = [0, 1, 2, 3, 4] * 20
+        hits = {}
+        for policy in ("lru", "lfu"):
+            store, banks = self.make_store(policy, cache_experts=3)
+            for expert in pattern:
+                gate_up, down = store.get("layer", expert)
+                torch.testing.assert_close(gate_up, banks["layer.gate_up_proj"][expert])
+                torch.testing.assert_close(down, banks["layer.down_proj"][expert])
+            hits[policy] = store.hits
+            self.assertLessEqual(store.resident_bytes, store.cache_bytes)
+        self.assertEqual(hits["lru"], 0)
+        # LFU keeps 3 of the 5 experts resident after warm-up: about 3 hits per cycle of 5.
+        self.assertGreater(hits["lfu"], 0.5 * len(pattern))
+
+    def test_lfu_follows_a_new_hot_set(self):
+        store, _ = self.make_store("lfu", cache_experts=2, experts=6)
+        store.decay_every = 8
+        for expert in [0, 1] * 20 + [4, 5] * 20:
+            store.get("layer", expert)
+        self.assertEqual({key[1] for key in store.cache}, {4, 5})
+
+    def test_cold_calls_keep_the_decode_set(self):
+        store, _ = self.make_store("lru", cache_experts=3)
+        for expert in (0, 1):
+            store.acquire("layer", expert)
+        store.cold = True  # a long prefill: installs 2, hits 0 without refreshing it
+        store.acquire("layer", 2)
+        store.acquire("layer", 0)
+        store.demote("layer")
+        store.cold = False
+        store.acquire("layer", 3)  # evicts the cold install, not the decode set
+        self.assertEqual({key[1] for key in store.cache}, {0, 1, 3})