Convert to the diffusers modular layout

.gitattributes

@@ -34,3 +34,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 text_encoder/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+processor/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text

audio_scheduler/scheduler_config.json

@@ -0,0 +1,5 @@
+{
+  "_class_name": "MiniMaxH3Scheduler",
+  "_diffusers_version": "0.42.0.dev0",
+  "shift": 12.0
+}

modular_model_index.json

@@ -0,0 +1,129 @@
+{
+  "_class_name": "PH3ImageModularPipeline",
+  "_diffusers_version": "0.42.0.dev0",
+  "_blocks_class_name": "PH3ImageBlocks",
+  "edit_canvas_pixels": 1000000,
+  "sample_sigmas": [
+    1.0,
+    0.994168,
+    0.989138,
+    0.978136,
+    0.951688,
+    0.925372,
+    0.810083,
+    0.528718,
+    0.0
+  ],
+  "text_encoder": [
+    "transformers",
+    "Qwen3VLForConditionalGeneration",
+    {
+      "type_hint": [
+        "transformers",
+        "Qwen3VLForConditionalGeneration"
+      ],
+      "pretrained_model_name_or_path": "PrunaAI/p-h3-image",
+      "subfolder": "text_encoder",
+      "variant": null,
+      "revision": null
+    }
+  ],
+  "text_connector": [
+    "diffusers",
+    "PH3ImageTextConnector",
+    {
+      "type_hint": [
+        "diffusers",
+        "PH3ImageTextConnector"
+      ],
+      "pretrained_model_name_or_path": "PrunaAI/p-h3-image",
+      "subfolder": "text_connector",
+      "variant": null,
+      "revision": null
+    }
+  ],
+  "tokenizer": [
+    "transformers",
+    "Qwen2TokenizerFast",
+    {
+      "type_hint": [
+        "transformers",
+        "Qwen2TokenizerFast"
+      ],
+      "pretrained_model_name_or_path": "PrunaAI/p-h3-image",
+      "subfolder": "tokenizer",
+      "variant": null,
+      "revision": null
+    }
+  ],
+  "processor": [
+    "transformers",
+    "Qwen3VLProcessor",
+    {
+      "type_hint": [
+        "transformers",
+        "Qwen3VLProcessor"
+      ],
+      "pretrained_model_name_or_path": "PrunaAI/p-h3-image",
+      "subfolder": "processor",
+      "variant": null,
+      "revision": null
+    }
+  ],
+  "vae": [
+    "diffusers",
+    "AutoencoderKLPH3Image",
+    {
+      "type_hint": [
+        "diffusers",
+        "AutoencoderKLPH3Image"
+      ],
+      "pretrained_model_name_or_path": "PrunaAI/p-h3-image",
+      "subfolder": "vae",
+      "variant": null,
+      "revision": null
+    }
+  ],
+  "transformer": [
+    "diffusers",
+    "MiniMaxH3Transformer3DModel",
+    {
+      "type_hint": [
+        "diffusers",
+        "MiniMaxH3Transformer3DModel"
+      ],
+      "pretrained_model_name_or_path": "PrunaAI/p-h3-image",
+      "subfolder": "transformer",
+      "variant": null,
+      "revision": null
+    }
+  ],
+  "scheduler": [
+    "diffusers",
+    "MiniMaxH3Scheduler",
+    {
+      "type_hint": [
+        "diffusers",
+        "MiniMaxH3Scheduler"
+      ],
+      "pretrained_model_name_or_path": "PrunaAI/p-h3-image",
+      "subfolder": "scheduler",
+      "variant": null,
+      "revision": null
+    }
+  ],
+  "audio_scheduler": [
+    "diffusers",
+    "MiniMaxH3Scheduler",
+    {
+      "type_hint": [
+        "diffusers",
+        "MiniMaxH3Scheduler"
+      ],
+      "pretrained_model_name_or_path": "PrunaAI/p-h3-image",
+      "subfolder": "audio_scheduler",
+      "variant": null,
+      "revision": null
+    }
+  ]
+}

processor/preprocessor_config.json

@@ -1,21 +0,0 @@
-{
-    "size": {
-        "longest_edge": 16777216,
-        "shortest_edge": 65536
-    },
-    "patch_size": 16,
-    "temporal_patch_size": 2,
-    "merge_size": 2,
-    "image_mean": [
-        0.5,
-        0.5,
-        0.5
-    ],
-    "image_std": [
-        0.5,
-        0.5,
-        0.5
-    ],
-    "processor_class": "Qwen3VLProcessor",
-    "image_processor_type": "Qwen2VLImageProcessorFast"
-}

{text_encoder → processor}/tokenizer_config.json

@@ -20,7 +20,7 @@
     "<|image_pad|>",
     "<|video_pad|>"
   ],
-  "is_local": false,
+  "is_local": true,
   "local_files_only": false,
   "model_max_length": 262144,
   "pad_token": "<|endoftext|>",

scheduler/scheduler_config.json

@@ -0,0 +1,5 @@
+{
+  "_class_name": "MiniMaxH3Scheduler",
+  "_diffusers_version": "0.42.0.dev0",
+  "shift": 12.0
+}

text_connector/config.json

@@ -0,0 +1,13 @@
+{
+  "_class_name": "PH3ImageTextConnector",
+  "_diffusers_version": "0.42.0.dev0",
+  "hidden_state_indices": [
+    13,
+    19,
+    25,
+    31,
+    36
+  ],
+  "out_channels": 5120,
+  "text_hidden_size": 2560
+}

vae/encoder.safetensors → text_connector/diffusion_pytorch_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e682abc411a0627be598b5bc5cfc6aef47acc7dc5c2a4e5b245be4e10c76dec8
-size 721365016
+oid sha256:4c4ad8cbf37db8a2da22cdd9905e566e259cf3477ea242f6aee1cf3fb695f070
+size 262308352

tokenizer/chat_template.jinja

@@ -0,0 +1,120 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {%- if messages[0].content is string %}
+            {{- messages[0].content }}
+        {%- else %}
+            {%- for content in messages[0].content %}
+                {%- if 'text' in content %}
+                    {{- content.text }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        {{- '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' }}
+        {%- if messages[0].content is string %}
+            {{- messages[0].content }}
+        {%- else %}
+            {%- for content in messages[0].content %}
+                {%- if 'text' in content %}
+                    {{- content.text }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- for message in messages %}
+    {%- if message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' }}
+        {%- if message.content is string %}
+            {{- message.content }}
+        {%- else %}
+            {%- for content in message.content %}
+                {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                    {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}
+                    <|vision_start|><|image_pad|><|vision_end|>
+                {%- elif content.type == 'video' or 'video' in content %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                    {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}
+                    <|vision_start|><|video_pad|><|vision_end|>
+                {%- elif 'text' in content %}
+                    {{- content.text }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role + '\n' }}
+        {%- if message.content is string %}
+            {{- message.content }}
+        {%- else %}
+            {%- for content_item in message.content %}
+                {%- if 'text' in content_item %}
+                    {{- content_item.text }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and message.content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {%- if message.content is string %}
+            {{- message.content }}
+        {%- else %}
+            {%- for content in message.content %}
+                {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                    {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}
+                    <|vision_start|><|image_pad|><|vision_end|>
+                {%- elif content.type == 'video' or 'video' in content %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                    {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}
+                    <|vision_start|><|video_pad|><|vision_end|>
+                {%- elif 'text' in content %}
+                    {{- content.text }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

text_encoder/h3_connector.safetensors → tokenizer/tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f76325b95e9a5b282970a6127dcef55fc3a5dffce77fac9f6ff7fa2546b98d2
-size 262308368
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

tokenizer/tokenizer_config.json

@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 262144,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "Qwen3VLProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

transformer/config.json

@@ -1,7 +1,6 @@
 {
   "_class_name": "MiniMaxH3Transformer3DModel",
   "_diffusers_version": "0.42.0.dev0",
-  "_name_or_path": "MiniMaxAI/MiniMax-H3",
   "attention_head_dim": 128,
   "audio_in_channels": 32,
   "ffn_dim": 14336,
@@ -23,5 +22,6 @@
   "rope_theta": 10000.0,
   "text_dim": 5120,
   "time_embed_dim": 2688,
-  "time_embed_hidden_dim": 5376
-}
+  "time_embed_hidden_dim": 5376,
+  "adaln_rank": 16
+}

transformer/diffusion_pytorch_model-00001-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e9d924673c9a11c1f2f8c386e63ca709aadbd8e0c113884fdd06bd96aa804ecc
-size 9813196128
+oid sha256:ee04c671142cf1d01db9915cac121ed03e66a5ac57f5045a27f2ba365f5a29bb
+size 9811366304

transformer/diffusion_pytorch_model-00002-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a1215c287f1880e7dea317aa613118a0161f40ce71d720f3d6396b04245a14b6
-size 9953228896
+oid sha256:baac3a6236b98b6985d704e61416e03acd384cb74c27577467b12b1036f1496e
+size 9950850184

transformer/diffusion_pytorch_model-00003-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a3acb8740d0fe9591797df4115c1b25ef11143f546ce0d24d273144cc495bfa9
-size 9953228896
+oid sha256:ab323a00c5ba68503a39c604b062ddf637e3a6c4501d7c3ed35bc48f8e1e08b5
+size 9950850184

transformer/diffusion_pytorch_model-00004-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b4ba24808bf090b2407f408330d4ac4d2d425d984498919fef71a6140a6d3d3f
-size 9946443064
+oid sha256:34905c0ab5543e72779c5490569ba0d5250c428b2b5c4bd07f67c4aae45632f5
+size 9944247336

transformer/diffusion_pytorch_model-00005-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f3bc5d39fd65d6a6f703d8d6a462fddecf56c34e38c5683a5b31c93bd3968696
-size 931652480
+oid sha256:41899fb95e0333b74dedcecf8e35f66046c58f850d0b42607f8c0256253ec31d
+size 931469504

transformer/fact_adaln.json

@@ -1,36 +0,0 @@
-{
- "rank": 16,
- "out_dtype": "bfloat16",
- "rel_err_max": 6.322517265289207e-07,
- "rel_err_mean": 6.02509869622736e-07,
- "dense_bf16_rel_err_max": 0.0017569654155522585,
- "dense_bf16_rel_err_mean": 0.0016905990871600807,
- "spectrum": [
-  1.0,
-  0.0870301851788273,
-  0.010991975064251243,
-  0.004357309018194422,
-  4.318734045821265e-05,
-  4.41002669873321e-06,
-  3.040825669553829e-07,
-  8.305876252014749e-08,
-  1.328276543180485e-09,
-  7.322764526377398e-10,
-  1.700334232753885e-10,
-  1.763900462704404e-12,
-  8.614475412219741e-14,
-  2.3521526272522684e-14,
-  1.4375947084497254e-14,
-  1.4100802766553918e-14,
-  8.544104132112073e-15,
-  6.684395506601792e-15,
-  5.591993323991046e-15,
-  5.049919930769181e-15,
-  4.781708032946834e-15,
-  4.484825333740503e-15,
-  4.331219812921819e-15,
-  4.317491273965274e-15
- ],
- "n_fit": 200001,
- "n_val": 20014
-}

vae/config.json

@@ -1,9 +1,6 @@
 {
-  "_class_name": "AutoencoderKLMiniMaxH3",
-  "_diffusers_version": "0.36.0.dev0",
-  "in_channels": 3,
-  "out_channels": 3,
-  "latent_channels": 24,
+  "_class_name": "AutoencoderKLPH3Image",
+  "_diffusers_version": "0.42.0.dev0",
   "block_out_channels": [
     128,
     256,
@@ -12,36 +9,16 @@
     512,
     1024
   ],
-  "layers_per_block": 2,
-  "spatial_downsample_factors": [
-    2,
-    2,
-    2,
-    2,
-    1,
-    1
-  ],
-  "temporal_downsample_factors": [
-    1,
-    2,
-    2,
-    1,
-    1,
-    1
-  ],
-  "norm_num_groups": 32,
-  "norm_eps": 1e-06,
-  "spatial_padding_mode": "reflect",
-  "decoder_num_layers": 36,
-  "decoder_num_attention_heads": 32,
   "decoder_attention_head_dim": 64,
-  "decoder_num_register_tokens": 4,
   "decoder_ffn_mult": 4,
-  "decoder_rope_theta": 100.0,
-  "decoder_rope_dim_ratio": 0.75,
   "decoder_norm_eps": 1e-05,
-  "clip_length": 17,
-  "token_drop": 3,
+  "decoder_num_attention_heads": 32,
+  "decoder_num_layers": 36,
+  "decoder_num_register_tokens": 4,
+  "decoder_rope_dim_ratio": 0.75,
+  "decoder_rope_theta": 100.0,
+  "in_channels": 3,
+  "latent_channels": 24,
   "latents_mean": [
     0.858090341091156,
     -0.9606591463088989,
@@ -93,5 +70,26 @@
     3.1627357006073,
     2.2816812992095947,
     2.6127843856811523
+  ],
+  "layers_per_block": 2,
+  "norm_eps": 1e-06,
+  "norm_num_groups": 32,
+  "out_channels": 3,
+  "spatial_downsample_factors": [
+    2,
+    2,
+    2,
+    2,
+    1,
+    1
+  ],
+  "spatial_padding_mode": "reflect",
+  "temporal_downsample_factors": [
+    1,
+    2,
+    2,
+    1,
+    1,
+    1
   ]
-}
+}

decoder/decoder.safetensors → vae/diffusion_pytorch_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3747750d303aa9da3201aaadd4af7c60d9e31e3e803504a08611649d60956dac
-size 4837685440
+oid sha256:264964208bea153ca6366570b66d793c75b0fc64716fb6515a41b54e3dce6bf1
+size 5559055392