diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..a6344aac8c09253b3b630fb776ae94478aa0275b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/EXPORT_VALIDATION.json b/EXPORT_VALIDATION.json new file mode 100644 index 0000000000000000000000000000000000000000..fcfcc86680860e2e2096c33a994477a471abb496 --- /dev/null +++ b/EXPORT_VALIDATION.json @@ -0,0 +1,30 @@ +{ + "status": "passed", + "checked_at_utc": "2026-09-28 11:10:30 UTC", + "source_checkpoint_sha256": "16d38697d3bc9caf53ca1d347073eea33158dbd3d3ede03dc60bbdce53c00220", + "export": { + "file": "generator_lora.pt", + "sha256": "465bd51317544c4d0d9bfb84776927fac599c5e9be055989e9588de4172f3184", + "bytes": 2767464857, + "tensor_count": 624, + "parameters": 691798016, + "dtype": "float32", + "rank": 128, + "alpha": 128, + "exact_tensor_parity_verified": true, + "base_weights_included": false, + "critic_included": false, + "optimizer_state_included": false + }, + "checks": { + "all_624_student_tensors_exact_equal": true, + "source_snapshot_hashes_verified": true, + "cpu_native_runner_validation_exit_code": 0, + "checkpoint_step": 1000, + "model_evaluations_per_rollout": 4, + "sigma_endpoints_per_rollout": 5, + "prompts_validated": 4, + "base_revision": "fa9c8ab1eaa21c8ae25e7e40b83b2e6002f340af", + "new_gpu_generation_performed": false + } +} diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..c389b45855337ccec8ddeb389f4fe902abcd0b19 --- /dev/null +++ b/LICENSE @@ -0,0 +1,84 @@ +MiniMax H3 COMMUNITY LICENSE AGREEMENT +MiniMax H3 release date/License date: August 2, 2026. +The scope of this License Agreement (this “Agreement”) is expressly limited to the “Applicable Territory” as defined below. +By clicking to accept, or by using, reproducing, modifying, distributing, running, or displaying any portion or element of the MiniMax H3 Works (including through any Hosted Services) in any manner, you acknowledge and accept the terms of this Agreement, and this Agreement shall take immediate effect upon the occurrence of such act. +I. Definitions +1. “Acceptable Use Policy” means the policy published by MiniMax in Exhibit A. +2. “Agreement” means the terms and conditions set forth herein that govern the use, reproduction, distribution, modification, running, and display of the MiniMax H3 Works or any portion or element thereof. +3. “Applicable Territory” means worldwide, excluding the Excluded Territories. +4. “Documentation” means the specifications, manuals, and documentation concerning MiniMax H3 that are publicly released by MiniMax. +5. “Excluded Territories” means the European Union, the United Kingdom, the Republic of Korea and the United States of America. +6. “MiniMax H3” means the video generation model, together with its software and algorithms, including trained model weights, parameters (including optimizer states), machine-learning model code, inference-supporting code, and other elements thereof made publicly available by Us, as released at https://huggingface.co/MiniMaxAI/MiniMax-H3. +7. “MiniMax H3 Works” means (i) the Materials, (ii) the Model Derivatives, and (iii) all derivatives thereof. +8. “Hosted Services” means hosted services provided via application programming interfaces (APIs), web access, or any other electronic or remote means. +9. “Licensee,” “you,” or “your” means the natural or legal person exercising rights and/or using the MiniMax H3 Works for any purpose in any field of use under this Agreement. +10. “Materials” means, collectively, MiniMax H3 and the Documentation (and any portion thereof), in each case as made available by MiniMax under this Agreement and proprietary to MiniMax. +11. “Model Derivatives” means all of the following: (i) any modification of MiniMax H3 or any Model Derivative thereof; (ii) any work based on MiniMax H3 or any Model Derivative thereof; or (iii) any other machine learning model created by transferring the patterns of the weights, parameters, operational patterns, or Outputs of MiniMax H3 or any Model Derivative thereof to another model, such that the latter model exhibits behavior similar to MiniMax H3 or its Model Derivatives, including by distillation methods, methods using intermediate data representations, or methods based on training using synthetic-data Outputs generated by MiniMax H3 or its Model Derivatives. For the avoidance of doubt, Outputs are not deemed Model Derivatives. +12. “Output” means any result of operating or otherwise using MiniMax H3 or any Model Derivatives (including through Hosted Services). +13. “Third Party” means any natural or legal person that is not under common control with us or with you. +14. “Including” means “including but not limited to.” +15. “We,” “Us” or “MiniMax” means Nanonoble Pte. Ltd.. +II. Grant of Rights +Solely within the Applicable Territory, we grant you a non-exclusive, non-transferable, royalty-free, limited license to use, reproduce, distribute, create derivative works (including Model Derivatives), and modify the Materials in accordance with the terms of this Agreement and the Acceptable Use Policy, based on the intellectual property and other rights owned by MiniMax that are embodied in or used by the Materials. You shall not violate (or encourage or permit any person to violate) any term of this Agreement or the Acceptable Use Policy. +We will continuously evaluate the applicable laws, regulations and compliance requirements for the Excluded Territories. In the meantime, should any person in such Excluded Territories be interested in deploying our models, you are welcome to contact us about obtaining a license, which will be granted based on robust controls and guardrails for purposes of complying with the laws, regulations and compliance requirements of the Excluded Territories. +III. Distribution and Redistribution +Subject to and conditioned on your continuing compliance with this Agreement, including its territorial restrictions and the Acceptable Use Policy, and solely within the Applicable Territory, you may distribute or make available the MiniMax H3 Works to Third Parties within the Applicable Territory; provided, that all of the following conditions are met: +1. You must provide a copy of this Agreement to all such Third Parties who receive the MiniMax H3 Works or use your products or services related thereto; +2. You must cause any modified files to carry prominent notices stating that you have modified such files; +3. You are encouraged to: + a. display a notice on any product or service developed using MiniMax H3 indicating that the product or service is “Powered by MiniMax H3”; + b. add an AI-generation identifier to files produced using generative AI models including MiniMax H3; and + c. publish at least one technical blog post or a public statement describing your experience using MiniMax H3 Works; +4. All distributions to Third Parties (other than through Hosted Services) must be accompanied by a “NOTICE” text file containing the following notice: + “MiniMax H3 is licensed under the MiniMax H3 Community License Agreement, Copyright © 2026 MiniMax. All Rights Reserved.” +You may add your own copyright notices on your modifications; except as provided in this Section and in Section V, however, you may not impose additional or different terms and conditions on the use, reproduction, or distribution of your modifications or of any aggregate Model Derivatives, and your use, reproduction, modification, distribution, running, and display of the work must otherwise comply with the terms and conditions of this Agreement (including the provisions concerning the Applicable Territory). If you receive the MiniMax H3 Works from a Licensee as part of an integrated end-user product, the provisions of Section III of this Agreement do not apply to you, but Section V and Exhibit A remain applicable. +IV. Additional Commercial Terms +1. You shall obtain a separate, prior written authorization from MiniMax by contacting api@minimax.io with the subject line “MiniMax H3 licensing - authorization request”, if your commercial products and services generate more than 20 million US dollars (or equivalent in other currencies) in yearly revenue. +2. You shall prominently display “MiniMax H3”on the user interface of commercial product or service that uses MiniMax H3 or MiniMax H3 Works. +V. Use Restrictions +1. Your use of the MiniMax H3 Works must comply with applicable laws and regulations (including trade-compliance laws and regulations) and must comply with the Acceptable Use Policy for the MiniMax H3 Works, which is incorporated into this Agreement by reference. +2. Before providing access to the MiniMax H3 Works or any product, service, or Hosted Service incorporating them, you must bind each recipient or user to enforceable terms at least as protective as the use restrictions in this Section V and Exhibit A, and you must notify each recipient or user that those restrictions apply. +3. You may not use the MiniMax H3 Works or any of their Outputs or results to improve any other artificial intelligence model (other than MiniMax H3 or its Model Derivatives). +4. You may not use, reproduce, modify, distribute, or display the MiniMax H3 Works or any of their Outputs or results outside the Applicable Territory. Any such use outside the Applicable Territory is not authorized by this Agreement. +5. If you provide or make available to any Third Party a product, service, or Hosted Service that permits the generation of Outputs using MiniMax H3 or any Model Derivative, you must, before making that product or service available and throughout its operation, implement, maintain, test, and periodically review reasonable and proportionate technical and organizational safeguards designed to prevent and mitigate access, uses, and Outputs that violate this Section V or Exhibit A, including uses or Outputs that infringe, misappropriate, or otherwise violate any Third Party’s intellectual-property or other rights. You must not knowingly disable, materially weaken, or permit the circumvention of those safeguards. You must maintain a reasonably accessible mechanism for reporting suspected violations. Upon receiving a good-faith report or otherwise obtaining actual knowledge of a violation, you must promptly investigate and take reasonable steps within your control to stop or mitigate the violation, including removing or disabling access to offending content or services and suspending or terminating repeat violators where appropriate. You are responsible for implementing and enforcing these requirements with respect to your products, services, systems, users, and downstream recipients. +VI. Intellectual Property +1. Subject to MiniMax’s rights in the MiniMax H3 Works (and the intellectual property therein), and to your compliance with the terms and conditions of this Agreement, as between you and MiniMax, you will own the derivative works and modifications of the Materials that you have created or had created, as well as any Model Derivatives. +2. Except for the limited license expressly granted in this paragraph, no trademark license is granted under this Agreement; with respect to MiniMax H3 Works, the Licensee may not use any name or mark owned by or associated with MiniMax or any of its affiliates, except as reasonably and customarily necessary to describe and distribute the MiniMax H3 Works. MiniMax hereby grants you a license to use the “MiniMax H3” mark (the “Mark”) within the Applicable Territory solely for the purpose of complying with Section III.3; provided, that you comply with all applicable trademark-protection laws. All goodwill arising from your use of the Mark shall inure to the benefit of MiniMax. +3. If you bring or assert any suit or other legal proceeding (including a cross-claim or counterclaim in any action) against us or any other natural or legal person alleging that the Materials, any Output, or any portion of the foregoing infringes any intellectual property right or other right owned by you or for which you can obtain a license, all licenses granted to you under this Agreement will terminate as of the date such suit or proceeding is filed. You shall defend, indemnify, and hold us harmless against any Third-Party claim arising out of or related to the use or distribution of the MiniMax H3 Works by you or by any Third Party. +4. MiniMax claims no rights over the Outputs you generate. You and your users are entirely responsible for the Outputs and any subsequent use thereof. +VII. Disclaimers and Limitations of Liability +1. We have no obligation to support, update, provide training for, or develop any further version of the MiniMax H3 Works, or to grant any license with respect thereto. +2. UNLESS AND ONLY TO THE EXTENT REQUIRED BY APPLICABLE LAW, THE MINIMAX H3 WORKS AND ANY OUTPUT AND RESULTS THEREFROM ARE PROVIDED “AS IS” WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES OF ANY KIND INCLUDING ANY WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, COURSE OF DEALING, USAGE OF TRADE, OR FITNESS FOR A PARTICULAR PURPOSE. YOU ARE SOLELY RESPONSIBLE FOR DETERMINING THE APPROPRIATENESS OF USING, REPRODUCING, MODIFYING, PERFORMING, DISPLAYING OR DISTRIBUTING ANY OF THE MINIMAX H3 WORKS OR OUTPUTS AND ASSUME ANY AND ALL RISKS ASSOCIATED WITH YOUR OR A THIRD PARTY’S USE OR DISTRIBUTION OF ANY OF THE MINIMAX H3 WORKS OR OUTPUTS AND YOUR EXERCISE OF RIGHTS AND PERMISSIONS UNDER THIS AGREEMENT. +3. TO THE FULLEST EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT SHALL MINIMAX OR ITS AFFILIATES BE LIABLE UNDER ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, TORT, NEGLIGENCE, PRODUCTS LIABILITY, OR OTHERWISE, FOR ANY DAMAGES, INCLUDING ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, EXEMPLARY, CONSEQUENTIAL OR PUNITIVE DAMAGES, OR LOST PROFITS OF ANY KIND ARISING FROM THIS AGREEMENT OR RELATED TO ANY OF THE MINIMAX H3 WORKS OR OUTPUTS, EVEN IF MINIMAX OR ITS AFFILIATES HAVE BEEN ADVISED OF THE POSSIBILITY OF ANY OF THE FOREGOING. +VIII. Term and Termination +1. This Agreement is effective from the moment you accept this Agreement or begin accessing the Materials, and, subject to your compliance with its terms and conditions, will remain in effect until terminated as provided herein. +2. If you breach any term or condition of this Agreement, we have the right to terminate this Agreement. Upon termination, you must immediately cease accessing, using, and distributing the MiniMax H3 Works; delete or destroy all copies within your possession or control; and notify each downstream recipient that your authorization has ended. The obligations in the preceding sentence and Sections VI.1, VI.3, VII, and IX survive termination. +IX. Governing Law and Jurisdiction +1. This Agreement, and any dispute arising out of or related to this Agreement, shall be governed by the laws of the Hong Kong Special Administrative Region of the People’s Republic of China, without regard to its conflict-of-laws rules. The United Nations Convention on Contracts for the International Sale of Goods does not apply to this Agreement. +2. Any dispute arising out of or related to this Agreement shall be subject to the exclusive jurisdiction of the courts of the Hong Kong Special Administrative Region of the People’s Republic of China with competent jurisdiction. Both MiniMax and the Licensee hereby consent to the exclusive jurisdiction of such courts for any such dispute. +Additional Note: Please note that the encoder of MiniMax H3 uses Qwen3-VL-32B, which is licensed under Apache 2.0 License: https://github.com/QwenLM/Qwen3-VL/blob/main/LICENSE. + +Exhibit A — Acceptable Use Policy +MiniMax reserves the right to update this Acceptable Use Policy from time to time. +Last revised: August 2, 2026. +MiniMax is committed to promoting the safe and fair use of its tools and features, including MiniMax H3. You agree not to use MiniMax H3, any Model Derivatives, or any Output in any of the following ways: +1. Use outside the Applicable Territory; +2. Use in any manner that violates any applicable national, federal, state, local, or international law, regulation, or other legal requirement, or that infringes, misappropriates, or otherwise violates any Third Party’s intellectual-property or other proprietary rights, including through unauthorized reproduction, distribution, public display, public performance, or creation of derivative works; +3. Use in any manner that may harm yourself or others; +4. Use to repurpose or distribute the Outputs of MiniMax H3 or any Model Derivatives in order to harm yourself or others; +5. Use to circumvent or bypass any safety guardrails or safeguards we have implemented; +6. Use in any manner that exploits or harms, or intends to exploit or harm, minors; +7. Use to generate or disseminate verifiably false information and/or content for the purpose of harming others or influencing elections; +8. Use to manufacture or facilitate false online engagement, including fake reviews and other means of false online engagement; +9. Use to intentionally defame, disparage, or otherwise harass others; +10. Use to generate and/or disseminate malware (including ransomware) or any other content intended to damage electronic systems; +11. Use to generate or disseminate personally identifiable information for the purpose of harming others; +12. Use to generate or disseminate information (including images, code, posts, or articles) in or to any public environment (including via bot tweets or similar means) without clearly and prominently disclosing that such information and/or content is machine-generated; +13. Use to impersonate another person without that person’s consent, authorization, or lawful right to do so; +14. Use to make high-risk automated decisions in critical domains that affect individual safety, rights, or well-being (such as law enforcement, immigration, healthcare or medical services, critical-infrastructure management, product-safety components, essential services, credit, employment, housing, education, social scoring, or insurance); +15. Use in any manner that violates or disregards the social, ethical, or moral standards of other countries or regions; +16. Use to carry out, assist, threaten, incite, plan, advocate for, or encourage violent extremism or terrorism; +17. Use for any purpose intended to discriminate against, or harm, individuals or groups based on protected characteristics or categories, online or offline social behavior, or known or predicted personality traits; +18. Use to intentionally exploit the vulnerabilities of specific populations based on age, social, physical, or psychological characteristics, so as to materially distort the behavior of a member of that group in a manner that causes, or is likely to cause, physical or psychological harm to that person or to others; +19. Use for military purposes; +20. Use to engage in any unauthorized or unlicensed professional activity, including but not limited to financial, legal, medical or healthcare, or other professional practice. diff --git a/LICENSE-CODE b/LICENSE-CODE new file mode 100644 index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64 --- /dev/null +++ b/LICENSE-CODE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..dd7486f0d5175fa1e74c6e4df3e85f8b7641f3c7 --- /dev/null +++ b/NOTICE @@ -0,0 +1,3 @@ +MiniMax H3 is licensed under the MiniMax H3 Community License Agreement, Copyright © 2026 MiniMax. All Rights Reserved. + +This repository contains a modified MiniMax H3 Model Derivative: a four-step student LoRA trained using Audio:Video loss weights 2:1, 50% authored audio-description prompts, and teacher audio latent endpoint regression. Spectral supervision is disabled. Base-model weights are not included. diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1560b5b882b8143cbfa7305eff867795e688d091 --- /dev/null +++ b/README.md @@ -0,0 +1,137 @@ +--- +license: other +license_name: minimax-h3-community-license-agreement +license_link: LICENSE +base_model: MiniMaxAI/MiniMax-H3 +library_name: peft +pipeline_tag: text-to-video +tags: + - minimax-h3 + - lora + - dmd + - audio-video-generation + - four-step + - prompt-mixture +--- + +# MiniMax-H3 · 4-step · Audio 2:1 · PromptMix + teacher audio latent · iter1000 + +这是 **Prompt 混合+教师音频 latent 监督** 实验的 1000 外层 iteration checkpoint; +不是频谱组,也不是新启动的“频谱+Prompt 混合”组合组。 + +Private inference release for the requested experiment `audio_mixed_endpoint_latent`. +The export is lossless, student-only FP32. The 624 exported tensors were checked +for exact equality with the original dual-role checkpoint. Base weights, +critic weights, optimizer and RNG state are not included. + +## Identity and recipe + +| Item | Value | +|---|---| +| Source checkpoint | `checkpoint-001000` | +| Updates | 1000 outer iterations = 200 student updates; 1000 critic updates | +| Base | `MiniMaxAI/MiniMax-H3` @ `fa9c8ab1eaa21c8ae25e7e40b83b2e6002f340af` | +| LoRA | rank/alpha 128/128, FP32, 312 modules / 624 tensors | +| LoRA scope | 50 main blocks + 2 token-refiner blocks, attention + FFN | +| Initialization | Fresh from Base; no older distilled LoRA warm start | +| Training loss | Video DMD + 2 × (audio DMD + teacher audio endpoint MSE) | +| Endpoint | Frozen Base, 50 Euler calls from the selected student's noisy A/V state to zero noise; latent MSE coefficient 1 | +| Spectral loss | **Disabled: weight 0** | +| Training text | 50% original LongLive + 50% authored audio-description prompts | +| Authored pool | 32 core scenes × 8 framing variants = 256 texts; four balanced audio categories | +| Student / critic LR | 2e-5 / 5e-5; student updated every 5 outer iterations | +| Training shape | 1344×768, 124 frames, 16 GPUs | +| Validated inference | Direct-text T2VA; 4 actual transformer calls; native 32 kHz stereo audio | +| Evaluation shape | 960×544, 124 frames, 24 fps | +| CFG / IR / Regenerator | Off | + +The teacher's additional 50 calls are **training only**. Loading this adapter +for inference does not require a teacher rollout or an endpoint-loss calculation. + +## Download + +Authenticate with an account that can access this private organization repository: + +```bash +hf download Perflow-Shuai/MiniMax-H3-LongLive-LoRA-4Step-Audio2-PromptMix-iter1000 --local-dir ./h3-promptmix-iter1000 +cd ./h3-promptmix-iter1000 +``` + +## Environment + +Reuse the existing validated MiniMax-H3 Python 3.11 environment where possible. +`environment/observed_versions.json` records the actual package versions; +training used PyTorch **2.11.0+cu130 / CUDA 13.0**. It is a record, not a claim +that a clean environment was installed and tested during this export. + +The Diffusers integration must be pinned to: + +```text +https://github.com/huggingface/diffusers.git +b88b343e71e38bd404f7f082ac4a2cbde63713e1 +``` + +Use that installed checkout, or pass its `src` directory through +`--diffusers-src`. FFmpeg is required for native audio/video muxing. +No base-model weights are bundled: download the pinned Base snapshot separately +or pass an already downloaded copy with `--model-dir`. + +## Inference + +```bash +python run_inference.py \ + --model-dir /path/to/MiniMax-H3-Base-T2VA-fa9c8ab1 \ + --diffusers-src /path/to/diffusers-minimax-h3/src \ + --local-files-only \ + --prompts prompts.example.json \ + --expected-prompt-count 4 \ + --output-dir outputs \ + --height 544 --width 960 --num-frames 124 --device cuda:0 +``` + +Append `--validate-only` for a CPU-only checkpoint/config/prompt validation. +Without `--model-dir`, the runner uses the pinned `MiniMaxAI/MiniMax-H3` +Hub snapshot; omit `--local-files-only` if it needs downloading. + +**Do not simply set an ordinary pipeline to four steps.** This model uses the +bundled fresh-re-noise sampler: four model evaluations and three transitions +that predict clean latents and add fresh, independent video/audio noise. + +- Shared base grid: [1, 0.75, 0.5, 0.25, 0] +- Video sigma grid: [1, 0.972972973, 0.923076923, 0.8, 0] +- Audio sigma grid: [1, 0.9, 0.75, 0.5, 0] + +The wrapper checks weight and source hashes, then invokes +`minimax_h3.infer_student_4step`. `adapter_config.json` describes the PEFT +targets, but generic `load_lora_weights()` alone does not establish this sampler. +The native weight file contains `schema_version`, `step`, `config` and +`student_lora`; it is **not** a full optimizer-resume checkpoint. + +## Validation and limitations + +- Original source SHA-256: + `16d38697d3bc9caf53ca1d347073eea33158dbd3d3ede03dc60bbdce53c00220`. +- Published `generator_lora.pt` SHA-256: + `465bd51317544c4d0d9bfb84776927fac599c5e9be055989e9588de4172f3184`. +- Source checkpoint already has 47 paired evaluation cases. The fixed speech + subset had 23/29 exact ASR matches; orbital cockpit had 2/3 seeds matching. + ASR does not measure persistent background noise or voice clarity. +- User listening feedback indicated improvement for PromptMix, but equivalence + to Base50 audio quality has not been established. +- No new GPU generation was performed for this export; exact tensor parity and + CPU runner validation are the export checks. +- T2VA is the validated mode. Do not assume I2V, first/last-frame, or reference + workflows are validated. +- `training_config.json` preserves original provenance, including historical + filesystem paths. They are not required by the inference runner. +- `training_recipe/` is an audited recipe source snapshot, not a self-contained + resume-training package. Original LongLive prompt provenance is retained in + the config; authored texts and sampling are defined by the recipe sources. + +## License + +This adapter is a modified MiniMax-H3 Model Derivative, governed by the included +MiniMax H3 Community License Agreement (`LICENSE`), including its territorial +and acceptable-use restrictions. Private visibility does not replace these terms. +Integration code is separately covered by `LICENSE-CODE` (Apache-2.0). +See `NOTICE` and `provenance.json`. diff --git a/adapter_config.json b/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..968c151d533e72f6939739fd8f7dac7ea305dfa4 --- /dev/null +++ b/adapter_config.json @@ -0,0 +1,324 @@ +{ + "base_model_name_or_path": "MiniMaxAI/MiniMax-H3", + "revision": "fa9c8ab1eaa21c8ae25e7e40b83b2e6002f340af", + "peft_type": "LORA", + "r": 128, + "lora_alpha": 128, + "lora_dropout": 0, + "bias": "none", + "inference_mode": true, + "target_modules": [ + "token_refiner.refiner_blocks.0.attn.to_k", + "token_refiner.refiner_blocks.0.attn.to_out.0", + "token_refiner.refiner_blocks.0.attn.to_q", + "token_refiner.refiner_blocks.0.attn.to_v", + "token_refiner.refiner_blocks.0.ff.net.0.proj", + "token_refiner.refiner_blocks.0.ff.net.2", + "token_refiner.refiner_blocks.1.attn.to_k", + "token_refiner.refiner_blocks.1.attn.to_out.0", + "token_refiner.refiner_blocks.1.attn.to_q", + "token_refiner.refiner_blocks.1.attn.to_v", + "token_refiner.refiner_blocks.1.ff.net.0.proj", + "token_refiner.refiner_blocks.1.ff.net.2", + "transformer_blocks.0.attn.to_k", + "transformer_blocks.0.attn.to_out.0", + "transformer_blocks.0.attn.to_q", + "transformer_blocks.0.attn.to_v", + "transformer_blocks.0.ff.net.0.proj", + "transformer_blocks.0.ff.net.2", + "transformer_blocks.1.attn.to_k", + "transformer_blocks.1.attn.to_out.0", + "transformer_blocks.1.attn.to_q", + "transformer_blocks.1.attn.to_v", + "transformer_blocks.1.ff.net.0.proj", + "transformer_blocks.1.ff.net.2", + "transformer_blocks.10.attn.to_k", + "transformer_blocks.10.attn.to_out.0", + "transformer_blocks.10.attn.to_q", + "transformer_blocks.10.attn.to_v", + "transformer_blocks.10.ff.net.0.proj", + "transformer_blocks.10.ff.net.2", + "transformer_blocks.11.attn.to_k", + "transformer_blocks.11.attn.to_out.0", + "transformer_blocks.11.attn.to_q", + "transformer_blocks.11.attn.to_v", + "transformer_blocks.11.ff.net.0.proj", + "transformer_blocks.11.ff.net.2", + "transformer_blocks.12.attn.to_k", + "transformer_blocks.12.attn.to_out.0", + "transformer_blocks.12.attn.to_q", + "transformer_blocks.12.attn.to_v", + "transformer_blocks.12.ff.net.0.proj", + "transformer_blocks.12.ff.net.2", + "transformer_blocks.13.attn.to_k", + "transformer_blocks.13.attn.to_out.0", + "transformer_blocks.13.attn.to_q", + "transformer_blocks.13.attn.to_v", + "transformer_blocks.13.ff.net.0.proj", + "transformer_blocks.13.ff.net.2", + "transformer_blocks.14.attn.to_k", + "transformer_blocks.14.attn.to_out.0", + "transformer_blocks.14.attn.to_q", + "transformer_blocks.14.attn.to_v", + "transformer_blocks.14.ff.net.0.proj", + "transformer_blocks.14.ff.net.2", + "transformer_blocks.15.attn.to_k", + "transformer_blocks.15.attn.to_out.0", + "transformer_blocks.15.attn.to_q", + "transformer_blocks.15.attn.to_v", + "transformer_blocks.15.ff.net.0.proj", + "transformer_blocks.15.ff.net.2", + "transformer_blocks.16.attn.to_k", + "transformer_blocks.16.attn.to_out.0", + "transformer_blocks.16.attn.to_q", + "transformer_blocks.16.attn.to_v", + "transformer_blocks.16.ff.net.0.proj", + "transformer_blocks.16.ff.net.2", + "transformer_blocks.17.attn.to_k", + "transformer_blocks.17.attn.to_out.0", + "transformer_blocks.17.attn.to_q", + "transformer_blocks.17.attn.to_v", + "transformer_blocks.17.ff.net.0.proj", + "transformer_blocks.17.ff.net.2", + "transformer_blocks.18.attn.to_k", + "transformer_blocks.18.attn.to_out.0", + "transformer_blocks.18.attn.to_q", + "transformer_blocks.18.attn.to_v", + "transformer_blocks.18.ff.net.0.proj", + "transformer_blocks.18.ff.net.2", + "transformer_blocks.19.attn.to_k", + "transformer_blocks.19.attn.to_out.0", + "transformer_blocks.19.attn.to_q", + "transformer_blocks.19.attn.to_v", + "transformer_blocks.19.ff.net.0.proj", + "transformer_blocks.19.ff.net.2", + "transformer_blocks.2.attn.to_k", + "transformer_blocks.2.attn.to_out.0", + "transformer_blocks.2.attn.to_q", + "transformer_blocks.2.attn.to_v", + "transformer_blocks.2.ff.net.0.proj", + "transformer_blocks.2.ff.net.2", + "transformer_blocks.20.attn.to_k", + "transformer_blocks.20.attn.to_out.0", + "transformer_blocks.20.attn.to_q", + "transformer_blocks.20.attn.to_v", + "transformer_blocks.20.ff.net.0.proj", + "transformer_blocks.20.ff.net.2", + "transformer_blocks.21.attn.to_k", + "transformer_blocks.21.attn.to_out.0", + "transformer_blocks.21.attn.to_q", + "transformer_blocks.21.attn.to_v", + "transformer_blocks.21.ff.net.0.proj", + "transformer_blocks.21.ff.net.2", + "transformer_blocks.22.attn.to_k", + "transformer_blocks.22.attn.to_out.0", + "transformer_blocks.22.attn.to_q", + "transformer_blocks.22.attn.to_v", + "transformer_blocks.22.ff.net.0.proj", + "transformer_blocks.22.ff.net.2", + "transformer_blocks.23.attn.to_k", + "transformer_blocks.23.attn.to_out.0", + "transformer_blocks.23.attn.to_q", + "transformer_blocks.23.attn.to_v", + "transformer_blocks.23.ff.net.0.proj", + "transformer_blocks.23.ff.net.2", + "transformer_blocks.24.attn.to_k", + "transformer_blocks.24.attn.to_out.0", + "transformer_blocks.24.attn.to_q", + "transformer_blocks.24.attn.to_v", + "transformer_blocks.24.ff.net.0.proj", + "transformer_blocks.24.ff.net.2", + "transformer_blocks.25.attn.to_k", + "transformer_blocks.25.attn.to_out.0", + "transformer_blocks.25.attn.to_q", + "transformer_blocks.25.attn.to_v", + "transformer_blocks.25.ff.net.0.proj", + "transformer_blocks.25.ff.net.2", + "transformer_blocks.26.attn.to_k", + "transformer_blocks.26.attn.to_out.0", + "transformer_blocks.26.attn.to_q", + "transformer_blocks.26.attn.to_v", + "transformer_blocks.26.ff.net.0.proj", + "transformer_blocks.26.ff.net.2", + "transformer_blocks.27.attn.to_k", + "transformer_blocks.27.attn.to_out.0", + "transformer_blocks.27.attn.to_q", + "transformer_blocks.27.attn.to_v", + "transformer_blocks.27.ff.net.0.proj", + "transformer_blocks.27.ff.net.2", + "transformer_blocks.28.attn.to_k", + "transformer_blocks.28.attn.to_out.0", + "transformer_blocks.28.attn.to_q", + "transformer_blocks.28.attn.to_v", + "transformer_blocks.28.ff.net.0.proj", + "transformer_blocks.28.ff.net.2", + "transformer_blocks.29.attn.to_k", + "transformer_blocks.29.attn.to_out.0", + "transformer_blocks.29.attn.to_q", + "transformer_blocks.29.attn.to_v", + "transformer_blocks.29.ff.net.0.proj", + "transformer_blocks.29.ff.net.2", + "transformer_blocks.3.attn.to_k", + "transformer_blocks.3.attn.to_out.0", + "transformer_blocks.3.attn.to_q", + "transformer_blocks.3.attn.to_v", + "transformer_blocks.3.ff.net.0.proj", + "transformer_blocks.3.ff.net.2", + "transformer_blocks.30.attn.to_k", + "transformer_blocks.30.attn.to_out.0", + "transformer_blocks.30.attn.to_q", + "transformer_blocks.30.attn.to_v", + "transformer_blocks.30.ff.net.0.proj", + "transformer_blocks.30.ff.net.2", + "transformer_blocks.31.attn.to_k", + "transformer_blocks.31.attn.to_out.0", + "transformer_blocks.31.attn.to_q", + "transformer_blocks.31.attn.to_v", + "transformer_blocks.31.ff.net.0.proj", + "transformer_blocks.31.ff.net.2", + "transformer_blocks.32.attn.to_k", + "transformer_blocks.32.attn.to_out.0", + "transformer_blocks.32.attn.to_q", + "transformer_blocks.32.attn.to_v", + "transformer_blocks.32.ff.net.0.proj", + "transformer_blocks.32.ff.net.2", + "transformer_blocks.33.attn.to_k", + "transformer_blocks.33.attn.to_out.0", + "transformer_blocks.33.attn.to_q", + "transformer_blocks.33.attn.to_v", + "transformer_blocks.33.ff.net.0.proj", + "transformer_blocks.33.ff.net.2", + "transformer_blocks.34.attn.to_k", + "transformer_blocks.34.attn.to_out.0", + "transformer_blocks.34.attn.to_q", + "transformer_blocks.34.attn.to_v", + "transformer_blocks.34.ff.net.0.proj", + "transformer_blocks.34.ff.net.2", + "transformer_blocks.35.attn.to_k", + "transformer_blocks.35.attn.to_out.0", + "transformer_blocks.35.attn.to_q", + "transformer_blocks.35.attn.to_v", + "transformer_blocks.35.ff.net.0.proj", + "transformer_blocks.35.ff.net.2", + "transformer_blocks.36.attn.to_k", + "transformer_blocks.36.attn.to_out.0", + "transformer_blocks.36.attn.to_q", + "transformer_blocks.36.attn.to_v", + "transformer_blocks.36.ff.net.0.proj", + "transformer_blocks.36.ff.net.2", + "transformer_blocks.37.attn.to_k", + "transformer_blocks.37.attn.to_out.0", + "transformer_blocks.37.attn.to_q", + "transformer_blocks.37.attn.to_v", + "transformer_blocks.37.ff.net.0.proj", + "transformer_blocks.37.ff.net.2", + "transformer_blocks.38.attn.to_k", + "transformer_blocks.38.attn.to_out.0", + "transformer_blocks.38.attn.to_q", + "transformer_blocks.38.attn.to_v", + "transformer_blocks.38.ff.net.0.proj", + "transformer_blocks.38.ff.net.2", + "transformer_blocks.39.attn.to_k", + "transformer_blocks.39.attn.to_out.0", + "transformer_blocks.39.attn.to_q", + "transformer_blocks.39.attn.to_v", + "transformer_blocks.39.ff.net.0.proj", + "transformer_blocks.39.ff.net.2", + "transformer_blocks.4.attn.to_k", + "transformer_blocks.4.attn.to_out.0", + "transformer_blocks.4.attn.to_q", + "transformer_blocks.4.attn.to_v", + "transformer_blocks.4.ff.net.0.proj", + "transformer_blocks.4.ff.net.2", + "transformer_blocks.40.attn.to_k", + "transformer_blocks.40.attn.to_out.0", + "transformer_blocks.40.attn.to_q", + "transformer_blocks.40.attn.to_v", + "transformer_blocks.40.ff.net.0.proj", + "transformer_blocks.40.ff.net.2", + "transformer_blocks.41.attn.to_k", + "transformer_blocks.41.attn.to_out.0", + "transformer_blocks.41.attn.to_q", + "transformer_blocks.41.attn.to_v", + "transformer_blocks.41.ff.net.0.proj", + "transformer_blocks.41.ff.net.2", + "transformer_blocks.42.attn.to_k", + "transformer_blocks.42.attn.to_out.0", + "transformer_blocks.42.attn.to_q", + "transformer_blocks.42.attn.to_v", + "transformer_blocks.42.ff.net.0.proj", + "transformer_blocks.42.ff.net.2", + "transformer_blocks.43.attn.to_k", + "transformer_blocks.43.attn.to_out.0", + "transformer_blocks.43.attn.to_q", + "transformer_blocks.43.attn.to_v", + "transformer_blocks.43.ff.net.0.proj", + "transformer_blocks.43.ff.net.2", + "transformer_blocks.44.attn.to_k", + "transformer_blocks.44.attn.to_out.0", + "transformer_blocks.44.attn.to_q", + "transformer_blocks.44.attn.to_v", + "transformer_blocks.44.ff.net.0.proj", + "transformer_blocks.44.ff.net.2", + "transformer_blocks.45.attn.to_k", + "transformer_blocks.45.attn.to_out.0", + "transformer_blocks.45.attn.to_q", + "transformer_blocks.45.attn.to_v", + "transformer_blocks.45.ff.net.0.proj", + "transformer_blocks.45.ff.net.2", + "transformer_blocks.46.attn.to_k", + "transformer_blocks.46.attn.to_out.0", + "transformer_blocks.46.attn.to_q", + "transformer_blocks.46.attn.to_v", + "transformer_blocks.46.ff.net.0.proj", + "transformer_blocks.46.ff.net.2", + "transformer_blocks.47.attn.to_k", + "transformer_blocks.47.attn.to_out.0", + "transformer_blocks.47.attn.to_q", + "transformer_blocks.47.attn.to_v", + "transformer_blocks.47.ff.net.0.proj", + "transformer_blocks.47.ff.net.2", + "transformer_blocks.48.attn.to_k", + "transformer_blocks.48.attn.to_out.0", + "transformer_blocks.48.attn.to_q", + "transformer_blocks.48.attn.to_v", + "transformer_blocks.48.ff.net.0.proj", + "transformer_blocks.48.ff.net.2", + "transformer_blocks.49.attn.to_k", + "transformer_blocks.49.attn.to_out.0", + "transformer_blocks.49.attn.to_q", + "transformer_blocks.49.attn.to_v", + "transformer_blocks.49.ff.net.0.proj", + "transformer_blocks.49.ff.net.2", + "transformer_blocks.5.attn.to_k", + "transformer_blocks.5.attn.to_out.0", + "transformer_blocks.5.attn.to_q", + "transformer_blocks.5.attn.to_v", + "transformer_blocks.5.ff.net.0.proj", + "transformer_blocks.5.ff.net.2", + "transformer_blocks.6.attn.to_k", + "transformer_blocks.6.attn.to_out.0", + "transformer_blocks.6.attn.to_q", + "transformer_blocks.6.attn.to_v", + "transformer_blocks.6.ff.net.0.proj", + "transformer_blocks.6.ff.net.2", + "transformer_blocks.7.attn.to_k", + "transformer_blocks.7.attn.to_out.0", + "transformer_blocks.7.attn.to_q", + "transformer_blocks.7.attn.to_v", + "transformer_blocks.7.ff.net.0.proj", + "transformer_blocks.7.ff.net.2", + "transformer_blocks.8.attn.to_k", + "transformer_blocks.8.attn.to_out.0", + "transformer_blocks.8.attn.to_q", + "transformer_blocks.8.attn.to_v", + "transformer_blocks.8.ff.net.0.proj", + "transformer_blocks.8.ff.net.2", + "transformer_blocks.9.attn.to_k", + "transformer_blocks.9.attn.to_out.0", + "transformer_blocks.9.attn.to_q", + "transformer_blocks.9.attn.to_v", + "transformer_blocks.9.ff.net.0.proj", + "transformer_blocks.9.ff.net.2" + ] +} diff --git a/environment/observed_versions.json b/environment/observed_versions.json new file mode 100644 index 0000000000000000000000000000000000000000..a6b5b18509df190171826fef71e1e2be0ec7f283 --- /dev/null +++ b/environment/observed_versions.json @@ -0,0 +1,25 @@ +{ + "packages": { + "torch": "2.11.0", + "torchvision": "0.26.0", + "transformers": "5.12.1", + "huggingface_hub": "1.26.0", + "peft": "0.20.0", + "accelerate": "1.14.0", + "safetensors": "0.8.0", + "diffusers": "0.40.0.dev0", + "av": "16.1.0", + "numpy": "2.4.4", + "pillow": "12.2.0", + "sentencepiece": "0.2.2", + "imageio": "2.37.4", + "imageio-ffmpeg": "0.6.0", + "scipy": "1.17.1", + "einops": "0.8.2", + "omegaconf": "2.3.1", + "kernels": "0.12.3" + }, + "training_torch": "2.11.0+cu130", + "training_cuda": "13.0", + "python": "3.11.15 (main, Jun 11 2026, 15:11:53) [GCC 14.3.0]" +} diff --git a/generator_lora.pt b/generator_lora.pt new file mode 100644 index 0000000000000000000000000000000000000000..07502a0b21ff487416338c22f19ffecf53deeff1 --- /dev/null +++ b/generator_lora.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:465bd51317544c4d0d9bfb84776927fac599c5e9be055989e9588de4172f3184 +size 2767464857 diff --git a/prompts.example.json b/prompts.example.json new file mode 100644 index 0000000000000000000000000000000000000000..5202167bcff4cd780bd2f44fc4b4fdb183d5779c --- /dev/null +++ b/prompts.example.json @@ -0,0 +1,25 @@ +{ + "schema_version": 1, + "prompts": [ + { + "id": "orbital_launch_cockpit", + "prompt": "A high-detail cinematic view inside a near-future orbital launch cockpit during ascent. One locked wide camera faces two restrained astronauts and the forward windows as engine vibration begins, indicator lights flicker consistently, both bodies shake against their harnesses, a loose checklist trembles, and small droplets lift into brief microgravity near the end while Earth's blue horizon slides downward outside. Keep faces, suits, hands, switches, panels, window geometry, and motion physically coherent; no cuts, captions, logos, or unreadable generated labels. Synchronized stereo audio: a deep engine rumble builds through the seats, structural rattles match visible vibration, warning beeps occur with amber light pulses, ventilation remains audible, and the commander clearly says, \"Stage separation confirmed,\" over a crackling radio exactly as the vibration eases. No music.", + "seed": 2026081206 + }, + { + "id": "robotics_precision_assembly", + "prompt": "High-resolution industrial documentary footage inside a clean robotics laboratory. A single slow push-in camera observes two articulated robot arms assembling a compact metal gearbox: the left gripper places one toothed gear onto a fixed shaft, releases it, and withdraws; the right tool lowers three screws in sequence and tightens them without intersecting the housing. Keep joint counts, arm topology, grippers, gear teeth, shaft alignment, screw positions, reflections, and contact physics exact, with no cuts, labels, logos, sparks, or impossible tool changes. Synchronized stereo audio: servo whines move with each arm, the gear makes a precise metal seat click, three screwdriver bursts align with the three visible screws, a torque-complete beep follows the final stop, and ventilation hum remains constant. No music or speech.", + "seed": 2026081214 + }, + { + "id": "short_chinese_speech", + "seed": 2026091102, + "prompt": "固定中近景,一位年轻男子站在安静的书房里看向镜头,自然地用普通话说:“今天阳光很好,我们出去走走吧。”嘴型与声音同步,发音清楚,语气轻松,轻微自然呼吸声,安静室内环境声。无配乐,无字幕,连续镜头。" + }, + { + "id": "short_english_speech", + "seed": 2026091101, + "prompt": "One continuous medium close-up of a woman in a quiet recording studio, looking at the camera. She naturally says, \"The rain has stopped. Let us go outside.\" Her lips match the words. Clear natural English speech in stereo, quiet room tone, no music, no subtitles, no cuts." + } + ] +} diff --git a/provenance.json b/provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..e1fdf38766e7d914df6fd1b9ae5891f5858d79a7 --- /dev/null +++ b/provenance.json @@ -0,0 +1,119 @@ +{ + "repo_id": "Perflow-Shuai/MiniMax-H3-LongLive-LoRA-4Step-Audio2-PromptMix-iter1000", + "visibility": "private", + "arm": "audio_mixed_endpoint_latent", + "checkpoint_step": 1000, + "student_updates": 200, + "critic_updates": 1000, + "original_dual_role_checkpoint_sha256": "16d38697d3bc9caf53ca1d347073eea33158dbd3d3ede03dc60bbdce53c00220", + "original_config_sha256": "916da7041fe91de966917d8600ac7c1ec64a9b9d4171c457151c687efeb7bf5a", + "source_checkpoint_receipt": { + "artifacts": { + "lora.pt": { + "bytes": 5534893431, + "sha256": "16d38697d3bc9caf53ca1d347073eea33158dbd3d3ede03dc60bbdce53c00220" + }, + "manifest.json": { + "bytes": 189, + "sha256": "aae37edc9111198e59fc702d5ed869cedae53dc3c83e4b8669d7d12e69f63491" + }, + "training_state.pt": { + "bytes": 11070100855, + "sha256": "194d11018a0af758384caf93be04e652ebaf2e1baa6e4aabfb966cc9290c3c41" + } + }, + "base_weights_included": false, + "config_sha256": "916da7041fe91de966917d8600ac7c1ec64a9b9d4171c457151c687efeb7bf5a", + "created_at": "2026-09-28T00:05:51.237946Z", + "critic_update_count": 1000, + "global_step": 1000, + "kind": "minimax_h3_dmd_longrun_checkpoint", + "optimizer_layout": "joint_role_groups_per_role_adamw_v2", + "schema_version": 2, + "status": "success", + "student_update_count": 200, + "world_size": 16 + }, + "export": { + "file": "generator_lora.pt", + "sha256": "465bd51317544c4d0d9bfb84776927fac599c5e9be055989e9588de4172f3184", + "bytes": 2767464857, + "tensor_count": 624, + "parameters": 691798016, + "dtype": "float32", + "rank": 128, + "alpha": 128, + "exact_tensor_parity_verified": true, + "base_weights_included": false, + "critic_included": false, + "optimizer_state_included": false + }, + "inference": { + "base_model": "MiniMaxAI/MiniMax-H3", + "base_revision": "fa9c8ab1eaa21c8ae25e7e40b83b2e6002f340af", + "diffusers_revision": "b88b343e71e38bd404f7f082ac4a2cbde63713e1", + "nfe": 4, + "rollout_mode": "predict_x0_fresh_renoise", + "video_shift": 12, + "audio_shift": 3, + "cfg": false, + "ir": false, + "regenerator": false, + "audio_sample_rate": 32000 + }, + "evaluation": { + "existing_cases": 47, + "step": 1000, + "source_manifest_sha256": "97924e5e8d84a0c20753230b333b3a34204e04d7a67a5a0f3f5d3617d6fcd3d4", + "asr_exact_matches": 23, + "asr_reference_cases": 29, + "orbital_exact_matches": 2, + "orbital_seeds": 3, + "near_Base50_audio_quality_proven": false, + "export_new_gpu_inference_performed": false + }, + "source_hashes": { + "source_snapshot/minimax_h3/__init__.py": "7928c5364f079a90f05d14d9ef8c3949173f1e5caf9010f08779793b3fa50291", + "source_snapshot/minimax_h3/build_prompt_cache.py": "5ed06cb54ca28bc776d92f6da13b37aaa5baccb9e7e537fbcd37282f871cae98", + "source_snapshot/minimax_h3/cfg_guidance.py": "ee796ccfc5a03aef1e98054eaf30585d74b190e54308ba1ca192e2373e06df24", + "source_snapshot/minimax_h3/dmd_4step.py": "3d72c4dd3b3793f91fdcce3d37a809e18be805889fd3cc4bf57d96426eef8d0f", + "source_snapshot/minimax_h3/dmd_8step.py": "77ab7e2c7ce40cad12b141b1b24467406ff66a7fa420b5a752bdc5d5b74dc2b1", + "source_snapshot/minimax_h3/dmd_core.py": "4c2cc6909d41356d27c95275a0e32a5a3d96038634cb83e42a270424f6a86164", + "source_snapshot/minimax_h3/encode_longlive_prompt_cache.py": "7bdd4004bf4973158bac9866975c08b825579ee279c45916bd7789eacd57dc42", + "source_snapshot/minimax_h3/encode_longlive_prompt_cache_v4.py": "b39b37977235320b541e54d78833088cabd3199c8fe77059399ea8438cabcb29", + "source_snapshot/minimax_h3/extract_dmd8_inline5.py": "9f43ef1bad719a328f7ece6d14657534ec9cf76ebe91405760fed15d5bed821e", + "source_snapshot/minimax_h3/fresh_noise_scheduler.py": "404cbb1e6eb05e5a11222b95f86ba5eec88fb09718d32094a04dd4ac834fbcdb", + "source_snapshot/minimax_h3/fresh_noise_scheduler_4step.py": "ae6d4887471edb94096be0d65dfcd850131f819620d35450a26289dd90a5dc8e", + "source_snapshot/minimax_h3/full_h3_lora.py": "ad4b08ff2aae784452f5dd3468fc7e571cecb9665ada80ef5f60f04bf8cf4343", + "source_snapshot/minimax_h3/infer_base.py": "dd6a5ef8aad12ba76c65d74048726fd0f9623a116f216b4a300ef42272d09b93", + "source_snapshot/minimax_h3/infer_student.py": "c3dfa6456fcc1d49500c5a6445431e0254181f0d5342fa24353dc57e8c18c5d8", + "source_snapshot/minimax_h3/infer_student_4step.py": "3bbb180764ef744791065e172acf759b65f8ff7b8e85a3ac8cdc9ee59c738db3", + "source_snapshot/minimax_h3/infer_student_8step.py": "867252e18cb1c53c266d2a3a658453d63e7b9a38711a94ec36bcf88d7f944f5b", + "source_snapshot/minimax_h3/longlive_lora.py": "4d061332055b2b59dfa7632ca7502f98ecdace283333b28f872b1f74f2c1cb4e", + "source_snapshot/minimax_h3/longlive_prompt_distributed.py": "eff9f6e68cc17f4b0237990ac79478a45d5a6e846a4b9b812a508cc03faca48d", + "source_snapshot/minimax_h3/longlive_prompt_set.py": "ed80195faf3f1614c2ec5d3678298505eae00490456ebc365df995324cc33703", + "source_snapshot/minimax_h3/model_provenance.py": "8b5409504258657fdab9665dbd0a51a09e5503e468a9b0e7eb76ad0c2f55ab20", + "source_snapshot/minimax_h3/probe_student_lora_ab.py": "6ff009a751dc07c54cf8dedc98b4061f36080eec93c5068996c5bcec448f0150", + "source_snapshot/minimax_h3/train_dmd.py": "a2f2c71ead08514d96898019eee6c39536192eeb5b017bb2c7454d1cfb0e7172", + "source_snapshot/minimax_h3/train_dmd_longrun.py": "5843b7c57eea8c602bd06cae4580d4efae6fc15be4d370b4a2951cf60baf6ecb", + "source_snapshot/minimax_h3/train_dmd_longrun_4step.py": "5a751327012031d7a51e5004dc3d47094d7810e52ed72d05b04b513bedcee1bf", + "source_snapshot/minimax_h3/train_dmd_longrun_4step_autoresume.py": "42abbef8788b0d86ae6d2d6094bec6144d754d355748eefea6a7f189b7bd8918", + "source_snapshot/minimax_h3/train_dmd_longrun_8step.py": "2f0302f973ba0729040687741eed95edbf321bd4c85745ec1ca977fe91c9e7a0", + "source_snapshot/minimax_h3/train_dmd_longrun_8step_autoresume.py": "21b033ae059b73587257f6923db3587ce6b16c18aa4e8b6f5a91a615905d68dd", + "source_snapshot/minimax_h3/train_dmd_longrun_v2.py": "2ef1d89d400df60a86be10b8828ea7c4b87fb96112538bfd25e7baaf5e8bc357", + "source_snapshot/minimax_h3/train_dmd_longrun_v3.py": "ca1905890c1e3c849ccb7f189166d7d7ddaa22b9d6162c629beab8e0b5c65c81", + "source_snapshot/minimax_h3/train_dmd_longrun_v4.py": "e27328e359c6ff4fccd38623fb7eadb84c48f131fb547ca82bd7364c3f2dd44c", + "source_snapshot/minimax_h3/train_dmd_longrun_v5.py": "14778be13523721a38fb539d973c6d0ad7eff8054ab146a76adcb684bdd9c7cf", + "source_snapshot/minimax_h3/train_utils.py": "fd64c02074680470776fa9e8f58578bde7fdb90603d2cad3eab5e9216c6f8e9c", + "source_snapshot/minimax_h3/visualize_teacher_x0.py": "962f86b0fb699accecc36eb0b57fdc278835c22731f78f5e4d5fe36c3a33d49f", + "training_recipe/train_fresh.py": "6b478c00a64c5bb405823960f541fa3d0faf4cc14b17db5cc762043a71ddec10", + "training_recipe/common.py": "60d1bd3d7774dca1ac1c54234ac05fd2fcb7a0ad1a01450d789fe882f7ad2ef7", + "training_recipe/audio_objectives.py": "4e22fd00a67f5e7e7ce03bec2607911079aad7129fc6b50c88cae3385bc1c605", + "training_recipe/train_audio_mixed.py": "6193189881a1fef24144c4ddf57aa77965bf69308a1392724c826720e8d6c338", + "training_recipe/audio_mixed_conditions.py": "2b41ff3d1249ccdd7857d7b384476e8b65ec09593a44c810e931c6106d863dfd", + "training_recipe/audio_candidate_cache.py": "234d4a9cfad2952b2f98b76f70c77a46399483e9d6da5030ac80639e1fb4b616", + "training_recipe/audio_data_candidates.py": "28cebc27e0c8d914cd716401f9ebb9c03133c5803b362165ddb2e56aa7744274", + "LICENSE": "59b99642b95ea21630e311198ddbfffbfe05aadba0c2f5d884cbdf4efcc90f44", + "LICENSE-CODE": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4" + } +} diff --git a/run_inference.py b/run_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..bee9e5567192bcb0c5414fdeb2b5dba7848162a1 --- /dev/null +++ b/run_inference.py @@ -0,0 +1,56 @@ +"""Validate the exact exported student and invoke its pinned four-call runner.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys + +HERE = Path(__file__).resolve().parent + + +def sha(path): + digest = hashlib.sha256() + with path.open('rb') as stream: + for block in iter(lambda: stream.read(16*1024*1024), b''): + digest.update(block) + return digest.hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--model-dir', '--model-id', default='MiniMaxAI/MiniMax-H3') + parser.add_argument('--diffusers-src', type=Path) + parser.add_argument('--prompts', type=Path, default=HERE/'prompts.example.json') + parser.add_argument('--expected-prompt-count', type=int, default=4) + parser.add_argument('--output-dir', type=Path, default=Path('outputs')) + args, rest = parser.parse_known_args() + for argument in rest: + if argument.split('=')[0] in ('--checkpoint', '--revision', '--num-inference-steps'): + parser.error('Checkpoint, revision and four-call sampler are fixed by this release') + provenance = json.loads((HERE/'provenance.json').read_text()) + checkpoint = HERE/provenance['export']['file'] + if checkpoint.is_symlink() or sha(checkpoint) != provenance['export']['sha256']: + raise RuntimeError('Student checkpoint checksum mismatch') + for name, digest in json.loads((HERE/'source_hashes.json').read_text()).items(): + if sha(HERE/name) != digest: + raise RuntimeError('Source snapshot checksum mismatch: '+name) + paths = [str(HERE/'source_snapshot')] + if args.diffusers_src is not None: + if not (args.diffusers_src/'diffusers').is_dir(): + parser.error('--diffusers-src must contain the diffusers package') + paths.append(str(args.diffusers_src.resolve())) + env = os.environ.copy() + env['PYTHONPATH'] = os.pathsep.join(paths + ([env['PYTHONPATH']] if env.get('PYTHONPATH') else [])) + env.update(PYTHONDONTWRITEBYTECODE='1', PYTHONUNBUFFERED='1', TOKENIZERS_PARALLELISM='false') + env.setdefault('PYTORCH_CUDA_ALLOC_CONF', 'expandable_segments:True') + return subprocess.run([sys.executable, '-m', 'minimax_h3.infer_student_4step', + '--checkpoint', str(checkpoint), '--model-id', args.model_dir, + '--revision', provenance['inference']['base_revision'], + '--prompts', str(args.prompts), '--expected-prompt-count', str(args.expected_prompt_count), + '--output-dir', str(args.output_dir), *rest], env=env).returncode + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/source_hashes.json b/source_hashes.json new file mode 100644 index 0000000000000000000000000000000000000000..bc48fbf839203cce1d34a48aef9c60c5414ccfb2 --- /dev/null +++ b/source_hashes.json @@ -0,0 +1,44 @@ +{ + "source_snapshot/minimax_h3/__init__.py": "7928c5364f079a90f05d14d9ef8c3949173f1e5caf9010f08779793b3fa50291", + "source_snapshot/minimax_h3/build_prompt_cache.py": "5ed06cb54ca28bc776d92f6da13b37aaa5baccb9e7e537fbcd37282f871cae98", + "source_snapshot/minimax_h3/cfg_guidance.py": "ee796ccfc5a03aef1e98054eaf30585d74b190e54308ba1ca192e2373e06df24", + "source_snapshot/minimax_h3/dmd_4step.py": "3d72c4dd3b3793f91fdcce3d37a809e18be805889fd3cc4bf57d96426eef8d0f", + "source_snapshot/minimax_h3/dmd_8step.py": "77ab7e2c7ce40cad12b141b1b24467406ff66a7fa420b5a752bdc5d5b74dc2b1", + "source_snapshot/minimax_h3/dmd_core.py": "4c2cc6909d41356d27c95275a0e32a5a3d96038634cb83e42a270424f6a86164", + "source_snapshot/minimax_h3/encode_longlive_prompt_cache.py": "7bdd4004bf4973158bac9866975c08b825579ee279c45916bd7789eacd57dc42", + "source_snapshot/minimax_h3/encode_longlive_prompt_cache_v4.py": "b39b37977235320b541e54d78833088cabd3199c8fe77059399ea8438cabcb29", + "source_snapshot/minimax_h3/extract_dmd8_inline5.py": "9f43ef1bad719a328f7ece6d14657534ec9cf76ebe91405760fed15d5bed821e", + "source_snapshot/minimax_h3/fresh_noise_scheduler.py": "404cbb1e6eb05e5a11222b95f86ba5eec88fb09718d32094a04dd4ac834fbcdb", + "source_snapshot/minimax_h3/fresh_noise_scheduler_4step.py": "ae6d4887471edb94096be0d65dfcd850131f819620d35450a26289dd90a5dc8e", + "source_snapshot/minimax_h3/full_h3_lora.py": "ad4b08ff2aae784452f5dd3468fc7e571cecb9665ada80ef5f60f04bf8cf4343", + "source_snapshot/minimax_h3/infer_base.py": "dd6a5ef8aad12ba76c65d74048726fd0f9623a116f216b4a300ef42272d09b93", + "source_snapshot/minimax_h3/infer_student.py": "c3dfa6456fcc1d49500c5a6445431e0254181f0d5342fa24353dc57e8c18c5d8", + "source_snapshot/minimax_h3/infer_student_4step.py": "3bbb180764ef744791065e172acf759b65f8ff7b8e85a3ac8cdc9ee59c738db3", + "source_snapshot/minimax_h3/infer_student_8step.py": "867252e18cb1c53c266d2a3a658453d63e7b9a38711a94ec36bcf88d7f944f5b", + "source_snapshot/minimax_h3/longlive_lora.py": "4d061332055b2b59dfa7632ca7502f98ecdace283333b28f872b1f74f2c1cb4e", + "source_snapshot/minimax_h3/longlive_prompt_distributed.py": "eff9f6e68cc17f4b0237990ac79478a45d5a6e846a4b9b812a508cc03faca48d", + "source_snapshot/minimax_h3/longlive_prompt_set.py": "ed80195faf3f1614c2ec5d3678298505eae00490456ebc365df995324cc33703", + "source_snapshot/minimax_h3/model_provenance.py": "8b5409504258657fdab9665dbd0a51a09e5503e468a9b0e7eb76ad0c2f55ab20", + "source_snapshot/minimax_h3/probe_student_lora_ab.py": "6ff009a751dc07c54cf8dedc98b4061f36080eec93c5068996c5bcec448f0150", + "source_snapshot/minimax_h3/train_dmd.py": "a2f2c71ead08514d96898019eee6c39536192eeb5b017bb2c7454d1cfb0e7172", + "source_snapshot/minimax_h3/train_dmd_longrun.py": "5843b7c57eea8c602bd06cae4580d4efae6fc15be4d370b4a2951cf60baf6ecb", + "source_snapshot/minimax_h3/train_dmd_longrun_4step.py": "5a751327012031d7a51e5004dc3d47094d7810e52ed72d05b04b513bedcee1bf", + "source_snapshot/minimax_h3/train_dmd_longrun_4step_autoresume.py": "42abbef8788b0d86ae6d2d6094bec6144d754d355748eefea6a7f189b7bd8918", + "source_snapshot/minimax_h3/train_dmd_longrun_8step.py": "2f0302f973ba0729040687741eed95edbf321bd4c85745ec1ca977fe91c9e7a0", + "source_snapshot/minimax_h3/train_dmd_longrun_8step_autoresume.py": "21b033ae059b73587257f6923db3587ce6b16c18aa4e8b6f5a91a615905d68dd", + "source_snapshot/minimax_h3/train_dmd_longrun_v2.py": "2ef1d89d400df60a86be10b8828ea7c4b87fb96112538bfd25e7baaf5e8bc357", + "source_snapshot/minimax_h3/train_dmd_longrun_v3.py": "ca1905890c1e3c849ccb7f189166d7d7ddaa22b9d6162c629beab8e0b5c65c81", + "source_snapshot/minimax_h3/train_dmd_longrun_v4.py": "e27328e359c6ff4fccd38623fb7eadb84c48f131fb547ca82bd7364c3f2dd44c", + "source_snapshot/minimax_h3/train_dmd_longrun_v5.py": "14778be13523721a38fb539d973c6d0ad7eff8054ab146a76adcb684bdd9c7cf", + "source_snapshot/minimax_h3/train_utils.py": "fd64c02074680470776fa9e8f58578bde7fdb90603d2cad3eab5e9216c6f8e9c", + "source_snapshot/minimax_h3/visualize_teacher_x0.py": "962f86b0fb699accecc36eb0b57fdc278835c22731f78f5e4d5fe36c3a33d49f", + "training_recipe/train_fresh.py": "6b478c00a64c5bb405823960f541fa3d0faf4cc14b17db5cc762043a71ddec10", + "training_recipe/common.py": "60d1bd3d7774dca1ac1c54234ac05fd2fcb7a0ad1a01450d789fe882f7ad2ef7", + "training_recipe/audio_objectives.py": "4e22fd00a67f5e7e7ce03bec2607911079aad7129fc6b50c88cae3385bc1c605", + "training_recipe/train_audio_mixed.py": "6193189881a1fef24144c4ddf57aa77965bf69308a1392724c826720e8d6c338", + "training_recipe/audio_mixed_conditions.py": "2b41ff3d1249ccdd7857d7b384476e8b65ec09593a44c810e931c6106d863dfd", + "training_recipe/audio_candidate_cache.py": "234d4a9cfad2952b2f98b76f70c77a46399483e9d6da5030ac80639e1fb4b616", + "training_recipe/audio_data_candidates.py": "28cebc27e0c8d914cd716401f9ebb9c03133c5803b362165ddb2e56aa7744274", + "LICENSE": "59b99642b95ea21630e311198ddbfffbfe05aadba0c2f5d884cbdf4efcc90f44", + "LICENSE-CODE": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4" +} diff --git a/source_snapshot/minimax_h3/__init__.py b/source_snapshot/minimax_h3/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3b97762ccd73cad1c7c602e30a2202d63897f9e8 --- /dev/null +++ b/source_snapshot/minimax_h3/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""MiniMax-H3 integration helpers kept separate from LongLive's existing pipelines.""" + +__all__ = ["PromptSpec", "RunConfig", "load_prompt_specs", "run_batch"] + + +def __getattr__(name): + # Keep ``python -m minimax_h3.infer_base`` from pre-importing its own module. + if name in __all__: + from . import infer_base + + return getattr(infer_base, name) + raise AttributeError(name) diff --git a/source_snapshot/minimax_h3/build_prompt_cache.py b/source_snapshot/minimax_h3/build_prompt_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..6ac51647297e13a9e2769457fe344f77dc1fa555 --- /dev/null +++ b/source_snapshot/minimax_h3/build_prompt_cache.py @@ -0,0 +1,570 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Consolidate an audited four-prompt MiniMax-H3 inference run for DMD. + +The source of truth is the strict prompt JSON plus the artifact chain written by +``minimax_h3.infer_base``:: + + prompts JSON -> run manifest -> sample metadata -> training_inputs.pt + +Every link is checked before a byte is written. The resulting ``torch.save`` +payload contains only tensors and weights-only-safe Python primitives, uses the +schema accepted by :func:`minimax_h3.train_utils.load_prompt_cache`, and never +contains negative/CFG, Context-IR, or Regenerator conditioning. +""" + +from __future__ import annotations + +import argparse +import hashlib +import hmac +import json +import os +import re +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping, Sequence + + +EXPECTED_PROMPT_COUNT = 4 +WORKFLOW = "t2va" +TEXT_TOKEN_TAG = 1 +VIDEO_TOKEN_TAG = 0 +AUDIO_TOKEN_TAG = 2 +_COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$") +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_SAFE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$") + +_TRAINING_TENSOR_FIELDS = frozenset( + { + "prompt_embeds", + "text_token_tags", + "token_tags", + "position_ids", + "video_indices", + "audio_indices", + "text_indices", + } +) +_TRAINING_INTEGER_FIELDS = frozenset( + { + "num_condition_video_rows", + "num_condition_audio_rows", + "num_latent_frames", + "latent_height", + "latent_width", + "num_audio_latents", + } +) +_TRAINING_FIELDS = _TRAINING_TENSOR_FIELDS | _TRAINING_INTEGER_FIELDS + + +@dataclass(frozen=True) +class PromptRecord: + id: str + prompt: str + seed: int + + +def sha256_file(path: str | Path) -> str: + """Return a lowercase SHA-256 over the exact on-disk bytes.""" + + digest = hashlib.sha256() + with Path(path).open("rb") as source: + while chunk := source.read(16 * 1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _require_mapping(value: object, label: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError(f"{label} must be a mapping.") + if not all(isinstance(key, str) for key in value): + raise ValueError(f"{label} keys must all be strings.") + return value + + +def _require_exact_sha(value: object, actual: str, label: str) -> None: + if not isinstance(value, str) or not _SHA256.fullmatch(value): + raise ValueError(f"{label} must be a lowercase SHA-256 string.") + if not hmac.compare_digest(value, actual): + raise ValueError(f"{label} mismatch: expected {value}, computed {actual}.") + + +def _read_json(path: Path, label: str) -> Mapping[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ValueError(f"Could not read {label} as UTF-8 JSON: {path}") from error + return _require_mapping(value, label) + + +def _resolve_file_beneath( + root: Path, + raw_path: object, + label: str, + *, + base: Path, +) -> Path: + if not isinstance(raw_path, str) or not raw_path: + raise ValueError(f"{label} path must be a non-empty string.") + candidate = Path(raw_path) + if not candidate.is_absolute(): + candidate = base / candidate + try: + candidate = candidate.resolve(strict=True) + except OSError as error: + raise ValueError(f"{label} does not resolve to an existing file: {candidate}") from error + if candidate != root and root not in candidate.parents: + raise ValueError(f"{label} escapes the run output root: {candidate}") + if not candidate.is_file() or candidate.stat().st_size <= 0: + raise ValueError(f"{label} must be a non-empty regular file: {candidate}") + return candidate + + +def load_prompt_manifest(path: str | Path) -> tuple[list[PromptRecord], str]: + """Load the exact four-row prompt schema used by the official runner.""" + + source = Path(path).resolve(strict=True) + payload = _read_json(source, "prompt manifest") + if set(payload) != {"schema_version", "prompts"}: + raise ValueError("Prompt manifest fields must be exactly schema_version/prompts.") + if payload["schema_version"] != 1: + raise ValueError("Only prompt manifest schema_version=1 is supported.") + rows = payload["prompts"] + if not isinstance(rows, list) or len(rows) != EXPECTED_PROMPT_COUNT: + raise ValueError(f"Prompt manifest must contain exactly {EXPECTED_PROMPT_COUNT} rows.") + + prompts: list[PromptRecord] = [] + seen: set[str] = set() + for index, raw_row in enumerate(rows): + row = _require_mapping(raw_row, f"prompts[{index}]") + if set(row) != {"id", "prompt", "seed"}: + raise ValueError(f"prompts[{index}] fields must be exactly id/prompt/seed.") + prompt_id = row["id"] + prompt = row["prompt"] + seed = row["seed"] + if not isinstance(prompt_id, str) or not _SAFE_ID.fullmatch(prompt_id): + raise ValueError(f"prompts[{index}].id is not a safe artifact id: {prompt_id!r}") + if prompt_id in seen: + raise ValueError(f"Duplicate prompt id: {prompt_id!r}") + if not isinstance(prompt, str) or not prompt.strip(): + raise ValueError(f"prompts[{index}].prompt must be a non-empty string.") + if isinstance(seed, bool) or not isinstance(seed, int) or not 0 <= seed < 2**63: + raise ValueError(f"prompts[{index}].seed must be an integer in [0, 2**63).") + seen.add(prompt_id) + prompts.append(PromptRecord(prompt_id, prompt, seed)) + return prompts, sha256_file(source) + + +def _validate_run_manifest( + path: Path, + prompts: Sequence[PromptRecord], + *, + expected_sha256: str | None, +) -> tuple[Mapping[str, Any], str, str]: + manifest_sha = sha256_file(path) + if expected_sha256 is not None: + _require_exact_sha(expected_sha256, manifest_sha, "run manifest SHA-256") + payload = _read_json(path, "run manifest") + if payload.get("schema_version") != 1: + raise ValueError("Run manifest must use schema_version=1.") + if payload.get("runner") != "LongLive/minimax_h3/infer_base.py": + raise ValueError("Run manifest was not produced by the audited MiniMax-H3 runner.") + + invariants = _require_mapping(payload.get("invariants"), "run manifest invariants") + required_invariants = { + "official_diffusers_modular_pipeline": True, + "workflow": WORKFLOW, + "uses_context_ir": False, + "uses_regenerator": False, + "uses_cfg": False, + "prompt_count": EXPECTED_PROMPT_COUNT, + "official_to_longlive_identity_adapter": True, + } + for name, expected in required_invariants.items(): + if invariants.get(name) != expected: + raise ValueError(f"Run invariant {name!r} must be {expected!r}.") + + config = _require_mapping(payload.get("config"), "run manifest config") + revision = config.get("resolved_revision") + if not isinstance(revision, str) or not _COMMIT_SHA.fullmatch(revision): + raise ValueError("Run manifest resolved_revision must be a lowercase 40-character commit SHA.") + if config.get("dtype") != "bfloat16": + raise ValueError("The official prompt cache must originate from a bfloat16 parity run.") + + samples = payload.get("samples") + if not isinstance(samples, list) or len(samples) != EXPECTED_PROMPT_COUNT: + raise ValueError(f"Run manifest must contain exactly {EXPECTED_PROMPT_COUNT} samples.") + actual_ids = [] + for index, raw_sample in enumerate(samples): + sample = _require_mapping(raw_sample, f"run samples[{index}]") + prompt_id = sample.get("id") + if not isinstance(prompt_id, str): + raise ValueError(f"run samples[{index}].id must be a string.") + actual_ids.append(prompt_id) + expected_ids = [prompt.id for prompt in prompts] + if actual_ids != expected_ids: + raise ValueError( + f"Run sample order/ids differ from the prompt manifest: actual={actual_ids}, expected={expected_ids}." + ) + return payload, manifest_sha, revision + + +def _require_integer(value: object, name: str, *, allow_zero: bool = False) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"training_inputs[{name!r}] must be an integer.") + minimum = 0 if allow_zero else 1 + if value < minimum: + raise ValueError(f"training_inputs[{name!r}] must be >= {minimum}.") + return value + + +def _validate_training_inputs(value: object, prompt_id: str) -> tuple[dict[str, Any], tuple[int, ...]]: + """Validate the runner's exact positive embedding/layout state dict.""" + + import torch + + payload = _require_mapping(value, f"{prompt_id} training_inputs") + if set(payload) != _TRAINING_FIELDS: + missing = sorted(_TRAINING_FIELDS - set(payload)) + extra = sorted(set(payload) - _TRAINING_FIELDS) + raise ValueError( + f"{prompt_id} training_inputs fields differ from the official runner; " + f"missing={missing}, extra={extra}." + ) + for name in _TRAINING_TENSOR_FIELDS: + tensor = payload[name] + if not isinstance(tensor, torch.Tensor): + raise ValueError(f"{prompt_id} training_inputs[{name!r}] must be a tensor.") + if tensor.layout != torch.strided: + raise ValueError(f"{prompt_id} training_inputs[{name!r}] must be a dense strided tensor.") + + prompt_embeds = payload["prompt_embeds"] + text_token_tags = payload["text_token_tags"] + token_tags = payload["token_tags"] + position_ids = payload["position_ids"] + video_indices = payload["video_indices"] + audio_indices = payload["audio_indices"] + text_indices = payload["text_indices"] + if prompt_embeds.ndim != 3 or prompt_embeds.shape[0] != 1 or prompt_embeds.shape[1] < 1: + raise ValueError(f"{prompt_id} prompt_embeds must have shape [1, text_rows, text_dim].") + if not torch.is_floating_point(prompt_embeds): + raise ValueError(f"{prompt_id} prompt_embeds must use a floating-point dtype.") + if not bool(torch.isfinite(prompt_embeds).all()): + raise ValueError(f"{prompt_id} prompt_embeds contain non-finite values.") + if text_token_tags.dtype != torch.long or text_token_tags.shape != (prompt_embeds.shape[1],): + raise ValueError(f"{prompt_id} text_token_tags must be torch.long with one entry per text row.") + if not bool((text_token_tags == TEXT_TOKEN_TAG).all()): + raise ValueError(f"{prompt_id} text_token_tags must contain text rows only.") + + if token_tags.dtype != torch.long or token_tags.ndim != 1: + raise ValueError(f"{prompt_id} token_tags must be a one-dimensional torch.long tensor.") + sequence_length = token_tags.numel() + if position_ids.ndim != 2 or position_ids.shape != (sequence_length, 3): + raise ValueError(f"{prompt_id} position_ids must have shape [sequence, 3].") + if not (torch.is_floating_point(position_ids) or position_ids.dtype in (torch.int32, torch.int64)): + raise ValueError(f"{prompt_id} position_ids must use a numeric dtype.") + if torch.is_floating_point(position_ids) and not bool(torch.isfinite(position_ids).all()): + raise ValueError(f"{prompt_id} position_ids contain non-finite values.") + + for name, indices in ( + ("video_indices", video_indices), + ("audio_indices", audio_indices), + ("text_indices", text_indices), + ): + if indices.dtype != torch.long or indices.ndim != 1 or indices.numel() == 0: + raise ValueError(f"{prompt_id} {name} must be a non-empty one-dimensional torch.long tensor.") + if int(indices.min()) < 0 or int(indices.max()) >= sequence_length: + raise ValueError(f"{prompt_id} {name} contains an out-of-range row.") + all_indices = torch.cat((video_indices, audio_indices, text_indices)) + if all_indices.numel() != sequence_length or torch.unique(all_indices).numel() != sequence_length: + raise ValueError(f"{prompt_id} layout indices must partition every packed row exactly once.") + if text_indices.numel() != prompt_embeds.shape[1]: + raise ValueError(f"{prompt_id} text index count differs from prompt embedding rows.") + expected_tags = torch.empty_like(token_tags) + expected_tags[video_indices] = VIDEO_TOKEN_TAG + expected_tags[audio_indices] = AUDIO_TOKEN_TAG + expected_tags[text_indices] = TEXT_TOKEN_TAG + if not torch.equal(token_tags, expected_tags): + raise ValueError(f"{prompt_id} token_tags disagree with the video/audio/text indices.") + if not torch.equal(token_tags[text_indices], text_token_tags): + raise ValueError(f"{prompt_id} text_token_tags disagree with packed token_tags.") + + condition_video_rows = _require_integer( + payload["num_condition_video_rows"], "num_condition_video_rows", allow_zero=True + ) + condition_audio_rows = _require_integer( + payload["num_condition_audio_rows"], "num_condition_audio_rows", allow_zero=True + ) + if condition_video_rows != 0 or condition_audio_rows != 0: + raise ValueError(f"{prompt_id} contains Context-IR/reference conditioning rows.") + geometry = tuple( + _require_integer(payload[name], name) + for name in ("num_latent_frames", "latent_height", "latent_width", "num_audio_latents") + ) + condition = { + "id": prompt_id, + # prompt/seed are injected only after metadata identity validation. + "prompt_embeds": prompt_embeds.detach().to("cpu").contiguous(), + "text_token_tags": text_token_tags.detach().to("cpu").contiguous(), + } + return condition, geometry + + +def _validate_metadata_and_training_input( + *, + output_root: Path, + prompt: PromptRecord, + sample: Mapping[str, Any], + model_revision: str, +) -> tuple[dict[str, Any], dict[str, Any]]: + import torch + + metadata_path = _resolve_file_beneath( + output_root, + sample.get("metadata"), + f"{prompt.id} metadata", + base=output_root, + ) + metadata_sha = sha256_file(metadata_path) + _require_exact_sha(sample.get("metadata_sha256"), metadata_sha, f"{prompt.id} metadata SHA-256") + metadata = _read_json(metadata_path, f"{prompt.id} metadata") + if metadata.get("schema_version") != 1: + raise ValueError(f"{prompt.id} metadata must use schema_version=1.") + identity = _require_mapping(metadata.get("sample"), f"{prompt.id} metadata sample") + expected_identity = {"id": prompt.id, "prompt": prompt.prompt, "seed": prompt.seed} + if dict(identity) != expected_identity: + raise ValueError(f"{prompt.id} metadata id/prompt/seed do not exactly match the prompt manifest.") + + request = _require_mapping(metadata.get("request"), f"{prompt.id} request") + request_contract = { + "workflow": WORKFLOW, + "uses_context_ir": False, + "uses_regenerator": False, + "uses_cfg": False, + "dtype": "bfloat16", + } + for name, expected in request_contract.items(): + if request.get(name) != expected: + raise ValueError(f"{prompt.id} request field {name!r} must be {expected!r}.") + model = _require_mapping(metadata.get("model"), f"{prompt.id} model") + if model.get("resolved_revision") != model_revision: + raise ValueError(f"{prompt.id} metadata model revision differs from the run manifest.") + + artifacts = _require_mapping(metadata.get("artifacts"), f"{prompt.id} artifacts") + training_record = _require_mapping( + artifacts.get("training_inputs"), f"{prompt.id} training_inputs artifact" + ) + if training_record.get("format") != "torch tensor-only conditioning/layout state_dict": + raise ValueError(f"{prompt.id} training_inputs artifact has an unexpected format label.") + training_path = _resolve_file_beneath( + output_root, + training_record.get("path"), + f"{prompt.id} training_inputs", + base=metadata_path.parent, + ) + training_sha = sha256_file(training_path) + _require_exact_sha( + training_record.get("sha256"), training_sha, f"{prompt.id} metadata training_inputs SHA-256" + ) + _require_exact_sha( + sample.get("training_inputs_sha256"), training_sha, f"{prompt.id} manifest training_inputs SHA-256" + ) + try: + training_inputs = torch.load(training_path, map_location="cpu", weights_only=True) + except Exception as error: + raise ValueError(f"{prompt.id} training_inputs is not a weights-only-safe torch payload.") from error + condition, geometry = _validate_training_inputs(training_inputs, prompt.id) + condition["prompt"] = prompt.prompt + condition["seed"] = prompt.seed + # Keep the loader-facing condition schema exact and human-readable. + condition = { + "id": condition["id"], + "prompt": condition["prompt"], + "seed": condition["seed"], + "prompt_embeds": condition["prompt_embeds"], + "text_token_tags": condition["text_token_tags"], + } + evidence = { + "id": prompt.id, + "metadata": str(metadata_path), + "metadata_sha256": metadata_sha, + "training_inputs": str(training_path), + "training_inputs_sha256": training_sha, + "geometry": { + "num_latent_frames": geometry[0], + "latent_height": geometry[1], + "latent_width": geometry[2], + "num_audio_latents": geometry[3], + }, + } + return condition, evidence + + +def _atomic_torch_save(path: Path, payload: Mapping[str, Any]) -> None: + import torch + + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", suffix=".tmp", dir=path.parent + ) + os.close(descriptor) + temporary = Path(temporary_name) + try: + torch.save(dict(payload), temporary) + with temporary.open("rb") as handle: + os.fsync(handle.fileno()) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _atomic_json_write(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", suffix=".tmp", dir=path.parent + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + json.dump(payload, handle, ensure_ascii=False, indent=2, sort_keys=True) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def build_prompt_cache( + *, + prompts_json: str | Path, + run_manifest: str | Path, + output: str | Path, + sidecar: str | Path | None = None, + expected_run_manifest_sha256: str | None = None, +) -> dict[str, Any]: + """Validate the complete source chain, atomically save the cache, and return its sidecar.""" + + import torch + + prompt_path = Path(prompts_json).resolve(strict=True) + run_manifest_path = Path(run_manifest).resolve(strict=True) + output_path = Path(output).resolve() + sidecar_path = ( + Path(sidecar).resolve() + if sidecar is not None + else output_path.with_name(output_path.name + ".json") + ) + if output_path == sidecar_path: + raise ValueError("Prompt cache and JSON sidecar paths must be different.") + if output_path in {prompt_path, run_manifest_path} or sidecar_path in {prompt_path, run_manifest_path}: + raise ValueError("Output paths cannot overwrite either source manifest.") + + prompts, prompt_sha = load_prompt_manifest(prompt_path) + manifest, manifest_sha, revision = _validate_run_manifest( + run_manifest_path, + prompts, + expected_sha256=expected_run_manifest_sha256, + ) + output_root = run_manifest_path.parent.resolve() + samples = manifest["samples"] + conditions: list[dict[str, Any]] = [] + evidence: list[dict[str, Any]] = [] + seen_source_paths: set[str] = set() + common_geometry: Mapping[str, int] | None = None + for prompt, raw_sample in zip(prompts, samples, strict=True): + sample = _require_mapping(raw_sample, f"run sample {prompt.id}") + condition, sample_evidence = _validate_metadata_and_training_input( + output_root=output_root, + prompt=prompt, + sample=sample, + model_revision=revision, + ) + for name in ("metadata", "training_inputs"): + source_path = sample_evidence[name] + if source_path in seen_source_paths: + raise ValueError(f"Multiple prompts unexpectedly reuse source artifact {source_path}.") + seen_source_paths.add(source_path) + geometry = sample_evidence["geometry"] + if common_geometry is None: + common_geometry = geometry + elif geometry != common_geometry: + raise ValueError("All four prompt inputs must use the same latent/audio geometry.") + conditions.append(condition) + evidence.append(sample_evidence) + + source_paths = {Path(path).resolve() for path in seen_source_paths} + if output_path in source_paths or sidecar_path in source_paths: + raise ValueError("Output paths cannot overwrite metadata or training input artifacts.") + payload: dict[str, Any] = { + "schema_version": 1, + "workflow": WORKFLOW, + "cfg_enabled": False, + "model_revision": revision, + "prompt_manifest_sha256": prompt_sha, + "conditions": conditions, + } + _atomic_torch_save(output_path, payload) + try: + reloaded = torch.load(output_path, map_location="cpu", weights_only=True) + except Exception as error: + raise RuntimeError("Atomic prompt-cache output did not pass a weights-only reload.") from error + if not isinstance(reloaded, Mapping) or set(reloaded) != set(payload): + raise RuntimeError("Atomic prompt-cache output failed its schema reload check.") + output_sha = sha256_file(output_path) + sidecar_payload: dict[str, Any] = { + "schema_version": 1, + "artifact_type": "minimax_h3_positive_prompt_cache", + "workflow": WORKFLOW, + "cfg_enabled": False, + "model_revision": revision, + "prompt_manifest": str(prompt_path), + "prompt_manifest_sha256": prompt_sha, + "run_manifest": str(run_manifest_path), + "run_manifest_sha256": manifest_sha, + "output": str(output_path), + "output_bytes": output_path.stat().st_size, + "output_sha256": output_sha, + "conditions": evidence, + } + _atomic_json_write(sidecar_path, sidecar_payload) + return {**sidecar_payload, "sidecar": str(sidecar_path)} + + +def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--prompts-json", type=Path, required=True) + parser.add_argument("--run-manifest", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--sidecar", + type=Path, + help="JSON evidence path (default: OUTPUT.json).", + ) + parser.add_argument( + "--expected-run-manifest-sha256", + help="Optional external SHA-256 gate, e.g. from the official run SUCCESS.json.", + ) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parse_args(argv) + sidecar = build_prompt_cache( + prompts_json=args.prompts_json, + run_manifest=args.run_manifest, + output=args.output, + sidecar=args.sidecar, + expected_run_manifest_sha256=args.expected_run_manifest_sha256, + ) + print(json.dumps(sidecar, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/source_snapshot/minimax_h3/cfg_guidance.py b/source_snapshot/minimax_h3/cfg_guidance.py new file mode 100644 index 0000000000000000000000000000000000000000..3c2cfe9bf789cd786bfca393d8d6f3d9059c9d95 --- /dev/null +++ b/source_snapshot/minimax_h3/cfg_guidance.py @@ -0,0 +1,87 @@ +"""Experimental extra guidance; the released H3 base is already CFG-distilled.""" +from __future__ import annotations + +import math +import torch + +from minimax_h3.dmd_4step import H3DMD4StepTrainer +from minimax_h3.train_utils import ( + H3Condition, H3PackedLayout, disable_adapters_for_teacher, +) + + +def combine(conditional: torch.Tensor, reference: torch.Tensor, scale: float): + if not math.isfinite(scale) or scale < 1: + raise ValueError("Extra guidance scale must be finite and >= 1.") + if conditional.shape != reference.shape: + raise ValueError("Guidance predictions must have identical shapes.") + if scale == 1: + return conditional + # Accumulate the subtraction in fp32, then retain the caller's dtype. + return (conditional.float() + (scale - 1) * + (conditional.float() - reference.float())).to(conditional.dtype) + + +def reference_layout(layout: H3PackedLayout, reference_tags: torch.Tensor): + """Rebuild a text-only T2VA layout; preserve all media coordinates relative to text.""" + layout.validate() + device = layout.position_ids.device + old_text = layout.text_indices.numel() + new_text = reference_tags.numel() + if new_text < 1 or not bool((reference_tags == 1).all()): + raise ValueError("Reference must contain one or more text-only rows.") + if not torch.equal(layout.text_indices, torch.arange(old_text, device=device)): + raise ValueError("Expected the official text-prefix T2VA packing.") + shift = new_text - old_text + media_positions = layout.position_ids[old_text:].clone() + media_positions[:, 0] += shift + text_positions = torch.zeros((new_text, 3), device=device, + dtype=layout.position_ids.dtype) + text_positions[:, 0] = torch.arange(new_text, device=device) + result = H3PackedLayout( + position_ids=torch.cat((text_positions, media_positions)), + token_tags=torch.cat((reference_tags.to(device), layout.token_tags[old_text:])), + video_indices=layout.video_indices + shift, + audio_indices=layout.audio_indices + shift, + text_indices=torch.arange(new_text, device=device), + ) + result.validate() + return result + + +class GuidedTeacherTrainer(H3DMD4StepTrainer): + """Extra guidance changes only the real-score teacher, never student/fake-score sampling.""" + + def __init__(self, *, reference_condition: H3Condition, + teacher_video_scale: float, teacher_audio_scale: float, **kwargs): + reference_condition.validate() + for scale in (teacher_video_scale, teacher_audio_scale): + if not math.isfinite(scale) or scale < 1: + raise ValueError("Teacher guidance scales must be finite and >= 1.") + super().__init__(**kwargs) + self.reference_condition = reference_condition + self.teacher_video_scale = teacher_video_scale + self.teacher_audio_scale = teacher_audio_scale + self.guidance_metrics = {} + + def _score_pair(self, **kwargs): + video, audio, critic_video, critic_audio = super()._score_pair(**kwargs) + if self.teacher_video_scale == self.teacher_audio_scale == 1: + return video, audio, critic_video, critic_audio + reference = self.reference_condition.to(kwargs["noisy_video"].device) + layout = reference_layout(kwargs["layout"], reference.text_token_tags) + with torch.no_grad(), disable_adapters_for_teacher(self.role_model): + ref_video, ref_audio = self._forward( + role="teacher_reference", video_rows=kwargs["noisy_video"], + audio_rows=kwargs["noisy_audio"], condition=reference, layout=layout, + video_timestep=kwargs["video_timestep"], + audio_timestep=kwargs["audio_timestep"], adapters_enabled=False, + ) + self.guidance_metrics = { + "teacher_video_guidance_scale": self.teacher_video_scale, + "teacher_audio_guidance_scale": self.teacher_audio_scale, + "teacher_video_reference_delta_rms": float((video.float() - ref_video.float()).square().mean().sqrt()), + "teacher_audio_reference_delta_rms": float((audio.float() - ref_audio.float()).square().mean().sqrt()), + } + return (combine(video, ref_video, self.teacher_video_scale), + combine(audio, ref_audio, self.teacher_audio_scale), critic_video, critic_audio) diff --git a/source_snapshot/minimax_h3/dmd_4step.py b/source_snapshot/minimax_h3/dmd_4step.py new file mode 100644 index 0000000000000000000000000000000000000000..130b8346df57dc315d59f5920f9ab85c9ae9ec49 --- /dev/null +++ b/source_snapshot/minimax_h3/dmd_4step.py @@ -0,0 +1,891 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Isolated four-evaluation fresh-noise DMD core for MiniMax-H3. + +The older MiniMax-H3 trainers in this repository use a deterministic Euler +rollout. This module deliberately does not modify those trainers. It +implements the fresh-noise backward-simulation contract described by the H3 +DMD technical note, native four denoiser evaluations: + +* four H3 forwards and five sigma endpoints (including terminal zero), +* three predict-x0-then-re-noise transitions, +* independent video/audio transition noise, always drawn video first, and +* a fixed-cost target-stage path: three detached state-building forwards plus + exactly one differentiable forward at a uniformly sampled stage. + +The class inherits the existing adapter switching, H3 packed forward, score +pair, loss, clipping, and saved-tensor-offload machinery. The implementation +is otherwise self-contained so the existing v1-v5 training paths retain their +original four-step behavior. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass + +import torch + +from .dmd_core import ( + H3SigmaSchedules, + PositiveOnlyCallCounter, + build_h3_sigma_schedules, + critic_flow_matching_loss, + dmd_generator_pseudo_loss, + h3_add_noise, + h3_data_velocity_to_x0, +) +from .train_utils import ( + CRITIC_ADAPTER, + STUDENT_ADAPTER, + CriticStepOutput, + GeneratorStepOutput, + H3CallAudit, + H3Condition, + H3DMDTrainer, + H3PackedLayout, + _random_normal_like, + paired_score_timesteps, + use_adapter, +) + + +NUM_MODEL_EVALUATIONS = 4 +NUM_SIGMA_ENDPOINTS = NUM_MODEL_EVALUATIONS + 1 +NUM_FRESH_NOISE_TRANSITIONS = NUM_MODEL_EVALUATIONS - 1 +TARGET_STAGE_MIN = 0 +TARGET_STAGE_MAX = NUM_MODEL_EVALUATIONS - 1 + + +@dataclass(frozen=True) +class H3FreshNoiseStageState: + """Detached input rows and clocks cached for one denoiser stage.""" + + stage: int + video_rows: torch.Tensor + audio_rows: torch.Tensor + video_sigma: torch.Tensor + audio_sigma: torch.Tensor + video_timestep: torch.Tensor + audio_timestep: torch.Tensor + + +@dataclass(frozen=True) +class H3TargetStageRolloutOutput: + """One differentiable target-stage clean prediction plus its state cache.""" + + generated_video: torch.Tensor + generated_audio: torch.Tensor + target_stage: int + stage_states: tuple[H3FreshNoiseStageState, ...] + + +@dataclass(frozen=True) +class H3DMD4GeneratorStepOutput(GeneratorStepOutput): + """Generator metrics augmented with the sampled four-step target stage.""" + + target_stage: int + zero_grad_accepted: bool + + +def build_4step_sigma_schedules( + *, + video_shift: float = 12.0, + audio_shift: float = 3.0, + device: torch.device | str | None = None, + dtype: torch.dtype = torch.float32, +) -> H3SigmaSchedules: + """Return the fixed four-evaluation/five-endpoint H3 sigma grid.""" + + schedules = build_h3_sigma_schedules( + num_evaluations=NUM_MODEL_EVALUATIONS, + video_shift=video_shift, + audio_shift=audio_shift, + device=device, + dtype=dtype, + ) + if schedules.num_evaluations != NUM_MODEL_EVALUATIONS: + raise AssertionError("Four-step H3 schedule construction changed unexpectedly.") + if schedules.video_sigmas.numel() != NUM_SIGMA_ENDPOINTS: + raise AssertionError("Four-step H3 must have exactly five sigma endpoints.") + return schedules + + +def sample_target_stage( + *, + generator: torch.Generator | None = None, + device: torch.device | str | None = None, +) -> int: + """Sample ``Uniform({0, ..., 3})`` with an optionally replayable RNG.""" + + if device is None: + device = generator.device if generator is not None else torch.device("cpu") + return int( + torch.randint( + TARGET_STAGE_MIN, + TARGET_STAGE_MAX + 1, + (), + generator=generator, + device=device, + ).item() + ) + + +def _validate_target_stage(target_stage: int) -> int: + if isinstance(target_stage, bool) or not isinstance(target_stage, int): + raise TypeError("target_stage must be an integer in [0, 3].") + if not TARGET_STAGE_MIN <= target_stage <= TARGET_STAGE_MAX: + raise ValueError("target_stage must lie in [0, 3].") + return target_stage + + +def _fresh_transition( + *, + clean_video: torch.Tensor, + clean_audio: torch.Tensor, + next_video_timestep: torch.Tensor, + next_audio_timestep: torch.Tensor, + generator: torch.Generator | None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Re-noise both modalities, with the replay-critical video-first order.""" + + # Do not combine or reorder these draws. A/V shapes differ in real H3, + # and this precise order is part of seeded checkpoint/evaluation replay. + video_noise = _random_normal_like(clean_video, generator) + audio_noise = _random_normal_like(clean_audio, generator) + return ( + h3_add_noise(clean_video, video_noise, next_video_timestep), + h3_add_noise(clean_audio, audio_noise, next_audio_timestep), + ) + + +class H3DMD4StepTrainer(H3DMDTrainer): + """H3 DMD trainer with a four-forward fresh-noise student sampler.""" + + def __init__( + self, + *, + forward_model: torch.nn.Module, + role_model: torch.nn.Module, + student_optimizer: torch.optim.Optimizer, + critic_optimizer: torch.optim.Optimizer, + video_shift: float = 12.0, + audio_shift: float = 3.0, + video_loss_weight: float = 1.0, + audio_loss_weight: float = 1.0, + max_grad_norm: float = 1.0, + audit: H3CallAudit | None = None, + allow_zero_generator_grad: bool = False, + ) -> None: + if not isinstance(allow_zero_generator_grad, bool): + raise TypeError("allow_zero_generator_grad must be a bool.") + loss_weights = { + "video_loss_weight": video_loss_weight, + "audio_loss_weight": audio_loss_weight, + } + for name, value in loss_weights.items(): + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + or float(value) < 0.0 + ): + raise ValueError(f"{name} must be a finite non-negative number.") + if float(video_loss_weight) == 0.0 and float(audio_loss_weight) == 0.0: + raise ValueError("At least one modality loss weight must be positive.") + super().__init__( + forward_model=forward_model, + role_model=role_model, + student_optimizer=student_optimizer, + critic_optimizer=critic_optimizer, + video_shift=video_shift, + audio_shift=audio_shift, + max_grad_norm=max_grad_norm, + audit=audit, + ) + self.video_loss_weight = float(video_loss_weight) + self.audio_loss_weight = float(audio_loss_weight) + self.allow_zero_generator_grad = allow_zero_generator_grad + + def _schedules(self, reference: torch.Tensor) -> H3SigmaSchedules: + return build_4step_sigma_schedules( + video_shift=self.video_shift, + audio_shift=self.audio_shift, + device=reference.device, + dtype=torch.float32, + ) + + def _clip_gradient_subset( + self, + parameters: tuple[torch.nn.Parameter, ...] | list[torch.nn.Parameter], + *, + device: torch.device, + ) -> torch.Tensor: + """Clip one FSDP-sharded role without touching deferred other-role grads.""" + + gradients = [ + parameter.grad + for parameter in parameters + if parameter.grad is not None + ] + local_squared = torch.zeros((), device=device, dtype=torch.float32) + if gradients: + local_squared = torch.stack( + [ + torch.dot( + gradient.detach().float().reshape(-1), + gradient.detach().float().reshape(-1), + ) + for gradient in gradients + ] + ).sum() + # A valid FSDP rank may own no shard of this role. Keep a separate + # presence scalar and make every rank enter the same all-reduce anyway. + reduced = torch.stack( + [ + local_squared.to(device=device), + torch.tensor(float(bool(gradients)), device=device), + ] + ) + if self.forward_model is not self.role_model: + import torch.distributed as dist + + dist.all_reduce(reduced, op=dist.ReduceOp.SUM) + if float(reduced[1]) <= 0.0: + raise RuntimeError("No rank owns a gradient for the requested LoRA role.") + norm = reduced[0].sqrt() + if not bool(torch.isfinite(norm)): + raise FloatingPointError("The global role gradient norm is non-finite.") + coefficient = torch.clamp( + self.max_grad_norm / (norm + 1.0e-6), + max=1.0, + ) + for gradient in gradients: + gradient.mul_(coefficient.to(device=gradient.device, dtype=gradient.dtype)) + return norm + + @staticmethod + def _local_gradient_signature( + parameters: tuple[torch.nn.Parameter, ...] | list[torch.nn.Parameter], + *, + device: torch.device, + ) -> torch.Tensor: + """Return a small exact-change sentinel without cloning full gradients.""" + + gradients = [ + parameter.grad.detach().float() + for parameter in parameters + if parameter.grad is not None + ] + if not gradients: + return torch.zeros(3, device=device, dtype=torch.float32) + sums = torch.stack([gradient.sum() for gradient in gradients]).sum() + squared = torch.stack( + [ + torch.dot(gradient.reshape(-1), gradient.reshape(-1)) + for gradient in gradients + ] + ).sum() + return torch.stack( + [ + torch.tensor(float(len(gradients)), device=device), + sums.to(device=device), + squared.to(device=device), + ] + ) + + def _assert_global_signature_presence( + self, + signature: torch.Tensor, + *, + role: str, + ) -> None: + """Accept an empty local shard but require the role globally.""" + + presence = signature[0].detach().clone() + if self.forward_model is not self.role_model: + import torch.distributed as dist + + dist.all_reduce(presence, op=dist.ReduceOp.SUM) + if float(presence) <= 0.0: + raise RuntimeError(f"No deferred {role} gradients exist on any rank.") + + def _global_gradient_presence( + self, + parameter_groups: tuple[ + tuple[torch.nn.Parameter, ...] | list[torch.nn.Parameter], ... + ], + *, + device: torch.device, + ) -> tuple[bool, ...]: + """Return global role presence without branching on local shard layout.""" + + flags = torch.tensor( + [ + int(any(parameter.grad is not None for parameter in parameters)) + for parameters in parameter_groups + ], + device=device, + dtype=torch.int32, + ) + if self.forward_model is not self.role_model: + import torch.distributed as dist + + dist.all_reduce(flags, op=dist.ReduceOp.MAX) + return tuple(bool(value) for value in flags.tolist()) + + def _all_ranks_true(self, value: bool, *, device: torch.device) -> bool: + consensus = torch.tensor(int(value), device=device, dtype=torch.int32) + if self.forward_model is not self.role_model: + import torch.distributed as dist + + dist.all_reduce(consensus, op=dist.ReduceOp.MIN) + return bool(consensus.item()) + + @staticmethod + def _stage_state( + *, + stage: int, + video_rows: torch.Tensor, + audio_rows: torch.Tensor, + schedules: H3SigmaSchedules, + ) -> H3FreshNoiseStageState: + return H3FreshNoiseStageState( + stage=stage, + video_rows=video_rows.detach(), + audio_rows=audio_rows.detach(), + video_sigma=schedules.video_sigmas[stage], + audio_sigma=schedules.audio_sigmas[stage], + video_timestep=schedules.video_timesteps[stage], + audio_timestep=schedules.audio_timesteps[stage], + ) + + def detached_rollout( + self, + *, + video_noise: torch.Tensor, + audio_noise: torch.Tensor, + condition: H3Condition, + layout: H3PackedLayout, + target_stage: int, + generator: torch.Generator | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Run all four stages and return the selected detached clean prediction. + + All four positive forwards and all three fresh-noise transitions are + executed regardless of ``target_stage``. Besides matching the training + distribution, this fixed cost is required so data-parallel ranks enter + FSDP collectives in the same order when they sample different stages. + """ + + target_stage = _validate_target_stage(target_stage) + schedules = self._schedules(video_noise) + video = video_noise.detach() + audio = audio_noise.detach() + selected_video: torch.Tensor | None = None + selected_audio: torch.Tensor | None = None + with use_adapter(self.role_model, STUDENT_ADAPTER): + with torch.no_grad(): + with PositiveOnlyCallCounter( + expected_calls=NUM_MODEL_EVALUATIONS + ) as calls: + for stage in range(NUM_MODEL_EVALUATIONS): + video_velocity, audio_velocity = calls.call( + self._forward, + role="student_detached_stage", + video_rows=video, + audio_rows=audio, + condition=condition, + layout=layout, + video_timestep=schedules.video_timesteps[stage], + audio_timestep=schedules.audio_timesteps[stage], + adapters_enabled=True, + ) + clean_video = h3_data_velocity_to_x0( + video, + video_velocity, + schedules.video_timesteps[stage], + ) + clean_audio = h3_data_velocity_to_x0( + audio, + audio_velocity, + schedules.audio_timesteps[stage], + ) + if stage == target_stage: + selected_video = clean_video.detach() + selected_audio = clean_audio.detach() + if stage < TARGET_STAGE_MAX: + video, audio = _fresh_transition( + clean_video=clean_video, + clean_audio=clean_audio, + next_video_timestep=schedules.video_timesteps[stage + 1], + next_audio_timestep=schedules.audio_timesteps[stage + 1], + generator=generator, + ) + if selected_video is None or selected_audio is None: + raise AssertionError("The four-stage detached rollout missed its target stage.") + return selected_video, selected_audio + + def student_rollout( + self, + *, + video_noise: torch.Tensor, + audio_noise: torch.Tensor, + condition: H3Condition, + layout: H3PackedLayout, + target_stage: int, + generator: torch.Generator | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Compatibility name for fake-score data generation.""" + + return self.detached_rollout( + video_noise=video_noise, + audio_noise=audio_noise, + condition=condition, + layout=layout, + target_stage=target_stage, + generator=generator, + ) + + def target_stage_rollout( + self, + *, + video_noise: torch.Tensor, + audio_noise: torch.Tensor, + condition: H3Condition, + layout: H3PackedLayout, + target_stage: int | None = None, + generator: torch.Generator | None = None, + ) -> H3TargetStageRolloutOutput: + """Build all stage states detached, then differentiate one random stage. + + Stage selection consumes the first RNG draw when ``target_stage`` is + omitted. The next fourteen draws are the seven video/audio transition + pairs. Passing an explicit stage intentionally consumes no stage RNG. + """ + + if target_stage is None: + target_stage = sample_target_stage( + generator=generator, + device=video_noise.device, + ) + target_stage = _validate_target_stage(target_stage) + schedules = self._schedules(video_noise) + video = video_noise.detach() + audio = audio_noise.detach() + stage_states = [ + self._stage_state( + stage=0, + video_rows=video, + audio_rows=audio, + schedules=schedules, + ) + ] + + with use_adapter(self.role_model, STUDENT_ADAPTER): + with PositiveOnlyCallCounter( + expected_calls=NUM_MODEL_EVALUATIONS + ) as calls: + # Always materialize states 1..3. The three no-grad forward + # order is independent of the randomly selected target stage. + with torch.no_grad(): + for stage in range(NUM_FRESH_NOISE_TRANSITIONS): + video_velocity, audio_velocity = calls.call( + self._forward, + role="student_stage_cache", + video_rows=video, + audio_rows=audio, + condition=condition, + layout=layout, + video_timestep=schedules.video_timesteps[stage], + audio_timestep=schedules.audio_timesteps[stage], + adapters_enabled=True, + ) + clean_video = h3_data_velocity_to_x0( + video, + video_velocity, + schedules.video_timesteps[stage], + ) + clean_audio = h3_data_velocity_to_x0( + audio, + audio_velocity, + schedules.audio_timesteps[stage], + ) + video, audio = _fresh_transition( + clean_video=clean_video, + clean_audio=clean_audio, + next_video_timestep=schedules.video_timesteps[stage + 1], + next_audio_timestep=schedules.audio_timesteps[stage + 1], + generator=generator, + ) + video = video.detach() + audio = audio.detach() + stage_states.append( + self._stage_state( + stage=stage + 1, + video_rows=video, + audio_rows=audio, + schedules=schedules, + ) + ) + + selected = stage_states[target_stage] + video_velocity, audio_velocity = calls.call( + self._forward, + role="student_target_stage", + video_rows=selected.video_rows, + audio_rows=selected.audio_rows, + condition=condition, + layout=layout, + video_timestep=selected.video_timestep, + audio_timestep=selected.audio_timestep, + adapters_enabled=True, + ) + generated_video = h3_data_velocity_to_x0( + selected.video_rows, + video_velocity, + selected.video_timestep, + ) + generated_audio = h3_data_velocity_to_x0( + selected.audio_rows, + audio_velocity, + selected.audio_timestep, + ) + + if len(stage_states) != NUM_MODEL_EVALUATIONS: + raise AssertionError("Target-stage rollout must cache exactly four states.") + return H3TargetStageRolloutOutput( + generated_video=generated_video, + generated_audio=generated_audio, + target_stage=target_stage, + stage_states=tuple(stage_states), + ) + + def critic_step( + self, + *, + clean_video: torch.Tensor, + clean_audio: torch.Tensor, + condition: H3Condition, + layout: H3PackedLayout, + score_base_sigma: float, + score_video_noise: torch.Tensor | None = None, + score_audio_noise: torch.Tensor | None = None, + generator: torch.Generator | None = None, + defer_optimizer_step: bool = False, + ) -> CriticStepOutput: + """Backpropagate fake-score DSM, optionally deferring its AdamW step.""" + + if not isinstance(defer_optimizer_step, bool): + raise TypeError("defer_optimizer_step must be a bool.") + self.critic_optimizer.zero_grad(set_to_none=True) + self.student_optimizer.zero_grad(set_to_none=True) + clean_video = clean_video.detach() + clean_audio = clean_audio.detach() + video_timestep, audio_timestep = self._score_timesteps(score_base_sigma) + if score_video_noise is None: + score_video_noise = _random_normal_like(clean_video, generator) + if score_audio_noise is None: + score_audio_noise = _random_normal_like(clean_audio, generator) + noisy_video = h3_add_noise(clean_video, score_video_noise, video_timestep) + noisy_audio = h3_add_noise(clean_audio, score_audio_noise, audio_timestep) + with use_adapter(self.role_model, CRITIC_ADAPTER): + predicted_video, predicted_audio = self._forward( + role="critic_train", + video_rows=noisy_video, + audio_rows=noisy_audio, + condition=condition, + layout=layout, + video_timestep=video_timestep, + audio_timestep=audio_timestep, + adapters_enabled=True, + ) + losses = critic_flow_matching_loss( + predicted_video_velocity=predicted_video, + clean_video=clean_video, + video_noise=score_video_noise, + predicted_audio_velocity=predicted_audio, + clean_audio=clean_audio, + audio_noise=score_audio_noise, + video_weight=self.video_loss_weight, + audio_weight=self.audio_loss_weight, + ) + if losses.audio is None or not bool(torch.isfinite(losses.total)): + raise FloatingPointError("The joint H3 critic loss is not finite.") + losses.total.backward() + grad_norm = self._clip_active_gradients( + self.critic_parameters, losses.total.device + ) + if not bool(torch.isfinite(grad_norm)) or float(grad_norm) == 0.0: + raise FloatingPointError("The critic LoRA gradient is zero or non-finite.") + (student_has_grad,) = self._global_gradient_presence( + (self.student_parameters,), + device=losses.total.device, + ) + if student_has_grad: + raise AssertionError("Critic backward leaked into the student adapter.") + if not defer_optimizer_step: + self.critic_optimizer.step() + return CriticStepOutput( + loss=losses.total.detach(), + video_loss=losses.video.detach(), + audio_loss=losses.audio.detach(), + grad_norm=grad_norm.detach(), + score_video_timestep=video_timestep, + score_audio_timestep=audio_timestep, + ) + + def step_deferred_role_optimizer( + self, + *, + generator_update: bool, + device: torch.device, + ) -> None: + """Apply one joint AdamW step after all pre-update score targets exist.""" + + if not isinstance(generator_update, bool): + raise TypeError("generator_update must be a bool.") + if not isinstance(device, torch.device): + raise TypeError("device must be a torch.device.") + if self.student_optimizer is not self.critic_optimizer: + raise RuntimeError("Deferred role stepping requires one shared optimizer.") + critic_has_grad, student_has_grad = self._global_gradient_presence( + (self.critic_parameters, self.student_parameters), + device=device, + ) + if not critic_has_grad: + raise RuntimeError("Deferred critic gradients are missing.") + if student_has_grad != generator_update: + raise RuntimeError( + "Deferred student gradient presence differs from update cadence." + ) + self.student_optimizer.step() + + def generator_step( + self, + *, + video_noise: torch.Tensor, + audio_noise: torch.Tensor, + condition: H3Condition, + layout: H3PackedLayout, + score_base_sigma: float, + score_video_noise: torch.Tensor | None = None, + score_audio_noise: torch.Tensor | None = None, + target_stage: int | None = None, + generator: torch.Generator | None = None, + allow_zero_generator_grad: bool | None = None, + defer_optimizer_step: bool = False, + preserve_deferred_critic_gradients: bool = False, + ) -> H3DMD4GeneratorStepOutput: + """Update the student from one random stage and report that stage. + + ``allow_zero_generator_grad`` exists for the documented initialization + bootstrap where zero-initialized fake-score LoRA and the frozen teacher + produce the same DMD target. Non-finite gradients always remain fatal. + A caller can enable this only for the first update even if the trainer + default remains strict. + """ + + if allow_zero_generator_grad is None: + allow_zero_generator_grad = self.allow_zero_generator_grad + if not isinstance(allow_zero_generator_grad, bool): + raise TypeError("allow_zero_generator_grad must be a bool.") + if not isinstance(defer_optimizer_step, bool) or not isinstance( + preserve_deferred_critic_gradients, bool + ): + raise TypeError("Deferred optimizer controls must be bools.") + if preserve_deferred_critic_gradients and not defer_optimizer_step: + raise ValueError( + "Preserving deferred critic gradients requires a deferred step." + ) + + deferred_critic_signature: torch.Tensor | None = None + deferred_critic_gradients: tuple[torch.Tensor | None, ...] | None = None + if preserve_deferred_critic_gradients: + if self.student_optimizer is not self.critic_optimizer: + raise RuntimeError( + "Deferred joint stepping requires one shared role optimizer." + ) + deferred_critic_signature = self._local_gradient_signature( + self.critic_parameters, + device=video_noise.device, + ) + self._assert_global_signature_presence( + deferred_critic_signature, + role="critic", + ) + # FSDP ``use_orig_params=True`` exposes each original gradient as + # a view into its flat shard. PEFT then marks the inactive critic + # adapter ``requires_grad=False`` while the generator is active, + # so FSDP intentionally drops those original ``.grad`` views at + # the end of the next backward. Stash the already clipped local + # critic shards, clear the complete mixed-role flat gradients, + # and restore the exact critic values after the isolated generator + # backward. This also avoids re-reducing an existing critic shard. + deferred_critic_gradients = tuple( + parameter.grad.detach().clone() + if parameter.grad is not None + else None + for parameter in self.critic_parameters + ) + self.student_optimizer.zero_grad(set_to_none=True) + else: + self.student_optimizer.zero_grad(set_to_none=True) + self.critic_optimizer.zero_grad(set_to_none=True) + with self._offload_student_saved_tensors(video_noise.device): + rollout = self.target_stage_rollout( + video_noise=video_noise, + audio_noise=audio_noise, + condition=condition, + layout=layout, + target_stage=target_stage, + generator=generator, + ) + generated_video = rollout.generated_video + generated_audio = rollout.generated_audio + selected_stage = rollout.target_stage + # The seven other cached states are not loss inputs and can be released + # before the two score calls consume additional device memory. + del rollout + + video_timestep, audio_timestep = self._score_timesteps(score_base_sigma) + if score_video_noise is None: + score_video_noise = _random_normal_like(generated_video, generator) + if score_audio_noise is None: + score_audio_noise = _random_normal_like(generated_audio, generator) + noisy_video = h3_add_noise( + generated_video.detach(), score_video_noise, video_timestep + ) + noisy_audio = h3_add_noise( + generated_audio.detach(), score_audio_noise, audio_timestep + ) + teacher_video_v, teacher_audio_v, critic_video_v, critic_audio_v = ( + self._score_pair( + noisy_video=noisy_video, + noisy_audio=noisy_audio, + condition=condition, + layout=layout, + video_timestep=video_timestep, + audio_timestep=audio_timestep, + ) + ) + teacher_video_x0 = h3_data_velocity_to_x0( + noisy_video, teacher_video_v, video_timestep + ) + teacher_audio_x0 = h3_data_velocity_to_x0( + noisy_audio, teacher_audio_v, audio_timestep + ) + critic_video_x0 = h3_data_velocity_to_x0( + noisy_video, critic_video_v, video_timestep + ) + critic_audio_x0 = h3_data_velocity_to_x0( + noisy_audio, critic_audio_v, audio_timestep + ) + losses = dmd_generator_pseudo_loss( + generated_video=generated_video, + real_video_x0=teacher_video_x0, + fake_video_x0=critic_video_x0, + generated_audio=generated_audio, + real_audio_x0=teacher_audio_x0, + fake_audio_x0=critic_audio_x0, + video_weight=self.video_loss_weight, + audio_weight=self.audio_loss_weight, + ) + if losses.audio is None or not bool(torch.isfinite(losses.total)): + raise FloatingPointError("The joint H3 generator DMD loss is not finite.") + + self.role_model.set_adapter(STUDENT_ADAPTER) + if losses.total.device.type == "cuda": + torch.cuda.empty_cache() + self.cuda_cache_clears_before_generator_backward += 1 + losses.total.backward() + grad_norm = ( + self._clip_gradient_subset( + self.student_parameters, + device=losses.total.device, + ) + if preserve_deferred_critic_gradients + else self._clip_active_gradients( + self.student_parameters, losses.total.device + ) + ) + if not bool(torch.isfinite(grad_norm)): + raise FloatingPointError("The student LoRA gradient is non-finite.") + zero_grad = float(grad_norm) == 0.0 + if zero_grad and not allow_zero_generator_grad: + raise FloatingPointError("The student LoRA gradient is zero.") + if preserve_deferred_critic_gradients: + assert deferred_critic_signature is not None + assert deferred_critic_gradients is not None + (critic_has_generator_grad,) = self._global_gradient_presence( + (self.critic_parameters,), + device=losses.total.device, + ) + if critic_has_generator_grad: + raise AssertionError( + "Generator DMD backward leaked into the cleared critic adapter." + ) + for parameter, gradient in zip( + self.critic_parameters, deferred_critic_gradients + ): + parameter.grad = gradient + observed_signature = self._local_gradient_signature( + self.critic_parameters, + device=losses.total.device, + ) + signature_unchanged = torch.equal( + observed_signature, deferred_critic_signature + ) + if not self._all_ranks_true( + signature_unchanged, + device=losses.total.device, + ): + raise AssertionError( + "Generator DMD backward changed a deferred critic gradient." + ) + else: + (critic_has_grad,) = self._global_gradient_presence( + (self.critic_parameters,), + device=losses.total.device, + ) + if critic_has_grad: + raise AssertionError( + "Generator DMD backward leaked into the critic adapter." + ) + if not defer_optimizer_step: + self.student_optimizer.step() + return H3DMD4GeneratorStepOutput( + generated_video=generated_video.detach(), + generated_audio=generated_audio.detach(), + loss=losses.total.detach(), + video_loss=losses.video.detach(), + audio_loss=losses.audio.detach(), + grad_norm=grad_norm.detach(), + score_video_timestep=video_timestep, + score_audio_timestep=audio_timestep, + target_stage=selected_stage, + zero_grad_accepted=zero_grad and allow_zero_generator_grad, + ) + + def _score_timesteps(self, base_sigma: float) -> tuple[float, float]: + """Keep the inherited score convention local and easy to audit.""" + + return paired_score_timesteps( + base_sigma, + video_shift=self.video_shift, + audio_shift=self.audio_shift, + ) + + +__all__ = [ + "H3DMD4GeneratorStepOutput", + "H3DMD4StepTrainer", + "H3FreshNoiseStageState", + "H3TargetStageRolloutOutput", + "NUM_FRESH_NOISE_TRANSITIONS", + "NUM_MODEL_EVALUATIONS", + "NUM_SIGMA_ENDPOINTS", + "build_4step_sigma_schedules", + "sample_target_stage", +] diff --git a/source_snapshot/minimax_h3/dmd_8step.py b/source_snapshot/minimax_h3/dmd_8step.py new file mode 100644 index 0000000000000000000000000000000000000000..d44f082daffc4dca026c14c5177189365e705b1c --- /dev/null +++ b/source_snapshot/minimax_h3/dmd_8step.py @@ -0,0 +1,868 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Isolated eight-evaluation fresh-noise DMD core for MiniMax-H3. + +The older MiniMax-H3 trainers in this repository use a deterministic Euler +rollout. This module deliberately does not modify those trainers. It +implements the fresh-noise backward-simulation contract described by the H3 +DMD technical note, expanded from four to eight denoiser evaluations: + +* eight H3 forwards and nine sigma endpoints (including terminal zero), +* seven predict-x0-then-re-noise transitions, +* independent video/audio transition noise, always drawn video first, and +* a fixed-cost target-stage path: seven detached state-building forwards plus + exactly one differentiable forward at a uniformly sampled stage. + +The class inherits the existing adapter switching, H3 packed forward, score +pair, loss, clipping, and saved-tensor-offload machinery. The implementation +is otherwise self-contained so the existing v1-v5 training paths retain their +original four-step behavior. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import torch + +from .dmd_core import ( + H3SigmaSchedules, + PositiveOnlyCallCounter, + build_h3_sigma_schedules, + critic_flow_matching_loss, + dmd_generator_pseudo_loss, + h3_add_noise, + h3_data_velocity_to_x0, +) +from .train_utils import ( + CRITIC_ADAPTER, + STUDENT_ADAPTER, + CriticStepOutput, + GeneratorStepOutput, + H3CallAudit, + H3Condition, + H3DMDTrainer, + H3PackedLayout, + _random_normal_like, + paired_score_timesteps, + use_adapter, +) + + +NUM_MODEL_EVALUATIONS = 8 +NUM_SIGMA_ENDPOINTS = NUM_MODEL_EVALUATIONS + 1 +NUM_FRESH_NOISE_TRANSITIONS = NUM_MODEL_EVALUATIONS - 1 +TARGET_STAGE_MIN = 0 +TARGET_STAGE_MAX = NUM_MODEL_EVALUATIONS - 1 + + +@dataclass(frozen=True) +class H3FreshNoiseStageState: + """Detached input rows and clocks cached for one denoiser stage.""" + + stage: int + video_rows: torch.Tensor + audio_rows: torch.Tensor + video_sigma: torch.Tensor + audio_sigma: torch.Tensor + video_timestep: torch.Tensor + audio_timestep: torch.Tensor + + +@dataclass(frozen=True) +class H3TargetStageRolloutOutput: + """One differentiable target-stage clean prediction plus its state cache.""" + + generated_video: torch.Tensor + generated_audio: torch.Tensor + target_stage: int + stage_states: tuple[H3FreshNoiseStageState, ...] + + +@dataclass(frozen=True) +class H3DMD8GeneratorStepOutput(GeneratorStepOutput): + """Generator metrics augmented with the sampled eight-step target stage.""" + + target_stage: int + zero_grad_accepted: bool + + +def build_8step_sigma_schedules( + *, + video_shift: float = 12.0, + audio_shift: float = 3.0, + device: torch.device | str | None = None, + dtype: torch.dtype = torch.float32, +) -> H3SigmaSchedules: + """Return the fixed eight-evaluation/nine-endpoint H3 sigma grid.""" + + schedules = build_h3_sigma_schedules( + num_evaluations=NUM_MODEL_EVALUATIONS, + video_shift=video_shift, + audio_shift=audio_shift, + device=device, + dtype=dtype, + ) + if schedules.num_evaluations != NUM_MODEL_EVALUATIONS: + raise AssertionError("Eight-step H3 schedule construction changed unexpectedly.") + if schedules.video_sigmas.numel() != NUM_SIGMA_ENDPOINTS: + raise AssertionError("Eight-step H3 must have exactly nine sigma endpoints.") + return schedules + + +def sample_target_stage( + *, + generator: torch.Generator | None = None, + device: torch.device | str | None = None, +) -> int: + """Sample ``Uniform({0, ..., 7})`` with an optionally replayable RNG.""" + + if device is None: + device = generator.device if generator is not None else torch.device("cpu") + return int( + torch.randint( + TARGET_STAGE_MIN, + TARGET_STAGE_MAX + 1, + (), + generator=generator, + device=device, + ).item() + ) + + +def _validate_target_stage(target_stage: int) -> int: + if isinstance(target_stage, bool) or not isinstance(target_stage, int): + raise TypeError("target_stage must be an integer in [0, 7].") + if not TARGET_STAGE_MIN <= target_stage <= TARGET_STAGE_MAX: + raise ValueError("target_stage must lie in [0, 7].") + return target_stage + + +def _fresh_transition( + *, + clean_video: torch.Tensor, + clean_audio: torch.Tensor, + next_video_timestep: torch.Tensor, + next_audio_timestep: torch.Tensor, + generator: torch.Generator | None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Re-noise both modalities, with the replay-critical video-first order.""" + + # Do not combine or reorder these draws. A/V shapes differ in real H3, + # and this precise order is part of seeded checkpoint/evaluation replay. + video_noise = _random_normal_like(clean_video, generator) + audio_noise = _random_normal_like(clean_audio, generator) + return ( + h3_add_noise(clean_video, video_noise, next_video_timestep), + h3_add_noise(clean_audio, audio_noise, next_audio_timestep), + ) + + +class H3DMD8StepTrainer(H3DMDTrainer): + """H3 DMD trainer with an eight-forward fresh-noise student sampler.""" + + def __init__( + self, + *, + forward_model: torch.nn.Module, + role_model: torch.nn.Module, + student_optimizer: torch.optim.Optimizer, + critic_optimizer: torch.optim.Optimizer, + video_shift: float = 12.0, + audio_shift: float = 3.0, + max_grad_norm: float = 1.0, + audit: H3CallAudit | None = None, + allow_zero_generator_grad: bool = False, + ) -> None: + if not isinstance(allow_zero_generator_grad, bool): + raise TypeError("allow_zero_generator_grad must be a bool.") + super().__init__( + forward_model=forward_model, + role_model=role_model, + student_optimizer=student_optimizer, + critic_optimizer=critic_optimizer, + video_shift=video_shift, + audio_shift=audio_shift, + max_grad_norm=max_grad_norm, + audit=audit, + ) + self.allow_zero_generator_grad = allow_zero_generator_grad + + def _schedules(self, reference: torch.Tensor) -> H3SigmaSchedules: + return build_8step_sigma_schedules( + video_shift=self.video_shift, + audio_shift=self.audio_shift, + device=reference.device, + dtype=torch.float32, + ) + + def _clip_gradient_subset( + self, + parameters: tuple[torch.nn.Parameter, ...] | list[torch.nn.Parameter], + *, + device: torch.device, + ) -> torch.Tensor: + """Clip one FSDP-sharded role without touching deferred other-role grads.""" + + gradients = [ + parameter.grad + for parameter in parameters + if parameter.grad is not None + ] + local_squared = torch.zeros((), device=device, dtype=torch.float32) + if gradients: + local_squared = torch.stack( + [ + torch.dot( + gradient.detach().float().reshape(-1), + gradient.detach().float().reshape(-1), + ) + for gradient in gradients + ] + ).sum() + # A valid FSDP rank may own no shard of this role. Keep a separate + # presence scalar and make every rank enter the same all-reduce anyway. + reduced = torch.stack( + [ + local_squared.to(device=device), + torch.tensor(float(bool(gradients)), device=device), + ] + ) + if self.forward_model is not self.role_model: + import torch.distributed as dist + + dist.all_reduce(reduced, op=dist.ReduceOp.SUM) + if float(reduced[1]) <= 0.0: + raise RuntimeError("No rank owns a gradient for the requested LoRA role.") + norm = reduced[0].sqrt() + if not bool(torch.isfinite(norm)): + raise FloatingPointError("The global role gradient norm is non-finite.") + coefficient = torch.clamp( + self.max_grad_norm / (norm + 1.0e-6), + max=1.0, + ) + for gradient in gradients: + gradient.mul_(coefficient.to(device=gradient.device, dtype=gradient.dtype)) + return norm + + @staticmethod + def _local_gradient_signature( + parameters: tuple[torch.nn.Parameter, ...] | list[torch.nn.Parameter], + *, + device: torch.device, + ) -> torch.Tensor: + """Return a small exact-change sentinel without cloning full gradients.""" + + gradients = [ + parameter.grad.detach().float() + for parameter in parameters + if parameter.grad is not None + ] + if not gradients: + return torch.zeros(3, device=device, dtype=torch.float32) + sums = torch.stack([gradient.sum() for gradient in gradients]).sum() + squared = torch.stack( + [ + torch.dot(gradient.reshape(-1), gradient.reshape(-1)) + for gradient in gradients + ] + ).sum() + return torch.stack( + [ + torch.tensor(float(len(gradients)), device=device), + sums.to(device=device), + squared.to(device=device), + ] + ) + + def _assert_global_signature_presence( + self, + signature: torch.Tensor, + *, + role: str, + ) -> None: + """Accept an empty local shard but require the role globally.""" + + presence = signature[0].detach().clone() + if self.forward_model is not self.role_model: + import torch.distributed as dist + + dist.all_reduce(presence, op=dist.ReduceOp.SUM) + if float(presence) <= 0.0: + raise RuntimeError(f"No deferred {role} gradients exist on any rank.") + + def _global_gradient_presence( + self, + parameter_groups: tuple[ + tuple[torch.nn.Parameter, ...] | list[torch.nn.Parameter], ... + ], + *, + device: torch.device, + ) -> tuple[bool, ...]: + """Return global role presence without branching on local shard layout.""" + + flags = torch.tensor( + [ + int(any(parameter.grad is not None for parameter in parameters)) + for parameters in parameter_groups + ], + device=device, + dtype=torch.int32, + ) + if self.forward_model is not self.role_model: + import torch.distributed as dist + + dist.all_reduce(flags, op=dist.ReduceOp.MAX) + return tuple(bool(value) for value in flags.tolist()) + + def _all_ranks_true(self, value: bool, *, device: torch.device) -> bool: + consensus = torch.tensor(int(value), device=device, dtype=torch.int32) + if self.forward_model is not self.role_model: + import torch.distributed as dist + + dist.all_reduce(consensus, op=dist.ReduceOp.MIN) + return bool(consensus.item()) + + @staticmethod + def _stage_state( + *, + stage: int, + video_rows: torch.Tensor, + audio_rows: torch.Tensor, + schedules: H3SigmaSchedules, + ) -> H3FreshNoiseStageState: + return H3FreshNoiseStageState( + stage=stage, + video_rows=video_rows.detach(), + audio_rows=audio_rows.detach(), + video_sigma=schedules.video_sigmas[stage], + audio_sigma=schedules.audio_sigmas[stage], + video_timestep=schedules.video_timesteps[stage], + audio_timestep=schedules.audio_timesteps[stage], + ) + + def detached_rollout( + self, + *, + video_noise: torch.Tensor, + audio_noise: torch.Tensor, + condition: H3Condition, + layout: H3PackedLayout, + target_stage: int, + generator: torch.Generator | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Run all eight stages and return the selected detached clean prediction. + + All eight positive forwards and all seven fresh-noise transitions are + executed regardless of ``target_stage``. Besides matching the training + distribution, this fixed cost is required so data-parallel ranks enter + FSDP collectives in the same order when they sample different stages. + """ + + target_stage = _validate_target_stage(target_stage) + schedules = self._schedules(video_noise) + video = video_noise.detach() + audio = audio_noise.detach() + selected_video: torch.Tensor | None = None + selected_audio: torch.Tensor | None = None + with use_adapter(self.role_model, STUDENT_ADAPTER): + with torch.no_grad(): + with PositiveOnlyCallCounter( + expected_calls=NUM_MODEL_EVALUATIONS + ) as calls: + for stage in range(NUM_MODEL_EVALUATIONS): + video_velocity, audio_velocity = calls.call( + self._forward, + role="student_detached_stage", + video_rows=video, + audio_rows=audio, + condition=condition, + layout=layout, + video_timestep=schedules.video_timesteps[stage], + audio_timestep=schedules.audio_timesteps[stage], + adapters_enabled=True, + ) + clean_video = h3_data_velocity_to_x0( + video, + video_velocity, + schedules.video_timesteps[stage], + ) + clean_audio = h3_data_velocity_to_x0( + audio, + audio_velocity, + schedules.audio_timesteps[stage], + ) + if stage == target_stage: + selected_video = clean_video.detach() + selected_audio = clean_audio.detach() + if stage < TARGET_STAGE_MAX: + video, audio = _fresh_transition( + clean_video=clean_video, + clean_audio=clean_audio, + next_video_timestep=schedules.video_timesteps[stage + 1], + next_audio_timestep=schedules.audio_timesteps[stage + 1], + generator=generator, + ) + if selected_video is None or selected_audio is None: + raise AssertionError("The eight-stage detached rollout missed its target stage.") + return selected_video, selected_audio + + def student_rollout( + self, + *, + video_noise: torch.Tensor, + audio_noise: torch.Tensor, + condition: H3Condition, + layout: H3PackedLayout, + target_stage: int, + generator: torch.Generator | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Compatibility name for fake-score data generation.""" + + return self.detached_rollout( + video_noise=video_noise, + audio_noise=audio_noise, + condition=condition, + layout=layout, + target_stage=target_stage, + generator=generator, + ) + + def target_stage_rollout( + self, + *, + video_noise: torch.Tensor, + audio_noise: torch.Tensor, + condition: H3Condition, + layout: H3PackedLayout, + target_stage: int | None = None, + generator: torch.Generator | None = None, + ) -> H3TargetStageRolloutOutput: + """Build all stage states detached, then differentiate one random stage. + + Stage selection consumes the first RNG draw when ``target_stage`` is + omitted. The next fourteen draws are the seven video/audio transition + pairs. Passing an explicit stage intentionally consumes no stage RNG. + """ + + if target_stage is None: + target_stage = sample_target_stage( + generator=generator, + device=video_noise.device, + ) + target_stage = _validate_target_stage(target_stage) + schedules = self._schedules(video_noise) + video = video_noise.detach() + audio = audio_noise.detach() + stage_states = [ + self._stage_state( + stage=0, + video_rows=video, + audio_rows=audio, + schedules=schedules, + ) + ] + + with use_adapter(self.role_model, STUDENT_ADAPTER): + with PositiveOnlyCallCounter( + expected_calls=NUM_MODEL_EVALUATIONS + ) as calls: + # Always materialize states 1..7. The seven no-grad forward + # order is independent of the randomly selected target stage. + with torch.no_grad(): + for stage in range(NUM_FRESH_NOISE_TRANSITIONS): + video_velocity, audio_velocity = calls.call( + self._forward, + role="student_stage_cache", + video_rows=video, + audio_rows=audio, + condition=condition, + layout=layout, + video_timestep=schedules.video_timesteps[stage], + audio_timestep=schedules.audio_timesteps[stage], + adapters_enabled=True, + ) + clean_video = h3_data_velocity_to_x0( + video, + video_velocity, + schedules.video_timesteps[stage], + ) + clean_audio = h3_data_velocity_to_x0( + audio, + audio_velocity, + schedules.audio_timesteps[stage], + ) + video, audio = _fresh_transition( + clean_video=clean_video, + clean_audio=clean_audio, + next_video_timestep=schedules.video_timesteps[stage + 1], + next_audio_timestep=schedules.audio_timesteps[stage + 1], + generator=generator, + ) + video = video.detach() + audio = audio.detach() + stage_states.append( + self._stage_state( + stage=stage + 1, + video_rows=video, + audio_rows=audio, + schedules=schedules, + ) + ) + + selected = stage_states[target_stage] + video_velocity, audio_velocity = calls.call( + self._forward, + role="student_target_stage", + video_rows=selected.video_rows, + audio_rows=selected.audio_rows, + condition=condition, + layout=layout, + video_timestep=selected.video_timestep, + audio_timestep=selected.audio_timestep, + adapters_enabled=True, + ) + generated_video = h3_data_velocity_to_x0( + selected.video_rows, + video_velocity, + selected.video_timestep, + ) + generated_audio = h3_data_velocity_to_x0( + selected.audio_rows, + audio_velocity, + selected.audio_timestep, + ) + + if len(stage_states) != NUM_MODEL_EVALUATIONS: + raise AssertionError("Target-stage rollout must cache exactly eight states.") + return H3TargetStageRolloutOutput( + generated_video=generated_video, + generated_audio=generated_audio, + target_stage=target_stage, + stage_states=tuple(stage_states), + ) + + def critic_step( + self, + *, + clean_video: torch.Tensor, + clean_audio: torch.Tensor, + condition: H3Condition, + layout: H3PackedLayout, + score_base_sigma: float, + score_video_noise: torch.Tensor | None = None, + score_audio_noise: torch.Tensor | None = None, + generator: torch.Generator | None = None, + defer_optimizer_step: bool = False, + ) -> CriticStepOutput: + """Backpropagate fake-score DSM, optionally deferring its AdamW step.""" + + if not isinstance(defer_optimizer_step, bool): + raise TypeError("defer_optimizer_step must be a bool.") + self.critic_optimizer.zero_grad(set_to_none=True) + self.student_optimizer.zero_grad(set_to_none=True) + clean_video = clean_video.detach() + clean_audio = clean_audio.detach() + video_timestep, audio_timestep = self._score_timesteps(score_base_sigma) + if score_video_noise is None: + score_video_noise = _random_normal_like(clean_video, generator) + if score_audio_noise is None: + score_audio_noise = _random_normal_like(clean_audio, generator) + noisy_video = h3_add_noise(clean_video, score_video_noise, video_timestep) + noisy_audio = h3_add_noise(clean_audio, score_audio_noise, audio_timestep) + with use_adapter(self.role_model, CRITIC_ADAPTER): + predicted_video, predicted_audio = self._forward( + role="critic_train", + video_rows=noisy_video, + audio_rows=noisy_audio, + condition=condition, + layout=layout, + video_timestep=video_timestep, + audio_timestep=audio_timestep, + adapters_enabled=True, + ) + losses = critic_flow_matching_loss( + predicted_video_velocity=predicted_video, + clean_video=clean_video, + video_noise=score_video_noise, + predicted_audio_velocity=predicted_audio, + clean_audio=clean_audio, + audio_noise=score_audio_noise, + ) + if losses.audio is None or not bool(torch.isfinite(losses.total)): + raise FloatingPointError("The joint H3 critic loss is not finite.") + losses.total.backward() + grad_norm = self._clip_active_gradients( + self.critic_parameters, losses.total.device + ) + if not bool(torch.isfinite(grad_norm)) or float(grad_norm) == 0.0: + raise FloatingPointError("The critic LoRA gradient is zero or non-finite.") + (student_has_grad,) = self._global_gradient_presence( + (self.student_parameters,), + device=losses.total.device, + ) + if student_has_grad: + raise AssertionError("Critic backward leaked into the student adapter.") + if not defer_optimizer_step: + self.critic_optimizer.step() + return CriticStepOutput( + loss=losses.total.detach(), + video_loss=losses.video.detach(), + audio_loss=losses.audio.detach(), + grad_norm=grad_norm.detach(), + score_video_timestep=video_timestep, + score_audio_timestep=audio_timestep, + ) + + def step_deferred_role_optimizer( + self, + *, + generator_update: bool, + device: torch.device, + ) -> None: + """Apply one joint AdamW step after all pre-update score targets exist.""" + + if not isinstance(generator_update, bool): + raise TypeError("generator_update must be a bool.") + if not isinstance(device, torch.device): + raise TypeError("device must be a torch.device.") + if self.student_optimizer is not self.critic_optimizer: + raise RuntimeError("Deferred role stepping requires one shared optimizer.") + critic_has_grad, student_has_grad = self._global_gradient_presence( + (self.critic_parameters, self.student_parameters), + device=device, + ) + if not critic_has_grad: + raise RuntimeError("Deferred critic gradients are missing.") + if student_has_grad != generator_update: + raise RuntimeError( + "Deferred student gradient presence differs from update cadence." + ) + self.student_optimizer.step() + + def generator_step( + self, + *, + video_noise: torch.Tensor, + audio_noise: torch.Tensor, + condition: H3Condition, + layout: H3PackedLayout, + score_base_sigma: float, + score_video_noise: torch.Tensor | None = None, + score_audio_noise: torch.Tensor | None = None, + target_stage: int | None = None, + generator: torch.Generator | None = None, + allow_zero_generator_grad: bool | None = None, + defer_optimizer_step: bool = False, + preserve_deferred_critic_gradients: bool = False, + ) -> H3DMD8GeneratorStepOutput: + """Update the student from one random stage and report that stage. + + ``allow_zero_generator_grad`` exists for the documented initialization + bootstrap where zero-initialized fake-score LoRA and the frozen teacher + produce the same DMD target. Non-finite gradients always remain fatal. + A caller can enable this only for the first update even if the trainer + default remains strict. + """ + + if allow_zero_generator_grad is None: + allow_zero_generator_grad = self.allow_zero_generator_grad + if not isinstance(allow_zero_generator_grad, bool): + raise TypeError("allow_zero_generator_grad must be a bool.") + if not isinstance(defer_optimizer_step, bool) or not isinstance( + preserve_deferred_critic_gradients, bool + ): + raise TypeError("Deferred optimizer controls must be bools.") + if preserve_deferred_critic_gradients and not defer_optimizer_step: + raise ValueError( + "Preserving deferred critic gradients requires a deferred step." + ) + + deferred_critic_signature: torch.Tensor | None = None + deferred_critic_gradients: tuple[torch.Tensor | None, ...] | None = None + if preserve_deferred_critic_gradients: + if self.student_optimizer is not self.critic_optimizer: + raise RuntimeError( + "Deferred joint stepping requires one shared role optimizer." + ) + deferred_critic_signature = self._local_gradient_signature( + self.critic_parameters, + device=video_noise.device, + ) + self._assert_global_signature_presence( + deferred_critic_signature, + role="critic", + ) + # FSDP ``use_orig_params=True`` exposes each original gradient as + # a view into its flat shard. PEFT then marks the inactive critic + # adapter ``requires_grad=False`` while the generator is active, + # so FSDP intentionally drops those original ``.grad`` views at + # the end of the next backward. Stash the already clipped local + # critic shards, clear the complete mixed-role flat gradients, + # and restore the exact critic values after the isolated generator + # backward. This also avoids re-reducing an existing critic shard. + deferred_critic_gradients = tuple( + parameter.grad.detach().clone() + if parameter.grad is not None + else None + for parameter in self.critic_parameters + ) + self.student_optimizer.zero_grad(set_to_none=True) + else: + self.student_optimizer.zero_grad(set_to_none=True) + self.critic_optimizer.zero_grad(set_to_none=True) + with self._offload_student_saved_tensors(video_noise.device): + rollout = self.target_stage_rollout( + video_noise=video_noise, + audio_noise=audio_noise, + condition=condition, + layout=layout, + target_stage=target_stage, + generator=generator, + ) + generated_video = rollout.generated_video + generated_audio = rollout.generated_audio + selected_stage = rollout.target_stage + # The seven other cached states are not loss inputs and can be released + # before the two score calls consume additional device memory. + del rollout + + video_timestep, audio_timestep = self._score_timesteps(score_base_sigma) + if score_video_noise is None: + score_video_noise = _random_normal_like(generated_video, generator) + if score_audio_noise is None: + score_audio_noise = _random_normal_like(generated_audio, generator) + noisy_video = h3_add_noise( + generated_video.detach(), score_video_noise, video_timestep + ) + noisy_audio = h3_add_noise( + generated_audio.detach(), score_audio_noise, audio_timestep + ) + teacher_video_v, teacher_audio_v, critic_video_v, critic_audio_v = ( + self._score_pair( + noisy_video=noisy_video, + noisy_audio=noisy_audio, + condition=condition, + layout=layout, + video_timestep=video_timestep, + audio_timestep=audio_timestep, + ) + ) + teacher_video_x0 = h3_data_velocity_to_x0( + noisy_video, teacher_video_v, video_timestep + ) + teacher_audio_x0 = h3_data_velocity_to_x0( + noisy_audio, teacher_audio_v, audio_timestep + ) + critic_video_x0 = h3_data_velocity_to_x0( + noisy_video, critic_video_v, video_timestep + ) + critic_audio_x0 = h3_data_velocity_to_x0( + noisy_audio, critic_audio_v, audio_timestep + ) + losses = dmd_generator_pseudo_loss( + generated_video=generated_video, + real_video_x0=teacher_video_x0, + fake_video_x0=critic_video_x0, + generated_audio=generated_audio, + real_audio_x0=teacher_audio_x0, + fake_audio_x0=critic_audio_x0, + ) + if losses.audio is None or not bool(torch.isfinite(losses.total)): + raise FloatingPointError("The joint H3 generator DMD loss is not finite.") + + self.role_model.set_adapter(STUDENT_ADAPTER) + if losses.total.device.type == "cuda": + torch.cuda.empty_cache() + self.cuda_cache_clears_before_generator_backward += 1 + losses.total.backward() + grad_norm = ( + self._clip_gradient_subset( + self.student_parameters, + device=losses.total.device, + ) + if preserve_deferred_critic_gradients + else self._clip_active_gradients( + self.student_parameters, losses.total.device + ) + ) + if not bool(torch.isfinite(grad_norm)): + raise FloatingPointError("The student LoRA gradient is non-finite.") + zero_grad = float(grad_norm) == 0.0 + if zero_grad and not allow_zero_generator_grad: + raise FloatingPointError("The student LoRA gradient is zero.") + if preserve_deferred_critic_gradients: + assert deferred_critic_signature is not None + assert deferred_critic_gradients is not None + (critic_has_generator_grad,) = self._global_gradient_presence( + (self.critic_parameters,), + device=losses.total.device, + ) + if critic_has_generator_grad: + raise AssertionError( + "Generator DMD backward leaked into the cleared critic adapter." + ) + for parameter, gradient in zip( + self.critic_parameters, deferred_critic_gradients + ): + parameter.grad = gradient + observed_signature = self._local_gradient_signature( + self.critic_parameters, + device=losses.total.device, + ) + signature_unchanged = torch.equal( + observed_signature, deferred_critic_signature + ) + if not self._all_ranks_true( + signature_unchanged, + device=losses.total.device, + ): + raise AssertionError( + "Generator DMD backward changed a deferred critic gradient." + ) + else: + (critic_has_grad,) = self._global_gradient_presence( + (self.critic_parameters,), + device=losses.total.device, + ) + if critic_has_grad: + raise AssertionError( + "Generator DMD backward leaked into the critic adapter." + ) + if not defer_optimizer_step: + self.student_optimizer.step() + return H3DMD8GeneratorStepOutput( + generated_video=generated_video.detach(), + generated_audio=generated_audio.detach(), + loss=losses.total.detach(), + video_loss=losses.video.detach(), + audio_loss=losses.audio.detach(), + grad_norm=grad_norm.detach(), + score_video_timestep=video_timestep, + score_audio_timestep=audio_timestep, + target_stage=selected_stage, + zero_grad_accepted=zero_grad and allow_zero_generator_grad, + ) + + def _score_timesteps(self, base_sigma: float) -> tuple[float, float]: + """Keep the inherited score convention local and easy to audit.""" + + return paired_score_timesteps( + base_sigma, + video_shift=self.video_shift, + audio_shift=self.audio_shift, + ) + + +__all__ = [ + "H3DMD8GeneratorStepOutput", + "H3DMD8StepTrainer", + "H3FreshNoiseStageState", + "H3TargetStageRolloutOutput", + "NUM_FRESH_NOISE_TRANSITIONS", + "NUM_MODEL_EVALUATIONS", + "NUM_SIGMA_ENDPOINTS", + "build_8step_sigma_schedules", + "sample_target_stage", +] diff --git a/source_snapshot/minimax_h3/dmd_core.py b/source_snapshot/minimax_h3/dmd_core.py new file mode 100644 index 0000000000000000000000000000000000000000..003e4aea0a0748418d027f4fc79b05b21546899d --- /dev/null +++ b/source_snapshot/minimax_h3/dmd_core.py @@ -0,0 +1,631 @@ +"""Small, weight-free MiniMax-H3 full-sequence DMD math helpers. + +This module intentionally has no dependency on the MiniMax-H3 checkpoint or on +Diffusers. It isolates the conventions that differ from the Wan paths in +LongLive so they can be unit-tested before a 33B model is loaded: + +* H3 uses ``t = 1 - sigma`` with ``t=1`` denoting clean data. +* H3 predicts a data-ward velocity and reconstructs + ``x0 = x_t + (1 - t) * velocity``. +* A four-evaluation schedule therefore contains five sigma endpoints. +* The released checkpoint is guidance-distilled and must use positive-only + model calls. + +The loss helpers accept video and optional audio tensors. They deliberately +stop gradients through teacher/critic targets; only the generated sample is +updated by the DMD pseudo-loss and only the predicted velocity is updated by +the critic flow-matching loss. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Any, Callable, TypeVar + +import torch + + +Tensor = torch.Tensor +_T = TypeVar("_T") + +H3_VIDEO_TAG = 0 +H3_TEXT_TAG = 1 +H3_AUDIO_TAG = 2 +H3_PADDING_TAG = -1 +DMD_NORMALIZATION = "per_modality_full_sequence" + + +def _broadcast_time_like( + timestep: float | Tensor, + reference: Tensor, + *, + name: str = "timestep", +) -> Tensor: + """Convert a scalar or leading-dimension timestep tensor for broadcasting.""" + + value = torch.as_tensor(timestep, device=reference.device, dtype=reference.dtype) + if value.ndim > reference.ndim: + raise ValueError( + f"{name} has {value.ndim} dimensions but the reference has " + f"only {reference.ndim}." + ) + if not bool(torch.isfinite(value).all()): + raise ValueError(f"{name} must contain only finite values.") + if bool(((value < 0) | (value > 1)).any()): + raise ValueError(f"{name} must lie in [0, 1].") + + # H3 callers commonly provide [B] or [B, F] timesteps for [B, F, ...] + # latents. Treat the timestep dimensions as leading dimensions. + value = value.reshape(*value.shape, *((1,) * (reference.ndim - value.ndim))) + try: + torch.broadcast_shapes(value.shape, reference.shape) + except RuntimeError as error: + raise ValueError( + f"{name} with shape {tuple(value.shape)} cannot broadcast to " + f"reference shape {tuple(reference.shape)}." + ) from error + return value + + +def _require_same_shape(name: str, *tensors: Tensor) -> None: + shapes = {tuple(tensor.shape) for tensor in tensors} + if len(shapes) != 1: + raise ValueError( + f"{name} tensors must have identical shapes, got " + f"{sorted(shapes)}." + ) + + +def h3_add_noise(clean: Tensor, noise: Tensor, timestep: float | Tensor) -> Tensor: + """Apply MiniMax-H3's rectified-flow forward process. + + H3's time convention is clean-ward: ``t=0`` is noise and ``t=1`` is + clean. Consequently, ``x_t = t * x0 + (1 - t) * epsilon``. + """ + + _require_same_shape("clean/noise", clean, noise) + t = _broadcast_time_like(timestep, clean) + return t * clean + (1.0 - t) * noise + + +def h3_data_velocity_to_x0( + noisy_sample: Tensor, + data_velocity: Tensor, + timestep: float | Tensor, +) -> Tensor: + """Convert H3's data-ward velocity prediction to a clean estimate. + + The plus sign is essential: ``x0 = x_t + (1 - t) * v``. This is the + opposite sign from the noise-ward velocity used by several other flow + implementations. + """ + + _require_same_shape("sample/velocity", noisy_sample, data_velocity) + t = _broadcast_time_like(timestep, noisy_sample) + return noisy_sample + (1.0 - t) * data_velocity + + +def h3_flow_velocity_target(clean: Tensor, noise: Tensor) -> Tensor: + """Return H3's straight-path data-ward velocity target ``x0 - noise``.""" + + _require_same_shape("clean/noise", clean, noise) + return clean - noise + + +def _shift_sigmas(base_sigmas: Tensor, shift: float) -> Tensor: + if ( + isinstance(shift, bool) + or not isinstance(shift, (int, float)) + or not math.isfinite(float(shift)) + or shift <= 0 + ): + raise ValueError(f"sigma shift must be positive, got {shift!r}.") + shifted = float(shift) * base_sigmas / ( + 1.0 + (float(shift) - 1.0) * base_sigmas + ) + # Preserve the mathematically exact boundary values even in low precision. + shifted = shifted.clone() + shifted[0] = 1.0 + shifted[-1] = 0.0 + if not bool((shifted[:-1] > shifted[1:]).all()): + raise ValueError( + "The requested dtype/evaluation count collapsed adjacent sigma " + "endpoints; use float32 or float64." + ) + return shifted + + +@dataclass(frozen=True) +class H3SigmaSchedules: + """Paired video/audio sigma endpoints for one H3 denoising trajectory.""" + + video_sigmas: Tensor + audio_sigmas: Tensor + + def __post_init__(self) -> None: + if self.video_sigmas.ndim != 1 or self.audio_sigmas.ndim != 1: + raise ValueError("video_sigmas and audio_sigmas must be 1-D tensors.") + if self.video_sigmas.shape != self.audio_sigmas.shape: + raise ValueError("Video and audio schedules must have equal lengths.") + if self.video_sigmas.numel() < 2: + raise ValueError("A schedule needs at least two sigma endpoints.") + + @property + def num_evaluations(self) -> int: + """Number of model calls, one less than the endpoint count.""" + + return self.video_sigmas.numel() - 1 + + @property + def video_timesteps(self) -> Tensor: + """H3 video timesteps for model calls; the terminal sigma is excluded.""" + + return 1.0 - self.video_sigmas[:-1] + + @property + def audio_timesteps(self) -> Tensor: + """H3 audio timesteps for model calls; the terminal sigma is excluded.""" + + return 1.0 - self.audio_sigmas[:-1] + + +def build_h3_sigma_schedules( + *, + num_evaluations: int = 4, + video_shift: float = 12.0, + audio_shift: float = 3.0, + device: torch.device | str | None = None, + dtype: torch.dtype = torch.float32, +) -> H3SigmaSchedules: + """Build H3's shifted video/audio endpoint schedules. + + ``num_evaluations=4`` intentionally creates five endpoints. H3 includes + the terminal ``sigma=0`` in the configured grid, but that endpoint does + not trigger another transformer call. + """ + + if isinstance(num_evaluations, bool) or not isinstance(num_evaluations, int): + raise TypeError("num_evaluations must be an integer.") + if num_evaluations < 1: + raise ValueError("num_evaluations must be at least one.") + if not dtype.is_floating_point: + raise TypeError("Sigma schedules require a floating-point dtype.") + + base = torch.linspace( + 1.0, + 0.0, + num_evaluations + 1, + device=device, + dtype=dtype, + ) + return H3SigmaSchedules( + video_sigmas=_shift_sigmas(base, video_shift), + audio_sigmas=_shift_sigmas(base, audio_shift), + ) + + +def assert_positive_only_calls( + *, + positive_calls: int, + unconditional_calls: int, + expected_positive_calls: int, +) -> None: + """Assert the call counts required by H3's CFG-distilled checkpoint.""" + + if unconditional_calls != 0: + raise AssertionError( + "MiniMax-H3 is guidance-distilled: unconditional/CFG calls must be zero, " + f"got {unconditional_calls}." + ) + if positive_calls != expected_positive_calls: + raise AssertionError( + f"Expected {expected_positive_calls} positive calls, got {positive_calls}." + ) + + +class PositiveOnlyCallCounter: + """Context-managed model-call counter that rejects CFG branches. + + Example:: + + with PositiveOnlyCallCounter(expected_calls=4) as calls: + for timestep in timesteps: + prediction = calls.call(model, noisy, timestep=timestep) + """ + + def __init__(self, expected_calls: int) -> None: + if isinstance(expected_calls, bool) or not isinstance(expected_calls, int): + raise TypeError("expected_calls must be an integer.") + if expected_calls < 0: + raise ValueError("expected_calls cannot be negative.") + self.expected_calls = expected_calls + self.positive_calls = 0 + self.unconditional_calls = 0 + + def record(self, *, branch: str = "positive") -> None: + if branch != "positive": + self.unconditional_calls += 1 + raise AssertionError( + "MiniMax-H3 accepts only the positive branch; an unconditional/CFG " + f"call was requested as branch={branch!r}." + ) + self.positive_calls += 1 + if self.positive_calls > self.expected_calls: + raise AssertionError( + f"Positive call count exceeded {self.expected_calls}: " + f"got {self.positive_calls}." + ) + + def call( + self, + function: Callable[..., _T], + *args: Any, + branch: str = "positive", + **kwargs: Any, + ) -> _T: + """Record one positive call, then invoke ``function``.""" + + self.record(branch=branch) + return function(*args, **kwargs) + + def assert_complete(self) -> None: + assert_positive_only_calls( + positive_calls=self.positive_calls, + unconditional_calls=self.unconditional_calls, + expected_positive_calls=self.expected_calls, + ) + + def __enter__(self) -> "PositiveOnlyCallCounter": + return self + + def __exit__(self, exc_type, exc_value, traceback) -> bool: + if exc_type is None: + self.assert_complete() + return False + + +@dataclass(frozen=True) +class MultimodalLoss: + """Total loss and its independently reduced video/audio components.""" + + total: Tensor + video: Tensor + audio: Tensor | None = None + + +def _masked_mean(values: Tensor, mask: Tensor | None, *, name: str) -> Tensor: + if mask is None: + return values.mean() + mask = torch.as_tensor(mask, device=values.device, dtype=torch.bool) + try: + mask = torch.broadcast_to(mask, values.shape) + except RuntimeError as error: + raise ValueError( + f"{name} mask shape {tuple(mask.shape)} cannot broadcast to " + f"{tuple(values.shape)}." + ) from error + selected = values.masked_select(mask) + if selected.numel() == 0: + raise ValueError(f"{name} mask selects no elements.") + return selected.mean() + + +def per_sample_normalizer( + generated: Tensor, + real_x0: Tensor, + *, + mask: Tensor | None = None, + eps: float = 1e-6, + name: str = "modality", +) -> Tensor: + """Return one DMD normalizer per sample and modality. + + ``generated`` and ``real_x0`` use ``[batch, packed_row, ...features]``. + For every batch item this computes + + ``mean(abs(generated - real_x0))`` + + over all selected packed rows and feature elements. Video and audio call + this helper separately, so neither modality can influence the other's + scale. There is deliberately no temporal grouping: the revised H3 student + denoises the complete native sequence at every step. + + A mask is applied to both the normalizer statistic and the final loss. + Invalid, non-broadcastable, or empty masks fail closed. + """ + + _require_same_shape(f"{name} generated/real", generated, real_x0) + if generated.ndim < 2: + raise ValueError( + f"{name} packed rows require shape [batch, rows, ...], got " + f"{tuple(generated.shape)}." + ) + if generated.shape[0] == 0 or generated.shape[1] == 0: + raise ValueError(f"{name} packed rows require non-empty batch and row axes.") + if ( + isinstance(eps, bool) + or not isinstance(eps, (int, float)) + or not math.isfinite(float(eps)) + or eps <= 0 + ): + raise ValueError(f"eps must be positive, got {eps!r}.") + + if mask is None: + selected = torch.ones_like(generated, dtype=torch.bool) + else: + selected = torch.as_tensor(mask, device=generated.device, dtype=torch.bool) + try: + selected = torch.broadcast_to(selected, generated.shape) + except RuntimeError as error: + raise ValueError( + f"{name} mask shape {tuple(selected.shape)} cannot broadcast to " + f"{tuple(generated.shape)}." + ) from error + if not bool(selected.any()): + raise ValueError(f"{name} mask selects no elements.") + + batch_size = generated.shape[0] + # Promote both operands *before* subtraction. Casting a BF16 residual after + # the subtraction has already rounded away information does not satisfy the + # DMD FP32-normalization contract. + generated_fp32 = generated.detach().to(dtype=torch.float32) + real_x0_fp32 = real_x0.detach().to(dtype=torch.float32) + residual = (generated_fp32 - real_x0_fp32).abs() + residual = residual.reshape(batch_size, -1) + selected = selected.reshape(batch_size, -1) + sums = torch.where(selected, residual, 0.0).sum(dim=1) + counts = selected.sum(dim=1, dtype=torch.float32) + if bool((counts == 0).any()): + raise ValueError(f"{name} mask leaves at least one batch sample empty.") + means = (sums / counts).clamp_min(float(eps)) + return means.reshape(batch_size, *((1,) * (generated.ndim - 1))) + + +def _validate_weight(value: float, name: str) -> float: + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + or value < 0 + ): + raise ValueError(f"{name} must be a non-negative number, got {value!r}.") + return float(value) + + +def _combine_multimodal_losses( + video_loss: Tensor, + audio_loss: Tensor | None, + *, + video_weight: float, + audio_weight: float, +) -> MultimodalLoss: + video_weight = _validate_weight(video_weight, "video_weight") + audio_weight = _validate_weight(audio_weight, "audio_weight") + if video_weight == 0 and (audio_loss is None or audio_weight == 0): + raise ValueError("At least one present modality must have a positive weight.") + total = video_loss * video_weight + if audio_loss is not None: + total = total + audio_loss * audio_weight + return MultimodalLoss(total=total, video=video_loss, audio=audio_loss) + + +def _require_complete_audio_triplet( + first: Tensor | None, + second: Tensor | None, + third: Tensor | None, + *, + names: tuple[str, str, str], +) -> bool: + present = (first is not None, second is not None, third is not None) + if any(present) and not all(present): + missing = [name for name, is_present in zip(names, present) if not is_present] + raise ValueError( + "Audio loss requires all audio tensors; missing " + ", ".join(missing) + "." + ) + return all(present) + + +def _generator_pseudo_loss_one_modality( + generated: Tensor, + real_x0: Tensor, + fake_x0: Tensor, + *, + mask: Tensor | None, + normalize: bool, + eps: float, + name: str, +) -> Tensor: + _require_same_shape(f"{name} generated/real/fake", generated, real_x0, fake_x0) + if not isinstance(eps, (int, float)) or not math.isfinite(float(eps)) or eps <= 0: + raise ValueError(f"eps must be positive, got {eps}.") + + # DMD Eq. 7 direction in LongLive's x0-prediction form: fake - real. + # Teacher and fake-score predictions are targets and must not receive grads. + # Every operand is promoted before arithmetic: doing the subtraction in + # BF16 and casting its already-rounded result would violate the document's + # FP32 direction contract. + generated_fp32 = generated.to(dtype=torch.float32) + real_x0_fp32 = real_x0.detach().to(dtype=torch.float32) + fake_x0_fp32 = fake_x0.detach().to(dtype=torch.float32) + dmd_gradient = fake_x0_fp32 - real_x0_fp32 + if normalize: + normalizer = per_sample_normalizer( + generated, + real_x0, + mask=mask, + eps=eps, + name=name, + ) + dmd_gradient = dmd_gradient / normalizer + dmd_gradient = torch.nan_to_num(dmd_gradient) + + # 0.5*MSE against a detached pseudo-target gives the desired DMD direction + # as the gradient with respect to `generated` (up to mean reduction). + pseudo_target = (generated_fp32 - dmd_gradient).detach() + squared_error = (generated_fp32 - pseudo_target).square() + return 0.5 * _masked_mean(squared_error, mask, name=name) + + +def dmd_generator_pseudo_loss( + *, + generated_video: Tensor, + real_video_x0: Tensor, + fake_video_x0: Tensor, + generated_audio: Tensor | None = None, + real_audio_x0: Tensor | None = None, + fake_audio_x0: Tensor | None = None, + video_mask: Tensor | None = None, + audio_mask: Tensor | None = None, + normalize: bool = True, + eps: float = 1e-6, + video_weight: float = 1.0, + audio_weight: float = 1.0, +) -> MultimodalLoss: + """Compute the detached-target DMD generator pseudo-loss. + + ``real_*_x0`` is the frozen teacher's clean prediction and + ``fake_*_x0`` is the trainable fake-score/critic clean prediction. If the + score networks produce H3 velocities, convert them with + :func:`h3_data_velocity_to_x0` before calling this function. + """ + + video_loss = _generator_pseudo_loss_one_modality( + generated_video, + real_video_x0, + fake_video_x0, + mask=video_mask, + normalize=normalize, + eps=eps, + name="video", + ) + + has_audio = _require_complete_audio_triplet( + generated_audio, + real_audio_x0, + fake_audio_x0, + names=("generated_audio", "real_audio_x0", "fake_audio_x0"), + ) + audio_loss = None + if has_audio: + audio_loss = _generator_pseudo_loss_one_modality( + generated_audio, + real_audio_x0, + fake_audio_x0, + mask=audio_mask, + normalize=normalize, + eps=eps, + name="audio", + ) + elif audio_mask is not None: + raise ValueError("audio_mask was provided without audio tensors.") + + return _combine_multimodal_losses( + video_loss, + audio_loss, + video_weight=video_weight, + audio_weight=audio_weight, + ) + + +def _critic_flow_loss_one_modality( + predicted_velocity: Tensor, + clean: Tensor, + noise: Tensor, + *, + mask: Tensor | None, + name: str, +) -> Tensor: + _require_same_shape( + f"{name} predicted_velocity/clean/noise", + predicted_velocity, + clean, + noise, + ) + # As for the generator loss, promote the DSM target operands before their + # subtraction so the final MSE is formed entirely in FP32. + target = h3_flow_velocity_target( + clean.detach().to(dtype=torch.float32), + noise.detach().to(dtype=torch.float32), + ) + predicted_velocity_fp32 = predicted_velocity.to(dtype=torch.float32) + squared_error = (predicted_velocity_fp32 - target).square() + return _masked_mean(squared_error, mask, name=name) + + +def critic_flow_matching_loss( + *, + predicted_video_velocity: Tensor, + clean_video: Tensor, + video_noise: Tensor, + predicted_audio_velocity: Tensor | None = None, + clean_audio: Tensor | None = None, + audio_noise: Tensor | None = None, + video_mask: Tensor | None = None, + audio_mask: Tensor | None = None, + video_weight: float = 1.0, + audio_weight: float = 1.0, +) -> MultimodalLoss: + """Train the critic/fake-score with H3's data-ward flow target. + + For the straight H3 interpolation ``x_t=t*x0+(1-t)*noise``, the target is + constant over time: ``x0-noise``. This function intentionally uses that + sign rather than LongLive's existing noise-ward ``noise-x0`` target. + """ + + video_loss = _critic_flow_loss_one_modality( + predicted_video_velocity, + clean_video, + video_noise, + mask=video_mask, + name="video", + ) + + has_audio = _require_complete_audio_triplet( + predicted_audio_velocity, + clean_audio, + audio_noise, + names=( + "predicted_audio_velocity", + "clean_audio", + "audio_noise", + ), + ) + audio_loss = None + if has_audio: + audio_loss = _critic_flow_loss_one_modality( + predicted_audio_velocity, + clean_audio, + audio_noise, + mask=audio_mask, + name="audio", + ) + elif audio_mask is not None: + raise ValueError("audio_mask was provided without audio tensors.") + + return _combine_multimodal_losses( + video_loss, + audio_loss, + video_weight=video_weight, + audio_weight=audio_weight, + ) + + +__all__ = [ + "DMD_NORMALIZATION", + "H3_AUDIO_TAG", + "H3_PADDING_TAG", + "H3_TEXT_TAG", + "H3_VIDEO_TAG", + "H3SigmaSchedules", + "MultimodalLoss", + "PositiveOnlyCallCounter", + "assert_positive_only_calls", + "build_h3_sigma_schedules", + "critic_flow_matching_loss", + "dmd_generator_pseudo_loss", + "h3_add_noise", + "h3_data_velocity_to_x0", + "h3_flow_velocity_target", + "per_sample_normalizer", +] diff --git a/source_snapshot/minimax_h3/encode_longlive_prompt_cache.py b/source_snapshot/minimax_h3/encode_longlive_prompt_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..13e8ee9505bfb04e31e3558c9a7fcf1ce2979e5e --- /dev/null +++ b/source_snapshot/minimax_h3/encode_longlive_prompt_cache.py @@ -0,0 +1,361 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Distributed, text-only MiniMax-H3 cache builder for LongLive prompts. + +Each torchrun rank owns a stable strided shard. The only model component +loaded is the released Qwen3-VL text conditioner used by MiniMax-H3 Base; no +IR, reference transformer, Regenerator, negative prompt, or CFG branch is +constructed. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping, Sequence + +import torch + +from .infer_base import sha256_file +from .longlive_prompt_set import ( + CACHE_ARTIFACT_TYPE, + SELECTED_MANIFEST_SHA256, + SOURCE_BYTES, + SOURCE_PATH, + SOURCE_REPO_ID, + SOURCE_REVISION, + SOURCE_SHA256, + load_prompt_cache, +) +from .train_dmd import validate_local_model_provenance + + +MODEL_ID = "MiniMaxAI/MiniMax-H3" +WORKFLOW = "t2va" +TEXT_TAG = 1 +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _atomic_torch_save(payload: Any, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=path.parent, suffix=".tmp", delete=False) as handle: + temporary = Path(handle.name) + try: + torch.save(payload, temporary) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _atomic_json(payload: Any, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + dir=path.parent, suffix=".tmp", mode="w", encoding="utf-8", delete=False + ) as handle: + temporary = Path(handle.name) + json.dump(payload, handle, ensure_ascii=False, indent=2, sort_keys=True) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + try: + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _load_prompt_manifest(path: Path, expected_count: int) -> tuple[dict[str, Any], list[dict[str, Any]]]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict) or payload.get("schema_version") != 1: + raise ValueError("Prompt manifest must be a schema-v1 JSON object.") + rows = payload.get("prompts") + if not isinstance(rows, list) or len(rows) != expected_count: + raise ValueError( + f"Prompt manifest must contain exactly {expected_count} prompts, got " + f"{len(rows) if isinstance(rows, list) else 'non-list'}." + ) + observed_ids: set[str] = set() + normalized = [] + for index, row in enumerate(rows): + if not isinstance(row, Mapping): + raise ValueError(f"Prompt row {index} is not a mapping.") + prompt_id, prompt, seed = row.get("id"), row.get("prompt"), row.get("seed") + if not isinstance(prompt_id, str) or not prompt_id or prompt_id in observed_ids: + raise ValueError(f"Prompt row {index} has a missing or duplicate id.") + if not isinstance(prompt, str) or not prompt or prompt != prompt.strip(): + raise ValueError(f"Prompt row {index} is empty or not canonicalized.") + if not isinstance(seed, int) or isinstance(seed, bool) or seed < 0: + raise ValueError(f"Prompt row {index} has an invalid seed.") + observed_ids.add(prompt_id) + normalized.append({"id": prompt_id, "prompt": prompt, "seed": seed}) + return payload, normalized + + +def _validate_condition(row: Mapping[str, Any]) -> None: + embeds = row.get("prompt_embeds") + tags = row.get("text_token_tags") + if not isinstance(embeds, torch.Tensor) or embeds.ndim != 3 or embeds.shape[0] != 1: + raise ValueError(f"Malformed prompt embeddings for {row.get('id')!r}.") + if not isinstance(tags, torch.Tensor) or tags.ndim != 1: + raise ValueError(f"Malformed text tags for {row.get('id')!r}.") + if embeds.shape[1] != tags.shape[0] or tags.numel() == 0: + raise ValueError(f"Embedding/tag length mismatch for {row.get('id')!r}.") + if tags.dtype != torch.long or not bool(torch.all(tags == TEXT_TAG)): + raise ValueError(f"Condition {row.get('id')!r} is not text-only.") + if embeds.device.type != "cpu" or tags.device.type != "cpu": + raise ValueError("Cache tensors must be stored on CPU.") + + +def _load_completed_shard(path: Path, expected_ids: list[str]) -> list[dict[str, Any]]: + if not path.is_file(): + return [] + payload = torch.load(path, map_location="cpu", weights_only=True) + if not isinstance(payload, Mapping) or payload.get("schema_version") != 1: + raise ValueError(f"Invalid prompt-cache shard: {path}") + rows = payload.get("conditions") + if not isinstance(rows, list) or len(rows) > len(expected_ids): + raise ValueError(f"Invalid condition list in shard: {path}") + if [row.get("id") for row in rows] != expected_ids[: len(rows)]: + raise ValueError(f"Shard ordering/provenance mismatch: {path}") + for row in rows: + _validate_condition(row) + return list(rows) + + +def _encode_rank( + *, + rows: list[dict[str, Any]], + shard_path: Path, + model_dir: Path, + revision: str, + device: torch.device, + checkpoint_every: int, +) -> list[dict[str, Any]]: + conditions = _load_completed_shard(shard_path, [row["id"] for row in rows]) + if len(conditions) == len(rows): + return conditions + + from diffusers.modular_pipelines.minimax_h3.encoders import MiniMaxH3TextEncoderStep + + block = MiniMaxH3TextEncoderStep() + pipe = block.init_pipeline(str(model_dir)) + pipe.load_components( + dtype=torch.bfloat16, + pretrained_model_name_or_path=str(model_dir), + revision=revision, + local_files_only=True, + ) + loaded = set(name for name, value in pipe.components.items() if value is not None) + if loaded != {"text_encoder", "tokenizer", "processor"}: + raise RuntimeError(f"Text-only pipeline loaded unexpected components: {sorted(loaded)}") + pipe.to(device) + + for position in range(len(conditions), len(rows)): + spec = rows[position] + output = pipe( + prompt=spec["prompt"], + output=["prompt_embeds", "text_token_tags"], + ) + condition = { + **spec, + "prompt_embeds": output["prompt_embeds"].detach().cpu(), + "text_token_tags": output["text_token_tags"].detach().cpu(), + } + _validate_condition(condition) + conditions.append(condition) + if len(conditions) % checkpoint_every == 0 or len(conditions) == len(rows): + _atomic_torch_save( + { + "schema_version": 1, + "rank": int(os.environ.get("RANK", "0")), + "world_size": int(os.environ.get("WORLD_SIZE", "1")), + "conditions": conditions, + }, + shard_path, + ) + print( + json.dumps( + { + "event": "prompt_cache_shard_progress", + "rank": int(os.environ.get("RANK", "0")), + "completed": len(conditions), + "assigned": len(rows), + }, + sort_keys=True, + ), + flush=True, + ) + return conditions + + +def _merge( + *, + rows: list[dict[str, Any]], + manifest: Path, + manifest_payload: Mapping[str, Any], + output: Path, + shard_dir: Path, + world_size: int, + model_dir: Path, + revision: str, + smoke: bool, +) -> None: + by_id: dict[str, dict[str, Any]] = {} + for rank in range(world_size): + path = shard_dir / f"shard-{rank:05d}-of-{world_size:05d}.pt" + payload = torch.load(path, map_location="cpu", weights_only=True) + conditions = payload.get("conditions") if isinstance(payload, Mapping) else None + expected = rows[rank::world_size] + if not isinstance(conditions, list) or [r.get("id") for r in conditions] != [r["id"] for r in expected]: + raise RuntimeError(f"Incomplete or misordered cache shard: {path}") + for condition in conditions: + _validate_condition(condition) + if condition["id"] in by_id: + raise RuntimeError(f"Duplicate condition id while merging: {condition['id']}") + by_id[condition["id"]] = condition + ordered = [by_id[row["id"]] for row in rows] + source = { + "repo_id": SOURCE_REPO_ID, + "revision": SOURCE_REVISION, + "path": SOURCE_PATH, + "bytes": SOURCE_BYTES, + "sha256": SOURCE_SHA256, + } + manifest_sha256 = sha256_file(manifest) + if not smoke and manifest_sha256 != SELECTED_MANIFEST_SHA256: + raise RuntimeError( + "Formal manifest differs from the pinned LongLive 1,000-prompt selection: " + f"{manifest_sha256} != {SELECTED_MANIFEST_SHA256}" + ) + payload = { + "schema_version": 1, + "artifact_type": CACHE_ARTIFACT_TYPE, + "workflow": WORKFLOW, + "cfg_enabled": False, + "ir": False, + "regenerator": False, + "model_revision": revision, + "prompt_manifest_sha256": manifest_sha256, + "source": source, + "conditions": ordered, + } + _atomic_torch_save(payload, output) + output_sha = sha256_file(output) + sidecar = { + "schema_version": 1, + "status": "success", + "created_at": _utc_now(), + "prompt_cache": str(output.resolve()), + "prompt_cache_sha256": output_sha, + "prompt_manifest": str(manifest.resolve()), + "prompt_manifest_sha256": manifest_sha256, + "prompt_count": len(ordered), + "conditioner_calls": len(ordered), + "workflow": WORKFLOW, + "cfg_enabled": False, + "ir_enabled": False, + "regenerator_enabled": False, + "text_only": True, + "smoke_only": smoke, + "model_revision": revision, + "source": source, + "selection": { + "trim": "python_str_strip", + "deduplication": "exact_keep_first_source_line", + "ordering": "sha256_utf8_ascending_then_utf8_bytes", + "selected": len(ordered), + }, + } + _atomic_json(sidecar, Path(str(output) + ".json")) + if not smoke: + _, validated = load_prompt_cache(output) + if len(validated) != len(ordered): + raise RuntimeError("Strict LongLive prompt-cache readback count mismatch.") + print(json.dumps(sidecar, sort_keys=True), flush=True) + + +def build_cache(args: argparse.Namespace) -> int: + model_dir, manifest, output = Path(args.model_dir), Path(args.manifest), Path(args.output) + validate_local_model_provenance(str(model_dir), args.revision) + manifest_payload, rows = _load_prompt_manifest(manifest, args.expected_prompts) + if args.smoke: + rows = rows[:1] + if output.exists() or Path(str(output) + ".json").exists(): + raise FileExistsError(f"Refusing to overwrite an existing final cache: {output}") + + world_size = int(os.environ.get("WORLD_SIZE", "1")) + rank = int(os.environ.get("RANK", "0")) + local_rank = int(os.environ.get("LOCAL_RANK", "0")) + if world_size > 1: + torch.cuda.set_device(local_rank) + torch.distributed.init_process_group("nccl") + elif not torch.cuda.is_available(): + raise RuntimeError("MiniMax-H3 prompt encoding requires CUDA.") + else: + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + shard_dir = Path(args.shard_dir) + shard_path = shard_dir / f"shard-{rank:05d}-of-{world_size:05d}.pt" + assigned = rows[rank::world_size] + if not assigned: + _atomic_torch_save( + {"schema_version": 1, "rank": rank, "world_size": world_size, "conditions": []}, + shard_path, + ) + else: + _encode_rank( + rows=assigned, + shard_path=shard_path, + model_dir=model_dir, + revision=args.revision, + device=device, + checkpoint_every=args.checkpoint_every, + ) + if world_size > 1: + torch.distributed.barrier() + if rank == 0: + _merge( + rows=rows, + manifest=manifest, + manifest_payload=manifest_payload, + output=output, + shard_dir=shard_dir, + world_size=world_size, + model_dir=model_dir, + revision=args.revision, + smoke=args.smoke, + ) + if world_size > 1: + torch.distributed.barrier() + torch.distributed.destroy_process_group() + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model-dir", required=True) + parser.add_argument("--revision", required=True) + parser.add_argument("--manifest", required=True) + parser.add_argument("--expected-prompts", type=int, default=1000) + parser.add_argument("--output", required=True) + parser.add_argument("--shard-dir", required=True) + parser.add_argument("--checkpoint-every", type=int, default=10) + parser.add_argument("--smoke", action="store_true") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.expected_prompts < 1 or args.checkpoint_every < 1: + raise ValueError("expected-prompts and checkpoint-every must be positive.") + return build_cache(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/source_snapshot/minimax_h3/encode_longlive_prompt_cache_v4.py b/source_snapshot/minimax_h3/encode_longlive_prompt_cache_v4.py new file mode 100644 index 0000000000000000000000000000000000000000..b84f7860c2d4b7683be776467064a051da745e50 --- /dev/null +++ b/source_snapshot/minimax_h3/encode_longlive_prompt_cache_v4.py @@ -0,0 +1,443 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Encode MiniMax-H3 v4's rank-local LongLive prompt schedules. + +The formal v4 default remains 32 torchrun ranks. ``--expected-world-size`` +allows a derived experiment to bind the same strict cache format to another +explicit world size. Each rank independently reproduces its +``DistributedSampler`` stream and writes one cache file containing its first +2,000 conditions. Rank 0 writes only a small collection manifest; it never +merges the rank-local tensors into one artifact. With the production world +size of 16 this yields 32,000 globally distinct scheduled conditions. +""" + +from __future__ import annotations + +import argparse +import json +import os +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping, Sequence + +import torch + +from .longlive_prompt_distributed import ( + CACHE_ARTIFACT_TYPE, + COLLECTION_ARTIFACT_TYPE, + COLLECTION_SCHEMA_VERSION, + FORMAL_EPOCH, + FORMAL_GLOBAL_STEPS, + FORMAL_SEED, + FORMAL_WORLD_SIZE, + TEXT_TOKEN_TAG, + PromptRow, + build_rank_schedule, + collection_sampler_descriptor, + load_prompt_cache_collection, + load_rank_prompt_cache, + make_rank_cache_payload, + read_pinned_dataset, + sha256_file, + source_descriptor, + validate_cross_rank_schedule, +) +from .train_dmd import validate_local_model_provenance + + +MODEL_ID = "MiniMaxAI/MiniMax-H3" +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def rank_cache_name(rank: int, world_size: int) -> str: + return f"rank-{rank:05d}-of-{world_size:05d}.pt" + + +def _atomic_torch_save(payload: Any, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=path.parent, suffix=".tmp", delete=False) as handle: + temporary = Path(handle.name) + try: + torch.save(payload, temporary) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _atomic_json(payload: Any, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + dir=path.parent, + suffix=".tmp", + mode="w", + encoding="utf-8", + delete=False, + ) as handle: + temporary = Path(handle.name) + json.dump(payload, handle, ensure_ascii=False, indent=2, sort_keys=True) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + try: + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _condition_spec(row: PromptRow, global_step: int) -> dict[str, Any]: + return { + "global_step": global_step, + "id": row.id, + "source_index": row.source_index, + "source_line": row.source_line, + "prompt": row.prompt, + "prompt_sha256": row.prompt_sha256, + } + + +def _validate_encoded_condition(row: Mapping[str, Any]) -> None: + embeds = row.get("prompt_embeds") + tags = row.get("text_token_tags") + if ( + not isinstance(embeds, torch.Tensor) + or embeds.ndim != 3 + or embeds.shape[0] != 1 + or embeds.shape[1] < 1 + or embeds.shape[2] < 1 + or embeds.dtype != torch.bfloat16 + or not torch.is_floating_point(embeds) + or not bool(torch.isfinite(embeds).all()) + ): + raise ValueError(f"Malformed prompt embeddings for {row.get('id')!r}.") + if ( + not isinstance(tags, torch.Tensor) + or tags.dtype != torch.long + or tags.shape != (embeds.shape[1],) + or not bool((tags == TEXT_TOKEN_TAG).all()) + ): + raise ValueError(f"Condition {row.get('id')!r} is not positive text-only.") + if embeds.device.type != "cpu" or tags.device.type != "cpu": + raise ValueError("Cache tensors must be stored on CPU.") + + +def _load_partial(path: Path, schedule: list[PromptRow], rank: int) -> list[dict[str, Any]]: + if not path.is_file(): + return [] + payload = torch.load(path, map_location="cpu", weights_only=True) + if not isinstance(payload, Mapping): + raise ValueError(f"Malformed partial rank cache: {path}") + expected_fields = {"schema_version", "rank", "conditions"} + if set(payload) != expected_fields or payload.get("schema_version") != 1: + raise ValueError(f"Malformed partial rank cache schema: {path}") + if payload.get("rank") != rank: + raise ValueError(f"Partial rank cache belongs to another rank: {path}") + conditions = payload.get("conditions") + if not isinstance(conditions, list) or len(conditions) > len(schedule): + raise ValueError(f"Malformed partial condition list: {path}") + for position, condition in enumerate(conditions): + expected = _condition_spec(schedule[position], position + 1) + if not isinstance(condition, Mapping) or any( + condition.get(field) != value for field, value in expected.items() + ): + raise ValueError(f"Partial rank cache schedule mismatch at position {position}") + _validate_encoded_condition(condition) + return list(conditions) + + +def _encode_rank( + *, + schedule: list[PromptRow], + partial_path: Path, + model_dir: Path, + revision: str, + device: torch.device, + checkpoint_every: int, + rank: int, +) -> list[dict[str, Any]]: + conditions = _load_partial(partial_path, schedule, rank) + if len(conditions) == len(schedule): + return conditions + + from diffusers.modular_pipelines.minimax_h3.encoders import MiniMaxH3TextEncoderStep + + block = MiniMaxH3TextEncoderStep() + pipe = block.init_pipeline(str(model_dir)) + pipe.load_components( + dtype=torch.bfloat16, + pretrained_model_name_or_path=str(model_dir), + revision=revision, + local_files_only=True, + ) + loaded = {name for name, value in pipe.components.items() if value is not None} + if loaded != {"text_encoder", "tokenizer", "processor"}: + raise RuntimeError(f"Text-only pipeline loaded unexpected components: {sorted(loaded)}") + pipe.to(device) + + for position in range(len(conditions), len(schedule)): + source_row = schedule[position] + output = pipe( + prompt=source_row.prompt, + output=["prompt_embeds", "text_token_tags"], + ) + condition = { + **_condition_spec(source_row, position + 1), + "prompt_embeds": output["prompt_embeds"].detach().cpu(), + "text_token_tags": output["text_token_tags"].detach().cpu(), + } + _validate_encoded_condition(condition) + conditions.append(condition) + if len(conditions) % checkpoint_every == 0 or len(conditions) == len(schedule): + _atomic_torch_save( + {"schema_version": 1, "rank": rank, "conditions": conditions}, + partial_path, + ) + print( + json.dumps( + { + "event": "v4_prompt_cache_rank_progress", + "rank": rank, + "completed": len(conditions), + "assigned": len(schedule), + }, + sort_keys=True, + ), + flush=True, + ) + return conditions + + +def _write_rank_sidecar( + *, + cache_path: Path, + rank: int, + world_size: int, + seed: int, + epoch: int, + global_steps: int, + model_revision: str, +) -> dict[str, Any]: + payload = { + "schema_version": 1, + "status": "success", + "artifact_type": CACHE_ARTIFACT_TYPE, + "created_at": _utc_now(), + "rank": rank, + "world_size": world_size, + "seed": seed, + "epoch": epoch, + "global_steps": global_steps, + "conditioner_calls": global_steps, + "cache": str(cache_path.resolve()), + "cache_bytes": cache_path.stat().st_size, + "cache_sha256": sha256_file(cache_path), + "model_revision": model_revision, + "workflow": "t2va", + "cfg_enabled": False, + "ir_enabled": False, + "regenerator_enabled": False, + "text_only": True, + } + _atomic_json(payload, Path(str(cache_path) + ".json")) + return payload + + +def _write_collection( + output_dir: Path, + world_size: int, + model_revision: str, + *, + global_steps: int = FORMAL_GLOBAL_STEPS, +) -> dict[str, Any]: + shards = [] + for rank in range(world_size): + cache_path = output_dir / rank_cache_name(rank, world_size) + sidecar_path = Path(str(cache_path) + ".json") + if not cache_path.is_file() or not sidecar_path.is_file(): + raise RuntimeError(f"Missing completed rank cache or sidecar for rank {rank}") + sidecar = json.loads(sidecar_path.read_text(encoding="utf-8")) + if sidecar.get("rank") != rank or sidecar.get("status") != "success": + raise RuntimeError(f"Malformed rank-cache sidecar: {sidecar_path}") + if ( + sidecar.get("world_size") != world_size + or sidecar.get("seed") != FORMAL_SEED + or sidecar.get("epoch") != FORMAL_EPOCH + or sidecar.get("global_steps") != global_steps + or sidecar.get("model_revision") != model_revision + ): + raise RuntimeError(f"Rank-cache sidecar contract drift: {sidecar_path}") + if sidecar.get("cache_bytes") != cache_path.stat().st_size: + raise RuntimeError(f"Rank-cache byte count drift: {cache_path}") + shards.append( + { + "rank": rank, + "path": cache_path.name, + "bytes": sidecar["cache_bytes"], + "sha256": sidecar["cache_sha256"], + "conditions": sidecar["global_steps"], + } + ) + schedule_audit = validate_cross_rank_schedule( + world_size=world_size, + seed=FORMAL_SEED, + epoch=FORMAL_EPOCH, + global_steps=global_steps, + ) + collection = { + "schema_version": COLLECTION_SCHEMA_VERSION, + "artifact_type": COLLECTION_ARTIFACT_TYPE, + "status": "success", + "created_at": _utc_now(), + "storage": "rank_local_no_merged_tensor_cache", + "workflow": "t2va", + "cfg_enabled": False, + "ir": False, + "regenerator": False, + "text_only": True, + "model_revision": model_revision, + "dataset": source_descriptor(), + "sampler": collection_sampler_descriptor( + world_size=world_size, + seed=FORMAL_SEED, + epoch=FORMAL_EPOCH, + global_steps=global_steps, + ), + "schedule_audit": schedule_audit, + "shards": shards, + } + collection_path = output_dir / "collection.json" + _atomic_json(collection, collection_path) + load_prompt_cache_collection( + collection_path, + verify_shards=False, + expected_world_size=world_size, + expected_seed=FORMAL_SEED, + expected_epoch=FORMAL_EPOCH, + expected_global_steps=global_steps, + ) + return collection + + +def build_cache(args: argparse.Namespace) -> int: + model_dir = Path(args.model_dir) + raw_txt = Path(args.raw_txt) + output_dir = Path(args.output_dir) + validate_local_model_provenance(str(model_dir), args.revision) + + world_size = int(os.environ.get("WORLD_SIZE", "1")) + rank = int(os.environ.get("RANK", "0")) + local_rank = int(os.environ.get("LOCAL_RANK", "0")) + if args.expected_world_size < 1: + raise ValueError("expected-world-size must be positive") + if not args.smoke and world_size != args.expected_world_size: + raise ValueError( + "Formal v4 cache encoding requires " + f"WORLD_SIZE={args.expected_world_size}, got {world_size}" + ) + global_steps = 1 if args.smoke else FORMAL_GLOBAL_STEPS + rows = read_pinned_dataset(raw_txt) + schedule = build_rank_schedule( + rows, + rank=rank, + world_size=world_size, + seed=FORMAL_SEED, + epoch=FORMAL_EPOCH, + global_steps=global_steps, + ) + output_dir.mkdir(parents=True, exist_ok=True) + final_path = output_dir / rank_cache_name(rank, world_size) + partial_path = output_dir / f".{rank_cache_name(rank, world_size)}.partial" + if final_path.exists() or Path(str(final_path) + ".json").exists(): + raise FileExistsError(f"Refusing to overwrite completed rank cache: {final_path}") + + if world_size > 1: + torch.cuda.set_device(local_rank) + torch.distributed.init_process_group("nccl") + elif not torch.cuda.is_available(): + raise RuntimeError("MiniMax-H3 prompt encoding requires CUDA") + else: + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + conditions = _encode_rank( + schedule=schedule, + partial_path=partial_path, + model_dir=model_dir, + revision=args.revision, + device=device, + checkpoint_every=args.checkpoint_every, + rank=rank, + ) + payload = make_rank_cache_payload( + rank=rank, + model_revision=args.revision, + conditions=conditions, + world_size=world_size, + seed=FORMAL_SEED, + epoch=FORMAL_EPOCH, + global_steps=global_steps, + ) + _atomic_torch_save(payload, final_path) + load_rank_prompt_cache( + final_path, + expected_rank=rank, + expected_world_size=world_size, + expected_seed=FORMAL_SEED, + expected_epoch=FORMAL_EPOCH, + expected_global_steps=global_steps, + ) + sidecar = _write_rank_sidecar( + cache_path=final_path, + rank=rank, + world_size=world_size, + seed=FORMAL_SEED, + epoch=FORMAL_EPOCH, + global_steps=global_steps, + model_revision=args.revision, + ) + partial_path.unlink(missing_ok=True) + print(json.dumps(sidecar, sort_keys=True), flush=True) + + if world_size > 1: + torch.distributed.barrier() + if rank == 0 and not args.smoke: + print( + json.dumps( + _write_collection( + output_dir, + world_size, + args.revision, + global_steps=global_steps, + ), + sort_keys=True, + ), + flush=True, + ) + if world_size > 1: + torch.distributed.barrier() + torch.distributed.destroy_process_group() + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model-dir", required=True) + parser.add_argument("--revision", required=True) + parser.add_argument("--raw-txt", required=True) + parser.add_argument("--output-dir", required=True) + parser.add_argument("--expected-world-size", type=int, default=FORMAL_WORLD_SIZE) + parser.add_argument("--checkpoint-every", type=int, default=10) + parser.add_argument("--smoke", action="store_true") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.checkpoint_every < 1: + raise ValueError("checkpoint-every must be positive") + return build_cache(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/source_snapshot/minimax_h3/extract_dmd8_inline5.py b/source_snapshot/minimax_h3/extract_dmd8_inline5.py new file mode 100644 index 0000000000000000000000000000000000000000..3eca2c787d0081f04b41685c3ae95c066312bc4b --- /dev/null +++ b/source_snapshot/minimax_h3/extract_dmd8_inline5.py @@ -0,0 +1,434 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Bind the DMD8 inline-five evaluation to the first five rank-0 cache rows. + +The formal LongLive cache is rank-local and has no merged condition tensor. +For an unambiguous, replayable definition, "first five" means rank 0, +``conditions[0:5]`` (global steps 1 through 5) from the collection's first +shard. This helper never mutates the cache. It verifies the committed prompt +JSON byte-for-field against those rows, then writes a tensor-only conditioning +payload plus a provenance receipt. ``infer_student_8step`` consumes that +payload directly: it does not tokenize the prompt or run Qwen3-VL again. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +from typing import Any, Mapping, Sequence + +import torch + +from .infer_base import load_prompt_specs, sha256_file, tensor_sha256 +from .longlive_prompt_distributed import ( + CACHE_ARTIFACT_TYPE, + CACHE_SCHEMA_VERSION, + SOURCE_LINE_COUNT, + load_prompt_cache_collection, +) + + +INLINE_RANK = 0 +INLINE_COUNT = 5 +EXPECTED_WORLD_SIZE = 16 +EXPECTED_CACHE_SEED = 20260806 +EXPECTED_CACHE_EPOCH = 0 +EXPECTED_GLOBAL_STEPS = 1000 +INLINE_EVALUATION_SEEDS = tuple(range(2026081251, 2026081256)) +INLINE_SELECTION_POLICY = "rank_00000_conditions_0000_through_0004" +INLINE_CONDITIONING_MODE = "exact_cached_prompt_embeds_and_text_token_tags" +INLINE_PAYLOAD_SCHEMA_VERSION = 1 +INLINE_PAYLOAD_ARTIFACT_TYPE = "minimax_h3_dmd8_inline5_cached_conditioning" +_CONDITION_FIELDS = { + "global_step", + "id", + "source_index", + "source_line", + "prompt", + "prompt_sha256", + "prompt_embeds", + "text_token_tags", +} +_PAYLOAD_CONDITION_FIELDS = { + "cache_rank", + "cache_condition_position", + "cache_global_step", + "source_index", + "source_line", + "id", + "prompt", + "prompt_sha256", + "evaluation_seed", + "prompt_embeds", + "text_token_tags", +} + + +def _tensor_descriptor(value: torch.Tensor) -> dict[str, Any]: + return { + "shape": list(value.shape), + "dtype": str(value.dtype), + "sha256": tensor_sha256(value), + } + + +def build_inline_prompt_payload( + conditions: Sequence[Mapping[str, Any]], +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Validate five cache rows and derive prompt/provenance JSON objects.""" + + if len(conditions) != INLINE_COUNT: + raise ValueError(f"Inline cache selection must contain exactly {INLINE_COUNT} rows.") + prompts: list[dict[str, Any]] = [] + provenance: list[dict[str, Any]] = [] + for position, (row, evaluation_seed) in enumerate( + zip(conditions, INLINE_EVALUATION_SEEDS) + ): + label = f"conditions[{position}]" + if not isinstance(row, Mapping) or set(row) != _CONDITION_FIELDS: + raise ValueError(f"{label} does not match the rank-cache condition schema.") + global_step = row["global_step"] + source_index = row["source_index"] + source_line = row["source_line"] + prompt_id = row["id"] + prompt = row["prompt"] + prompt_sha = row["prompt_sha256"] + if global_step != position + 1: + raise ValueError(f"{label}.global_step is not rank-0 step {position + 1}.") + if ( + isinstance(source_index, bool) + or not isinstance(source_index, int) + or not 0 <= source_index < SOURCE_LINE_COUNT + or source_line != source_index + 1 + ): + raise ValueError(f"{label} source index/line binding is invalid.") + if not isinstance(prompt, str) or not prompt.strip() or prompt != prompt.rstrip(): + raise ValueError(f"{label}.prompt is not normalized LongLive text.") + observed_prompt_sha = hashlib.sha256(prompt.encode("utf-8")).hexdigest() + expected_id = f"vidprom-{source_line:06d}-{observed_prompt_sha[:12]}" + if prompt_sha != observed_prompt_sha or prompt_id != expected_id: + raise ValueError(f"{label} ID/prompt SHA binding is invalid.") + embeds = row["prompt_embeds"] + tags = row["text_token_tags"] + if ( + not isinstance(embeds, torch.Tensor) + or embeds.device.type != "cpu" + or embeds.layout != torch.strided + or embeds.dtype != torch.bfloat16 + or embeds.ndim != 3 + or embeds.shape[0] != 1 + or embeds.shape[1] < 1 + or embeds.shape[2] != 5120 + or not bool(torch.isfinite(embeds).all()) + ): + raise ValueError(f"{label}.prompt_embeds is not finite CPU BF16 [1,N,5120].") + if ( + not isinstance(tags, torch.Tensor) + or tags.device.type != "cpu" + or tags.layout != torch.strided + or tags.dtype != torch.long + or tags.shape != (embeds.shape[1],) + or not bool((tags == 1).all()) + ): + raise ValueError(f"{label}.text_token_tags is not the positive text layout.") + prompts.append({"id": prompt_id, "prompt": prompt, "seed": evaluation_seed}) + provenance.append( + { + "cache_rank": INLINE_RANK, + "cache_condition_position": position, + "cache_global_step": global_step, + "source_index": source_index, + "source_line": source_line, + "id": prompt_id, + "prompt": prompt, + "prompt_sha256": prompt_sha, + "evaluation_seed": evaluation_seed, + "evaluation_seed_policy": "fixed_dmd8_inline5_v1", + "cached_prompt_embeds": _tensor_descriptor(embeds), + "cached_text_token_tags": _tensor_descriptor(tags), + } + ) + return {"schema_version": 1, "prompts": prompts}, provenance + + +def build_inline_conditioning_payload( + conditions: Sequence[Mapping[str, Any]], + *, + model_revision: str, + collection_sha256: str, + rank_cache_shard_sha256: str, + checkpoint_sha256: str, +) -> dict[str, Any]: + """Copy the five validated mmap rows into a weights-only-safe payload.""" + + _, provenance = build_inline_prompt_payload(conditions) + payload_conditions: list[dict[str, Any]] = [] + for row, descriptor in zip(conditions, provenance): + payload_row = { + "cache_rank": descriptor["cache_rank"], + "cache_condition_position": descriptor["cache_condition_position"], + "cache_global_step": descriptor["cache_global_step"], + "source_index": descriptor["source_index"], + "source_line": descriptor["source_line"], + "id": descriptor["id"], + "prompt": descriptor["prompt"], + "prompt_sha256": descriptor["prompt_sha256"], + "evaluation_seed": descriptor["evaluation_seed"], + # Clone both tensors so the derived artifact is independent of the + # read-only mmap backing the formal rank cache. + "prompt_embeds": row["prompt_embeds"].detach().cpu().contiguous().clone(), + "text_token_tags": row["text_token_tags"].detach().cpu().contiguous().clone(), + } + if set(payload_row) != _PAYLOAD_CONDITION_FIELDS: + raise AssertionError("Inline conditioning payload schema drifted.") + payload_conditions.append(payload_row) + return { + "schema_version": INLINE_PAYLOAD_SCHEMA_VERSION, + "artifact_type": INLINE_PAYLOAD_ARTIFACT_TYPE, + "selection_policy": INLINE_SELECTION_POLICY, + "conditioning_mode": INLINE_CONDITIONING_MODE, + "model_revision": model_revision, + "collection_sha256": collection_sha256, + "rank_cache_shard_sha256": rank_cache_shard_sha256, + "checkpoint_sha256": checkpoint_sha256, + "conditions": payload_conditions, + } + + +def extract_and_verify_inline5( + *, + collection_path: str | Path, + expected_prompts_path: str | Path, + checkpoint_path: str | Path, + output_payload_path: str | Path | None = None, +) -> dict[str, Any]: + """Verify the world-16 collection/shard and return the binding receipt.""" + + collection_path = Path(collection_path).resolve() + expected_prompts_path = Path(expected_prompts_path).resolve() + checkpoint_path = Path(checkpoint_path).resolve() + collection = load_prompt_cache_collection( + collection_path, + verify_shards=False, + expected_world_size=EXPECTED_WORLD_SIZE, + expected_seed=EXPECTED_CACHE_SEED, + expected_epoch=EXPECTED_CACHE_EPOCH, + expected_global_steps=EXPECTED_GLOBAL_STEPS, + ) + collection_sha = sha256_file(collection_path) + try: + checkpoint = torch.load( + checkpoint_path, + map_location="cpu", + weights_only=True, + mmap=True, + ) + except Exception as error: + raise ValueError(f"Unable to mmap eight-step checkpoint: {checkpoint_path}") from error + checkpoint_config = checkpoint.get("config") if isinstance(checkpoint, Mapping) else None + if not isinstance(checkpoint_config, Mapping): + raise ValueError("Eight-step checkpoint has no embedded training config.") + cached_path = checkpoint_config.get("prompt_cache") + cached_path = Path(cached_path).resolve() if isinstance(cached_path, str) else None + collection_provenance = checkpoint_config.get("prompt_collection_provenance") + eight_step_contract = ( + checkpoint_config.get("rollout_mode"), + checkpoint_config.get("model_evaluations_per_rollout"), + checkpoint_config.get("sigma_endpoints_per_rollout"), + checkpoint_config.get("fresh_noise_transitions_per_rollout"), + ) + if ( + checkpoint.get("schema_version") != 1 + or eight_step_contract != ("predict_x0_fresh_renoise", 8, 9, 7) + or cached_path != collection_path + or checkpoint_config.get("prompt_cache_sha256") != collection_sha + or checkpoint_config.get("resolved_revision") != collection["model_revision"] + or not isinstance(collection_provenance, Mapping) + or Path(collection_provenance.get("path", "")).resolve() != collection_path + or collection_provenance.get("sha256") != collection_sha + ): + raise ValueError( + "Prompt-cache collection is not the exact collection embedded in the checkpoint config." + ) + shard_record = collection["shards"][INLINE_RANK] + shard_path = collection_path.parent / shard_record["path"] + observed_shard_sha = sha256_file(shard_path) + if observed_shard_sha != shard_record["sha256"]: + raise ValueError("Rank-0 cache shard SHA-256 differs from collection.json.") + try: + shard = torch.load( + shard_path, + map_location="cpu", + weights_only=True, + mmap=True, + ) + except Exception as error: + raise ValueError(f"Unable to mmap rank-0 prompt cache: {shard_path}") from error + if not isinstance(shard, Mapping): + raise ValueError("Rank-0 prompt cache payload is not a mapping.") + if ( + shard.get("schema_version") != CACHE_SCHEMA_VERSION + or shard.get("artifact_type") != CACHE_ARTIFACT_TYPE + or shard.get("workflow") != "t2va" + or shard.get("cfg_enabled") is not False + or shard.get("ir") is not False + or shard.get("regenerator") is not False + or shard.get("model_revision") != collection["model_revision"] + or shard.get("dataset") != collection["dataset"] + ): + raise ValueError("Rank-0 prompt cache header differs from its collection contract.") + expected_sampler = {**collection["sampler"], "rank": INLINE_RANK} + if shard.get("sampler") != expected_sampler: + raise ValueError("Rank-0 prompt-cache sampler differs from collection.json.") + conditions = shard.get("conditions") + if not isinstance(conditions, list) or len(conditions) != EXPECTED_GLOBAL_STEPS: + raise ValueError("Rank-0 prompt cache does not contain 1,000 conditions.") + inline_conditions = conditions[:INLINE_COUNT] + prompt_payload, condition_provenance = build_inline_prompt_payload(inline_conditions) + expected_prompts = load_prompt_specs( + expected_prompts_path, expected_count=INLINE_COUNT + ) + expected_payload = { + "schema_version": 1, + "prompts": [ + {"id": row.id, "prompt": row.prompt, "seed": row.seed} + for row in expected_prompts + ], + } + if prompt_payload != expected_payload: + raise ValueError( + "Committed inline-five prompts do not exactly match rank-0 cache rows 0..4." + ) + checkpoint_sha = sha256_file(checkpoint_path) + conditioning_payload = build_inline_conditioning_payload( + inline_conditions, + model_revision=collection["model_revision"], + collection_sha256=collection_sha, + rank_cache_shard_sha256=observed_shard_sha, + checkpoint_sha256=checkpoint_sha, + ) + payload_record = None + if output_payload_path is not None: + output_payload_path = Path(output_payload_path).resolve() + _atomic_torch_save(output_payload_path, conditioning_payload) + payload_record = { + "path": str(output_payload_path), + "bytes": output_payload_path.stat().st_size, + "sha256": sha256_file(output_payload_path), + "schema_version": INLINE_PAYLOAD_SCHEMA_VERSION, + "artifact_type": INLINE_PAYLOAD_ARTIFACT_TYPE, + } + receipt = { + "schema_version": 1, + "status": "verified", + "selection_policy": INLINE_SELECTION_POLICY, + "conditioning_mode": INLINE_CONDITIONING_MODE, + "cached_prompt_embeds_used_by_inference": output_payload_path is not None, + "text_encoder_must_not_run": True, + "conditioning_payload": payload_record, + "collection": { + "path": str(collection_path), + "sha256": collection_sha, + "model_revision": collection["model_revision"], + "dataset": collection["dataset"], + "sampler": collection["sampler"], + }, + "rank_cache_shard": { + "rank": INLINE_RANK, + "path": str(shard_path.resolve()), + "bytes": shard_path.stat().st_size, + "sha256": observed_shard_sha, + }, + "prompt_file": { + "path": str(expected_prompts_path), + "sha256": sha256_file(expected_prompts_path), + }, + "checkpoint_binding": { + "path": str(checkpoint_path), + "sha256": checkpoint_sha, + "step": checkpoint.get("step"), + "training_config_sha256": checkpoint_config.get("config_sha256"), + "training_prompt_cache_path": str(cached_path), + "training_prompt_cache_sha256": checkpoint_config.get( + "prompt_cache_sha256" + ), + }, + "conditions": condition_provenance, + } + return receipt + + +def _atomic_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + os.replace(temporary, path) + + +def _atomic_torch_save(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + torch.save(value, temporary) + os.replace(temporary, path) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--collection", type=Path, required=True) + parser.add_argument("--checkpoint", type=Path, required=True) + parser.add_argument("--expected-prompts", type=Path, required=True) + parser.add_argument("--output-payload", type=Path, required=True) + parser.add_argument("--output-provenance", type=Path, required=True) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + receipt = extract_and_verify_inline5( + collection_path=args.collection, + expected_prompts_path=args.expected_prompts, + checkpoint_path=args.checkpoint, + output_payload_path=args.output_payload, + ) + _atomic_json(args.output_provenance, receipt) + print( + json.dumps( + { + "status": receipt["status"], + "selection_policy": receipt["selection_policy"], + "conditioning_mode": receipt["conditioning_mode"], + "conditions": len(receipt["conditions"]), + "output_payload": str(args.output_payload.resolve()), + "output_provenance": str(args.output_provenance.resolve()), + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + + +__all__ = [ + "INLINE_CONDITIONING_MODE", + "INLINE_COUNT", + "INLINE_EVALUATION_SEEDS", + "INLINE_PAYLOAD_ARTIFACT_TYPE", + "INLINE_PAYLOAD_SCHEMA_VERSION", + "INLINE_RANK", + "INLINE_SELECTION_POLICY", + "build_inline_prompt_payload", + "build_inline_conditioning_payload", + "extract_and_verify_inline5", + "main", +] diff --git a/source_snapshot/minimax_h3/fresh_noise_scheduler.py b/source_snapshot/minimax_h3/fresh_noise_scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..c6a8e80dc320191197de68c6ba900c707451bce9 --- /dev/null +++ b/source_snapshot/minimax_h3/fresh_noise_scheduler.py @@ -0,0 +1,332 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Fresh-noise MiniMax-H3 scheduler used by the eight-step DMD student. + +The released :class:`diffusers.MiniMaxH3Scheduler` takes deterministic Euler +steps. The DMD student was trained with a different transition: predict +``x0`` at the current H3 timestep, then independently re-noise that prediction +at the next modality-specific sigma. This module changes only ``step`` and +keeps the released schedule construction, timestep convention, conditioning +noise helper, and scheduler configuration intact. + +One T2VA request owns two scheduler instances. They must be bound to the +same ``torch.Generator``. The official H3 denoise block calls the video +scheduler first and the audio scheduler second, producing the replay-critical +draw order ``video, audio`` for each of the seven non-terminal transitions. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import torch +from diffusers.schedulers.scheduling_minimax_h3 import ( + MiniMaxH3Scheduler, + MiniMaxH3SchedulerOutput, +) +from diffusers.utils.torch_utils import randn_tensor + + +NUM_MODEL_EVALUATIONS = 8 +NUM_SIGMA_ENDPOINTS = 9 +NUM_FRESH_NOISE_TRANSITIONS = 7 +_STREAMS = ("video", "audio") + + +@dataclass +class FreshNoiseDrawAudit: + """Shared, request-local record of the two schedulers' RNG consumption.""" + + generator: torch.Generator + events: list[str] = field(default_factory=list) + + def record(self, stream: str) -> None: + if stream not in _STREAMS: + raise ValueError(f"Unknown H3 fresh-noise stream: {stream!r}.") + self.events.append(stream) + + @property + def expected_events(self) -> list[str]: + return list(_STREAMS) * NUM_FRESH_NOISE_TRANSITIONS + + def validate_complete(self) -> None: + if self.events != self.expected_events: + raise RuntimeError( + "H3 fresh-noise draw order/count changed: " + f"observed={self.events}, expected={self.expected_events}." + ) + + +class FreshNoiseMiniMaxH3Scheduler(MiniMaxH3Scheduler): + """Eight-forward predict-x0/fresh-re-noise scheduler for one modality.""" + + def __init__(self, shift: float = 12.0): + super().__init__(shift=shift) + self._fresh_noise_generator: torch.Generator | None = None + self._fresh_noise_audit: FreshNoiseDrawAudit | None = None + self._fresh_noise_stream: str | None = None + self.step_call_count = 0 + self.fresh_noise_draw_count = 0 + self.terminal_clean_output_count = 0 + + @property + def fresh_noise_generator(self) -> torch.Generator | None: + return self._fresh_noise_generator + + @property + def fresh_noise_stream(self) -> str | None: + return self._fresh_noise_stream + + def bind_generator( + self, + generator: torch.Generator, + *, + stream: str, + audit: FreshNoiseDrawAudit, + ) -> None: + """Bind one request RNG and its shared video/audio draw audit.""" + + if not isinstance(generator, torch.Generator): + raise TypeError("Fresh-noise inference requires one torch.Generator.") + if stream not in _STREAMS: + raise ValueError(f"Fresh-noise stream must be one of {_STREAMS}, got {stream!r}.") + if not isinstance(audit, FreshNoiseDrawAudit): + raise TypeError("Fresh-noise scheduler audit has the wrong type.") + if audit.generator is not generator: + raise ValueError("Both H3 schedulers must share the audit's exact generator object.") + self._fresh_noise_generator = generator + self._fresh_noise_audit = audit + self._fresh_noise_stream = stream + + def set_timesteps( + self, + num_inference_steps: int | None = None, + device: str | torch.device | None = None, + sigmas: list[float] | torch.Tensor | None = None, + ) -> None: + """Build and strictly validate the eight-forward/nine-endpoint grid.""" + + super().set_timesteps( + num_inference_steps=num_inference_steps, + device=device, + sigmas=sigmas, + ) + if self.sigmas is None or self.timesteps is None: + raise AssertionError("MiniMax-H3 scheduler did not construct its grid.") + if self.sigmas.numel() != NUM_SIGMA_ENDPOINTS: + raise ValueError( + "Eight-step fresh-noise H3 requires exactly nine sigma endpoints " + f"including terminal zero, found {self.sigmas.numel()}." + ) + if self.timesteps.numel() != NUM_MODEL_EVALUATIONS: + raise AssertionError("Nine H3 endpoints must expose exactly eight timesteps.") + if float(self.sigmas[-1]) != 0.0: + raise AssertionError("The ninth H3 sigma endpoint must be terminal zero.") + self.step_call_count = 0 + self.fresh_noise_draw_count = 0 + self.terminal_clean_output_count = 0 + + def step( + self, + model_output: torch.FloatTensor, + timestep: float | torch.FloatTensor, + sample: torch.FloatTensor, + return_dict: bool = True, + ) -> MiniMaxH3SchedulerOutput | tuple: + r"""Predict ``x0`` and fresh re-noise it at the next schedule point. + + For current ``t = 1 - sigma`` and H3's data-ward velocity ``v``:: + + x0 = x_t + (1 - t) * v + x_next = t_next * x0 + (1 - t_next) * fresh_noise + + The terminal transition has ``sigma_next = 0`` and therefore returns + ``x0`` without drawing noise. + """ + + if isinstance(timestep, int) or ( + isinstance(timestep, torch.Tensor) and not timestep.is_floating_point() + ): + raise ValueError( + "Passing integer indices to FreshNoiseMiniMaxH3Scheduler.step() " + "is unsupported; pass a value from scheduler.timesteps." + ) + if sample.shape != model_output.shape: + raise ValueError( + "MiniMax-H3 fresh-noise sample/model_output shapes differ: " + f"{tuple(sample.shape)} != {tuple(model_output.shape)}." + ) + if self.sigmas is None or self.timesteps is None: + raise RuntimeError("Call set_timesteps() before fresh-noise step().") + if self._step_index is None: + self._step_index = ( + self.index_for_timestep(timestep) + if self._begin_index is None + else self._begin_index + ) + if not 0 <= self._step_index < NUM_MODEL_EVALUATIONS: + raise RuntimeError( + f"Fresh-noise H3 step index {self._step_index} lies outside 0..7." + ) + + if not isinstance(timestep, torch.Tensor): + timestep = torch.tensor(timestep, dtype=sample.dtype) + timestep = timestep.to(device=sample.device, dtype=sample.dtype) + sigma_from_timestep = 1.0 - timestep + while sigma_from_timestep.ndim < sample.ndim: + sigma_from_timestep = sigma_from_timestep.unsqueeze(-1) + clean = sample + sigma_from_timestep * model_output + + next_index = self._step_index + 1 + if next_index == NUM_MODEL_EVALUATIONS: + # The terminal sigma is exactly zero. Drawing and multiplying by + # zero would still advance the shared RNG and break seed replay. + prev_sample = clean + self.terminal_clean_output_count += 1 + else: + generator = self._fresh_noise_generator + audit = self._fresh_noise_audit + stream = self._fresh_noise_stream + if generator is None or audit is None or stream is None: + raise RuntimeError( + "Bind the request's shared generator to both fresh-noise " + "H3 schedulers before pipeline inference." + ) + if audit.generator is not generator: + raise RuntimeError("Fresh-noise scheduler/audit generator identity changed.") + # randn_tensor preserves Diffusers' CPU-generator/CUDA-sample + # behavior, which torch.randn(device='cuda', generator=cpu_rng) + # does not support. + noise = randn_tensor( + clean.shape, + generator=generator, + device=clean.device, + dtype=clean.dtype, + ) + next_timestep = self.timesteps[next_index].to( + device=clean.device, dtype=clean.dtype + ) + while next_timestep.ndim < clean.ndim: + next_timestep = next_timestep.unsqueeze(-1) + prev_sample = next_timestep * clean + (1.0 - next_timestep) * noise + audit.record(stream) + self.fresh_noise_draw_count += 1 + + self._step_index += 1 + self.step_call_count += 1 + prev_sample = prev_sample.to(dtype=sample.dtype) + if not return_dict: + return (prev_sample,) + return MiniMaxH3SchedulerOutput(prev_sample=prev_sample) + + +def install_fresh_noise_schedulers(pipe: object) -> None: + """Replace an already-validated official H3 scheduler pair in-place.""" + + video = getattr(pipe, "scheduler", None) + audio = getattr(pipe, "audio_scheduler", None) + if video is None or audio is None: + raise TypeError("MiniMax-H3 pipeline is missing its video/audio schedulers.") + if video.__class__.__name__ != "MiniMaxH3Scheduler" or audio.__class__.__name__ != "MiniMaxH3Scheduler": + raise TypeError( + "Fresh-noise installation requires the untouched official " + "MiniMaxH3Scheduler pair." + ) + + def shift(component: object) -> float: + config = getattr(component, "config", None) + value = getattr(config, "shift", None) + if value is None and isinstance(config, dict): + value = config.get("shift") + return float(value) + + video_fresh = FreshNoiseMiniMaxH3Scheduler(shift=shift(video)) + audio_fresh = FreshNoiseMiniMaxH3Scheduler(shift=shift(audio)) + if (video_fresh.shift, audio_fresh.shift) != (12.0, 3.0): + raise RuntimeError( + "Unexpected H3 fresh-noise shifts: " + f"video={video_fresh.shift}, audio={audio_fresh.shift}." + ) + register = getattr(pipe, "register_components", None) + if callable(register): + register(scheduler=video_fresh, audio_scheduler=audio_fresh) + else: + setattr(pipe, "scheduler", video_fresh) + setattr(pipe, "audio_scheduler", audio_fresh) + + +def bind_pipeline_fresh_noise_generator( + pipe: object, generator: torch.Generator +) -> FreshNoiseDrawAudit: + """Bind the same request RNG to video then audio scheduler instances.""" + + video = getattr(pipe, "scheduler", None) + audio = getattr(pipe, "audio_scheduler", None) + if not isinstance(video, FreshNoiseMiniMaxH3Scheduler) or not isinstance( + audio, FreshNoiseMiniMaxH3Scheduler + ): + raise TypeError("Pipeline does not hold the fresh-noise H3 scheduler pair.") + audit = FreshNoiseDrawAudit(generator=generator) + video.bind_generator(generator, stream="video", audit=audit) + audio.bind_generator(generator, stream="audio", audit=audit) + if video.fresh_noise_generator is not audio.fresh_noise_generator: + raise AssertionError("Video/audio H3 schedulers do not share one generator.") + return audit + + +def validate_completed_pipeline_rollout( + pipe: object, audit: FreshNoiseDrawAudit +) -> dict[str, object]: + """Fail closed on the 8/9/7 sampler and seeded draw-order contract.""" + + video = getattr(pipe, "scheduler", None) + audio = getattr(pipe, "audio_scheduler", None) + if not isinstance(video, FreshNoiseMiniMaxH3Scheduler) or not isinstance( + audio, FreshNoiseMiniMaxH3Scheduler + ): + raise TypeError("Pipeline fresh-noise schedulers disappeared during rollout.") + audit.validate_complete() + for name, scheduler in (("video", video), ("audio", audio)): + observed = ( + scheduler.sigmas.numel() if scheduler.sigmas is not None else None, + scheduler.step_call_count, + scheduler.fresh_noise_draw_count, + scheduler.terminal_clean_output_count, + ) + expected = ( + NUM_SIGMA_ENDPOINTS, + NUM_MODEL_EVALUATIONS, + NUM_FRESH_NOISE_TRANSITIONS, + 1, + ) + if observed != expected: + raise RuntimeError( + f"{name} fresh-noise rollout contract changed: " + f"observed={observed}, expected={expected}." + ) + if video.fresh_noise_generator is not audio.fresh_noise_generator: + raise RuntimeError("Video/audio schedulers did not retain their shared RNG.") + return { + "scheduler_class": video.__class__.__name__, + "model_evaluations": NUM_MODEL_EVALUATIONS, + "sigma_endpoints": NUM_SIGMA_ENDPOINTS, + "fresh_noise_transitions": NUM_FRESH_NOISE_TRANSITIONS, + "video_noise_draws": video.fresh_noise_draw_count, + "audio_noise_draws": audio.fresh_noise_draw_count, + "draw_order": list(audit.events), + "shared_generator": True, + "terminal_sigma_zero_draws_noise": False, + } + + +__all__ = [ + "FreshNoiseDrawAudit", + "FreshNoiseMiniMaxH3Scheduler", + "NUM_FRESH_NOISE_TRANSITIONS", + "NUM_MODEL_EVALUATIONS", + "NUM_SIGMA_ENDPOINTS", + "bind_pipeline_fresh_noise_generator", + "install_fresh_noise_schedulers", + "validate_completed_pipeline_rollout", +] diff --git a/source_snapshot/minimax_h3/fresh_noise_scheduler_4step.py b/source_snapshot/minimax_h3/fresh_noise_scheduler_4step.py new file mode 100644 index 0000000000000000000000000000000000000000..c8e2dfd189d83735d99910926f232822261577f5 --- /dev/null +++ b/source_snapshot/minimax_h3/fresh_noise_scheduler_4step.py @@ -0,0 +1,332 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Fresh-noise MiniMax-H3 scheduler used by the four-step DMD student. + +The released :class:`diffusers.MiniMaxH3Scheduler` takes deterministic Euler +steps. The DMD student was trained with a different transition: predict +``x0`` at the current H3 timestep, then independently re-noise that prediction +at the next modality-specific sigma. This module changes only ``step`` and +keeps the released schedule construction, timestep convention, conditioning +noise helper, and scheduler configuration intact. + +One T2VA request owns two scheduler instances. They must be bound to the +same ``torch.Generator``. The official H3 denoise block calls the video +scheduler first and the audio scheduler second, producing the replay-critical +draw order ``video, audio`` for each of the three non-terminal transitions. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import torch +from diffusers.schedulers.scheduling_minimax_h3 import ( + MiniMaxH3Scheduler, + MiniMaxH3SchedulerOutput, +) +from diffusers.utils.torch_utils import randn_tensor + + +NUM_MODEL_EVALUATIONS = 4 +NUM_SIGMA_ENDPOINTS = 5 +NUM_FRESH_NOISE_TRANSITIONS = 3 +_STREAMS = ("video", "audio") + + +@dataclass +class FreshNoiseDrawAudit: + """Shared, request-local record of the two schedulers' RNG consumption.""" + + generator: torch.Generator + events: list[str] = field(default_factory=list) + + def record(self, stream: str) -> None: + if stream not in _STREAMS: + raise ValueError(f"Unknown H3 fresh-noise stream: {stream!r}.") + self.events.append(stream) + + @property + def expected_events(self) -> list[str]: + return list(_STREAMS) * NUM_FRESH_NOISE_TRANSITIONS + + def validate_complete(self) -> None: + if self.events != self.expected_events: + raise RuntimeError( + "H3 fresh-noise draw order/count changed: " + f"observed={self.events}, expected={self.expected_events}." + ) + + +class FreshNoiseMiniMaxH3Scheduler(MiniMaxH3Scheduler): + """Four-forward predict-x0/fresh-re-noise scheduler for one modality.""" + + def __init__(self, shift: float = 12.0): + super().__init__(shift=shift) + self._fresh_noise_generator: torch.Generator | None = None + self._fresh_noise_audit: FreshNoiseDrawAudit | None = None + self._fresh_noise_stream: str | None = None + self.step_call_count = 0 + self.fresh_noise_draw_count = 0 + self.terminal_clean_output_count = 0 + + @property + def fresh_noise_generator(self) -> torch.Generator | None: + return self._fresh_noise_generator + + @property + def fresh_noise_stream(self) -> str | None: + return self._fresh_noise_stream + + def bind_generator( + self, + generator: torch.Generator, + *, + stream: str, + audit: FreshNoiseDrawAudit, + ) -> None: + """Bind one request RNG and its shared video/audio draw audit.""" + + if not isinstance(generator, torch.Generator): + raise TypeError("Fresh-noise inference requires one torch.Generator.") + if stream not in _STREAMS: + raise ValueError(f"Fresh-noise stream must be one of {_STREAMS}, got {stream!r}.") + if not isinstance(audit, FreshNoiseDrawAudit): + raise TypeError("Fresh-noise scheduler audit has the wrong type.") + if audit.generator is not generator: + raise ValueError("Both H3 schedulers must share the audit's exact generator object.") + self._fresh_noise_generator = generator + self._fresh_noise_audit = audit + self._fresh_noise_stream = stream + + def set_timesteps( + self, + num_inference_steps: int | None = None, + device: str | torch.device | None = None, + sigmas: list[float] | torch.Tensor | None = None, + ) -> None: + """Build and strictly validate the four-forward/five-endpoint grid.""" + + super().set_timesteps( + num_inference_steps=num_inference_steps, + device=device, + sigmas=sigmas, + ) + if self.sigmas is None or self.timesteps is None: + raise AssertionError("MiniMax-H3 scheduler did not construct its grid.") + if self.sigmas.numel() != NUM_SIGMA_ENDPOINTS: + raise ValueError( + "Four-step fresh-noise H3 requires exactly five sigma endpoints " + f"including terminal zero, found {self.sigmas.numel()}." + ) + if self.timesteps.numel() != NUM_MODEL_EVALUATIONS: + raise AssertionError("Five H3 endpoints must expose exactly four timesteps.") + if float(self.sigmas[-1]) != 0.0: + raise AssertionError("The fifth H3 sigma endpoint must be terminal zero.") + self.step_call_count = 0 + self.fresh_noise_draw_count = 0 + self.terminal_clean_output_count = 0 + + def step( + self, + model_output: torch.FloatTensor, + timestep: float | torch.FloatTensor, + sample: torch.FloatTensor, + return_dict: bool = True, + ) -> MiniMaxH3SchedulerOutput | tuple: + r"""Predict ``x0`` and fresh re-noise it at the next schedule point. + + For current ``t = 1 - sigma`` and H3's data-ward velocity ``v``:: + + x0 = x_t + (1 - t) * v + x_next = t_next * x0 + (1 - t_next) * fresh_noise + + The terminal transition has ``sigma_next = 0`` and therefore returns + ``x0`` without drawing noise. + """ + + if isinstance(timestep, int) or ( + isinstance(timestep, torch.Tensor) and not timestep.is_floating_point() + ): + raise ValueError( + "Passing integer indices to FreshNoiseMiniMaxH3Scheduler.step() " + "is unsupported; pass a value from scheduler.timesteps." + ) + if sample.shape != model_output.shape: + raise ValueError( + "MiniMax-H3 fresh-noise sample/model_output shapes differ: " + f"{tuple(sample.shape)} != {tuple(model_output.shape)}." + ) + if self.sigmas is None or self.timesteps is None: + raise RuntimeError("Call set_timesteps() before fresh-noise step().") + if self._step_index is None: + self._step_index = ( + self.index_for_timestep(timestep) + if self._begin_index is None + else self._begin_index + ) + if not 0 <= self._step_index < NUM_MODEL_EVALUATIONS: + raise RuntimeError( + f"Fresh-noise H3 step index {self._step_index} lies outside 0..3." + ) + + if not isinstance(timestep, torch.Tensor): + timestep = torch.tensor(timestep, dtype=sample.dtype) + timestep = timestep.to(device=sample.device, dtype=sample.dtype) + sigma_from_timestep = 1.0 - timestep + while sigma_from_timestep.ndim < sample.ndim: + sigma_from_timestep = sigma_from_timestep.unsqueeze(-1) + clean = sample + sigma_from_timestep * model_output + + next_index = self._step_index + 1 + if next_index == NUM_MODEL_EVALUATIONS: + # The terminal sigma is exactly zero. Drawing and multiplying by + # zero would still advance the shared RNG and break seed replay. + prev_sample = clean + self.terminal_clean_output_count += 1 + else: + generator = self._fresh_noise_generator + audit = self._fresh_noise_audit + stream = self._fresh_noise_stream + if generator is None or audit is None or stream is None: + raise RuntimeError( + "Bind the request's shared generator to both fresh-noise " + "H3 schedulers before pipeline inference." + ) + if audit.generator is not generator: + raise RuntimeError("Fresh-noise scheduler/audit generator identity changed.") + # randn_tensor preserves Diffusers' CPU-generator/CUDA-sample + # behavior, which torch.randn(device='cuda', generator=cpu_rng) + # does not support. + noise = randn_tensor( + clean.shape, + generator=generator, + device=clean.device, + dtype=clean.dtype, + ) + next_timestep = self.timesteps[next_index].to( + device=clean.device, dtype=clean.dtype + ) + while next_timestep.ndim < clean.ndim: + next_timestep = next_timestep.unsqueeze(-1) + prev_sample = next_timestep * clean + (1.0 - next_timestep) * noise + audit.record(stream) + self.fresh_noise_draw_count += 1 + + self._step_index += 1 + self.step_call_count += 1 + prev_sample = prev_sample.to(dtype=sample.dtype) + if not return_dict: + return (prev_sample,) + return MiniMaxH3SchedulerOutput(prev_sample=prev_sample) + + +def install_fresh_noise_schedulers(pipe: object) -> None: + """Replace an already-validated official H3 scheduler pair in-place.""" + + video = getattr(pipe, "scheduler", None) + audio = getattr(pipe, "audio_scheduler", None) + if video is None or audio is None: + raise TypeError("MiniMax-H3 pipeline is missing its video/audio schedulers.") + if video.__class__.__name__ != "MiniMaxH3Scheduler" or audio.__class__.__name__ != "MiniMaxH3Scheduler": + raise TypeError( + "Fresh-noise installation requires the untouched official " + "MiniMaxH3Scheduler pair." + ) + + def shift(component: object) -> float: + config = getattr(component, "config", None) + value = getattr(config, "shift", None) + if value is None and isinstance(config, dict): + value = config.get("shift") + return float(value) + + video_fresh = FreshNoiseMiniMaxH3Scheduler(shift=shift(video)) + audio_fresh = FreshNoiseMiniMaxH3Scheduler(shift=shift(audio)) + if (video_fresh.shift, audio_fresh.shift) != (12.0, 3.0): + raise RuntimeError( + "Unexpected H3 fresh-noise shifts: " + f"video={video_fresh.shift}, audio={audio_fresh.shift}." + ) + register = getattr(pipe, "register_components", None) + if callable(register): + register(scheduler=video_fresh, audio_scheduler=audio_fresh) + else: + setattr(pipe, "scheduler", video_fresh) + setattr(pipe, "audio_scheduler", audio_fresh) + + +def bind_pipeline_fresh_noise_generator( + pipe: object, generator: torch.Generator +) -> FreshNoiseDrawAudit: + """Bind the same request RNG to video then audio scheduler instances.""" + + video = getattr(pipe, "scheduler", None) + audio = getattr(pipe, "audio_scheduler", None) + if not isinstance(video, FreshNoiseMiniMaxH3Scheduler) or not isinstance( + audio, FreshNoiseMiniMaxH3Scheduler + ): + raise TypeError("Pipeline does not hold the fresh-noise H3 scheduler pair.") + audit = FreshNoiseDrawAudit(generator=generator) + video.bind_generator(generator, stream="video", audit=audit) + audio.bind_generator(generator, stream="audio", audit=audit) + if video.fresh_noise_generator is not audio.fresh_noise_generator: + raise AssertionError("Video/audio H3 schedulers do not share one generator.") + return audit + + +def validate_completed_pipeline_rollout( + pipe: object, audit: FreshNoiseDrawAudit +) -> dict[str, object]: + """Fail closed on the 4/5/3 sampler and seeded draw-order contract.""" + + video = getattr(pipe, "scheduler", None) + audio = getattr(pipe, "audio_scheduler", None) + if not isinstance(video, FreshNoiseMiniMaxH3Scheduler) or not isinstance( + audio, FreshNoiseMiniMaxH3Scheduler + ): + raise TypeError("Pipeline fresh-noise schedulers disappeared during rollout.") + audit.validate_complete() + for name, scheduler in (("video", video), ("audio", audio)): + observed = ( + scheduler.sigmas.numel() if scheduler.sigmas is not None else None, + scheduler.step_call_count, + scheduler.fresh_noise_draw_count, + scheduler.terminal_clean_output_count, + ) + expected = ( + NUM_SIGMA_ENDPOINTS, + NUM_MODEL_EVALUATIONS, + NUM_FRESH_NOISE_TRANSITIONS, + 1, + ) + if observed != expected: + raise RuntimeError( + f"{name} fresh-noise rollout contract changed: " + f"observed={observed}, expected={expected}." + ) + if video.fresh_noise_generator is not audio.fresh_noise_generator: + raise RuntimeError("Video/audio schedulers did not retain their shared RNG.") + return { + "scheduler_class": video.__class__.__name__, + "model_evaluations": NUM_MODEL_EVALUATIONS, + "sigma_endpoints": NUM_SIGMA_ENDPOINTS, + "fresh_noise_transitions": NUM_FRESH_NOISE_TRANSITIONS, + "video_noise_draws": video.fresh_noise_draw_count, + "audio_noise_draws": audio.fresh_noise_draw_count, + "draw_order": list(audit.events), + "shared_generator": True, + "terminal_sigma_zero_draws_noise": False, + } + + +__all__ = [ + "FreshNoiseDrawAudit", + "FreshNoiseMiniMaxH3Scheduler", + "NUM_FRESH_NOISE_TRANSITIONS", + "NUM_MODEL_EVALUATIONS", + "NUM_SIGMA_ENDPOINTS", + "bind_pipeline_fresh_noise_generator", + "install_fresh_noise_schedulers", + "validate_completed_pipeline_rollout", +] diff --git a/source_snapshot/minimax_h3/full_h3_lora.py b/source_snapshot/minimax_h3/full_h3_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..491a48230a9a2709bcafd7e1d2acfe2288c03e14 --- /dev/null +++ b/source_snapshot/minimax_h3/full_h3_lora.py @@ -0,0 +1,184 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Technical-document LoRA policy for the released MiniMax-H3 backbone. + +The DMD recipe adapts the six attention/FFN projections in all 50 main +transformer blocks and both token-refiner blocks. Full PEFT target names are +used deliberately so no similarly named top-level projection can be selected +by suffix matching. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import torch + + +H3_DMD_LORA_POLICY = "h3_full_50_main_plus_2_token_refiner_attention_ffn_v1" +H3_DMD_MAIN_BLOCK_COUNT = 50 +H3_DMD_REFINER_BLOCK_COUNT = 2 +H3_DMD_TARGETS_PER_BLOCK = 6 +H3_DMD_TARGET_MODULE_COUNT = ( + (H3_DMD_MAIN_BLOCK_COUNT + H3_DMD_REFINER_BLOCK_COUNT) + * H3_DMD_TARGETS_PER_BLOCK +) +H3_DMD_RELATIVE_TARGETS = ( + "attn.to_q", + "attn.to_k", + "attn.to_v", + "attn.to_out.0", + "ff.net.0.proj", + "ff.net.2", +) + + +def released_full_h3_target_modules() -> list[str]: + """Return the exact 312 PEFT targets required by the DMD document.""" + + main = [ + f"transformer_blocks.{index}.{relative}" + for index in range(H3_DMD_MAIN_BLOCK_COUNT) + for relative in H3_DMD_RELATIVE_TARGETS + ] + refiner = [ + f"token_refiner.refiner_blocks.{index}.{relative}" + for index in range(H3_DMD_REFINER_BLOCK_COUNT) + for relative in H3_DMD_RELATIVE_TARGETS + ] + return sorted([*main, *refiner]) + + +def collect_full_h3_target_modules(model: torch.nn.Module) -> list[str]: + """Validate and collect the released model's 50+2 block projections.""" + + try: + from diffusers.models.transformers.transformer_minimax_h3 import ( + MiniMaxH3TokenRefinerBlock, + MiniMaxH3TransformerBlock, + ) + except ImportError as error: + raise RuntimeError( + "The pinned MiniMax-H3 Diffusers checkout is required for LoRA selection." + ) from error + + expected_relative = set(H3_DMD_RELATIVE_TARGETS) + blocks = [ + (name, module) + for name, module in model.named_modules() + if name + and isinstance( + module, (MiniMaxH3TransformerBlock, MiniMaxH3TokenRefinerBlock) + ) + ] + main_count = sum( + isinstance(module, MiniMaxH3TransformerBlock) for _, module in blocks + ) + refiner_count = sum( + isinstance(module, MiniMaxH3TokenRefinerBlock) for _, module in blocks + ) + if (main_count, refiner_count) != ( + H3_DMD_MAIN_BLOCK_COUNT, + H3_DMD_REFINER_BLOCK_COUNT, + ): + raise RuntimeError( + "MiniMax-H3 block topology changed: " + f"observed main/refiner={main_count}/{refiner_count}, expected " + f"{H3_DMD_MAIN_BLOCK_COUNT}/{H3_DMD_REFINER_BLOCK_COUNT}." + ) + + targets: list[str] = [] + for block_name, block in blocks: + observed_relative: set[str] = set() + for branch_name in ("attn", "ff"): + branch = getattr(block, branch_name, None) + if branch is None: + raise RuntimeError(f"H3 block {block_name!r} lacks {branch_name!r}.") + for relative_name, module in branch.named_modules(): + if isinstance(module, torch.nn.Linear): + relative = ( + branch_name + if not relative_name + else f"{branch_name}.{relative_name}" + ) + observed_relative.add(relative) + targets.append(f"{block_name}.{relative}") + if observed_relative != expected_relative: + raise RuntimeError( + f"H3 block {block_name!r} Linear layout changed: " + f"observed={sorted(observed_relative)}, " + f"expected={sorted(expected_relative)}." + ) + + targets.sort() + expected = released_full_h3_target_modules() + if targets != expected: + raise RuntimeError( + "Collected H3 LoRA targets differ from the exact released 312-target set." + ) + return targets + + +def install_full_h3_lora_adapters( + model: torch.nn.Module, + *, + rank: int = 128, + alpha: int = 128, + dropout: float = 0.0, + master_dtype: torch.dtype = torch.float32, +) -> dict[str, list[torch.nn.Parameter]]: + """Install generator/fake-score adapters and keep their masters in FP32.""" + + from .train_utils import install_independent_lora_adapters + + targets = collect_full_h3_target_modules(model) + adapters = install_independent_lora_adapters( + model, + rank=rank, + alpha=alpha, + dropout=dropout, + target_modules=targets, + ) + if master_dtype != torch.float32: + raise ValueError("The technical-document LoRA master dtype must be float32.") + for parameters in adapters.values(): + for parameter in parameters: + parameter.data = parameter.data.to(dtype=master_dtype) + observed_dtypes = { + parameter.dtype + for parameters in adapters.values() + for parameter in parameters + } + if observed_dtypes != {torch.float32}: + raise RuntimeError( + f"H3 LoRA master dtype conversion failed: {sorted(map(str, observed_dtypes))}." + ) + model._h3_lora_policy = H3_DMD_LORA_POLICY + model._h3_lora_target_modules = tuple(targets) + model._h3_lora_master_dtype = "float32" + return adapters + + +def validate_full_h3_checkpoint_targets(targets: Sequence[str]) -> None: + """Reject checkpoints that do not name the exact 312-target policy.""" + + expected = released_full_h3_target_modules() + if list(targets) != expected: + raise ValueError( + "Checkpoint H3 LoRA targets do not match the released 312-module " + f"contract: observed={len(targets)}, expected={len(expected)}." + ) + + +__all__ = [ + "H3_DMD_LORA_POLICY", + "H3_DMD_MAIN_BLOCK_COUNT", + "H3_DMD_REFINER_BLOCK_COUNT", + "H3_DMD_RELATIVE_TARGETS", + "H3_DMD_TARGET_MODULE_COUNT", + "collect_full_h3_target_modules", + "install_full_h3_lora_adapters", + "released_full_h3_target_modules", + "validate_full_h3_checkpoint_targets", +] diff --git a/source_snapshot/minimax_h3/infer_base.py b/source_snapshot/minimax_h3/infer_base.py new file mode 100644 index 0000000000000000000000000000000000000000..bb24250a272eed544d1cb6c06e2a2a5f11bf597c --- /dev/null +++ b/source_snapshot/minimax_h3/infer_base.py @@ -0,0 +1,762 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Auditable MiniMax-H3 Base T2VA inference through the official Diffusers blocks. + +This module is deliberately a thin runner. It does not copy a model, scheduler, prompt +encoder, latent layout, or denoising loop into LongLive. The production path constructs +``diffusers.ModularPipeline`` with the official ``t2va`` workflow and calls it once per +prompt. Small dependency-injection seams exist only so artifact handling can be tested +without downloading the checkpoint. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import subprocess +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence + +from .model_provenance import validate_local_model_provenance + + +MODEL_ID = "MiniMaxAI/MiniMax-H3" +WORKFLOW = "t2va" +FPS = 24 +EXPECTED_PROMPT_COUNT = 4 +_SAFE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$") +_COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$", re.IGNORECASE) +_PIPELINE_OUTPUTS = ( + "videos", + "audio", + "sampling_rate", + "latents", + "audio_latents", + # Persist the exact official conditioning and packed layout used by the + # successful baseline. The DMD smoke run can then unload the 32B Qwen + # conditioner and still train against byte-audited, prompt-specific inputs. + "prompt_embeds", + "text_token_tags", + "token_tags", + "position_ids", + "video_indices", + "audio_indices", + "text_indices", + "num_condition_video_rows", + "num_condition_audio_rows", + "num_latent_frames", + "latent_height", + "latent_width", + "num_audio_latents", +) +_FORBIDDEN_CFG_INPUTS = ("guidance_scale", "negative_prompt", "guidance_rescale", "guider") + + +@dataclass(frozen=True) +class PromptSpec: + """One independently seeded request from the four-prompt input file.""" + + id: str + prompt: str + seed: int + + +@dataclass(frozen=True) +class RunConfig: + """All numeric and provenance knobs required to reproduce a Base T2VA run.""" + + model_id: str = MODEL_ID + requested_revision: str = "main" + resolved_revision: str | None = None + height: int = 768 + width: int = 1344 + num_frames: int = 124 + num_inference_steps: int = 30 + dtype: str = "bfloat16" + device: str = "cuda" + memory_reserve_margin: str = "12GB" + local_files_only: bool = False + + def validate(self) -> None: + if not self.model_id: + raise ValueError("`model_id` cannot be empty.") + if not isinstance(self.requested_revision, str) or not self.requested_revision: + raise ValueError("`requested_revision` must be a non-empty Hub revision.") + if self.resolved_revision is not None and not isinstance(self.resolved_revision, str): + raise ValueError("`resolved_revision` must be a string or None.") + if self.dtype != "bfloat16": + raise ValueError("Parity runs intentionally support only the released bfloat16 precision.") + if self.height <= 0 or self.width <= 0 or self.height % 32 or self.width % 32: + raise ValueError("MiniMax-H3 height and width must be positive multiples of 32.") + if not 1 / 4 <= self.width / self.height <= 4: + raise ValueError("MiniMax-H3 supports aspect ratios from 1:4 through 4:1.") + if self.num_frames < 1 or self.num_frames % 17 != 5: + raise ValueError("`num_frames` must already be aligned as `17 * n + 5`.") + duration = self.num_frames / FPS + if not 5 <= duration <= 15: + raise ValueError("The aligned MiniMax-H3 duration must be between 5 and 15 seconds.") + if self.num_inference_steps < 2: + raise ValueError("At least two sigma grid points are required (the terminal zero is included).") + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(temporary, path) + + +def sha256_file(path: str | Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def tensor_sha256(tensor: Any) -> str: + """Hash tensor values plus dtype/shape, independent of ``torch.save`` bytes.""" + + import torch + + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"Expected a torch.Tensor, got {type(tensor).__name__}.") + value = tensor.detach().to("cpu").contiguous() + header = json.dumps( + {"dtype": str(value.dtype), "shape": list(value.shape)}, sort_keys=True, separators=(",", ":") + ).encode("utf-8") + raw = memoryview(value.view(torch.uint8).numpy().reshape(-1)) + digest = hashlib.sha256() + digest.update(header) + digest.update(b"\0") + digest.update(raw) + return digest.hexdigest() + + +def _tensor_descriptor(tensor: Any) -> dict[str, Any]: + return { + "shape": list(tensor.shape), + "dtype": str(tensor.dtype), + "sha256": tensor_sha256(tensor), + } + + +def _jsonable(value: Any) -> Any: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, Mapping): + return {str(key): _jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_jsonable(item) for item in value] + if hasattr(value, "to_dict"): + return _jsonable(value.to_dict()) + return str(value) + + +def _component_descriptor(component: Any) -> dict[str, Any]: + description = { + "class": component.__class__.__name__, + "module": component.__class__.__module__, + } + load_id = getattr(component, "_diffusers_load_id", None) + if load_id is not None: + description["diffusers_load_id"] = load_id + config = getattr(component, "config", None) + if config is not None: + description["config"] = _jsonable(config) + return description + + +def _find_git_head(module_file: str | None) -> str | None: + if not module_file: + return None + candidate = Path(module_file).resolve() + for parent in (candidate.parent, *candidate.parents): + if (parent / ".git").exists(): + result = subprocess.run( + ["git", "-C", str(parent), "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + ) + if result.returncode == 0: + return result.stdout.strip() + return None + + +def load_prompt_specs(path: str | Path, expected_count: int = EXPECTED_PROMPT_COUNT) -> list[PromptSpec]: + """Read the documented JSON schema and reject ambiguous or unsafe samples.""" + + source = Path(path) + payload = json.loads(source.read_text(encoding="utf-8")) + if isinstance(payload, Mapping): + unknown = set(payload) - {"schema_version", "prompts"} + if unknown: + raise ValueError(f"Unknown top-level prompt JSON fields: {sorted(unknown)}") + if payload.get("schema_version", 1) != 1: + raise ValueError("Only prompt schema_version=1 is supported.") + rows = payload.get("prompts") + else: + rows = payload + if not isinstance(rows, list): + raise ValueError("Prompt JSON must be a list or an object containing a `prompts` list.") + if len(rows) != expected_count: + raise ValueError(f"Expected exactly {expected_count} prompts, found {len(rows)}.") + + prompts: list[PromptSpec] = [] + seen_ids: set[str] = set() + for index, row in enumerate(rows): + if not isinstance(row, Mapping): + raise ValueError(f"prompts[{index}] must be an object.") + unknown = set(row) - {"id", "prompt", "seed"} + missing = {"id", "prompt", "seed"} - set(row) + if unknown or missing: + raise ValueError( + f"prompts[{index}] fields must be exactly id/prompt/seed; missing={sorted(missing)}, " + f"unknown={sorted(unknown)}." + ) + sample_id = row["id"] + prompt = row["prompt"] + seed = row["seed"] + if not isinstance(sample_id, str) or not _SAFE_ID.fullmatch(sample_id): + raise ValueError(f"prompts[{index}].id is not a safe artifact id: {sample_id!r}") + if sample_id in seen_ids: + raise ValueError(f"Duplicate prompt id: {sample_id!r}") + if not isinstance(prompt, str) or not prompt.strip(): + raise ValueError(f"prompts[{index}].prompt must be a non-empty string.") + if isinstance(seed, bool) or not isinstance(seed, int) or not 0 <= seed < 2**63: + raise ValueError(f"prompts[{index}].seed must be an integer in [0, 2**63).") + seen_ids.add(sample_id) + prompts.append(PromptSpec(id=sample_id, prompt=prompt, seed=seed)) + return prompts + + +def resolve_hub_revision(model_id: str, requested_revision: str, local_files_only: bool = False) -> str | None: + """Resolve a branch/tag to the immutable Hub commit used by every component.""" + + if Path(model_id).exists(): + return requested_revision or None + if _COMMIT_SHA.fullmatch(requested_revision): + return requested_revision.lower() + if local_files_only: + raise ValueError("Offline parity runs require `--revision` to be a 40-character commit SHA.") + try: + from huggingface_hub import HfApi + except ImportError as error: + raise RuntimeError("Install `huggingface_hub` to resolve the model revision before loading.") from error + info = HfApi().model_info(model_id, revision=requested_revision) + if not info.sha or not _COMMIT_SHA.fullmatch(info.sha): + raise RuntimeError(f"Could not resolve {model_id}@{requested_revision} to an immutable commit SHA.") + return info.sha.lower() + + +def build_official_pipeline( + config: RunConfig, + *, + transformer_setup: Callable[[Any], Any] | None = None, +) -> Any: + """Load only the official Diffusers MiniMax-H3 Base T2VA workflow. + + ``transformer_setup`` is an optional pre-offload hook used by student + inference to install a validated LoRA before ComponentsManager attaches + its automatic CPU-offload hooks. Base parity calls leave it unset. + """ + + config.validate() + try: + import torch + import diffusers + from diffusers import ComponentsManager, ModularPipeline + except ImportError as error: + raise RuntimeError( + "MiniMax-H3 requires the shared `diffusers_h3` checkout. Install it editable or put its `src/` on " + "PYTHONPATH; a released diffusers 0.31 environment is not sufficient." + ) from error + + required_exports = ( + "MiniMaxH3ModularPipeline", + "MiniMaxH3Transformer3DModel", + "MiniMaxH3Scheduler", + ) + missing = [name for name in required_exports if not hasattr(diffusers, name)] + if missing: + raise RuntimeError(f"The installed Diffusers build lacks MiniMax-H3 support: {missing}") + if config.device.startswith("cuda") and not torch.cuda.is_available(): + raise RuntimeError(f"Requested {config.device}, but torch.cuda.is_available() is false.") + + revision = config.resolved_revision or config.requested_revision + local_model_root = Path(config.model_id) + if not local_model_root.exists() and not _COMMIT_SHA.fullmatch(revision): + raise ValueError( + "Remote pipeline construction requires an immutable commit SHA. Call `resolve_hub_revision` first and " + "store it in RunConfig.resolved_revision." + ) + if local_model_root.exists(): + validate_local_model_provenance(config.model_id, revision) + manager = ComponentsManager() + pipe = ModularPipeline.from_pretrained( + config.model_id, + workflow=WORKFLOW, + components_manager=manager, + revision=revision, + local_files_only=config.local_files_only, + ) + # Passing the resolved SHA here is essential: component specs in a modular index may otherwise carry revision=None. + pipe.load_components( + dtype=torch.bfloat16, + # The modular index records the public Hub ID. Override it with the + # verified local root for an actually offline run; otherwise the + # component loaders would silently reach back to the Hub. + pretrained_model_name_or_path=config.model_id, + revision=revision, + local_files_only=config.local_files_only, + ) + if transformer_setup is not None: + transformer_setup(pipe.transformer) + manager.enable_auto_cpu_offload( + device=config.device, + memory_reserve_margin=config.memory_reserve_margin, + ) + assert_official_t2va_contract(pipe, expected_revision=revision) + return pipe + + +def assert_official_t2va_contract(pipe: Any, expected_revision: str | None = None) -> None: + """Fail closed if a stale or accidentally customized pipeline is supplied.""" + + if pipe.__class__.__name__ != "MiniMaxH3ModularPipeline": + raise RuntimeError(f"Expected MiniMaxH3ModularPipeline, found {pipe.__class__.__name__}.") + expected_classes = { + "transformer": ("MiniMaxH3Transformer3DModel",), + "scheduler": ("MiniMaxH3Scheduler",), + "audio_scheduler": ("MiniMaxH3Scheduler",), + "vae": ("AutoencoderKLMiniMaxH3",), + "audio_vae": ("AutoencoderKLMiniMaxH3Audio",), + "text_encoder": ("Qwen3VLForConditionalGeneration",), + # The pinned modular index declares Qwen2TokenizerFast, while its pinned + # tokenizer_config.json explicitly declares Qwen2Tokenizer. Current + # Transformers therefore resolves the official payload to the slow + # implementation. Both consume the exact same checked tokenizer files. + "tokenizer": ("Qwen2Tokenizer", "Qwen2TokenizerFast"), + "processor": ("Qwen3VLProcessor",), + "video_processor": ("VideoProcessor",), + } + for name, accepted_classes in expected_classes.items(): + component = getattr(pipe, name, None) + actual_class = component.__class__.__name__ if component is not None else None + if actual_class not in accepted_classes: + accepted = "/".join(accepted_classes) + raise RuntimeError(f"Expected official {name}={accepted}, found {actual_class}.") + + def config_value(component: Any, name: str) -> Any: + component_config = component.config + if hasattr(component_config, name): + return getattr(component_config, name) + if isinstance(component_config, Mapping): + return component_config.get(name) + return None + + scheduler_shift = float(config_value(pipe.scheduler, "shift")) + audio_shift = float(config_value(pipe.audio_scheduler, "shift")) + if scheduler_shift != 12.0 or audio_shift != 3.0: + raise RuntimeError( + f"Unexpected MiniMax-H3 scheduler shifts: video={scheduler_shift}, audio={audio_shift}; expected 12/3." + ) + + inputs = set(getattr(pipe, "default_call_parameters", {})) + forbidden = inputs.intersection(_FORBIDDEN_CFG_INPUTS) + if forbidden: + raise RuntimeError(f"The selected pipeline unexpectedly exposes CFG inputs: {sorted(forbidden)}") + if getattr(pipe, "transformer_ref", None) is not None: + raise RuntimeError("T2VA must not load the Ref2VA transformer partition.") + + if expected_revision is not None and _COMMIT_SHA.fullmatch(expected_revision): + for name in ( + "transformer", + "text_encoder", + "tokenizer", + "processor", + "vae", + "audio_vae", + "scheduler", + "audio_scheduler", + ): + load_id = getattr(getattr(pipe, name), "_diffusers_load_id", None) + loaded_revision = load_id.rsplit("|", 1)[-1] if isinstance(load_id, str) else None + if loaded_revision != expected_revision: + raise RuntimeError( + f"{name} was not loaded from the pinned revision {expected_revision}: load_id={load_id!r}." + ) + + +def _runtime_provenance(pipe: Any) -> dict[str, Any]: + import torch + + try: + import diffusers + + diffusers_version = getattr(diffusers, "__version__", None) + diffusers_file = getattr(diffusers, "__file__", None) + except ImportError: + diffusers_version = None + diffusers_file = None + + cuda_device = None + if torch.cuda.is_available(): + try: + cuda_device = torch.cuda.get_device_name(torch.cuda.current_device()) + except Exception: + cuda_device = None + return { + "torch_version": torch.__version__, + "torch_cuda_version": torch.version.cuda, + "cudnn_version": torch.backends.cudnn.version(), + "cuda_device": cuda_device, + "diffusers_version": diffusers_version, + "diffusers_module": diffusers_file, + "diffusers_git_commit": _find_git_head(diffusers_file), + "pipeline_class": pipe.__class__.__name__, + } + + +def _prepare_video_for_encoder(video: Any) -> Any: + """Convert Diffusers ``output_type='pt'`` video to uint8 FHWC for ``encode_video``. + + The detached CPU view is scaled in place after its float fingerprint has been captured by the caller. A full + 768p H3 result is large, so avoiding several simultaneous float32 conversion temporaries matters here. + """ + + import torch + + if not isinstance(video, torch.Tensor): + raise TypeError(f"Expected tensor video output, got {type(video).__name__}.") + frames = video.detach().to("cpu") + if frames.ndim == 5: + if frames.shape[0] != 1: + raise ValueError(f"MiniMax-H3 T2VA must return one video, got shape {tuple(frames.shape)}.") + frames = frames[0] + if frames.ndim != 4 or frames.shape[1] != 3: + raise ValueError(f"Expected FCHW video output, got shape {tuple(frames.shape)}.") + if not torch.is_floating_point(frames): + raise ValueError(f"Expected floating-point decoded video output, got {frames.dtype}.") + frames.clamp_(0, 1).mul_(255).round_() + return frames.permute(0, 2, 3, 1).to(torch.uint8).contiguous() + + +def _save_latents(path: Path, video_latents: Any, audio_latents: Any) -> None: + import torch + + # Tensor-only state dict; numeric fingerprints in metadata are independent of container serialization. + torch.save( + { + "video_latents": video_latents.detach().to("cpu").contiguous(), + "audio_latents": audio_latents.detach().to("cpu").contiguous(), + }, + path, + ) + + +def _save_training_inputs(path: Path, results: Mapping[str, Any]) -> None: + """Save only tensors and integer geometry needed by the H3 DMD runner.""" + + import torch + + tensor_names = ( + "prompt_embeds", + "text_token_tags", + "token_tags", + "position_ids", + "video_indices", + "audio_indices", + "text_indices", + ) + integer_names = ( + "num_condition_video_rows", + "num_condition_audio_rows", + "num_latent_frames", + "latent_height", + "latent_width", + "num_audio_latents", + ) + payload = { + name: results[name].detach().to("cpu").contiguous() for name in tensor_names + } + payload.update({name: int(results[name]) for name in integer_names}) + torch.save(payload, path) + + +def _default_video_encoder(video: Any, fps: int, output_path: str, audio: Any, audio_sample_rate: int) -> None: + from diffusers.utils.export_utils import encode_video + + encode_video( + video=video, + fps=fps, + output_path=output_path, + audio=audio, + audio_sample_rate=audio_sample_rate, + ) + + +def _run_one( + pipe: Any, + prompt: PromptSpec, + config: RunConfig, + sample_dir: Path, + video_encoder: Callable[..., None], + runtime: Mapping[str, Any], +) -> dict[str, Any]: + import torch + + sample_dir.mkdir(parents=True, exist_ok=True) + generator = torch.Generator().manual_seed(prompt.seed) + call_kwargs = { + "prompt": prompt.prompt, + "height": config.height, + "width": config.width, + "num_frames": config.num_frames, + "num_inference_steps": config.num_inference_steps, + "generator": generator, + "output_type": "pt", + "output": list(_PIPELINE_OUTPUTS), + } + if set(call_kwargs).intersection(_FORBIDDEN_CFG_INPUTS): + raise AssertionError("CFG parameters must never enter a MiniMax-H3 request.") + results = pipe(**call_kwargs) + if not isinstance(results, Mapping): + raise TypeError(f"Expected a mapping from ModularPipeline output=..., got {type(results).__name__}.") + missing = [name for name in _PIPELINE_OUTPUTS if results.get(name) is None] + if missing: + raise RuntimeError(f"MiniMax-H3 pipeline did not return required outputs: {missing}") + + video = results["videos"] + audio = results["audio"] + video_latents = results["latents"] + audio_latents = results["audio_latents"] + sampling_rate = int(results["sampling_rate"]) + decoded_video_descriptor = _tensor_descriptor(video) + encoder_frames = _prepare_video_for_encoder(video) + + latent_path = sample_dir / "latents.pt" + training_inputs_path = sample_dir / "training_inputs.pt" + video_path = sample_dir / "output.mp4" + metadata_path = sample_dir / "metadata.json" + _save_latents(latent_path, video_latents, audio_latents) + _save_training_inputs(training_inputs_path, results) + # This LongLive backend is intentionally an identity adapter around the + # official ModularPipeline result. Reload the persisted tensors and prove + # that the hand-off did not introduce a dtype/layout/value conversion. + persisted_latents = torch.load(latent_path, map_location="cpu", weights_only=True) + official_video_cpu = video_latents.detach().to("cpu").contiguous() + official_audio_cpu = audio_latents.detach().to("cpu").contiguous() + if not torch.equal(persisted_latents["video_latents"], official_video_cpu): + raise RuntimeError("LongLive persistence changed the official video latent tensor.") + if not torch.equal(persisted_latents["audio_latents"], official_audio_cpu): + raise RuntimeError("LongLive persistence changed the official audio latent tensor.") + video_encoder( + video=encoder_frames, + fps=FPS, + output_path=str(video_path), + audio=audio[0].detach().to("cpu"), + audio_sample_rate=sampling_rate, + ) + + components = { + name: _component_descriptor(getattr(pipe, name)) + for name in ( + "transformer", + "text_encoder", + "tokenizer", + "processor", + "vae", + "audio_vae", + "scheduler", + "audio_scheduler", + "video_processor", + ) + } + metadata = { + "schema_version": 1, + "created_at": _utc_now(), + "sample": asdict(prompt), + "request": { + "workflow": WORKFLOW, + "height": config.height, + "width": config.width, + "num_frames": config.num_frames, + "fps": FPS, + "num_inference_steps": config.num_inference_steps, + "model_evaluations": config.num_inference_steps - 1, + "dtype": config.dtype, + "output_type": "pt", + "uses_context_ir": False, + "uses_regenerator": False, + "uses_cfg": False, + }, + "model": { + "id": config.model_id, + "requested_revision": config.requested_revision, + "resolved_revision": config.resolved_revision, + }, + "runtime": dict(runtime), + "parity": { + "definition": "official ModularPipeline output -> LongLive identity adapter artifact", + "video_latents_exact": True, + "audio_latents_exact": True, + "video_latents_max_abs": 0.0, + "audio_latents_max_abs": 0.0, + }, + "components": components, + "numeric_outputs": { + "decoded_video": decoded_video_descriptor, + "mux_video_uint8": _tensor_descriptor(encoder_frames), + "decoded_audio": _tensor_descriptor(audio), + "video_latents": _tensor_descriptor(video_latents), + "audio_latents": _tensor_descriptor(audio_latents), + "sampling_rate": sampling_rate, + }, + "artifacts": { + "video": {"path": video_path.name, "sha256": sha256_file(video_path)}, + "latents": { + "path": latent_path.name, + "format": "torch tensor-only state_dict", + "sha256": sha256_file(latent_path), + }, + "training_inputs": { + "path": training_inputs_path.name, + "format": "torch tensor-only conditioning/layout state_dict", + "sha256": sha256_file(training_inputs_path), + }, + }, + } + _write_json(metadata_path, metadata) + return { + "id": prompt.id, + "metadata": str(metadata_path), + "metadata_sha256": sha256_file(metadata_path), + "video_sha256": metadata["artifacts"]["video"]["sha256"], + "latents_sha256": metadata["artifacts"]["latents"]["sha256"], + "training_inputs_sha256": metadata["artifacts"]["training_inputs"]["sha256"], + "video_latents_numeric_sha256": metadata["numeric_outputs"]["video_latents"]["sha256"], + "audio_latents_numeric_sha256": metadata["numeric_outputs"]["audio_latents"]["sha256"], + } + + +def run_batch( + pipe: Any, + prompts: Sequence[PromptSpec], + config: RunConfig, + output_dir: str | Path, + *, + video_encoder: Callable[..., None] | None = None, + validate_pipeline: bool = True, +) -> dict[str, Any]: + """Run four independent requests and write a manifest plus per-sample evidence.""" + + config.validate() + if len(prompts) != EXPECTED_PROMPT_COUNT: + raise ValueError(f"Expected exactly {EXPECTED_PROMPT_COUNT} prompts, got {len(prompts)}.") + if validate_pipeline: + pinned_revision = config.resolved_revision or config.requested_revision + expected_revision = pinned_revision if _COMMIT_SHA.fullmatch(pinned_revision) else None + assert_official_t2va_contract(pipe, expected_revision=expected_revision) + encoder = video_encoder or _default_video_encoder + destination = Path(output_dir).resolve() + destination.mkdir(parents=True, exist_ok=True) + runtime = _runtime_provenance(pipe) + started_at = _utc_now() + samples = [] + for index, prompt in enumerate(prompts): + samples.append( + _run_one( + pipe, + prompt, + config, + destination / f"{index:02d}_{prompt.id}", + encoder, + runtime, + ) + ) + + for sample in samples: + sample["metadata"] = str(Path(sample["metadata"]).relative_to(destination)) + manifest = { + "schema_version": 1, + "started_at": started_at, + "completed_at": _utc_now(), + "runner": "LongLive/minimax_h3/infer_base.py", + "config": asdict(config), + "invariants": { + "official_diffusers_modular_pipeline": True, + "workflow": WORKFLOW, + "uses_context_ir": False, + "uses_regenerator": False, + "uses_cfg": False, + "prompt_count": EXPECTED_PROMPT_COUNT, + "official_to_longlive_identity_adapter": True, + }, + "samples": samples, + } + _write_json(destination / "manifest.json", manifest) + return manifest + + +def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--prompts", type=Path, default=Path(__file__).with_name("prompts.json")) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--model-id", default=MODEL_ID) + parser.add_argument("--revision", default="main", help="Hub commit, tag, or branch; resolved to a SHA first.") + parser.add_argument("--height", type=int, default=768) + parser.add_argument("--width", type=int, default=1344) + parser.add_argument("--num-frames", type=int, default=124) + parser.add_argument("--num-inference-steps", type=int, default=30) + parser.add_argument("--device", default="cuda") + parser.add_argument("--memory-reserve-margin", default="12GB") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument( + "--validate-only", + action="store_true", + help="Validate JSON and numeric constraints without importing Diffusers or loading weights.", + ) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parse_args(argv) + prompts = load_prompt_specs(args.prompts) + config = RunConfig( + model_id=args.model_id, + requested_revision=args.revision, + height=args.height, + width=args.width, + num_frames=args.num_frames, + num_inference_steps=args.num_inference_steps, + device=args.device, + memory_reserve_margin=args.memory_reserve_margin, + local_files_only=args.local_files_only, + ) + config.validate() + if args.validate_only: + print(json.dumps({"config": asdict(config), "prompts": [asdict(item) for item in prompts]}, indent=2)) + return 0 + + revision = resolve_hub_revision(config.model_id, config.requested_revision, config.local_files_only) + config = RunConfig(**{**asdict(config), "resolved_revision": revision}) + pipe = build_official_pipeline(config) + manifest = run_batch(pipe, prompts, config, args.output_dir) + print(json.dumps(manifest, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/source_snapshot/minimax_h3/infer_student.py b/source_snapshot/minimax_h3/infer_student.py new file mode 100644 index 0000000000000000000000000000000000000000..f6aa131a7887a699eb8cd0684a05937c00745abb --- /dev/null +++ b/source_snapshot/minimax_h3/infer_student.py @@ -0,0 +1,531 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Decode MiniMax-H3 student rollouts from a LongLive LoRA-only checkpoint. + +The runner keeps the released H3 topology and stock full-sequence, +``is_causal=False`` attention. It installs only the checkpoint's student LoRA, +then asks the official Base T2VA ModularPipeline for five sigma endpoints, +which execute exactly four denoiser evaluations. There is no CFG, temporal AR, +IR, Regenerator, or attention-processor replacement. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from dataclasses import asdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping, Sequence + +import torch + +from .infer_base import ( + EXPECTED_PROMPT_COUNT, + FPS, + MODEL_ID, + WORKFLOW, + RunConfig, + _default_video_encoder, + _prepare_video_for_encoder, + _tensor_descriptor, + assert_official_t2va_contract, + build_official_pipeline, + load_prompt_specs, + resolve_hub_revision, + sha256_file, +) +from .longlive_lora import ( + LONGLIVE_H3_LORA_POLICY, + LONGLIVE_H3_TARGET_MODULE_COUNT, + validate_longlive_h3_checkpoint_targets, +) +from .train_utils import ( + NUM_MODEL_EVALUATIONS, + NUM_SIGMA_ENDPOINTS, + RELEASED_MAIN_BLOCKS, + RELEASED_REFINER_BLOCKS, + STUDENT_ADAPTER, + assert_released_h3_architecture, +) + + +_REQUIRED_CONTRACT = { + "workflow": WORKFLOW, + "cfg": False, + "rollout_mode": "full_sequence", + "attention_mode": "native_non_causal", + "attention_is_causal": False, + "flex_attention_backend": "native", + "temporal_ar_blocks": 0, + "model_evaluations_per_rollout": NUM_MODEL_EVALUATIONS, + "sigma_endpoints_per_rollout": NUM_SIGMA_ENDPOINTS, + "student_positive_forwards_per_rollout": NUM_MODEL_EVALUATIONS, + "transformer_block_count": RELEASED_MAIN_BLOCKS, + "token_refiner_block_count": RELEASED_REFINER_BLOCKS, + "normalization": "per_modality_full_sequence", +} +_LEGACY_ATTENTION_ONLY_TARGETS = ["to_q", "to_k", "to_v", "to_out.0"] + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _atomic_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + os.replace(temporary, path) + + +def _lora_state_digest(state: Mapping[str, torch.Tensor]) -> str: + """Hash adapter names plus canonical tensor descriptors in key order.""" + + digest = hashlib.sha256() + for name in sorted(state): + descriptor = _tensor_descriptor(state[name]) + digest.update(name.encode("utf-8")) + digest.update(b"\0") + digest.update( + json.dumps( + descriptor, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + ) + digest.update(b"\n") + return digest.hexdigest() + + +def load_student_checkpoint(path: str | Path) -> dict[str, Any]: + """Load and fail-closed validate one revised full-sequence DMD checkpoint.""" + + checkpoint_path = Path(path).resolve() + if not checkpoint_path.is_file(): + raise FileNotFoundError(f"Student checkpoint does not exist: {checkpoint_path}") + payload = torch.load(checkpoint_path, map_location="cpu", weights_only=True) + if not isinstance(payload, Mapping) or payload.get("schema_version") != 1: + raise ValueError("Unsupported or malformed H3 LoRA checkpoint.") + config = payload.get("config") + if not isinstance(config, Mapping): + raise ValueError("H3 LoRA checkpoint is missing its training config.") + mismatches = { + name: {"actual": config.get(name), "expected": expected} + for name, expected in _REQUIRED_CONTRACT.items() + if config.get(name) != expected + } + if mismatches: + raise ValueError( + "Checkpoint is not a revised native full-sequence H3 student: " + + json.dumps(mismatches, sort_keys=True) + ) + if config.get("ir") is not False or config.get("regenerator") is not False: + raise ValueError("Student rollout supports Base T2VA only (no IR/Regenerator).") + if config.get("teacher_adapter_state") != "disabled": + raise ValueError("Checkpoint does not record an adapter-disabled Base teacher.") + if config.get("student_adapter") != STUDENT_ADAPTER: + raise ValueError("Checkpoint student adapter name is not 'student'.") + + rank = config.get("lora_rank") + alpha = config.get("lora_alpha") + targets = config.get("lora_targets") + if isinstance(rank, bool) or not isinstance(rank, int) or rank < 1: + raise ValueError("Checkpoint LoRA rank must be a positive integer.") + if isinstance(alpha, bool) or not isinstance(alpha, int) or alpha < 1: + raise ValueError("Checkpoint LoRA alpha must be a positive integer.") + target_policy = config.get("lora_target_policy") + if target_policy is None: + if targets != _LEGACY_ATTENTION_ONLY_TARGETS: + raise ValueError(f"Unexpected legacy student LoRA targets: {targets!r}.") + elif target_policy == LONGLIVE_H3_LORA_POLICY: + if not isinstance(targets, list) or not all( + isinstance(name, str) and name for name in targets + ): + raise ValueError("LongLive-style student LoRA targets must be strings.") + validate_longlive_h3_checkpoint_targets(targets) + if config.get("lora_target_module_count") != LONGLIVE_H3_TARGET_MODULE_COUNT: + raise ValueError("LongLive-style student LoRA target count is invalid.") + else: + raise ValueError(f"Unsupported student LoRA target policy: {target_policy!r}.") + + state = payload.get("student_lora") + if not isinstance(state, Mapping) or not state: + raise ValueError("Checkpoint has no student_lora tensor state.") + if any(not isinstance(name, str) or not isinstance(value, torch.Tensor) for name, value in state.items()): + raise TypeError("Every student_lora entry must map a string to a tensor.") + if any("lora_" not in name for name in state): + raise ValueError("student_lora unexpectedly contains a non-LoRA tensor.") + lora_b = [value for name, value in state.items() if ".lora_B." in name] + if not lora_b or not any(bool(torch.count_nonzero(value)) for value in lora_b): + raise ValueError("Student checkpoint has no nonzero trained LoRA-B tensor.") + + return { + "path": checkpoint_path, + "sha256": sha256_file(checkpoint_path), + "step": int(payload.get("step", 0)), + "config": dict(config), + "student_lora": dict(state), + } + + +def install_student_lora( + transformer: torch.nn.Module, + checkpoint: Mapping[str, Any], +) -> dict[str, Any]: + """Install one student adapter and verify an exact tensor round trip.""" + + architecture = assert_released_h3_architecture(transformer) + config = checkpoint["config"] + state = checkpoint["student_lora"] + try: + from peft import LoraConfig, get_peft_model_state_dict, set_peft_model_state_dict + from peft.tuners.tuners_utils import cast_adapter_dtype + except ImportError as error: + raise RuntimeError("PEFT is required for H3 student rollout inference.") from error + + transformer.requires_grad_(False) + lora = LoraConfig( + r=int(config["lora_rank"]), + lora_alpha=int(config["lora_alpha"]), + lora_dropout=0.0, + target_modules=list(config["lora_targets"]), + init_lora_weights=True, + bias="none", + ) + transformer.add_adapter(lora, adapter_name=STUDENT_ADAPTER) + # The released pipeline loads the base transformer in bfloat16. PEFT may + # therefore create newly attached LoRA parameters in the base-weight dtype, + # while the DMD checkpoint contract stores FP32 LoRA masters. Promote the + # adapter before loading so set/get is an exact, lossless round trip instead + # of silently quantizing the checkpoint to bfloat16. + adapter_parameters = [ + (name, parameter) + for name, parameter in transformer.named_parameters() + if "lora_" in name and f".{STUDENT_ADAPTER}." in name + ] + if not adapter_parameters: + raise RuntimeError("Student LoRA attachment created no adapter parameters.") + adapter_names = {name for name, _ in adapter_parameters} + non_adapter_dtypes = { + name: parameter.dtype + for name, parameter in transformer.named_parameters() + if name not in adapter_names + } + cast_adapter_dtype( + transformer, + adapter_name=STUDENT_ADAPTER, + autocast_adapter_dtype=True, + ) + promoted_parameters = [ + (name, parameter) + for name, parameter in transformer.named_parameters() + if "lora_" in name and f".{STUDENT_ADAPTER}." in name + ] + if {name for name, _ in promoted_parameters} != adapter_names: + raise RuntimeError("Student LoRA parameter names changed during FP32 promotion.") + if {parameter.dtype for _, parameter in promoted_parameters} != {torch.float32}: + raise RuntimeError("Student LoRA parameters could not be promoted to FP32.") + if { + name: parameter.dtype + for name, parameter in transformer.named_parameters() + if name not in adapter_names + } != non_adapter_dtypes: + raise RuntimeError("Student LoRA FP32 promotion changed a backbone dtype.") + incompatible = set_peft_model_state_dict( + transformer, state, adapter_name=STUDENT_ADAPTER + ) + unexpected = sorted(getattr(incompatible, "unexpected_keys", ())) + missing_lora = sorted( + name + for name in getattr(incompatible, "missing_keys", ()) + if "lora_" in name + ) + if unexpected or missing_lora: + raise RuntimeError( + "Student LoRA PEFT load reported incompatible adapter keys: " + f"missing_lora={missing_lora[:5]}, unexpected={unexpected[:5]}." + ) + transformer.set_adapter(STUDENT_ADAPTER) + transformer.enable_adapters() + transformer.requires_grad_(False) + + reloaded = get_peft_model_state_dict(transformer, adapter_name=STUDENT_ADAPTER) + if set(reloaded) != set(state): + missing = sorted(set(state) - set(reloaded)) + extra = sorted(set(reloaded) - set(state)) + raise RuntimeError( + f"Student LoRA key round trip failed: missing={missing[:5]}, extra={extra[:5]}." + ) + mismatched = [ + name + for name, expected in state.items() + if not torch.equal(reloaded[name].detach().cpu(), expected.detach().cpu()) + ] + if mismatched: + raise RuntimeError(f"Student LoRA value round trip failed: {mismatched[:5]}.") + if tuple(transformer.active_adapters()) != (STUDENT_ADAPTER,): + raise RuntimeError("Student rollout failed to activate only the student adapter.") + + state_digest = _lora_state_digest(state) + reloaded_digest = _lora_state_digest(reloaded) + if reloaded_digest != state_digest: + raise RuntimeError("Student LoRA post-load tensor digest changed.") + + return { + **architecture, + "adapter": STUDENT_ADAPTER, + "lora_rank": int(config["lora_rank"]), + "lora_alpha": int(config["lora_alpha"]), + "lora_target_policy": config.get("lora_target_policy", "legacy_attention_only"), + "lora_target_module_count": len(config["lora_targets"]), + "lora_tensor_count": len(state), + "lora_checkpoint_tensor_digest": state_digest, + "lora_post_load_tensor_digest": reloaded_digest, + "lora_exact_round_trip": True, + "peft_missing_lora_keys": missing_lora, + "peft_unexpected_keys": unexpected, + "active_adapters": list(transformer.active_adapters()), + "checkpoint_step": int(checkpoint["step"]), + "checkpoint_sha256": checkpoint["sha256"], + } + + +def run_student_batch( + pipe: Any, + prompts: Sequence[Any], + config: RunConfig, + checkpoint: Mapping[str, Any], + output_dir: str | Path, + *, + adapter_metadata: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Generate MP4 visualizations for four positive prompts.""" + + if len(prompts) != EXPECTED_PROMPT_COUNT: + raise ValueError(f"Expected exactly four prompts, found {len(prompts)}.") + if config.num_inference_steps != NUM_SIGMA_ENDPOINTS: + raise ValueError("Student rollout must use five endpoints/four denoiser evaluations.") + assert_official_t2va_contract(pipe, expected_revision=config.resolved_revision) + if adapter_metadata is None: + adapter = install_student_lora(pipe.transformer, checkpoint) + else: + adapter = dict(adapter_metadata) + if tuple(pipe.transformer.active_adapters()) != (STUDENT_ADAPTER,): + raise RuntimeError("Pre-offload setup did not leave only student LoRA active.") + if adapter.get("checkpoint_sha256") != checkpoint["sha256"]: + raise RuntimeError("Pre-offload student metadata/checkpoint hash mismatch.") + + destination = Path(output_dir).resolve() + destination.mkdir(parents=True, exist_ok=True) + samples = [] + for index, prompt in enumerate(prompts): + sample_dir = destination / f"{index:02d}_{prompt.id}" + sample_dir.mkdir(parents=True, exist_ok=True) + generator = torch.Generator().manual_seed(prompt.seed) + results = pipe( + prompt=prompt.prompt, + height=config.height, + width=config.width, + num_frames=config.num_frames, + num_inference_steps=NUM_SIGMA_ENDPOINTS, + generator=generator, + output_type="pt", + output=["videos", "audio", "sampling_rate", "latents", "audio_latents"], + ) + if not isinstance(results, Mapping): + raise TypeError("Official H3 pipeline did not return a mapping.") + required = ("videos", "audio", "sampling_rate", "latents", "audio_latents") + missing = [name for name in required if results.get(name) is None] + if missing: + raise RuntimeError(f"Student H3 pipeline output is missing {missing}.") + + video_path = sample_dir / "student.mp4" + latent_path = sample_dir / "student_latents.pt" + frames = _prepare_video_for_encoder(results["videos"]) + sampling_rate = int(results["sampling_rate"]) + _default_video_encoder( + video=frames, + fps=FPS, + output_path=str(video_path), + audio=results["audio"][0].detach().cpu(), + audio_sample_rate=sampling_rate, + ) + torch.save( + { + "video_latents": results["latents"].detach().cpu().contiguous(), + "audio_latents": results["audio_latents"].detach().cpu().contiguous(), + }, + latent_path, + ) + metadata = { + "schema_version": 1, + "created_at": _utc_now(), + "sample": asdict(prompt), + "request": { + "workflow": WORKFLOW, + "height": config.height, + "width": config.width, + "num_frames": config.num_frames, + "fps": FPS, + "rollout_mode": "full_sequence", + "attention_mode": "native_non_causal", + "attention_is_causal": False, + "temporal_ar_blocks": 0, + "model_evaluations": NUM_MODEL_EVALUATIONS, + "sigma_endpoints": NUM_SIGMA_ENDPOINTS, + "uses_cfg": False, + "uses_context_ir": False, + "uses_regenerator": False, + }, + "model": { + "id": config.model_id, + "resolved_revision": config.resolved_revision, + "transformer_block_count": RELEASED_MAIN_BLOCKS, + "token_refiner_block_count": RELEASED_REFINER_BLOCKS, + }, + "student": adapter, + "numeric_outputs": { + "decoded_video": _tensor_descriptor(results["videos"]), + "decoded_audio": _tensor_descriptor(results["audio"]), + "video_latents": _tensor_descriptor(results["latents"]), + "audio_latents": _tensor_descriptor(results["audio_latents"]), + "sampling_rate": sampling_rate, + }, + "artifacts": { + "video": {"path": video_path.name, "sha256": sha256_file(video_path)}, + "latents": {"path": latent_path.name, "sha256": sha256_file(latent_path)}, + }, + } + metadata_path = sample_dir / "metadata.json" + _atomic_json(metadata_path, metadata) + samples.append( + { + "id": prompt.id, + "prompt": prompt.prompt, + "seed": prompt.seed, + "video": str(video_path.relative_to(destination)), + "video_sha256": metadata["artifacts"]["video"]["sha256"], + "metadata": str(metadata_path.relative_to(destination)), + "metadata_sha256": sha256_file(metadata_path), + } + ) + + manifest = { + "schema_version": 1, + "status": "success", + "created_at": _utc_now(), + "runner": "LongLive/minimax_h3/infer_student.py", + "config": asdict(config), + "contract": dict(_REQUIRED_CONTRACT), + "checkpoint": { + "path": str(checkpoint["path"]), + "sha256": checkpoint["sha256"], + "step": checkpoint["step"], + }, + "adapter": adapter, + "samples": samples, + } + manifest_path = destination / "manifest.json" + _atomic_json(manifest_path, manifest) + _atomic_json( + destination / "SUCCESS.json", + { + "schema_version": 1, + "status": "success", + "manifest": manifest_path.name, + "manifest_sha256": sha256_file(manifest_path), + }, + ) + return manifest + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--checkpoint", type=Path, required=True) + parser.add_argument("--prompts", type=Path, default=Path(__file__).with_name("prompts.json")) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--model-id", "--model-dir", dest="model_id", default=MODEL_ID) + parser.add_argument("--revision", default="main") + parser.add_argument("--height", type=int, default=768) + parser.add_argument("--width", type=int, default=1344) + parser.add_argument("--num-frames", type=int, default=124) + parser.add_argument("--device", default="cuda") + parser.add_argument("--memory-reserve-margin", default="12GB") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--validate-only", action="store_true") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + prompts = load_prompt_specs(args.prompts) + checkpoint = load_student_checkpoint(args.checkpoint) + revision = resolve_hub_revision( + args.model_id, args.revision, local_files_only=args.local_files_only + ) + checkpoint_revision = checkpoint["config"].get("resolved_revision") + if checkpoint_revision != revision: + raise ValueError( + "Student checkpoint/model revision mismatch: " + f"{checkpoint_revision!r} != {revision!r}." + ) + config = RunConfig( + model_id=args.model_id, + requested_revision=args.revision, + resolved_revision=revision, + height=args.height, + width=args.width, + num_frames=args.num_frames, + num_inference_steps=NUM_SIGMA_ENDPOINTS, + device=args.device, + memory_reserve_margin=args.memory_reserve_margin, + local_files_only=args.local_files_only, + ) + config.validate() + if args.validate_only: + print( + json.dumps( + { + "checkpoint": { + "path": str(checkpoint["path"]), + "sha256": checkpoint["sha256"], + "step": checkpoint["step"], + }, + "contract": _REQUIRED_CONTRACT, + "config": asdict(config), + "prompts": [asdict(prompt) for prompt in prompts], + }, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + ) + return 0 + adapter_metadata: dict[str, Any] = {} + + def setup_student(transformer: torch.nn.Module) -> None: + adapter_metadata.update(install_student_lora(transformer, checkpoint)) + + pipe = build_official_pipeline(config, transformer_setup=setup_student) + manifest = run_student_batch( + pipe, + prompts, + config, + checkpoint, + args.output_dir, + adapter_metadata=adapter_metadata, + ) + print(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/source_snapshot/minimax_h3/infer_student_4step.py b/source_snapshot/minimax_h3/infer_student_4step.py new file mode 100644 index 0000000000000000000000000000000000000000..f9be1b76fd925813b32eeba4d858974c60f91689 --- /dev/null +++ b/source_snapshot/minimax_h3/infer_student_4step.py @@ -0,0 +1,1063 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Infer the four-step MiniMax-H3 DMD student with fresh re-noising. + +This runner intentionally leaves ``infer_student.py`` and the pinned +Diffusers checkout unchanged. It reuses the audited Base T2VA pipeline, +prompt encoding, LoRA round-trip, VAE decoding, and MP4 muxing paths, while +replacing only the two scheduler instances after the official pipeline has +been validated. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from contextlib import contextmanager +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Iterator, Mapping, Sequence + +import torch + +from .dmd_core import DMD_NORMALIZATION +from .fresh_noise_scheduler_4step import ( + FreshNoiseMiniMaxH3Scheduler, + NUM_FRESH_NOISE_TRANSITIONS, + NUM_MODEL_EVALUATIONS, + NUM_SIGMA_ENDPOINTS, + bind_pipeline_fresh_noise_generator, + install_fresh_noise_schedulers, + validate_completed_pipeline_rollout, +) +from .full_h3_lora import ( + H3_DMD_LORA_POLICY, + H3_DMD_TARGET_MODULE_COUNT, + released_full_h3_target_modules, + validate_full_h3_checkpoint_targets, +) +from .extract_dmd8_inline5 import ( + INLINE_CONDITIONING_MODE, + INLINE_COUNT, + INLINE_PAYLOAD_ARTIFACT_TYPE, + INLINE_PAYLOAD_SCHEMA_VERSION, + INLINE_SELECTION_POLICY, +) +from .infer_base import ( + FPS, + MODEL_ID, + WORKFLOW, + RunConfig, + _default_video_encoder, + _prepare_video_for_encoder, + _tensor_descriptor, + assert_official_t2va_contract, + build_official_pipeline, + load_prompt_specs, + resolve_hub_revision, + sha256_file, +) +from .infer_student import ( + _atomic_json, + install_student_lora, +) +from .train_utils import ( + RELEASED_MAIN_BLOCKS, + RELEASED_REFINER_BLOCKS, + STUDENT_ADAPTER, +) + + +LORA_RANK = 128 +LORA_ALPHA = 128 +ROLLOUT_MODE = "predict_x0_fresh_renoise" +_EXPECTED_DRAW_ORDER = ["video", "audio"] +_REQUIRED_CONTRACT = { + "workflow": WORKFLOW, + "ir": False, + "regenerator": False, + "student_extra_cfg": False, + "extra_guidance_target": "real_score_teacher_only", + "teacher_adapter_state": "disabled", + "student_adapter": STUDENT_ADAPTER, + "model_variant": "standard_full_adaln", + "adaln_curve_rank": None, + "rollout_mode": ROLLOUT_MODE, + "attention_mode": "native_non_causal", + "attention_is_causal": False, + "temporal_ar_blocks": 0, + "model_evaluations_per_rollout": NUM_MODEL_EVALUATIONS, + "sigma_endpoints_per_rollout": NUM_SIGMA_ENDPOINTS, + "fresh_noise_transitions_per_rollout": NUM_FRESH_NOISE_TRANSITIONS, + "student_positive_forwards_per_rollout": NUM_MODEL_EVALUATIONS, + "fresh_noise_draw_order": _EXPECTED_DRAW_ORDER, + "video_audio_transition_noise_independent": True, + "sampler_matches_training_and_inference": True, + "transformer_block_count": RELEASED_MAIN_BLOCKS, + "token_refiner_block_count": RELEASED_REFINER_BLOCKS, + "normalization": DMD_NORMALIZATION, + "lora_rank": LORA_RANK, + "lora_alpha": LORA_ALPHA, + "lora_dropout": 0.0, + "lora_master_dtype": "float32", + "lora_target_policy": H3_DMD_LORA_POLICY, + "lora_target_module_count": H3_DMD_TARGET_MODULE_COUNT, +} +_CACHED_PAYLOAD_FIELDS = { + "schema_version", + "artifact_type", + "selection_policy", + "conditioning_mode", + "model_revision", + "collection_sha256", + "rank_cache_shard_sha256", + "checkpoint_sha256", + "conditions", +} +_CACHED_CONDITION_FIELDS = { + "cache_rank", + "cache_condition_position", + "cache_global_step", + "source_index", + "source_line", + "id", + "prompt", + "prompt_sha256", + "evaluation_seed", + "prompt_embeds", + "text_token_tags", +} +_PROVENANCE_CONDITION_FIELDS = { + "cache_rank", + "cache_condition_position", + "cache_global_step", + "source_index", + "source_line", + "id", + "prompt", + "prompt_sha256", + "evaluation_seed", + "evaluation_seed_policy", + "cached_prompt_embeds", + "cached_text_token_tags", +} + + +def _is_sha256(value: Any) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and all(character in "0123456789abcdef" for character in value) + ) + + +@dataclass(frozen=True) +class CachedConditioningSample: + """One cache row after fail-closed payload/provenance validation.""" + + cache_rank: int + cache_condition_position: int + cache_global_step: int + source_index: int + source_line: int + id: str + prompt: str + prompt_sha256: str + evaluation_seed: int + prompt_embeds: torch.Tensor + text_token_tags: torch.Tensor + prompt_embeds_descriptor: Mapping[str, Any] + text_token_tags_descriptor: Mapping[str, Any] + + +@dataclass(frozen=True) +class CachedConditioningBundle: + """Cryptographic binding of an inline payload to its JSON provenance.""" + + payload_path: Path + payload_sha256: str + payload_bytes: int + provenance_path: Path + provenance_sha256: str + model_revision: str + collection_sha256: str + rank_cache_shard_sha256: str + checkpoint_sha256: str + conditions: tuple[CachedConditioningSample, ...] + + def manifest_summary(self) -> dict[str, Any]: + return { + "mode": INLINE_CONDITIONING_MODE, + "selection_policy": INLINE_SELECTION_POLICY, + "cached_prompt_embeds_used_by_inference": True, + "text_encoder_forward_calls_required": 0, + "payload": { + "path": str(self.payload_path), + "bytes": self.payload_bytes, + "sha256": self.payload_sha256, + }, + "provenance": { + "path": str(self.provenance_path), + "sha256": self.provenance_sha256, + }, + "conditions": len(self.conditions), + } + + +def _expected_student_state_keys() -> set[str]: + keys: set[str] = set() + for target in released_full_h3_target_modules(): + keys.add(f"{target}.lora_A.weight") + keys.add(f"{target}.lora_B.weight") + return keys + + +def load_cached_conditioning( + payload_path: str | Path, + provenance_path: str | Path, + prompts: Sequence[Any], + *, + expected_model_revision: str, + expected_checkpoint_sha256: str, +) -> CachedConditioningBundle: + """Load exact inline cache tensors and validate every JSON/tensor binding.""" + + payload_path = Path(payload_path).resolve() + provenance_path = Path(provenance_path).resolve() + for label, path in ( + ("cached conditioning payload", payload_path), + ("cached conditioning provenance", provenance_path), + ): + if not path.is_file(): + raise FileNotFoundError(f"{label} does not exist: {path}") + payload_sha = sha256_file(payload_path) + provenance_sha = sha256_file(provenance_path) + try: + payload = torch.load( + payload_path, + map_location="cpu", + weights_only=True, + mmap=True, + ) + except Exception as error: + raise ValueError(f"Unable to mmap cached conditioning payload: {payload_path}") from error + try: + provenance = json.loads(provenance_path.read_text(encoding="utf-8")) + except Exception as error: + raise ValueError(f"Unable to parse cached conditioning provenance: {provenance_path}") from error + if not isinstance(payload, Mapping) or set(payload) != _CACHED_PAYLOAD_FIELDS: + raise ValueError("Cached conditioning payload has an unknown top-level schema.") + if ( + payload.get("schema_version") != INLINE_PAYLOAD_SCHEMA_VERSION + or payload.get("artifact_type") != INLINE_PAYLOAD_ARTIFACT_TYPE + or payload.get("selection_policy") != INLINE_SELECTION_POLICY + or payload.get("conditioning_mode") != INLINE_CONDITIONING_MODE + or payload.get("model_revision") != expected_model_revision + or payload.get("checkpoint_sha256") != expected_checkpoint_sha256 + ): + raise ValueError("Cached conditioning payload differs from the inline-five contract.") + if not isinstance(provenance, Mapping): + raise ValueError("Cached conditioning provenance must be a JSON object.") + payload_record = provenance.get("conditioning_payload") + checkpoint_binding = provenance.get("checkpoint_binding") + collection_record = provenance.get("collection") + shard_record = provenance.get("rank_cache_shard") + payload_record_path = ( + payload_record.get("path") if isinstance(payload_record, Mapping) else None + ) + if ( + provenance.get("schema_version") != 1 + or provenance.get("status") != "verified" + or provenance.get("selection_policy") != INLINE_SELECTION_POLICY + or provenance.get("conditioning_mode") != INLINE_CONDITIONING_MODE + or provenance.get("cached_prompt_embeds_used_by_inference") is not True + or provenance.get("text_encoder_must_not_run") is not True + or not isinstance(payload_record, Mapping) + or not isinstance(payload_record_path, str) + or Path(payload_record_path).resolve() != payload_path + or payload_record.get("bytes") != payload_path.stat().st_size + or payload_record.get("sha256") != payload_sha + or payload_record.get("schema_version") != INLINE_PAYLOAD_SCHEMA_VERSION + or payload_record.get("artifact_type") != INLINE_PAYLOAD_ARTIFACT_TYPE + or not isinstance(checkpoint_binding, Mapping) + or checkpoint_binding.get("sha256") != expected_checkpoint_sha256 + or not isinstance(collection_record, Mapping) + or collection_record.get("model_revision") != expected_model_revision + or not isinstance(shard_record, Mapping) + or not _is_sha256(expected_checkpoint_sha256) + or not _is_sha256(payload.get("collection_sha256")) + or not _is_sha256(payload.get("rank_cache_shard_sha256")) + ): + raise ValueError("Cached conditioning provenance does not bind the requested payload/checkpoint.") + if ( + payload.get("collection_sha256") != collection_record.get("sha256") + or payload.get("rank_cache_shard_sha256") != shard_record.get("sha256") + ): + raise ValueError("Cached conditioning payload does not bind the provenance collection/shard.") + + raw_conditions = payload.get("conditions") + provenance_conditions = provenance.get("conditions") + if ( + not isinstance(raw_conditions, list) + or not isinstance(provenance_conditions, list) + or len(raw_conditions) != INLINE_COUNT + or len(provenance_conditions) != INLINE_COUNT + or len(prompts) != INLINE_COUNT + ): + raise ValueError("Cached conditioning requires five payload, provenance, and prompt rows.") + + validated: list[CachedConditioningSample] = [] + for position, (raw, descriptor, prompt) in enumerate( + zip(raw_conditions, provenance_conditions, prompts) + ): + label = f"cached conditions[{position}]" + if not isinstance(raw, Mapping) or set(raw) != _CACHED_CONDITION_FIELDS: + raise ValueError(f"{label} has an unknown tensor payload schema.") + if not isinstance(descriptor, Mapping) or set(descriptor) != _PROVENANCE_CONDITION_FIELDS: + raise ValueError(f"{label} has an unknown provenance schema.") + metadata_names = ( + "cache_rank", + "cache_condition_position", + "cache_global_step", + "source_index", + "source_line", + "id", + "prompt", + "prompt_sha256", + "evaluation_seed", + ) + if any(raw.get(name) != descriptor.get(name) for name in metadata_names): + raise ValueError(f"{label} payload/provenance metadata differs.") + integer_names = ( + "cache_rank", + "cache_condition_position", + "cache_global_step", + "source_index", + "source_line", + "evaluation_seed", + ) + if any( + isinstance(raw.get(name), bool) or not isinstance(raw.get(name), int) + for name in integer_names + ): + raise ValueError(f"{label} integer metadata has an invalid type.") + prompt_text = raw.get("prompt") + observed_prompt_sha = ( + hashlib.sha256(prompt_text.encode("utf-8")).hexdigest() + if isinstance(prompt_text, str) + else None + ) + if ( + raw.get("cache_rank") != 0 + or raw.get("cache_condition_position") != position + or raw.get("cache_global_step") != position + 1 + or isinstance(raw.get("source_index"), bool) + or not isinstance(raw.get("source_index"), int) + or raw.get("source_index") < 0 + or raw.get("source_line") != raw.get("source_index") + 1 + or not isinstance(prompt_text, str) + or raw.get("prompt_sha256") != observed_prompt_sha + or raw.get("id") + != f"vidprom-{raw.get('source_line'):06d}-{observed_prompt_sha[:12]}" + or raw.get("id") != prompt.id + or prompt_text != prompt.prompt + or raw.get("evaluation_seed") != prompt.seed + or descriptor.get("evaluation_seed_policy") != "fixed_dmd8_inline5_v1" + ): + raise ValueError(f"{label} is not the exact committed inline prompt/cache row.") + embeds = raw.get("prompt_embeds") + tags = raw.get("text_token_tags") + if ( + not isinstance(embeds, torch.Tensor) + or embeds.device.type != "cpu" + or embeds.layout != torch.strided + or embeds.dtype != torch.bfloat16 + or embeds.ndim != 3 + or embeds.shape[0] != 1 + or embeds.shape[1] < 1 + or embeds.shape[2] != 5120 + or not bool(torch.isfinite(embeds).all()) + ): + raise ValueError(f"{label}.prompt_embeds must be finite CPU BF16 [1,N,5120].") + if ( + not isinstance(tags, torch.Tensor) + or tags.device.type != "cpu" + or tags.layout != torch.strided + or tags.dtype != torch.long + or tags.shape != (embeds.shape[1],) + or not bool((tags == 1).all()) + ): + raise ValueError(f"{label}.text_token_tags must be CPU int64 text tags [N].") + embeds_descriptor = _tensor_descriptor(embeds) + tags_descriptor = _tensor_descriptor(tags) + if ( + descriptor.get("cached_prompt_embeds") != embeds_descriptor + or descriptor.get("cached_text_token_tags") != tags_descriptor + ): + raise ValueError(f"{label} tensor dtype/shape/SHA differs from provenance.") + validated.append( + CachedConditioningSample( + cache_rank=raw["cache_rank"], + cache_condition_position=raw["cache_condition_position"], + cache_global_step=raw["cache_global_step"], + source_index=raw["source_index"], + source_line=raw["source_line"], + id=raw["id"], + prompt=raw["prompt"], + prompt_sha256=raw["prompt_sha256"], + evaluation_seed=raw["evaluation_seed"], + prompt_embeds=embeds.detach().cpu().contiguous().clone(), + text_token_tags=tags.detach().cpu().contiguous().clone(), + prompt_embeds_descriptor=embeds_descriptor, + text_token_tags_descriptor=tags_descriptor, + ) + ) + return CachedConditioningBundle( + payload_path=payload_path, + payload_sha256=payload_sha, + payload_bytes=payload_path.stat().st_size, + provenance_path=provenance_path, + provenance_sha256=provenance_sha, + model_revision=expected_model_revision, + collection_sha256=payload["collection_sha256"], + rank_cache_shard_sha256=payload["rank_cache_shard_sha256"], + checkpoint_sha256=expected_checkpoint_sha256, + conditions=tuple(validated), + ) + + +def load_4step_student_checkpoint(path: str | Path) -> dict[str, Any]: + """Load and fail-closed validate a document-aligned four-step LoRA.""" + + checkpoint_path = Path(path).resolve() + if not checkpoint_path.is_file(): + raise FileNotFoundError(f"Student checkpoint does not exist: {checkpoint_path}") + payload = torch.load(checkpoint_path, map_location="cpu", weights_only=True) + if not isinstance(payload, Mapping) or payload.get("schema_version") != 1: + raise ValueError("Unsupported or malformed four-step H3 LoRA checkpoint.") + step = payload.get("step") + if isinstance(step, bool) or not isinstance(step, int) or step < 1: + raise ValueError("Four-step H3 checkpoint step must be a positive integer.") + config = payload.get("config") + if not isinstance(config, Mapping): + raise ValueError("Four-step H3 checkpoint is missing its training config.") + mismatches = { + name: {"actual": config.get(name), "expected": expected} + for name, expected in _REQUIRED_CONTRACT.items() + if name not in config or config.get(name) != expected + } + if mismatches: + raise ValueError( + "Checkpoint does not satisfy the four-step fresh-noise H3 contract: " + + json.dumps(mismatches, sort_keys=True) + ) + if (float(config.get("video_shift", -1)), float(config.get("audio_shift", -1))) != ( + 12.0, + 3.0, + ): + raise ValueError("Four-step H3 checkpoint scheduler shifts must be video=12/audio=3.") + + targets = config.get("lora_targets") + if not isinstance(targets, list) or not all( + isinstance(name, str) and name for name in targets + ): + raise ValueError("Four-step H3 checkpoint LoRA targets must be strings.") + validate_full_h3_checkpoint_targets(targets) + + state = payload.get("student_lora") + if not isinstance(state, Mapping) or not state: + raise ValueError("Four-step H3 checkpoint has no student_lora tensor state.") + if any( + not isinstance(name, str) or not isinstance(value, torch.Tensor) + for name, value in state.items() + ): + raise TypeError("Every student_lora entry must map a string to a tensor.") + expected_keys = _expected_student_state_keys() + if set(state) != expected_keys: + missing = sorted(expected_keys - set(state)) + extra = sorted(set(state) - expected_keys) + raise ValueError( + "Student LoRA tensor keys do not cover the exact 312-target A/B state: " + f"missing={missing[:5]}, extra={extra[:5]}." + ) + if len(state) != 2 * H3_DMD_TARGET_MODULE_COUNT: + raise AssertionError("A 312-target LoRA must contain exactly 624 A/B tensors.") + wrong_dtype = sorted(name for name, value in state.items() if value.dtype != torch.float32) + if wrong_dtype: + raise ValueError( + "Four-step checkpoint violates the FP32 LoRA-master contract: " + f"{wrong_dtype[:5]}." + ) + rank_mismatches = sorted( + name + for name, value in state.items() + if value.ndim != 2 + or (".lora_A." in name and value.shape[0] != LORA_RANK) + or (".lora_B." in name and value.shape[1] != LORA_RANK) + ) + if rank_mismatches: + raise ValueError( + "Four-step checkpoint LoRA tensors do not have rank 128: " + f"{rank_mismatches[:5]}." + ) + lora_b = [value for name, value in state.items() if ".lora_B." in name] + if not any(bool(torch.count_nonzero(value)) for value in lora_b): + raise ValueError("Four-step student checkpoint has no trained nonzero LoRA-B tensor.") + + return { + "path": checkpoint_path, + "sha256": sha256_file(checkpoint_path), + "step": step, + "config": dict(config), + "student_lora": dict(state), + } + + +def install_4step_student_lora( + transformer: torch.nn.Module, + checkpoint: Mapping[str, Any], +) -> dict[str, Any]: + """Install the generator LoRA and verify 312 FP32 runtime branches.""" + + if getattr(getattr(transformer, "config", None), "adaln_curve_rank", None) is not None: + raise RuntimeError("Four-step inference requires the standard full-AdaLN H3 backbone.") + metadata = install_student_lora(transformer, checkpoint) + adapter_parameters = [ + (name, parameter) + for name, parameter in transformer.named_parameters() + if "lora_" in name and f".{STUDENT_ADAPTER}." in name + ] + lora_a_modules = { + name.split(".lora_A.", 1)[0] + for name, _ in adapter_parameters + if ".lora_A." in name + } + lora_b_modules = { + name.split(".lora_B.", 1)[0] + for name, _ in adapter_parameters + if ".lora_B." in name + } + if ( + len(lora_a_modules) != H3_DMD_TARGET_MODULE_COUNT + or lora_a_modules != lora_b_modules + ): + raise RuntimeError( + "Installed student adapter does not expose 312 matched LoRA A/B modules." + ) + runtime_dtypes = {parameter.dtype for _, parameter in adapter_parameters} + if runtime_dtypes != {torch.float32}: + raise RuntimeError( + "Installed student adapter did not retain FP32 LoRA weights: " + f"{sorted(map(str, runtime_dtypes))}." + ) + return { + **metadata, + "lora_target_module_count": H3_DMD_TARGET_MODULE_COUNT, + "lora_tensor_count": 2 * H3_DMD_TARGET_MODULE_COUNT, + "lora_master_dtype": "float32", + "runtime_lora_dtype": "float32", + "rollout_mode": ROLLOUT_MODE, + } + + +def _assert_fresh_noise_pipeline(pipe: Any) -> None: + if pipe.__class__.__name__ != "MiniMaxH3ModularPipeline": + raise RuntimeError(f"Expected MiniMaxH3ModularPipeline, found {pipe.__class__.__name__}.") + if not isinstance(pipe.scheduler, FreshNoiseMiniMaxH3Scheduler) or not isinstance( + pipe.audio_scheduler, FreshNoiseMiniMaxH3Scheduler + ): + raise RuntimeError("Four-step pipeline is missing its fresh-noise scheduler pair.") + if (pipe.scheduler.shift, pipe.audio_scheduler.shift) != (12.0, 3.0): + raise RuntimeError("Four-step fresh-noise scheduler shifts are not video=12/audio=3.") + if getattr(pipe, "transformer_ref", None) is not None: + raise RuntimeError("Four-step student inference is Base T2VA, not Ref2VA.") + forbidden = {"guidance_scale", "negative_prompt", "guidance_rescale", "guider"} + exposed = forbidden.intersection(getattr(pipe, "default_call_parameters", {})) + if exposed: + raise RuntimeError(f"Four-step T2VA unexpectedly exposes CFG inputs: {sorted(exposed)}") + + +def _text_encoder_forward_module(pipe: Any) -> torch.nn.Module: + text_encoder = getattr(pipe, "text_encoder", None) + target = getattr(text_encoder, "model", None) + if target is None: + target = text_encoder + if not isinstance(target, torch.nn.Module): + raise RuntimeError("Unable to install a forward audit on the H3 Qwen3-VL conditioner.") + return target + + +@contextmanager +def _text_encoder_forward_audit( + pipe: Any, *, forbid: bool +) -> Iterator[dict[str, int]]: + """Count Qwen model forwards and optionally make any call fail closed.""" + + audit = {"forward_calls": 0} + + def observe(_module: Any, _inputs: Any) -> None: + audit["forward_calls"] += 1 + if forbid: + raise RuntimeError( + "Cached-conditioning inference attempted to invoke the Qwen3-VL text encoder." + ) + + handle = _text_encoder_forward_module(pipe).register_forward_pre_hook(observe) + try: + yield audit + finally: + handle.remove() + + +@contextmanager +def _cached_text_encoder_bypass(pipe: Any) -> Iterator[dict[str, int]]: + """Temporarily remove the encoder block so cached tensors are real inputs. + + Merely passing ``prompt_embeds`` to the stock MiniMax-H3 modular pipeline is + insufficient: while its text-encoder block is present, those names are + intermediate outputs and unexpected top-level kwargs are ignored. Removing + exactly that block makes the downstream denoise block expose both tensors as + pipeline inputs. The original ordered block graph is restored in ``finally``. + """ + + blocks = getattr(getattr(pipe, "_blocks", None), "sub_blocks", None) + if blocks is None or "text_encoder" not in blocks: + raise RuntimeError("Official H3 block graph has no removable text_encoder block.") + original_items = list(blocks.items()) + blocks.pop("text_encoder") + try: + input_names = set(getattr(pipe._blocks, "input_names", ())) + required = {"prompt_embeds", "text_token_tags"} + if not required.issubset(input_names) or "prompt" in input_names: + raise RuntimeError( + "Removing the H3 text encoder did not expose the exact cached-conditioning inputs." + ) + with _text_encoder_forward_audit(pipe, forbid=True) as audit: + yield audit + finally: + blocks.clear() + blocks.update(original_items) + + +def _cached_conditioning_call_kwargs( + pipe: Any, condition: CachedConditioningSample +) -> dict[str, torch.Tensor]: + """Build a prompt-free call payload from one validated cache snapshot.""" + + execution_device = getattr(pipe, "_execution_device", None) + if execution_device is None: + raise RuntimeError("Official H3 pipeline has no resolved execution device.") + return { + # Qwen's official encoder step emits this tensor on the execution + # device. Preserve BF16 exactly while reproducing that placement. + "prompt_embeds": condition.prompt_embeds.to( + device=execution_device, dtype=torch.bfloat16 + ), + # The official layout builder consumes tags on CPU and constructs the + # packed structural tensors there, so retain the cached CPU int64 tags. + "text_token_tags": condition.text_token_tags, + } + + +def run_student_batch_4step( + pipe: Any, + prompts: Sequence[Any], + config: RunConfig, + checkpoint: Mapping[str, Any], + output_dir: str | Path, + *, + adapter_metadata: Mapping[str, Any] | None = None, + cached_conditioning: CachedConditioningBundle | None = None, +) -> dict[str, Any]: + """Generate and audit one independently seeded fresh-noise request per prompt.""" + + if not prompts: + raise ValueError("Four-step visualization requires at least one prompt.") + if config.num_inference_steps != NUM_SIGMA_ENDPOINTS: + raise ValueError("Four-step H3 must request five endpoints/four denoiser evaluations.") + if cached_conditioning is not None and len(cached_conditioning.conditions) != len(prompts): + raise ValueError("Cached conditioning count differs from the prompt count.") + _assert_fresh_noise_pipeline(pipe) + if adapter_metadata is None: + adapter = install_4step_student_lora(pipe.transformer, checkpoint) + else: + adapter = dict(adapter_metadata) + if tuple(pipe.transformer.active_adapters()) != (STUDENT_ADAPTER,): + raise RuntimeError("Pre-offload setup did not leave only student LoRA active.") + if adapter.get("checkpoint_sha256") != checkpoint["sha256"]: + raise RuntimeError("Pre-offload student metadata/checkpoint hash mismatch.") + + destination = Path(output_dir).resolve() + destination.mkdir(parents=True, exist_ok=True) + samples = [] + for index, prompt in enumerate(prompts): + sample_dir = destination / f"{index:02d}_{prompt.id}" + sample_dir.mkdir(parents=True, exist_ok=True) + generator = torch.Generator().manual_seed(prompt.seed) + draw_audit = bind_pipeline_fresh_noise_generator(pipe, generator) + transformer_calls = 0 + cached_sample = ( + cached_conditioning.conditions[index] + if cached_conditioning is not None + else None + ) + if cached_sample is not None and ( + cached_sample.id, + cached_sample.prompt, + cached_sample.evaluation_seed, + ) != (prompt.id, prompt.prompt, prompt.seed): + raise RuntimeError("Cached conditioning row no longer matches its prompt request.") + + def count_transformer_forward(_module: Any, _inputs: Any) -> None: + nonlocal transformer_calls + transformer_calls += 1 + + hook = pipe.transformer.register_forward_pre_hook(count_transformer_forward) + if cached_sample is None: + conditioning_kwargs: dict[str, Any] = {"prompt": prompt.prompt} + conditioning_context = _text_encoder_forward_audit(pipe, forbid=False) + else: + conditioning_kwargs = _cached_conditioning_call_kwargs(pipe, cached_sample) + conditioning_context = _cached_text_encoder_bypass(pipe) + try: + with conditioning_context as text_encoder_audit: + results = pipe( + **conditioning_kwargs, + height=config.height, + width=config.width, + num_frames=config.num_frames, + num_inference_steps=NUM_SIGMA_ENDPOINTS, + generator=generator, + output_type="pt", + output=["videos", "audio", "sampling_rate", "latents", "audio_latents"], + ) + finally: + hook.remove() + expected_text_encoder_calls = 1 if cached_sample is None else 0 + if text_encoder_audit["forward_calls"] != expected_text_encoder_calls: + raise RuntimeError( + "H3 text-encoder execution count differs from the conditioning mode: " + f"observed={text_encoder_audit['forward_calls']}, " + f"expected={expected_text_encoder_calls}." + ) + if transformer_calls != NUM_MODEL_EVALUATIONS: + raise RuntimeError( + "Official H3 loop did not execute exactly four positive forwards: " + f"observed={transformer_calls}." + ) + sampler_audit = validate_completed_pipeline_rollout(pipe, draw_audit) + sampler_audit["transformer_positive_forwards"] = transformer_calls + + if cached_sample is None: + conditioning_audit = { + "mode": "independent_direct_text", + "cached_prompt_embeds_used_by_inference": False, + "prompt_argument_supplied": True, + "prompt_embeds_argument_supplied": False, + "text_token_tags_argument_supplied": False, + "text_encoder_block_removed": False, + "text_encoder_execution_forbidden": False, + "text_encoder_forward_calls": text_encoder_audit["forward_calls"], + } + else: + passed_embeds = _tensor_descriptor(conditioning_kwargs["prompt_embeds"]) + passed_tags = _tensor_descriptor(conditioning_kwargs["text_token_tags"]) + if ( + passed_embeds != cached_sample.prompt_embeds_descriptor + or passed_tags != cached_sample.text_token_tags_descriptor + ): + raise RuntimeError("Tensors passed to H3 differ from the validated cache snapshot.") + conditioning_audit = { + "mode": INLINE_CONDITIONING_MODE, + "selection_policy": INLINE_SELECTION_POLICY, + "cached_prompt_embeds_used_by_inference": True, + "prompt_argument_supplied": False, + "prompt_embeds_argument_supplied": True, + "text_token_tags_argument_supplied": True, + "text_encoder_block_removed": True, + "text_encoder_execution_forbidden": True, + "text_encoder_forward_calls": text_encoder_audit["forward_calls"], + "payload_sha256": cached_conditioning.payload_sha256, + "provenance_sha256": cached_conditioning.provenance_sha256, + "cache_rank": cached_sample.cache_rank, + "cache_condition_position": cached_sample.cache_condition_position, + "cache_global_step": cached_sample.cache_global_step, + "source_index": cached_sample.source_index, + "source_line": cached_sample.source_line, + "prompt_sha256": cached_sample.prompt_sha256, + "passed_prompt_embeds": passed_embeds, + "passed_text_token_tags": passed_tags, + } + + if not isinstance(results, Mapping): + raise TypeError("Official H3 pipeline did not return a mapping.") + required = ("videos", "audio", "sampling_rate", "latents", "audio_latents") + missing = [name for name in required if results.get(name) is None] + if missing: + raise RuntimeError(f"Four-step H3 pipeline output is missing {missing}.") + + video_path = sample_dir / "student.mp4" + latent_path = sample_dir / "student_latents.pt" + frames = _prepare_video_for_encoder(results["videos"]) + sampling_rate = int(results["sampling_rate"]) + _default_video_encoder( + video=frames, + fps=FPS, + output_path=str(video_path), + audio=results["audio"][0].detach().cpu(), + audio_sample_rate=sampling_rate, + ) + torch.save( + { + "video_latents": results["latents"].detach().cpu().contiguous(), + "audio_latents": results["audio_latents"].detach().cpu().contiguous(), + }, + latent_path, + ) + metadata = { + "schema_version": 1, + "sample": asdict(prompt), + "request": { + "workflow": WORKFLOW, + "height": config.height, + "width": config.width, + "num_frames": config.num_frames, + "fps": FPS, + "rollout_mode": ROLLOUT_MODE, + "attention_mode": "native_non_causal", + "model_evaluations": NUM_MODEL_EVALUATIONS, + "sigma_endpoints": NUM_SIGMA_ENDPOINTS, + "fresh_noise_transitions": NUM_FRESH_NOISE_TRANSITIONS, + "uses_cfg": False, + "uses_context_ir": False, + "uses_regenerator": False, + }, + "sampler_audit": sampler_audit, + "conditioning_audit": conditioning_audit, + "model": { + "id": config.model_id, + "resolved_revision": config.resolved_revision, + "transformer_block_count": RELEASED_MAIN_BLOCKS, + "token_refiner_block_count": RELEASED_REFINER_BLOCKS, + }, + "student": adapter, + "numeric_outputs": { + "decoded_video": _tensor_descriptor(results["videos"]), + "decoded_audio": _tensor_descriptor(results["audio"]), + "video_latents": _tensor_descriptor(results["latents"]), + "audio_latents": _tensor_descriptor(results["audio_latents"]), + "sampling_rate": sampling_rate, + }, + "artifacts": { + "video": {"path": video_path.name, "sha256": sha256_file(video_path)}, + "latents": {"path": latent_path.name, "sha256": sha256_file(latent_path)}, + }, + } + metadata_path = sample_dir / "metadata.json" + _atomic_json(metadata_path, metadata) + samples.append( + { + "id": prompt.id, + "prompt": prompt.prompt, + "seed": prompt.seed, + "video": str(video_path.relative_to(destination)), + "video_sha256": metadata["artifacts"]["video"]["sha256"], + "metadata": str(metadata_path.relative_to(destination)), + "metadata_sha256": sha256_file(metadata_path), + "sampler_contract_verified": True, + "conditioning_mode": conditioning_audit["mode"], + "cached_prompt_embeds_used_by_inference": conditioning_audit[ + "cached_prompt_embeds_used_by_inference" + ], + "text_encoder_forward_calls": conditioning_audit[ + "text_encoder_forward_calls" + ], + } + ) + + manifest = { + "schema_version": 1, + "status": "success", + "runner": "LongLive/minimax_h3/infer_student_4step.py", + "config": asdict(config), + "contract": dict(_REQUIRED_CONTRACT), + "checkpoint": { + "path": str(checkpoint["path"]), + "sha256": checkpoint["sha256"], + "step": checkpoint["step"], + }, + "adapter": adapter, + "conditioning": ( + cached_conditioning.manifest_summary() + if cached_conditioning is not None + else { + "mode": "independent_direct_text", + "cached_prompt_embeds_used_by_inference": False, + "text_encoder_forward_calls_required_per_sample": 1, + } + ), + "samples": samples, + } + manifest_path = destination / "manifest.json" + _atomic_json(manifest_path, manifest) + _atomic_json( + destination / "SUCCESS.json", + { + "schema_version": 1, + "status": "success", + "manifest": manifest_path.name, + "manifest_sha256": sha256_file(manifest_path), + "samples": len(samples), + "fresh_noise_sampler_verified": True, + "conditioning_mode": ( + INLINE_CONDITIONING_MODE + if cached_conditioning is not None + else "independent_direct_text" + ), + "cached_prompt_embeds_used_by_inference": cached_conditioning is not None, + }, + ) + return manifest + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--checkpoint", type=Path, required=True) + parser.add_argument("--prompts", type=Path, default=Path(__file__).with_name("prompts.json")) + parser.add_argument("--expected-prompt-count", type=int, default=4) + parser.add_argument( + "--cached-conditioning", + type=Path, + help="Tensor payload emitted by extract_dmd8_inline5 (inline5 only).", + ) + parser.add_argument( + "--cached-conditioning-provenance", + type=Path, + help="JSON provenance emitted alongside --cached-conditioning.", + ) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--model-id", "--model-dir", dest="model_id", default=MODEL_ID) + parser.add_argument("--revision", default="main") + parser.add_argument("--height", type=int, default=768) + parser.add_argument("--width", type=int, default=1344) + parser.add_argument("--num-frames", type=int, default=124) + parser.add_argument("--device", default="cuda") + parser.add_argument("--memory-reserve-margin", default="12GB") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--validate-only", action="store_true") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.expected_prompt_count < 1: + raise ValueError("--expected-prompt-count must be positive.") + prompts = load_prompt_specs(args.prompts, expected_count=args.expected_prompt_count) + checkpoint = load_4step_student_checkpoint(args.checkpoint) + if (args.cached_conditioning is None) != ( + args.cached_conditioning_provenance is None + ): + raise ValueError( + "--cached-conditioning and --cached-conditioning-provenance must be supplied together." + ) + revision = resolve_hub_revision( + args.model_id, args.revision, local_files_only=args.local_files_only + ) + checkpoint_revision = checkpoint["config"].get("resolved_revision") + if checkpoint_revision != revision: + raise ValueError( + "Student checkpoint/model revision mismatch: " + f"{checkpoint_revision!r} != {revision!r}." + ) + cached_conditioning = None + if args.cached_conditioning is not None: + cached_conditioning = load_cached_conditioning( + args.cached_conditioning, + args.cached_conditioning_provenance, + prompts, + expected_model_revision=revision, + expected_checkpoint_sha256=checkpoint["sha256"], + ) + config = RunConfig( + model_id=args.model_id, + requested_revision=args.revision, + resolved_revision=revision, + height=args.height, + width=args.width, + num_frames=args.num_frames, + num_inference_steps=NUM_SIGMA_ENDPOINTS, + device=args.device, + memory_reserve_margin=args.memory_reserve_margin, + local_files_only=args.local_files_only, + ) + config.validate() + if args.validate_only: + print( + json.dumps( + { + "checkpoint": { + "path": str(checkpoint["path"]), + "sha256": checkpoint["sha256"], + "step": checkpoint["step"], + }, + "contract": _REQUIRED_CONTRACT, + "config": asdict(config), + "prompts": [asdict(prompt) for prompt in prompts], + "conditioning": ( + cached_conditioning.manifest_summary() + if cached_conditioning is not None + else { + "mode": "independent_direct_text", + "cached_prompt_embeds_used_by_inference": False, + } + ), + }, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + ) + return 0 + + adapter_metadata: dict[str, Any] = {} + + def setup_student(transformer: torch.nn.Module) -> None: + adapter_metadata.update(install_4step_student_lora(transformer, checkpoint)) + + pipe = build_official_pipeline(config, transformer_setup=setup_student) + # build_official_pipeline has already fail-closed validated the untouched + # official components, including their pinned revision and 12/3 shifts. + assert_official_t2va_contract(pipe, expected_revision=revision) + install_fresh_noise_schedulers(pipe) + manifest = run_student_batch_4step( + pipe, + prompts, + config, + checkpoint, + args.output_dir, + adapter_metadata=adapter_metadata, + cached_conditioning=cached_conditioning, + ) + print(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + + +__all__ = [ + "CachedConditioningBundle", + "CachedConditioningSample", + "LORA_ALPHA", + "LORA_RANK", + "ROLLOUT_MODE", + "build_parser", + "install_4step_student_lora", + "load_4step_student_checkpoint", + "load_cached_conditioning", + "main", + "run_student_batch_4step", +] diff --git a/source_snapshot/minimax_h3/infer_student_8step.py b/source_snapshot/minimax_h3/infer_student_8step.py new file mode 100644 index 0000000000000000000000000000000000000000..ddf4c4d0b307bda1cce555a952498d4398f5de0b --- /dev/null +++ b/source_snapshot/minimax_h3/infer_student_8step.py @@ -0,0 +1,1062 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Infer the eight-step MiniMax-H3 DMD student with fresh re-noising. + +This runner intentionally leaves ``infer_student.py`` and the pinned +Diffusers checkout unchanged. It reuses the audited Base T2VA pipeline, +prompt encoding, LoRA round-trip, VAE decoding, and MP4 muxing paths, while +replacing only the two scheduler instances after the official pipeline has +been validated. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from contextlib import contextmanager +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Iterator, Mapping, Sequence + +import torch + +from .dmd_core import DMD_NORMALIZATION +from .fresh_noise_scheduler import ( + FreshNoiseMiniMaxH3Scheduler, + NUM_FRESH_NOISE_TRANSITIONS, + NUM_MODEL_EVALUATIONS, + NUM_SIGMA_ENDPOINTS, + bind_pipeline_fresh_noise_generator, + install_fresh_noise_schedulers, + validate_completed_pipeline_rollout, +) +from .full_h3_lora import ( + H3_DMD_LORA_POLICY, + H3_DMD_TARGET_MODULE_COUNT, + released_full_h3_target_modules, + validate_full_h3_checkpoint_targets, +) +from .extract_dmd8_inline5 import ( + INLINE_CONDITIONING_MODE, + INLINE_COUNT, + INLINE_PAYLOAD_ARTIFACT_TYPE, + INLINE_PAYLOAD_SCHEMA_VERSION, + INLINE_SELECTION_POLICY, +) +from .infer_base import ( + FPS, + MODEL_ID, + WORKFLOW, + RunConfig, + _default_video_encoder, + _prepare_video_for_encoder, + _tensor_descriptor, + assert_official_t2va_contract, + build_official_pipeline, + load_prompt_specs, + resolve_hub_revision, + sha256_file, +) +from .infer_student import ( + _atomic_json, + install_student_lora, +) +from .train_utils import ( + RELEASED_MAIN_BLOCKS, + RELEASED_REFINER_BLOCKS, + STUDENT_ADAPTER, +) + + +LORA_RANK = 128 +LORA_ALPHA = 128 +ROLLOUT_MODE = "predict_x0_fresh_renoise" +_EXPECTED_DRAW_ORDER = ["video", "audio"] +_REQUIRED_CONTRACT = { + "workflow": WORKFLOW, + "ir": False, + "regenerator": False, + "cfg": False, + "teacher_adapter_state": "disabled", + "student_adapter": STUDENT_ADAPTER, + "model_variant": "standard_full_adaln", + "adaln_curve_rank": None, + "rollout_mode": ROLLOUT_MODE, + "attention_mode": "native_non_causal", + "attention_is_causal": False, + "temporal_ar_blocks": 0, + "model_evaluations_per_rollout": NUM_MODEL_EVALUATIONS, + "sigma_endpoints_per_rollout": NUM_SIGMA_ENDPOINTS, + "fresh_noise_transitions_per_rollout": NUM_FRESH_NOISE_TRANSITIONS, + "student_positive_forwards_per_rollout": NUM_MODEL_EVALUATIONS, + "fresh_noise_draw_order": _EXPECTED_DRAW_ORDER, + "video_audio_transition_noise_independent": True, + "sampler_matches_training_and_inference": True, + "transformer_block_count": RELEASED_MAIN_BLOCKS, + "token_refiner_block_count": RELEASED_REFINER_BLOCKS, + "normalization": DMD_NORMALIZATION, + "lora_rank": LORA_RANK, + "lora_alpha": LORA_ALPHA, + "lora_dropout": 0.0, + "lora_master_dtype": "float32", + "lora_target_policy": H3_DMD_LORA_POLICY, + "lora_target_module_count": H3_DMD_TARGET_MODULE_COUNT, +} +_CACHED_PAYLOAD_FIELDS = { + "schema_version", + "artifact_type", + "selection_policy", + "conditioning_mode", + "model_revision", + "collection_sha256", + "rank_cache_shard_sha256", + "checkpoint_sha256", + "conditions", +} +_CACHED_CONDITION_FIELDS = { + "cache_rank", + "cache_condition_position", + "cache_global_step", + "source_index", + "source_line", + "id", + "prompt", + "prompt_sha256", + "evaluation_seed", + "prompt_embeds", + "text_token_tags", +} +_PROVENANCE_CONDITION_FIELDS = { + "cache_rank", + "cache_condition_position", + "cache_global_step", + "source_index", + "source_line", + "id", + "prompt", + "prompt_sha256", + "evaluation_seed", + "evaluation_seed_policy", + "cached_prompt_embeds", + "cached_text_token_tags", +} + + +def _is_sha256(value: Any) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and all(character in "0123456789abcdef" for character in value) + ) + + +@dataclass(frozen=True) +class CachedConditioningSample: + """One cache row after fail-closed payload/provenance validation.""" + + cache_rank: int + cache_condition_position: int + cache_global_step: int + source_index: int + source_line: int + id: str + prompt: str + prompt_sha256: str + evaluation_seed: int + prompt_embeds: torch.Tensor + text_token_tags: torch.Tensor + prompt_embeds_descriptor: Mapping[str, Any] + text_token_tags_descriptor: Mapping[str, Any] + + +@dataclass(frozen=True) +class CachedConditioningBundle: + """Cryptographic binding of an inline payload to its JSON provenance.""" + + payload_path: Path + payload_sha256: str + payload_bytes: int + provenance_path: Path + provenance_sha256: str + model_revision: str + collection_sha256: str + rank_cache_shard_sha256: str + checkpoint_sha256: str + conditions: tuple[CachedConditioningSample, ...] + + def manifest_summary(self) -> dict[str, Any]: + return { + "mode": INLINE_CONDITIONING_MODE, + "selection_policy": INLINE_SELECTION_POLICY, + "cached_prompt_embeds_used_by_inference": True, + "text_encoder_forward_calls_required": 0, + "payload": { + "path": str(self.payload_path), + "bytes": self.payload_bytes, + "sha256": self.payload_sha256, + }, + "provenance": { + "path": str(self.provenance_path), + "sha256": self.provenance_sha256, + }, + "conditions": len(self.conditions), + } + + +def _expected_student_state_keys() -> set[str]: + keys: set[str] = set() + for target in released_full_h3_target_modules(): + keys.add(f"{target}.lora_A.weight") + keys.add(f"{target}.lora_B.weight") + return keys + + +def load_cached_conditioning( + payload_path: str | Path, + provenance_path: str | Path, + prompts: Sequence[Any], + *, + expected_model_revision: str, + expected_checkpoint_sha256: str, +) -> CachedConditioningBundle: + """Load exact inline cache tensors and validate every JSON/tensor binding.""" + + payload_path = Path(payload_path).resolve() + provenance_path = Path(provenance_path).resolve() + for label, path in ( + ("cached conditioning payload", payload_path), + ("cached conditioning provenance", provenance_path), + ): + if not path.is_file(): + raise FileNotFoundError(f"{label} does not exist: {path}") + payload_sha = sha256_file(payload_path) + provenance_sha = sha256_file(provenance_path) + try: + payload = torch.load( + payload_path, + map_location="cpu", + weights_only=True, + mmap=True, + ) + except Exception as error: + raise ValueError(f"Unable to mmap cached conditioning payload: {payload_path}") from error + try: + provenance = json.loads(provenance_path.read_text(encoding="utf-8")) + except Exception as error: + raise ValueError(f"Unable to parse cached conditioning provenance: {provenance_path}") from error + if not isinstance(payload, Mapping) or set(payload) != _CACHED_PAYLOAD_FIELDS: + raise ValueError("Cached conditioning payload has an unknown top-level schema.") + if ( + payload.get("schema_version") != INLINE_PAYLOAD_SCHEMA_VERSION + or payload.get("artifact_type") != INLINE_PAYLOAD_ARTIFACT_TYPE + or payload.get("selection_policy") != INLINE_SELECTION_POLICY + or payload.get("conditioning_mode") != INLINE_CONDITIONING_MODE + or payload.get("model_revision") != expected_model_revision + or payload.get("checkpoint_sha256") != expected_checkpoint_sha256 + ): + raise ValueError("Cached conditioning payload differs from the inline-five contract.") + if not isinstance(provenance, Mapping): + raise ValueError("Cached conditioning provenance must be a JSON object.") + payload_record = provenance.get("conditioning_payload") + checkpoint_binding = provenance.get("checkpoint_binding") + collection_record = provenance.get("collection") + shard_record = provenance.get("rank_cache_shard") + payload_record_path = ( + payload_record.get("path") if isinstance(payload_record, Mapping) else None + ) + if ( + provenance.get("schema_version") != 1 + or provenance.get("status") != "verified" + or provenance.get("selection_policy") != INLINE_SELECTION_POLICY + or provenance.get("conditioning_mode") != INLINE_CONDITIONING_MODE + or provenance.get("cached_prompt_embeds_used_by_inference") is not True + or provenance.get("text_encoder_must_not_run") is not True + or not isinstance(payload_record, Mapping) + or not isinstance(payload_record_path, str) + or Path(payload_record_path).resolve() != payload_path + or payload_record.get("bytes") != payload_path.stat().st_size + or payload_record.get("sha256") != payload_sha + or payload_record.get("schema_version") != INLINE_PAYLOAD_SCHEMA_VERSION + or payload_record.get("artifact_type") != INLINE_PAYLOAD_ARTIFACT_TYPE + or not isinstance(checkpoint_binding, Mapping) + or checkpoint_binding.get("sha256") != expected_checkpoint_sha256 + or not isinstance(collection_record, Mapping) + or collection_record.get("model_revision") != expected_model_revision + or not isinstance(shard_record, Mapping) + or not _is_sha256(expected_checkpoint_sha256) + or not _is_sha256(payload.get("collection_sha256")) + or not _is_sha256(payload.get("rank_cache_shard_sha256")) + ): + raise ValueError("Cached conditioning provenance does not bind the requested payload/checkpoint.") + if ( + payload.get("collection_sha256") != collection_record.get("sha256") + or payload.get("rank_cache_shard_sha256") != shard_record.get("sha256") + ): + raise ValueError("Cached conditioning payload does not bind the provenance collection/shard.") + + raw_conditions = payload.get("conditions") + provenance_conditions = provenance.get("conditions") + if ( + not isinstance(raw_conditions, list) + or not isinstance(provenance_conditions, list) + or len(raw_conditions) != INLINE_COUNT + or len(provenance_conditions) != INLINE_COUNT + or len(prompts) != INLINE_COUNT + ): + raise ValueError("Cached conditioning requires five payload, provenance, and prompt rows.") + + validated: list[CachedConditioningSample] = [] + for position, (raw, descriptor, prompt) in enumerate( + zip(raw_conditions, provenance_conditions, prompts) + ): + label = f"cached conditions[{position}]" + if not isinstance(raw, Mapping) or set(raw) != _CACHED_CONDITION_FIELDS: + raise ValueError(f"{label} has an unknown tensor payload schema.") + if not isinstance(descriptor, Mapping) or set(descriptor) != _PROVENANCE_CONDITION_FIELDS: + raise ValueError(f"{label} has an unknown provenance schema.") + metadata_names = ( + "cache_rank", + "cache_condition_position", + "cache_global_step", + "source_index", + "source_line", + "id", + "prompt", + "prompt_sha256", + "evaluation_seed", + ) + if any(raw.get(name) != descriptor.get(name) for name in metadata_names): + raise ValueError(f"{label} payload/provenance metadata differs.") + integer_names = ( + "cache_rank", + "cache_condition_position", + "cache_global_step", + "source_index", + "source_line", + "evaluation_seed", + ) + if any( + isinstance(raw.get(name), bool) or not isinstance(raw.get(name), int) + for name in integer_names + ): + raise ValueError(f"{label} integer metadata has an invalid type.") + prompt_text = raw.get("prompt") + observed_prompt_sha = ( + hashlib.sha256(prompt_text.encode("utf-8")).hexdigest() + if isinstance(prompt_text, str) + else None + ) + if ( + raw.get("cache_rank") != 0 + or raw.get("cache_condition_position") != position + or raw.get("cache_global_step") != position + 1 + or isinstance(raw.get("source_index"), bool) + or not isinstance(raw.get("source_index"), int) + or raw.get("source_index") < 0 + or raw.get("source_line") != raw.get("source_index") + 1 + or not isinstance(prompt_text, str) + or raw.get("prompt_sha256") != observed_prompt_sha + or raw.get("id") + != f"vidprom-{raw.get('source_line'):06d}-{observed_prompt_sha[:12]}" + or raw.get("id") != prompt.id + or prompt_text != prompt.prompt + or raw.get("evaluation_seed") != prompt.seed + or descriptor.get("evaluation_seed_policy") != "fixed_dmd8_inline5_v1" + ): + raise ValueError(f"{label} is not the exact committed inline prompt/cache row.") + embeds = raw.get("prompt_embeds") + tags = raw.get("text_token_tags") + if ( + not isinstance(embeds, torch.Tensor) + or embeds.device.type != "cpu" + or embeds.layout != torch.strided + or embeds.dtype != torch.bfloat16 + or embeds.ndim != 3 + or embeds.shape[0] != 1 + or embeds.shape[1] < 1 + or embeds.shape[2] != 5120 + or not bool(torch.isfinite(embeds).all()) + ): + raise ValueError(f"{label}.prompt_embeds must be finite CPU BF16 [1,N,5120].") + if ( + not isinstance(tags, torch.Tensor) + or tags.device.type != "cpu" + or tags.layout != torch.strided + or tags.dtype != torch.long + or tags.shape != (embeds.shape[1],) + or not bool((tags == 1).all()) + ): + raise ValueError(f"{label}.text_token_tags must be CPU int64 text tags [N].") + embeds_descriptor = _tensor_descriptor(embeds) + tags_descriptor = _tensor_descriptor(tags) + if ( + descriptor.get("cached_prompt_embeds") != embeds_descriptor + or descriptor.get("cached_text_token_tags") != tags_descriptor + ): + raise ValueError(f"{label} tensor dtype/shape/SHA differs from provenance.") + validated.append( + CachedConditioningSample( + cache_rank=raw["cache_rank"], + cache_condition_position=raw["cache_condition_position"], + cache_global_step=raw["cache_global_step"], + source_index=raw["source_index"], + source_line=raw["source_line"], + id=raw["id"], + prompt=raw["prompt"], + prompt_sha256=raw["prompt_sha256"], + evaluation_seed=raw["evaluation_seed"], + prompt_embeds=embeds.detach().cpu().contiguous().clone(), + text_token_tags=tags.detach().cpu().contiguous().clone(), + prompt_embeds_descriptor=embeds_descriptor, + text_token_tags_descriptor=tags_descriptor, + ) + ) + return CachedConditioningBundle( + payload_path=payload_path, + payload_sha256=payload_sha, + payload_bytes=payload_path.stat().st_size, + provenance_path=provenance_path, + provenance_sha256=provenance_sha, + model_revision=expected_model_revision, + collection_sha256=payload["collection_sha256"], + rank_cache_shard_sha256=payload["rank_cache_shard_sha256"], + checkpoint_sha256=expected_checkpoint_sha256, + conditions=tuple(validated), + ) + + +def load_8step_student_checkpoint(path: str | Path) -> dict[str, Any]: + """Load and fail-closed validate a document-aligned eight-step LoRA.""" + + checkpoint_path = Path(path).resolve() + if not checkpoint_path.is_file(): + raise FileNotFoundError(f"Student checkpoint does not exist: {checkpoint_path}") + payload = torch.load(checkpoint_path, map_location="cpu", weights_only=True) + if not isinstance(payload, Mapping) or payload.get("schema_version") != 1: + raise ValueError("Unsupported or malformed eight-step H3 LoRA checkpoint.") + step = payload.get("step") + if isinstance(step, bool) or not isinstance(step, int) or step < 1: + raise ValueError("Eight-step H3 checkpoint step must be a positive integer.") + config = payload.get("config") + if not isinstance(config, Mapping): + raise ValueError("Eight-step H3 checkpoint is missing its training config.") + mismatches = { + name: {"actual": config.get(name), "expected": expected} + for name, expected in _REQUIRED_CONTRACT.items() + if name not in config or config.get(name) != expected + } + if mismatches: + raise ValueError( + "Checkpoint does not satisfy the eight-step fresh-noise H3 contract: " + + json.dumps(mismatches, sort_keys=True) + ) + if (float(config.get("video_shift", -1)), float(config.get("audio_shift", -1))) != ( + 12.0, + 3.0, + ): + raise ValueError("Eight-step H3 checkpoint scheduler shifts must be video=12/audio=3.") + + targets = config.get("lora_targets") + if not isinstance(targets, list) or not all( + isinstance(name, str) and name for name in targets + ): + raise ValueError("Eight-step H3 checkpoint LoRA targets must be strings.") + validate_full_h3_checkpoint_targets(targets) + + state = payload.get("student_lora") + if not isinstance(state, Mapping) or not state: + raise ValueError("Eight-step H3 checkpoint has no student_lora tensor state.") + if any( + not isinstance(name, str) or not isinstance(value, torch.Tensor) + for name, value in state.items() + ): + raise TypeError("Every student_lora entry must map a string to a tensor.") + expected_keys = _expected_student_state_keys() + if set(state) != expected_keys: + missing = sorted(expected_keys - set(state)) + extra = sorted(set(state) - expected_keys) + raise ValueError( + "Student LoRA tensor keys do not cover the exact 312-target A/B state: " + f"missing={missing[:5]}, extra={extra[:5]}." + ) + if len(state) != 2 * H3_DMD_TARGET_MODULE_COUNT: + raise AssertionError("A 312-target LoRA must contain exactly 624 A/B tensors.") + wrong_dtype = sorted(name for name, value in state.items() if value.dtype != torch.float32) + if wrong_dtype: + raise ValueError( + "Eight-step checkpoint violates the FP32 LoRA-master contract: " + f"{wrong_dtype[:5]}." + ) + rank_mismatches = sorted( + name + for name, value in state.items() + if value.ndim != 2 + or (".lora_A." in name and value.shape[0] != LORA_RANK) + or (".lora_B." in name and value.shape[1] != LORA_RANK) + ) + if rank_mismatches: + raise ValueError( + "Eight-step checkpoint LoRA tensors do not have rank 128: " + f"{rank_mismatches[:5]}." + ) + lora_b = [value for name, value in state.items() if ".lora_B." in name] + if not any(bool(torch.count_nonzero(value)) for value in lora_b): + raise ValueError("Eight-step student checkpoint has no trained nonzero LoRA-B tensor.") + + return { + "path": checkpoint_path, + "sha256": sha256_file(checkpoint_path), + "step": step, + "config": dict(config), + "student_lora": dict(state), + } + + +def install_8step_student_lora( + transformer: torch.nn.Module, + checkpoint: Mapping[str, Any], +) -> dict[str, Any]: + """Install the generator LoRA and verify 312 FP32 runtime branches.""" + + if getattr(getattr(transformer, "config", None), "adaln_curve_rank", None) is not None: + raise RuntimeError("Eight-step inference requires the standard full-AdaLN H3 backbone.") + metadata = install_student_lora(transformer, checkpoint) + adapter_parameters = [ + (name, parameter) + for name, parameter in transformer.named_parameters() + if "lora_" in name and f".{STUDENT_ADAPTER}." in name + ] + lora_a_modules = { + name.split(".lora_A.", 1)[0] + for name, _ in adapter_parameters + if ".lora_A." in name + } + lora_b_modules = { + name.split(".lora_B.", 1)[0] + for name, _ in adapter_parameters + if ".lora_B." in name + } + if ( + len(lora_a_modules) != H3_DMD_TARGET_MODULE_COUNT + or lora_a_modules != lora_b_modules + ): + raise RuntimeError( + "Installed student adapter does not expose 312 matched LoRA A/B modules." + ) + runtime_dtypes = {parameter.dtype for _, parameter in adapter_parameters} + if runtime_dtypes != {torch.float32}: + raise RuntimeError( + "Installed student adapter did not retain FP32 LoRA weights: " + f"{sorted(map(str, runtime_dtypes))}." + ) + return { + **metadata, + "lora_target_module_count": H3_DMD_TARGET_MODULE_COUNT, + "lora_tensor_count": 2 * H3_DMD_TARGET_MODULE_COUNT, + "lora_master_dtype": "float32", + "runtime_lora_dtype": "float32", + "rollout_mode": ROLLOUT_MODE, + } + + +def _assert_fresh_noise_pipeline(pipe: Any) -> None: + if pipe.__class__.__name__ != "MiniMaxH3ModularPipeline": + raise RuntimeError(f"Expected MiniMaxH3ModularPipeline, found {pipe.__class__.__name__}.") + if not isinstance(pipe.scheduler, FreshNoiseMiniMaxH3Scheduler) or not isinstance( + pipe.audio_scheduler, FreshNoiseMiniMaxH3Scheduler + ): + raise RuntimeError("Eight-step pipeline is missing its fresh-noise scheduler pair.") + if (pipe.scheduler.shift, pipe.audio_scheduler.shift) != (12.0, 3.0): + raise RuntimeError("Eight-step fresh-noise scheduler shifts are not video=12/audio=3.") + if getattr(pipe, "transformer_ref", None) is not None: + raise RuntimeError("Eight-step student inference is Base T2VA, not Ref2VA.") + forbidden = {"guidance_scale", "negative_prompt", "guidance_rescale", "guider"} + exposed = forbidden.intersection(getattr(pipe, "default_call_parameters", {})) + if exposed: + raise RuntimeError(f"Eight-step T2VA unexpectedly exposes CFG inputs: {sorted(exposed)}") + + +def _text_encoder_forward_module(pipe: Any) -> torch.nn.Module: + text_encoder = getattr(pipe, "text_encoder", None) + target = getattr(text_encoder, "model", None) + if target is None: + target = text_encoder + if not isinstance(target, torch.nn.Module): + raise RuntimeError("Unable to install a forward audit on the H3 Qwen3-VL conditioner.") + return target + + +@contextmanager +def _text_encoder_forward_audit( + pipe: Any, *, forbid: bool +) -> Iterator[dict[str, int]]: + """Count Qwen model forwards and optionally make any call fail closed.""" + + audit = {"forward_calls": 0} + + def observe(_module: Any, _inputs: Any) -> None: + audit["forward_calls"] += 1 + if forbid: + raise RuntimeError( + "Cached-conditioning inference attempted to invoke the Qwen3-VL text encoder." + ) + + handle = _text_encoder_forward_module(pipe).register_forward_pre_hook(observe) + try: + yield audit + finally: + handle.remove() + + +@contextmanager +def _cached_text_encoder_bypass(pipe: Any) -> Iterator[dict[str, int]]: + """Temporarily remove the encoder block so cached tensors are real inputs. + + Merely passing ``prompt_embeds`` to the stock MiniMax-H3 modular pipeline is + insufficient: while its text-encoder block is present, those names are + intermediate outputs and unexpected top-level kwargs are ignored. Removing + exactly that block makes the downstream denoise block expose both tensors as + pipeline inputs. The original ordered block graph is restored in ``finally``. + """ + + blocks = getattr(getattr(pipe, "_blocks", None), "sub_blocks", None) + if blocks is None or "text_encoder" not in blocks: + raise RuntimeError("Official H3 block graph has no removable text_encoder block.") + original_items = list(blocks.items()) + blocks.pop("text_encoder") + try: + input_names = set(getattr(pipe._blocks, "input_names", ())) + required = {"prompt_embeds", "text_token_tags"} + if not required.issubset(input_names) or "prompt" in input_names: + raise RuntimeError( + "Removing the H3 text encoder did not expose the exact cached-conditioning inputs." + ) + with _text_encoder_forward_audit(pipe, forbid=True) as audit: + yield audit + finally: + blocks.clear() + blocks.update(original_items) + + +def _cached_conditioning_call_kwargs( + pipe: Any, condition: CachedConditioningSample +) -> dict[str, torch.Tensor]: + """Build a prompt-free call payload from one validated cache snapshot.""" + + execution_device = getattr(pipe, "_execution_device", None) + if execution_device is None: + raise RuntimeError("Official H3 pipeline has no resolved execution device.") + return { + # Qwen's official encoder step emits this tensor on the execution + # device. Preserve BF16 exactly while reproducing that placement. + "prompt_embeds": condition.prompt_embeds.to( + device=execution_device, dtype=torch.bfloat16 + ), + # The official layout builder consumes tags on CPU and constructs the + # packed structural tensors there, so retain the cached CPU int64 tags. + "text_token_tags": condition.text_token_tags, + } + + +def run_student_batch_8step( + pipe: Any, + prompts: Sequence[Any], + config: RunConfig, + checkpoint: Mapping[str, Any], + output_dir: str | Path, + *, + adapter_metadata: Mapping[str, Any] | None = None, + cached_conditioning: CachedConditioningBundle | None = None, +) -> dict[str, Any]: + """Generate and audit one independently seeded fresh-noise request per prompt.""" + + if not prompts: + raise ValueError("Eight-step visualization requires at least one prompt.") + if config.num_inference_steps != NUM_SIGMA_ENDPOINTS: + raise ValueError("Eight-step H3 must request nine endpoints/eight denoiser evaluations.") + if cached_conditioning is not None and len(cached_conditioning.conditions) != len(prompts): + raise ValueError("Cached conditioning count differs from the prompt count.") + _assert_fresh_noise_pipeline(pipe) + if adapter_metadata is None: + adapter = install_8step_student_lora(pipe.transformer, checkpoint) + else: + adapter = dict(adapter_metadata) + if tuple(pipe.transformer.active_adapters()) != (STUDENT_ADAPTER,): + raise RuntimeError("Pre-offload setup did not leave only student LoRA active.") + if adapter.get("checkpoint_sha256") != checkpoint["sha256"]: + raise RuntimeError("Pre-offload student metadata/checkpoint hash mismatch.") + + destination = Path(output_dir).resolve() + destination.mkdir(parents=True, exist_ok=True) + samples = [] + for index, prompt in enumerate(prompts): + sample_dir = destination / f"{index:02d}_{prompt.id}" + sample_dir.mkdir(parents=True, exist_ok=True) + generator = torch.Generator().manual_seed(prompt.seed) + draw_audit = bind_pipeline_fresh_noise_generator(pipe, generator) + transformer_calls = 0 + cached_sample = ( + cached_conditioning.conditions[index] + if cached_conditioning is not None + else None + ) + if cached_sample is not None and ( + cached_sample.id, + cached_sample.prompt, + cached_sample.evaluation_seed, + ) != (prompt.id, prompt.prompt, prompt.seed): + raise RuntimeError("Cached conditioning row no longer matches its prompt request.") + + def count_transformer_forward(_module: Any, _inputs: Any) -> None: + nonlocal transformer_calls + transformer_calls += 1 + + hook = pipe.transformer.register_forward_pre_hook(count_transformer_forward) + if cached_sample is None: + conditioning_kwargs: dict[str, Any] = {"prompt": prompt.prompt} + conditioning_context = _text_encoder_forward_audit(pipe, forbid=False) + else: + conditioning_kwargs = _cached_conditioning_call_kwargs(pipe, cached_sample) + conditioning_context = _cached_text_encoder_bypass(pipe) + try: + with conditioning_context as text_encoder_audit: + results = pipe( + **conditioning_kwargs, + height=config.height, + width=config.width, + num_frames=config.num_frames, + num_inference_steps=NUM_SIGMA_ENDPOINTS, + generator=generator, + output_type="pt", + output=["videos", "audio", "sampling_rate", "latents", "audio_latents"], + ) + finally: + hook.remove() + expected_text_encoder_calls = 1 if cached_sample is None else 0 + if text_encoder_audit["forward_calls"] != expected_text_encoder_calls: + raise RuntimeError( + "H3 text-encoder execution count differs from the conditioning mode: " + f"observed={text_encoder_audit['forward_calls']}, " + f"expected={expected_text_encoder_calls}." + ) + if transformer_calls != NUM_MODEL_EVALUATIONS: + raise RuntimeError( + "Official H3 loop did not execute exactly eight positive forwards: " + f"observed={transformer_calls}." + ) + sampler_audit = validate_completed_pipeline_rollout(pipe, draw_audit) + sampler_audit["transformer_positive_forwards"] = transformer_calls + + if cached_sample is None: + conditioning_audit = { + "mode": "independent_direct_text", + "cached_prompt_embeds_used_by_inference": False, + "prompt_argument_supplied": True, + "prompt_embeds_argument_supplied": False, + "text_token_tags_argument_supplied": False, + "text_encoder_block_removed": False, + "text_encoder_execution_forbidden": False, + "text_encoder_forward_calls": text_encoder_audit["forward_calls"], + } + else: + passed_embeds = _tensor_descriptor(conditioning_kwargs["prompt_embeds"]) + passed_tags = _tensor_descriptor(conditioning_kwargs["text_token_tags"]) + if ( + passed_embeds != cached_sample.prompt_embeds_descriptor + or passed_tags != cached_sample.text_token_tags_descriptor + ): + raise RuntimeError("Tensors passed to H3 differ from the validated cache snapshot.") + conditioning_audit = { + "mode": INLINE_CONDITIONING_MODE, + "selection_policy": INLINE_SELECTION_POLICY, + "cached_prompt_embeds_used_by_inference": True, + "prompt_argument_supplied": False, + "prompt_embeds_argument_supplied": True, + "text_token_tags_argument_supplied": True, + "text_encoder_block_removed": True, + "text_encoder_execution_forbidden": True, + "text_encoder_forward_calls": text_encoder_audit["forward_calls"], + "payload_sha256": cached_conditioning.payload_sha256, + "provenance_sha256": cached_conditioning.provenance_sha256, + "cache_rank": cached_sample.cache_rank, + "cache_condition_position": cached_sample.cache_condition_position, + "cache_global_step": cached_sample.cache_global_step, + "source_index": cached_sample.source_index, + "source_line": cached_sample.source_line, + "prompt_sha256": cached_sample.prompt_sha256, + "passed_prompt_embeds": passed_embeds, + "passed_text_token_tags": passed_tags, + } + + if not isinstance(results, Mapping): + raise TypeError("Official H3 pipeline did not return a mapping.") + required = ("videos", "audio", "sampling_rate", "latents", "audio_latents") + missing = [name for name in required if results.get(name) is None] + if missing: + raise RuntimeError(f"Eight-step H3 pipeline output is missing {missing}.") + + video_path = sample_dir / "student.mp4" + latent_path = sample_dir / "student_latents.pt" + frames = _prepare_video_for_encoder(results["videos"]) + sampling_rate = int(results["sampling_rate"]) + _default_video_encoder( + video=frames, + fps=FPS, + output_path=str(video_path), + audio=results["audio"][0].detach().cpu(), + audio_sample_rate=sampling_rate, + ) + torch.save( + { + "video_latents": results["latents"].detach().cpu().contiguous(), + "audio_latents": results["audio_latents"].detach().cpu().contiguous(), + }, + latent_path, + ) + metadata = { + "schema_version": 1, + "sample": asdict(prompt), + "request": { + "workflow": WORKFLOW, + "height": config.height, + "width": config.width, + "num_frames": config.num_frames, + "fps": FPS, + "rollout_mode": ROLLOUT_MODE, + "attention_mode": "native_non_causal", + "model_evaluations": NUM_MODEL_EVALUATIONS, + "sigma_endpoints": NUM_SIGMA_ENDPOINTS, + "fresh_noise_transitions": NUM_FRESH_NOISE_TRANSITIONS, + "uses_cfg": False, + "uses_context_ir": False, + "uses_regenerator": False, + }, + "sampler_audit": sampler_audit, + "conditioning_audit": conditioning_audit, + "model": { + "id": config.model_id, + "resolved_revision": config.resolved_revision, + "transformer_block_count": RELEASED_MAIN_BLOCKS, + "token_refiner_block_count": RELEASED_REFINER_BLOCKS, + }, + "student": adapter, + "numeric_outputs": { + "decoded_video": _tensor_descriptor(results["videos"]), + "decoded_audio": _tensor_descriptor(results["audio"]), + "video_latents": _tensor_descriptor(results["latents"]), + "audio_latents": _tensor_descriptor(results["audio_latents"]), + "sampling_rate": sampling_rate, + }, + "artifacts": { + "video": {"path": video_path.name, "sha256": sha256_file(video_path)}, + "latents": {"path": latent_path.name, "sha256": sha256_file(latent_path)}, + }, + } + metadata_path = sample_dir / "metadata.json" + _atomic_json(metadata_path, metadata) + samples.append( + { + "id": prompt.id, + "prompt": prompt.prompt, + "seed": prompt.seed, + "video": str(video_path.relative_to(destination)), + "video_sha256": metadata["artifacts"]["video"]["sha256"], + "metadata": str(metadata_path.relative_to(destination)), + "metadata_sha256": sha256_file(metadata_path), + "sampler_contract_verified": True, + "conditioning_mode": conditioning_audit["mode"], + "cached_prompt_embeds_used_by_inference": conditioning_audit[ + "cached_prompt_embeds_used_by_inference" + ], + "text_encoder_forward_calls": conditioning_audit[ + "text_encoder_forward_calls" + ], + } + ) + + manifest = { + "schema_version": 1, + "status": "success", + "runner": "LongLive/minimax_h3/infer_student_8step.py", + "config": asdict(config), + "contract": dict(_REQUIRED_CONTRACT), + "checkpoint": { + "path": str(checkpoint["path"]), + "sha256": checkpoint["sha256"], + "step": checkpoint["step"], + }, + "adapter": adapter, + "conditioning": ( + cached_conditioning.manifest_summary() + if cached_conditioning is not None + else { + "mode": "independent_direct_text", + "cached_prompt_embeds_used_by_inference": False, + "text_encoder_forward_calls_required_per_sample": 1, + } + ), + "samples": samples, + } + manifest_path = destination / "manifest.json" + _atomic_json(manifest_path, manifest) + _atomic_json( + destination / "SUCCESS.json", + { + "schema_version": 1, + "status": "success", + "manifest": manifest_path.name, + "manifest_sha256": sha256_file(manifest_path), + "samples": len(samples), + "fresh_noise_sampler_verified": True, + "conditioning_mode": ( + INLINE_CONDITIONING_MODE + if cached_conditioning is not None + else "independent_direct_text" + ), + "cached_prompt_embeds_used_by_inference": cached_conditioning is not None, + }, + ) + return manifest + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--checkpoint", type=Path, required=True) + parser.add_argument("--prompts", type=Path, default=Path(__file__).with_name("prompts.json")) + parser.add_argument("--expected-prompt-count", type=int, default=4) + parser.add_argument( + "--cached-conditioning", + type=Path, + help="Tensor payload emitted by extract_dmd8_inline5 (inline5 only).", + ) + parser.add_argument( + "--cached-conditioning-provenance", + type=Path, + help="JSON provenance emitted alongside --cached-conditioning.", + ) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--model-id", "--model-dir", dest="model_id", default=MODEL_ID) + parser.add_argument("--revision", default="main") + parser.add_argument("--height", type=int, default=768) + parser.add_argument("--width", type=int, default=1344) + parser.add_argument("--num-frames", type=int, default=124) + parser.add_argument("--device", default="cuda") + parser.add_argument("--memory-reserve-margin", default="12GB") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--validate-only", action="store_true") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.expected_prompt_count < 1: + raise ValueError("--expected-prompt-count must be positive.") + prompts = load_prompt_specs(args.prompts, expected_count=args.expected_prompt_count) + checkpoint = load_8step_student_checkpoint(args.checkpoint) + if (args.cached_conditioning is None) != ( + args.cached_conditioning_provenance is None + ): + raise ValueError( + "--cached-conditioning and --cached-conditioning-provenance must be supplied together." + ) + revision = resolve_hub_revision( + args.model_id, args.revision, local_files_only=args.local_files_only + ) + checkpoint_revision = checkpoint["config"].get("resolved_revision") + if checkpoint_revision != revision: + raise ValueError( + "Student checkpoint/model revision mismatch: " + f"{checkpoint_revision!r} != {revision!r}." + ) + cached_conditioning = None + if args.cached_conditioning is not None: + cached_conditioning = load_cached_conditioning( + args.cached_conditioning, + args.cached_conditioning_provenance, + prompts, + expected_model_revision=revision, + expected_checkpoint_sha256=checkpoint["sha256"], + ) + config = RunConfig( + model_id=args.model_id, + requested_revision=args.revision, + resolved_revision=revision, + height=args.height, + width=args.width, + num_frames=args.num_frames, + num_inference_steps=NUM_SIGMA_ENDPOINTS, + device=args.device, + memory_reserve_margin=args.memory_reserve_margin, + local_files_only=args.local_files_only, + ) + config.validate() + if args.validate_only: + print( + json.dumps( + { + "checkpoint": { + "path": str(checkpoint["path"]), + "sha256": checkpoint["sha256"], + "step": checkpoint["step"], + }, + "contract": _REQUIRED_CONTRACT, + "config": asdict(config), + "prompts": [asdict(prompt) for prompt in prompts], + "conditioning": ( + cached_conditioning.manifest_summary() + if cached_conditioning is not None + else { + "mode": "independent_direct_text", + "cached_prompt_embeds_used_by_inference": False, + } + ), + }, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + ) + return 0 + + adapter_metadata: dict[str, Any] = {} + + def setup_student(transformer: torch.nn.Module) -> None: + adapter_metadata.update(install_8step_student_lora(transformer, checkpoint)) + + pipe = build_official_pipeline(config, transformer_setup=setup_student) + # build_official_pipeline has already fail-closed validated the untouched + # official components, including their pinned revision and 12/3 shifts. + assert_official_t2va_contract(pipe, expected_revision=revision) + install_fresh_noise_schedulers(pipe) + manifest = run_student_batch_8step( + pipe, + prompts, + config, + checkpoint, + args.output_dir, + adapter_metadata=adapter_metadata, + cached_conditioning=cached_conditioning, + ) + print(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + + +__all__ = [ + "CachedConditioningBundle", + "CachedConditioningSample", + "LORA_ALPHA", + "LORA_RANK", + "ROLLOUT_MODE", + "build_parser", + "install_8step_student_lora", + "load_8step_student_checkpoint", + "load_cached_conditioning", + "main", + "run_student_batch_8step", +] diff --git a/source_snapshot/minimax_h3/longlive_lora.py b/source_snapshot/minimax_h3/longlive_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..b47d9553395d5c3223b6145d2bbaa866cc095647 --- /dev/null +++ b/source_snapshot/minimax_h3/longlive_lora.py @@ -0,0 +1,159 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""LongLive-Wan-style LoRA selection for MiniMax-H3. + +LongLive-Wan selects every ``nn.Linear`` below each main Wan transformer +block. MiniMax-H3 has a separate text token-refiner stack and a learned +timestep/AdaLN projection inside each main block, so the architecture-faithful +mapping is deliberately narrower: every Linear below the ``attn`` and ``ff`` +branches of each main ``MiniMaxH3TransformerBlock`` only. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import torch + + +LONGLIVE_H3_LORA_POLICY = "longlive_wan_block_wide_main_attention_ffn_v1" +LONGLIVE_H3_MAIN_BLOCK_COUNT = 50 +LONGLIVE_H3_TARGETS_PER_BLOCK = 6 +LONGLIVE_H3_TARGET_MODULE_COUNT = ( + LONGLIVE_H3_MAIN_BLOCK_COUNT * LONGLIVE_H3_TARGETS_PER_BLOCK +) +LONGLIVE_H3_RELATIVE_TARGETS = ( + "attn.to_q", + "attn.to_k", + "attn.to_v", + "attn.to_out.0", + "ff.net.0.proj", + "ff.net.2", +) + + +def released_longlive_h3_target_modules() -> list[str]: + """Return the exact 300 PEFT target names for the released 50-block H3.""" + + return sorted( + [ + f"transformer_blocks.{block_index}.{relative_name}" + for block_index in range(LONGLIVE_H3_MAIN_BLOCK_COUNT) + for relative_name in LONGLIVE_H3_RELATIVE_TARGETS + ] + ) + + +def collect_longlive_h3_target_modules( + model: torch.nn.Module, + *, + expected_main_blocks: int | None = LONGLIVE_H3_MAIN_BLOCK_COUNT, +) -> list[str]: + """Collect all attention/FFN Linear modules in main H3 blocks. + + Full names are returned intentionally. Suffix-only PEFT matching would + also modify the two text token-refiner blocks, which LongLive-Wan's + main-block selector does not do. + """ + + try: + from diffusers.models.transformers.transformer_minimax_h3 import ( + MiniMaxH3TransformerBlock, + ) + except ImportError as error: + raise RuntimeError( + "The editable diffusers_h3 checkout is required for H3 LoRA selection." + ) from error + + blocks = [ + (name, module) + for name, module in model.named_modules() + if name and isinstance(module, MiniMaxH3TransformerBlock) + ] + if expected_main_blocks is not None and len(blocks) != expected_main_blocks: + raise RuntimeError( + "MiniMax-H3 main-block count changed: " + f"observed={len(blocks)}, expected={expected_main_blocks}." + ) + if not blocks: + raise RuntimeError("No MiniMaxH3TransformerBlock modules were found.") + + expected_relative = set(LONGLIVE_H3_RELATIVE_TARGETS) + targets: list[str] = [] + for block_name, block in blocks: + observed_relative: set[str] = set() + for branch_name in ("attn", "ff"): + branch = getattr(block, branch_name, None) + if branch is None: + raise RuntimeError(f"H3 block {block_name!r} lacks {branch_name!r}.") + for relative_name, module in branch.named_modules(): + if isinstance(module, torch.nn.Linear): + relative = ( + branch_name + if not relative_name + else f"{branch_name}.{relative_name}" + ) + observed_relative.add(relative) + targets.append(f"{block_name}.{relative}") + if observed_relative != expected_relative: + raise RuntimeError( + f"H3 block {block_name!r} Linear layout changed: " + f"observed={sorted(observed_relative)}, " + f"expected={sorted(expected_relative)}." + ) + + targets.sort() + if len(targets) != len(set(targets)): + raise RuntimeError("Duplicate H3 LoRA target module names were collected.") + return targets + + +def install_longlive_h3_lora_adapters( + model: torch.nn.Module, + *, + rank: int = 128, + alpha: int = 128, + dropout: float = 0.0, + expected_main_blocks: int | None = LONGLIVE_H3_MAIN_BLOCK_COUNT, +) -> dict[str, list[torch.nn.Parameter]]: + """Install independent student/critic adapters using the block-wide policy.""" + + from .train_utils import install_independent_lora_adapters + + targets = collect_longlive_h3_target_modules( + model, expected_main_blocks=expected_main_blocks + ) + adapters = install_independent_lora_adapters( + model, + rank=rank, + alpha=alpha, + dropout=dropout, + target_modules=targets, + ) + model._h3_lora_policy = LONGLIVE_H3_LORA_POLICY + model._h3_lora_target_modules = tuple(targets) + return adapters + + +def validate_longlive_h3_checkpoint_targets(targets: Sequence[str]) -> None: + """Fail closed unless checkpoint metadata names the exact released targets.""" + + expected = released_longlive_h3_target_modules() + if list(targets) != expected: + raise ValueError( + "Checkpoint LongLive-style H3 LoRA targets do not match the released " + f"300-module contract: observed={len(targets)}, expected={len(expected)}." + ) + + +__all__ = [ + "LONGLIVE_H3_LORA_POLICY", + "LONGLIVE_H3_MAIN_BLOCK_COUNT", + "LONGLIVE_H3_RELATIVE_TARGETS", + "LONGLIVE_H3_TARGET_MODULE_COUNT", + "collect_longlive_h3_target_modules", + "install_longlive_h3_lora_adapters", + "released_longlive_h3_target_modules", + "validate_longlive_h3_checkpoint_targets", +] diff --git a/source_snapshot/minimax_h3/longlive_prompt_distributed.py b/source_snapshot/minimax_h3/longlive_prompt_distributed.py new file mode 100644 index 0000000000000000000000000000000000000000..88eacb42a9efa298e4ab13102de0338d5c6c70d0 --- /dev/null +++ b/source_snapshot/minimax_h3/longlive_prompt_distributed.py @@ -0,0 +1,699 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Pinned, distributed LongLive prompt schedule for MiniMax-H3 v4. + +The v4 dataset is the complete, pinned LongLive-1.3B text file. Rows are not +deduplicated, sorted, or truncated. Rank assignment is delegated to PyTorch's +``DistributedSampler`` with the same arguments used by LongLive, so this file +also serves as an executable definition of the sampling contract. + +Only the first ``global_steps`` rows of each rank's sampler stream need to be +encoded for a fixed-length run. This experiment snapshot binds that prefix to +2,000 outer steps. The rows are stored in one cache file per rank; there is +deliberately no merged multi-rank cache. +""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping, Sequence + +import torch +from torch.utils.data.distributed import DistributedSampler + + +SOURCE_REPO_ID = "Efficient-Large-Model/LongLive-1.3B" +SOURCE_REVISION = "cda9138d93872ef109d9a74c270184eb3a5acc6e" +SOURCE_PATH = "prompts/vidprom_filtered_extended.txt" +SOURCE_BYTES = 145_875_068 +SOURCE_SHA256 = "7896742f468bc8aef9e4547424d1ce0a951acdb2a82233790155401a99bf5aa5" +SOURCE_LINE_COUNT = 248_221 + +FORMAL_WORLD_SIZE = 32 +FORMAL_GLOBAL_STEPS = 2_000 +FORMAL_SEED = 20_260_806 +FORMAL_EPOCH = 0 +TEXT_TOKEN_TAG = 1 + +CACHE_SCHEMA_VERSION = 1 +CACHE_ARTIFACT_TYPE = "minimax_h3_longlive_distributed_rank_prompt_cache" +COLLECTION_SCHEMA_VERSION = 1 +COLLECTION_ARTIFACT_TYPE = "minimax_h3_longlive_distributed_prompt_cache_collection" +SAMPLER_NAME = "torch.utils.data.distributed.DistributedSampler" + +_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$") +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_PROMPT_ID_RE = re.compile(r"^vidprom-([0-9]{6})-([0-9a-f]{12})$") +_PAYLOAD_FIELDS = { + "schema_version", + "artifact_type", + "workflow", + "cfg_enabled", + "ir", + "regenerator", + "model_revision", + "dataset", + "sampler", + "conditions", +} +_DATASET_FIELDS = { + "repo_id", + "revision", + "path", + "bytes", + "sha256", + "line_count", + "row_policy", +} +_SAMPLER_FIELDS = { + "name", + "dataset_size", + "world_size", + "rank", + "seed", + "epoch", + "shuffle", + "drop_last", + "batch_size_per_rank", + "global_steps", +} +_CONDITION_FIELDS = { + "global_step", + "id", + "source_index", + "source_line", + "prompt", + "prompt_sha256", + "prompt_embeds", + "text_token_tags", +} +_COLLECTION_FIELDS = { + "schema_version", + "artifact_type", + "status", + "created_at", + "storage", + "workflow", + "cfg_enabled", + "ir", + "regenerator", + "text_only", + "model_revision", + "dataset", + "sampler", + "schedule_audit", + "shards", +} +_COLLECTION_SAMPLER_FIELDS = _SAMPLER_FIELDS - {"rank"} +_SCHEDULE_AUDIT_FIELDS = { + "dataset_size", + "world_size", + "global_steps", + "scheduled_rows", + "unique_source_indices", +} +_SHARD_FIELDS = {"rank", "path", "bytes", "sha256", "conditions"} + + +@dataclass(frozen=True) +class PromptRow: + """One source row, preserving dataset position and duplicate text.""" + + source_index: int + source_line: int + id: str + prompt: str + prompt_sha256: str + + +@dataclass(frozen=True) +class RankPromptCondition: + """One validated text-only condition in a rank-local training schedule.""" + + global_step: int + id: str + source_index: int + source_line: int + prompt: str + prompt_sha256: str + prompt_embeds: torch.Tensor + text_token_tags: torch.Tensor + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def sha256_file(path: str | Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as source: + while chunk := source.read(16 * 1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _source_descriptor() -> dict[str, Any]: + return { + "repo_id": SOURCE_REPO_ID, + "revision": SOURCE_REVISION, + "path": SOURCE_PATH, + "bytes": SOURCE_BYTES, + "sha256": SOURCE_SHA256, + "line_count": SOURCE_LINE_COUNT, + # This is exactly MultiTextConcatDataset's txt-mode normalization. + "row_policy": "nonempty_python_str_strip_then_python_str_rstrip", + } + + +def source_descriptor() -> dict[str, Any]: + """Return a copy of the pinned full-dataset descriptor.""" + + return _source_descriptor() + + +def _prompt_id(source_line: int, prompt_sha256: str) -> str: + return f"vidprom-{source_line:06d}-{prompt_sha256[:12]}" + + +def read_pinned_dataset(path: str | Path) -> list[PromptRow]: + """Read all pinned rows with LongLive's txt-dataset normalization. + + The pinned object has no blank rows, but the explicit blank-row check makes + drift visible instead of silently changing source indices. Exact duplicate + prompts remain separate rows. + """ + + source = Path(path) + if source.is_symlink() or not source.is_file(): + raise ValueError(f"raw prompt source must be a non-symlink regular file: {source}") + raw = source.read_bytes() + if len(raw) != SOURCE_BYTES: + raise ValueError(f"raw prompt byte count mismatch: {len(raw)} != {SOURCE_BYTES}") + observed_sha = sha256_bytes(raw) + if observed_sha != SOURCE_SHA256: + raise ValueError(f"raw prompt SHA-256 mismatch: {observed_sha} != {SOURCE_SHA256}") + if raw.startswith(b"\xef\xbb\xbf"): + raise ValueError("raw prompt source must not contain a UTF-8 BOM") + if b"\x00" in raw: + raise ValueError("raw prompt source contains a NUL byte") + if b"\r" in raw or not raw.endswith(b"\n"): + raise ValueError("raw prompt source must use LF lines and end with LF") + try: + lines = raw.decode("utf-8", errors="strict").splitlines() + except UnicodeDecodeError as error: + raise ValueError("raw prompt source is not strict UTF-8") from error + if len(lines) != SOURCE_LINE_COUNT: + raise ValueError( + f"raw prompt line count mismatch: {len(lines)} != {SOURCE_LINE_COUNT}" + ) + + rows: list[PromptRow] = [] + for source_index, raw_prompt in enumerate(lines): + # LongLive uses: [line.rstrip() for line in f if line.strip()]. + if not raw_prompt.strip(): + raise ValueError( + f"pinned dataset unexpectedly contains a blank row at line {source_index + 1}" + ) + prompt = raw_prompt.rstrip() + prompt_sha = sha256_bytes(prompt.encode("utf-8")) + source_line = source_index + 1 + rows.append( + PromptRow( + source_index=source_index, + source_line=source_line, + id=_prompt_id(source_line, prompt_sha), + prompt=prompt, + prompt_sha256=prompt_sha, + ) + ) + return rows + + +def distributed_indices( + dataset_size: int, + *, + rank: int, + world_size: int, + seed: int, + epoch: int, + global_steps: int, +) -> list[int]: + """Return a rank stream by directly instantiating ``DistributedSampler``.""" + + for label, value in ( + ("dataset_size", dataset_size), + ("world_size", world_size), + ("global_steps", global_steps), + ): + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ValueError(f"{label} must be a positive integer") + if isinstance(rank, bool) or not isinstance(rank, int) or not 0 <= rank < world_size: + raise ValueError("rank must be an integer in [0, world_size)") + if isinstance(seed, bool) or not isinstance(seed, int) or seed < 0: + raise ValueError("seed must be a non-negative integer") + if isinstance(epoch, bool) or not isinstance(epoch, int) or epoch < 0: + raise ValueError("epoch must be a non-negative integer") + + sampler = DistributedSampler( + range(dataset_size), + num_replicas=world_size, + rank=rank, + shuffle=True, + seed=seed, + drop_last=True, + ) + sampler.set_epoch(epoch) + if len(sampler) < global_steps: + raise ValueError( + f"rank sampler has only {len(sampler)} rows, fewer than {global_steps} steps" + ) + return list(iter(sampler))[:global_steps] + + +def build_rank_schedule( + rows: Sequence[PromptRow], + *, + rank: int, + world_size: int = FORMAL_WORLD_SIZE, + seed: int = FORMAL_SEED, + epoch: int = FORMAL_EPOCH, + global_steps: int = FORMAL_GLOBAL_STEPS, +) -> list[PromptRow]: + indices = distributed_indices( + len(rows), + rank=rank, + world_size=world_size, + seed=seed, + epoch=epoch, + global_steps=global_steps, + ) + return [rows[index] for index in indices] + + +def validate_cross_rank_schedule( + *, + dataset_size: int = SOURCE_LINE_COUNT, + world_size: int = FORMAL_WORLD_SIZE, + seed: int = FORMAL_SEED, + epoch: int = FORMAL_EPOCH, + global_steps: int = FORMAL_GLOBAL_STEPS, +) -> dict[str, int]: + """Prove that every global step uses a distinct source row on every rank.""" + + by_rank = [ + distributed_indices( + dataset_size, + rank=rank, + world_size=world_size, + seed=seed, + epoch=epoch, + global_steps=global_steps, + ) + for rank in range(world_size) + ] + for position in range(global_steps): + step_indices = [by_rank[rank][position] for rank in range(world_size)] + if len(set(step_indices)) != world_size: + raise ValueError(f"source-index collision across ranks at global step {position + 1}") + flattened = [index for rank_rows in by_rank for index in rank_rows] + if len(set(flattened)) != len(flattened): + raise ValueError("source-index reuse found inside the scheduled prefix") + return { + "dataset_size": dataset_size, + "world_size": world_size, + "global_steps": global_steps, + "scheduled_rows": len(flattened), + "unique_source_indices": len(set(flattened)), + } + + +def sampler_descriptor( + *, rank: int, world_size: int, seed: int, epoch: int, global_steps: int +) -> dict[str, Any]: + return { + "name": SAMPLER_NAME, + "dataset_size": SOURCE_LINE_COUNT, + "world_size": world_size, + "rank": rank, + "seed": seed, + "epoch": epoch, + "shuffle": True, + "drop_last": True, + "batch_size_per_rank": 1, + "global_steps": global_steps, + } + + +def collection_sampler_descriptor( + *, world_size: int, seed: int, epoch: int, global_steps: int +) -> dict[str, Any]: + descriptor = sampler_descriptor( + rank=0, + world_size=world_size, + seed=seed, + epoch=epoch, + global_steps=global_steps, + ) + del descriptor["rank"] + return descriptor + + +def make_rank_cache_payload( + *, + rank: int, + model_revision: str, + conditions: list[Mapping[str, Any]], + world_size: int = FORMAL_WORLD_SIZE, + seed: int = FORMAL_SEED, + epoch: int = FORMAL_EPOCH, + global_steps: int = FORMAL_GLOBAL_STEPS, +) -> dict[str, Any]: + return { + "schema_version": CACHE_SCHEMA_VERSION, + "artifact_type": CACHE_ARTIFACT_TYPE, + "workflow": "t2va", + "cfg_enabled": False, + "ir": False, + "regenerator": False, + "model_revision": model_revision, + "dataset": _source_descriptor(), + "sampler": sampler_descriptor( + rank=rank, + world_size=world_size, + seed=seed, + epoch=epoch, + global_steps=global_steps, + ), + "conditions": conditions, + } + + +def _require_exact_fields(value: Mapping[str, Any], expected: set[str], label: str) -> None: + actual = set(value) + if actual != expected: + raise ValueError( + f"{label} fields mismatch: missing={sorted(expected - actual)}, " + f"extra={sorted(actual - expected)}" + ) + + +def load_rank_prompt_cache( + path: str | Path, + *, + expected_rank: int | None = None, + expected_world_size: int = FORMAL_WORLD_SIZE, + expected_seed: int = FORMAL_SEED, + expected_epoch: int = FORMAL_EPOCH, + expected_global_steps: int = FORMAL_GLOBAL_STEPS, +) -> tuple[dict[str, Any], list[RankPromptCondition]]: + """Strictly load one rank-local positive, text-only prompt cache.""" + + source = Path(path) + if source.is_symlink() or not source.is_file(): + raise ValueError(f"rank prompt cache must be a non-symlink regular file: {source}") + try: + payload = torch.load(source, map_location="cpu", weights_only=True) + except Exception as error: + raise ValueError(f"unable to load rank prompt cache: {source}") from error + if not isinstance(payload, Mapping): + raise ValueError("rank prompt cache must contain a mapping") + _require_exact_fields(payload, _PAYLOAD_FIELDS, "rank prompt cache") + if payload["schema_version"] != CACHE_SCHEMA_VERSION: + raise ValueError("rank prompt cache schema_version mismatch") + if payload["artifact_type"] != CACHE_ARTIFACT_TYPE: + raise ValueError("rank prompt cache artifact_type mismatch") + if payload["workflow"] != "t2va": + raise ValueError("rank prompt cache workflow must be t2va") + for field in ("cfg_enabled", "ir", "regenerator"): + if payload[field] is not False: + raise ValueError(f"rank prompt cache {field} must be false") + revision = payload["model_revision"] + if not isinstance(revision, str) or not _COMMIT_RE.fullmatch(revision): + raise ValueError("rank prompt cache model_revision must be a commit SHA") + + dataset = payload["dataset"] + if not isinstance(dataset, Mapping): + raise ValueError("rank prompt cache dataset must be a mapping") + _require_exact_fields(dataset, _DATASET_FIELDS, "rank prompt cache dataset") + if dict(dataset) != _source_descriptor(): + raise ValueError("rank prompt cache does not bind the pinned complete dataset") + + sampler = payload["sampler"] + if not isinstance(sampler, Mapping): + raise ValueError("rank prompt cache sampler must be a mapping") + _require_exact_fields(sampler, _SAMPLER_FIELDS, "rank prompt cache sampler") + rank = sampler["rank"] + if expected_rank is not None and rank != expected_rank: + raise ValueError(f"rank prompt cache rank mismatch: {rank} != {expected_rank}") + expected_sampler = sampler_descriptor( + rank=rank, + world_size=expected_world_size, + seed=expected_seed, + epoch=expected_epoch, + global_steps=expected_global_steps, + ) + if dict(sampler) != expected_sampler: + raise ValueError("rank prompt cache sampler contract mismatch") + expected_indices = distributed_indices( + SOURCE_LINE_COUNT, + rank=rank, + world_size=expected_world_size, + seed=expected_seed, + epoch=expected_epoch, + global_steps=expected_global_steps, + ) + + raw_conditions = payload["conditions"] + if not isinstance(raw_conditions, list) or len(raw_conditions) != expected_global_steps: + raise ValueError( + f"rank prompt cache must contain exactly {expected_global_steps} conditions" + ) + conditions: list[RankPromptCondition] = [] + seen_indices: set[int] = set() + for position, raw_condition in enumerate(raw_conditions): + label = f"conditions[{position}]" + if not isinstance(raw_condition, Mapping): + raise ValueError(f"{label} must be a mapping") + _require_exact_fields(raw_condition, _CONDITION_FIELDS, label) + global_step = raw_condition["global_step"] + source_index = raw_condition["source_index"] + source_line = raw_condition["source_line"] + prompt_id = raw_condition["id"] + prompt = raw_condition["prompt"] + prompt_sha = raw_condition["prompt_sha256"] + if global_step != position + 1: + raise ValueError(f"{label}.global_step is out of order") + if source_index != expected_indices[position]: + raise ValueError(f"{label}.source_index does not match DistributedSampler") + if source_index in seen_indices: + raise ValueError(f"duplicate source_index in rank prompt cache: {source_index}") + if source_line != source_index + 1: + raise ValueError(f"{label}.source_line does not match source_index") + if not isinstance(prompt, str) or not prompt.strip() or prompt != prompt.rstrip(): + raise ValueError(f"{label}.prompt does not match LongLive rstrip normalization") + observed_prompt_sha = sha256_bytes(prompt.encode("utf-8")) + if ( + not isinstance(prompt_sha, str) + or not _SHA256_RE.fullmatch(prompt_sha) + or prompt_sha != observed_prompt_sha + ): + raise ValueError(f"{label}.prompt_sha256 mismatch") + if not isinstance(prompt_id, str) or not _PROMPT_ID_RE.fullmatch(prompt_id): + raise ValueError(f"{label}.id is malformed") + if prompt_id != _prompt_id(source_line, prompt_sha): + raise ValueError(f"{label}.id does not bind source line and prompt") + + embeds = raw_condition["prompt_embeds"] + tags = raw_condition["text_token_tags"] + if not isinstance(embeds, torch.Tensor) or not isinstance(tags, torch.Tensor): + raise ValueError(f"{label} embeddings and text tags must be tensors") + if embeds.device.type != "cpu" or tags.device.type != "cpu": + raise ValueError(f"{label} tensors must be stored on CPU") + if ( + embeds.layout != torch.strided + or embeds.ndim != 3 + or embeds.shape[0] != 1 + or embeds.shape[1] < 1 + or embeds.shape[2] < 1 + or embeds.dtype != torch.bfloat16 + or not torch.is_floating_point(embeds) + or not bool(torch.isfinite(embeds).all()) + ): + raise ValueError( + f"{label}.prompt_embeds must be finite bfloat16 [1, rows, dim]" + ) + if ( + tags.layout != torch.strided + or tags.dtype != torch.long + or tags.shape != (embeds.shape[1],) + or not bool((tags == TEXT_TOKEN_TAG).all()) + ): + raise ValueError(f"{label} must contain positive text-only conditioning tags") + seen_indices.add(source_index) + conditions.append( + RankPromptCondition( + global_step=global_step, + id=prompt_id, + source_index=source_index, + source_line=source_line, + prompt=prompt, + prompt_sha256=prompt_sha, + prompt_embeds=embeds.contiguous(), + text_token_tags=tags.contiguous(), + ) + ) + metadata = {key: value for key, value in payload.items() if key != "conditions"} + return metadata, conditions + + +def load_prompt_cache_collection( + path: str | Path, + *, + verify_shards: bool = True, + expected_world_size: int = FORMAL_WORLD_SIZE, + expected_seed: int = FORMAL_SEED, + expected_epoch: int = FORMAL_EPOCH, + expected_global_steps: int = FORMAL_GLOBAL_STEPS, +) -> dict[str, Any]: + """Validate the small manifest for a set of rank-local cache files. + + ``verify_shards=True`` performs full SHA-256 readback of every tensor file. + It does not deserialize or merge them. + """ + + source = Path(path) + if source.is_symlink() or not source.is_file(): + raise ValueError(f"prompt-cache collection must be a regular file: {source}") + try: + import json + + payload = json.loads(source.read_text(encoding="utf-8")) + except Exception as error: + raise ValueError(f"unable to read prompt-cache collection: {source}") from error + if not isinstance(payload, Mapping): + raise ValueError("prompt-cache collection must contain a mapping") + _require_exact_fields(payload, _COLLECTION_FIELDS, "prompt-cache collection") + if payload["schema_version"] != COLLECTION_SCHEMA_VERSION: + raise ValueError("prompt-cache collection schema_version mismatch") + if payload["artifact_type"] != COLLECTION_ARTIFACT_TYPE: + raise ValueError("prompt-cache collection artifact_type mismatch") + if payload["status"] != "success": + raise ValueError("prompt-cache collection is not successful") + if not isinstance(payload["created_at"], str) or not payload["created_at"]: + raise ValueError("prompt-cache collection created_at is malformed") + if payload["storage"] != "rank_local_no_merged_tensor_cache": + raise ValueError("prompt-cache collection storage policy mismatch") + if payload["workflow"] != "t2va" or payload["text_only"] is not True: + raise ValueError("prompt-cache collection must be positive text-only T2VA") + for field in ("cfg_enabled", "ir", "regenerator"): + if payload[field] is not False: + raise ValueError(f"prompt-cache collection {field} must be false") + revision = payload["model_revision"] + if not isinstance(revision, str) or not _COMMIT_RE.fullmatch(revision): + raise ValueError("prompt-cache collection model_revision must be a commit SHA") + + dataset = payload["dataset"] + if not isinstance(dataset, Mapping): + raise ValueError("prompt-cache collection dataset must be a mapping") + _require_exact_fields(dataset, _DATASET_FIELDS, "prompt-cache collection dataset") + if dict(dataset) != _source_descriptor(): + raise ValueError("prompt-cache collection does not bind the pinned complete dataset") + sampler = payload["sampler"] + if not isinstance(sampler, Mapping): + raise ValueError("prompt-cache collection sampler must be a mapping") + _require_exact_fields( + sampler, _COLLECTION_SAMPLER_FIELDS, "prompt-cache collection sampler" + ) + expected_sampler = collection_sampler_descriptor( + world_size=expected_world_size, + seed=expected_seed, + epoch=expected_epoch, + global_steps=expected_global_steps, + ) + if dict(sampler) != expected_sampler: + raise ValueError("prompt-cache collection sampler contract mismatch") + expected_audit = validate_cross_rank_schedule( + dataset_size=SOURCE_LINE_COUNT, + world_size=expected_world_size, + seed=expected_seed, + epoch=expected_epoch, + global_steps=expected_global_steps, + ) + audit = payload["schedule_audit"] + if not isinstance(audit, Mapping): + raise ValueError("prompt-cache collection schedule_audit must be a mapping") + _require_exact_fields(audit, _SCHEDULE_AUDIT_FIELDS, "schedule_audit") + if dict(audit) != expected_audit: + raise ValueError("prompt-cache collection schedule audit mismatch") + + shards = payload["shards"] + if not isinstance(shards, list) or len(shards) != expected_world_size: + raise ValueError( + f"prompt-cache collection must contain {expected_world_size} shard records" + ) + root = source.parent + for rank, shard in enumerate(shards): + label = f"shards[{rank}]" + if not isinstance(shard, Mapping): + raise ValueError(f"{label} must be a mapping") + _require_exact_fields(shard, _SHARD_FIELDS, label) + expected_name = f"rank-{rank:05d}-of-{expected_world_size:05d}.pt" + if shard["rank"] != rank or shard["path"] != expected_name: + raise ValueError(f"{label} rank/path mismatch") + if ( + isinstance(shard["bytes"], bool) + or not isinstance(shard["bytes"], int) + or shard["bytes"] < 1 + ): + raise ValueError(f"{label}.bytes must be positive") + if ( + not isinstance(shard["sha256"], str) + or not _SHA256_RE.fullmatch(shard["sha256"]) + ): + raise ValueError(f"{label}.sha256 is malformed") + if shard["conditions"] != expected_global_steps: + raise ValueError(f"{label}.conditions mismatch") + shard_path = root / expected_name + if shard_path.is_symlink() or not shard_path.is_file(): + raise ValueError(f"{label} cache file is missing or a symlink") + if shard_path.stat().st_size != shard["bytes"]: + raise ValueError(f"{label} cache byte count mismatch") + if verify_shards and sha256_file(shard_path) != shard["sha256"]: + raise ValueError(f"{label} cache SHA-256 mismatch") + return dict(payload) + + +__all__ = [ + "CACHE_ARTIFACT_TYPE", + "CACHE_SCHEMA_VERSION", + "COLLECTION_ARTIFACT_TYPE", + "COLLECTION_SCHEMA_VERSION", + "FORMAL_EPOCH", + "FORMAL_GLOBAL_STEPS", + "FORMAL_SEED", + "FORMAL_WORLD_SIZE", + "PromptRow", + "RankPromptCondition", + "SOURCE_BYTES", + "SOURCE_LINE_COUNT", + "SOURCE_PATH", + "SOURCE_REPO_ID", + "SOURCE_REVISION", + "SOURCE_SHA256", + "build_rank_schedule", + "collection_sampler_descriptor", + "distributed_indices", + "load_rank_prompt_cache", + "load_prompt_cache_collection", + "make_rank_cache_payload", + "read_pinned_dataset", + "sampler_descriptor", + "sha256_file", + "source_descriptor", + "validate_cross_rank_schedule", +] diff --git a/source_snapshot/minimax_h3/longlive_prompt_set.py b/source_snapshot/minimax_h3/longlive_prompt_set.py new file mode 100644 index 0000000000000000000000000000000000000000..3e088375f91cb47528735f597133f69503009ed5 --- /dev/null +++ b/source_snapshot/minimax_h3/longlive_prompt_set.py @@ -0,0 +1,438 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Deterministic MiniMax-H3 prompt selection from LongLive-1.3B. + +This module deliberately does not download data. It accepts the already +downloaded raw text file and fails closed unless its bytes match the pinned +Hugging Face object. Conversion is deterministic: + +``strip -> exact keep-first dedup -> SHA-256 sort -> first 1,000``. + +The prompt-cache loader is separate from the existing four-prompt smoke-test +loader. Its schema binds the fixed source and permits only positive text +conditioning for Base T2VA: CFG, Context-IR, and Regenerator are all disabled. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + +import torch + + +SOURCE_REPO_ID = "Efficient-Large-Model/LongLive-1.3B" +SOURCE_REVISION = "cda9138d93872ef109d9a74c270184eb3a5acc6e" +SOURCE_PATH = "prompts/vidprom_filtered_extended.txt" +SOURCE_BYTES = 145_875_068 +SOURCE_SHA256 = "7896742f468bc8aef9e4547424d1ce0a951acdb2a82233790155401a99bf5aa5" +SOURCE_LINE_COUNT = 248_221 +SOURCE_UNIQUE_COUNT = 248_219 +SELECTED_PROMPT_COUNT = 1_000 +SELECTED_MANIFEST_SHA256 = ( + "767657671eb8695a9cfd9515cf89432492ea6400767d45e00e62c6b1e600b607" +) +TEXT_TOKEN_TAG = 1 + +MANIFEST_SCHEMA_VERSION = 1 +PROVENANCE_SCHEMA_VERSION = 1 +CACHE_SCHEMA_VERSION = 1 +CACHE_ARTIFACT_TYPE = "minimax_h3_longlive_1000_prompt_cache" + +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$") +_PROMPT_ID_RE = re.compile(r"^vidprom-([0-9]{6})-([0-9a-f]{12})$") +_CACHE_FIELDS = { + "schema_version", + "artifact_type", + "workflow", + "cfg_enabled", + "ir", + "regenerator", + "model_revision", + "prompt_manifest_sha256", + "source", + "conditions", +} +_CACHE_SOURCE_FIELDS = {"repo_id", "revision", "path", "bytes", "sha256"} +_CONDITION_FIELDS = {"id", "prompt", "seed", "prompt_embeds", "text_token_tags"} + + +@dataclass(frozen=True) +class PromptCacheCondition: + """One validated, positive-only cached text condition.""" + + id: str + prompt: str + seed: int + prompt_embeds: torch.Tensor + text_token_tags: torch.Tensor + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def sha256_file(path: str | Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as source: + while chunk := source.read(16 * 1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def encode_json(payload: Mapping[str, Any]) -> bytes: + """Return the canonical on-disk JSON representation used for SHA binding.""" + + return ( + json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + ).encode("utf-8") + + +def _source_descriptor() -> dict[str, Any]: + return { + "repo_id": SOURCE_REPO_ID, + "revision": SOURCE_REVISION, + "path": SOURCE_PATH, + "bytes": SOURCE_BYTES, + "sha256": SOURCE_SHA256, + } + + +def _prompt_digest(prompt: str) -> str: + return sha256_bytes(prompt.encode("utf-8")) + + +def _prompt_id(source_line: int, prompt_sha256: str) -> str: + return f"vidprom-{source_line:06d}-{prompt_sha256[:12]}" + + +def _read_pinned_source(path: str | Path) -> list[tuple[int, str]]: + source = Path(path) + if source.is_symlink() or not source.is_file(): + raise ValueError(f"raw prompt source must be a non-symlink regular file: {source}") + raw = source.read_bytes() + if len(raw) != SOURCE_BYTES: + raise ValueError( + f"raw prompt byte count mismatch: {len(raw)} != {SOURCE_BYTES}" + ) + observed_sha256 = sha256_bytes(raw) + if observed_sha256 != SOURCE_SHA256: + raise ValueError( + f"raw prompt SHA-256 mismatch: {observed_sha256} != {SOURCE_SHA256}" + ) + if raw.startswith(b"\xef\xbb\xbf"): + raise ValueError("raw prompt source must not contain a UTF-8 BOM") + if b"\x00" in raw: + raise ValueError("raw prompt source contains a NUL byte") + if b"\r" in raw or not raw.endswith(b"\n"): + raise ValueError("raw prompt source must use LF lines and end with LF") + try: + text = raw.decode("utf-8", errors="strict") + except UnicodeDecodeError as error: + raise ValueError("raw prompt source is not strict UTF-8") from error + lines = text.splitlines() + if len(lines) != SOURCE_LINE_COUNT: + raise ValueError( + f"raw prompt line count mismatch: {len(lines)} != {SOURCE_LINE_COUNT}" + ) + canonical: list[tuple[int, str]] = [] + for source_line, raw_prompt in enumerate(lines, start=1): + prompt = raw_prompt.strip() + if not prompt: + raise ValueError(f"raw prompt line {source_line} is empty after strip") + canonical.append((source_line, prompt)) + return canonical + + +def build_prompt_manifest( + raw_txt: str | Path, +) -> tuple[dict[str, Any], dict[str, Any]]: + """Build the pinned 1,000-row schema-1 manifest and provenance payloads.""" + + canonical = _read_pinned_source(raw_txt) + first_by_prompt: dict[str, int] = {} + duplicate_lines: dict[str, list[int]] = {} + for source_line, prompt in canonical: + if prompt in first_by_prompt: + duplicate_lines.setdefault(prompt, [first_by_prompt[prompt]]).append(source_line) + else: + first_by_prompt[prompt] = source_line + if len(first_by_prompt) != SOURCE_UNIQUE_COUNT: + raise ValueError( + f"unique prompt count mismatch: {len(first_by_prompt)} != {SOURCE_UNIQUE_COUNT}" + ) + if len(first_by_prompt) < SELECTED_PROMPT_COUNT: + raise ValueError("source does not contain enough unique prompts") + + candidates = [ + (_prompt_digest(prompt), prompt, source_line) + for prompt, source_line in first_by_prompt.items() + ] + candidates.sort(key=lambda row: (row[0], row[1].encode("utf-8"))) + selected = candidates[:SELECTED_PROMPT_COUNT] + prompts = [ + { + "id": _prompt_id(source_line, prompt_sha256), + "prompt": prompt, + "seed": source_line - 1, + } + for prompt_sha256, prompt, source_line in selected + ] + manifest: dict[str, Any] = { + "schema_version": MANIFEST_SCHEMA_VERSION, + "prompts": prompts, + } + manifest_sha256 = sha256_bytes(encode_json(manifest)) + provenance: dict[str, Any] = { + "schema_version": PROVENANCE_SCHEMA_VERSION, + "artifact_type": "longlive_hf_prompt_selection_provenance", + "source": _source_descriptor(), + "policy": { + "trim": "python_str_strip", + "deduplication": "exact_keep_first_source_line", + "ordering": "sha256_utf8_ascending_then_utf8_bytes", + "selection": f"first_{SELECTED_PROMPT_COUNT}", + "id": "vidprom-{source_line:06d}-{prompt_sha256[:12]}", + "seed": "source_line_minus_one", + }, + "counts": { + "raw_lines": len(canonical), + "nonempty_after_strip": len(canonical), + "unique_after_exact_dedup": len(first_by_prompt), + "duplicate_extra_rows": len(canonical) - len(first_by_prompt), + "selected": len(prompts), + }, + "duplicates": [ + { + "prompt_sha256": _prompt_digest(prompt), + "kept_source_line": lines[0], + "dropped_source_lines": lines[1:], + } + for prompt, lines in sorted( + duplicate_lines.items(), key=lambda row: (row[1][0], row[0]) + ) + ], + "manifest_sha256": manifest_sha256, + "selected": [ + { + "id": row["id"], + "source_line": row["seed"] + 1, + "prompt_sha256": _prompt_digest(row["prompt"]), + "seed": row["seed"], + } + for row in prompts + ], + } + return manifest, provenance + + +def _atomic_write(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_raw = tempfile.mkstemp( + prefix=f".{path.name}.", suffix=".tmp", dir=path.parent + ) + temporary = Path(temporary_raw) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def write_prompt_artifacts( + *, + raw_txt: str | Path, + manifest_output: str | Path, + provenance_output: str | Path, +) -> dict[str, Any]: + """Atomically write the generated manifest and its external provenance.""" + + manifest_path = Path(manifest_output) + provenance_path = Path(provenance_output) + if manifest_path.resolve() == provenance_path.resolve(): + raise ValueError("manifest and provenance outputs must differ") + manifest, provenance = build_prompt_manifest(raw_txt) + manifest_bytes = encode_json(manifest) + provenance_bytes = encode_json(provenance) + _atomic_write(manifest_path, manifest_bytes) + _atomic_write(provenance_path, provenance_bytes) + return { + "schema_version": 1, + "status": "written", + "manifest": str(manifest_path.resolve()), + "manifest_sha256": sha256_bytes(manifest_bytes), + "provenance": str(provenance_path.resolve()), + "provenance_sha256": sha256_bytes(provenance_bytes), + "prompt_count": SELECTED_PROMPT_COUNT, + } + + +def _require_exact_fields(value: Mapping[str, Any], expected: set[str], label: str) -> None: + actual = set(value) + if actual != expected: + raise ValueError( + f"{label} fields mismatch: missing={sorted(expected - actual)}, " + f"extra={sorted(actual - expected)}" + ) + + +def _validate_cache_source(value: object) -> None: + if not isinstance(value, Mapping): + raise ValueError("prompt cache source must be a mapping") + _require_exact_fields(value, _CACHE_SOURCE_FIELDS, "prompt cache source") + if dict(value) != _source_descriptor(): + raise ValueError("prompt cache source does not match the pinned LongLive prompt set") + + +def load_prompt_cache( + path: str | Path, +) -> tuple[dict[str, Any], list[PromptCacheCondition]]: + """Load and strictly validate a 1,000-condition LongLive prompt cache.""" + + source = Path(path) + if source.is_symlink() or not source.is_file(): + raise ValueError(f"prompt cache must be a non-symlink regular file: {source}") + try: + payload = torch.load(source, map_location="cpu", weights_only=True) + except Exception as error: + raise ValueError(f"unable to load prompt cache: {source}") from error + if not isinstance(payload, Mapping): + raise ValueError("prompt cache must contain a mapping") + _require_exact_fields(payload, _CACHE_FIELDS, "prompt cache") + if payload["schema_version"] != CACHE_SCHEMA_VERSION: + raise ValueError("prompt cache schema_version must be 1") + if payload["artifact_type"] != CACHE_ARTIFACT_TYPE: + raise ValueError("prompt cache artifact_type mismatch") + if payload["workflow"] != "t2va": + raise ValueError("prompt cache workflow must be t2va") + for field in ("cfg_enabled", "ir", "regenerator"): + if payload[field] is not False: + raise ValueError(f"prompt cache {field} must be false") + model_revision = payload["model_revision"] + if not isinstance(model_revision, str) or not _COMMIT_RE.fullmatch(model_revision): + raise ValueError("prompt cache model_revision must be a 40-character commit SHA") + manifest_sha256 = payload["prompt_manifest_sha256"] + if not isinstance(manifest_sha256, str) or not _SHA256_RE.fullmatch( + manifest_sha256 + ): + raise ValueError("prompt_manifest_sha256 must be lowercase SHA-256") + if manifest_sha256 != SELECTED_MANIFEST_SHA256: + raise ValueError("prompt cache does not bind the pinned 1,000-prompt manifest") + _validate_cache_source(payload["source"]) + + raw_conditions = payload["conditions"] + if not isinstance(raw_conditions, list) or len(raw_conditions) != SELECTED_PROMPT_COUNT: + raise ValueError( + f"prompt cache must contain exactly {SELECTED_PROMPT_COUNT} conditions" + ) + conditions: list[PromptCacheCondition] = [] + seen_ids: set[str] = set() + seen_prompts: set[str] = set() + seen_seeds: set[int] = set() + for index, raw_condition in enumerate(raw_conditions): + label = f"conditions[{index}]" + if not isinstance(raw_condition, Mapping): + raise ValueError(f"{label} must be a mapping") + _require_exact_fields(raw_condition, _CONDITION_FIELDS, label) + prompt_id = raw_condition["id"] + prompt = raw_condition["prompt"] + seed = raw_condition["seed"] + if not isinstance(prompt_id, str) or not (match := _PROMPT_ID_RE.fullmatch(prompt_id)): + raise ValueError(f"{label}.id is not a deterministic LongLive prompt id") + if prompt_id in seen_ids: + raise ValueError(f"duplicate prompt id: {prompt_id}") + if not isinstance(prompt, str) or not prompt or prompt != prompt.strip(): + raise ValueError(f"{label}.prompt must be nonempty and stripped") + if prompt in seen_prompts: + raise ValueError(f"duplicate prompt text at {label}") + if isinstance(seed, bool) or not isinstance(seed, int) or not 0 <= seed < 2**63: + raise ValueError(f"{label}.seed must be an integer in [0, 2**63)") + if seed in seen_seeds: + raise ValueError(f"duplicate prompt seed: {seed}") + source_line = int(match.group(1)) + if seed != source_line - 1: + raise ValueError(f"{label}.seed does not match its source-line id") + if match.group(2) != _prompt_digest(prompt)[:12]: + raise ValueError(f"{label}.id SHA prefix does not match its prompt") + + embeds = raw_condition["prompt_embeds"] + tags = raw_condition["text_token_tags"] + if not isinstance(embeds, torch.Tensor) or not isinstance(tags, torch.Tensor): + raise ValueError(f"{label} embeddings and text tags must be tensors") + if embeds.layout != torch.strided or tags.layout != torch.strided: + raise ValueError(f"{label} tensors must use strided layout") + if ( + embeds.ndim != 3 + or embeds.shape[0] != 1 + or embeds.shape[1] < 1 + or embeds.shape[2] < 1 + or not torch.is_floating_point(embeds) + ): + raise ValueError(f"{label}.prompt_embeds must be floating [1, rows, dim]") + if not bool(torch.isfinite(embeds).all()): + raise ValueError(f"{label}.prompt_embeds contains non-finite values") + if tags.dtype != torch.long or tags.shape != (embeds.shape[1],): + raise ValueError(f"{label}.text_token_tags must be long with one tag per row") + if not bool((tags == TEXT_TOKEN_TAG).all()): + raise ValueError(f"{label} contains non-text conditioning tags") + + seen_ids.add(prompt_id) + seen_prompts.add(prompt) + seen_seeds.add(seed) + conditions.append( + PromptCacheCondition( + id=prompt_id, + prompt=prompt, + seed=seed, + prompt_embeds=embeds.contiguous(), + text_token_tags=tags.contiguous(), + ) + ) + reconstructed_manifest = { + "schema_version": MANIFEST_SCHEMA_VERSION, + "prompts": [ + {"id": row.id, "prompt": row.prompt, "seed": row.seed} + for row in conditions + ], + } + reconstructed_sha256 = sha256_bytes(encode_json(reconstructed_manifest)) + if reconstructed_sha256 != manifest_sha256: + raise ValueError( + "prompt cache conditions do not match prompt_manifest_sha256: " + f"{reconstructed_sha256} != {manifest_sha256}" + ) + metadata = {key: value for key, value in payload.items() if key != "conditions"} + return metadata, conditions + + +__all__ = [ + "CACHE_ARTIFACT_TYPE", + "SELECTED_MANIFEST_SHA256", + "SELECTED_PROMPT_COUNT", + "SOURCE_BYTES", + "SOURCE_LINE_COUNT", + "SOURCE_PATH", + "SOURCE_REPO_ID", + "SOURCE_REVISION", + "SOURCE_SHA256", + "SOURCE_UNIQUE_COUNT", + "PromptCacheCondition", + "build_prompt_manifest", + "encode_json", + "load_prompt_cache", + "sha256_bytes", + "sha256_file", + "write_prompt_artifacts", +] diff --git a/source_snapshot/minimax_h3/model_provenance.py b/source_snapshot/minimax_h3/model_provenance.py new file mode 100644 index 0000000000000000000000000000000000000000..eb22f320eaf8b436b56dea847767d77e8a0b5931 --- /dev/null +++ b/source_snapshot/minimax_h3/model_provenance.py @@ -0,0 +1,136 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Fail-closed provenance validation for a local MiniMax-H3 Base payload.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping +from pathlib import Path +from typing import Any + + +MODEL_ID = "MiniMaxAI/MiniMax-H3" +MODEL_REVISION = "fa9c8ab1eaa21c8ae25e7e40b83b2e6002f340af" +DIFFUSERS_REVISION = "b88b343e71e38bd404f7f082ac4a2cbde63713e1" +MODEL_MANIFEST_SHA256 = "3ae5c8f528cd8c766e6ccdbf56486c77a5a624056bbb41c9f32126cd22db3fe4" +MODEL_PAYLOAD_BYTES = 144_051_143_211 +MODEL_PAYLOAD_FILES = 61 +EXCLUDED_ROOTS = ["transformer_ref", "FL2VA", "Ref2VA"] + +_COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$") +_SHA256 = re.compile(r"^[0-9a-f]{64}$") + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + while chunk := source.read(16 * 1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _load_json(path: Path, label: str) -> dict[str, Any]: + if path.is_symlink() or not path.is_file(): + raise RuntimeError(f"{label} must be a non-symlink regular file: {path}") + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise RuntimeError(f"Invalid {label}: {path}") from error + if not isinstance(payload, dict): + raise RuntimeError(f"{label} must contain a JSON object: {path}") + return payload + + +def _verification_payload( + sentinel: Mapping[str, Any], model_root: Path +) -> dict[str, Any]: + verification = sentinel.get("verification") + if isinstance(verification, Mapping): + return dict(verification) + if isinstance(verification, str) and verification: + path = Path(verification) + if not path.is_absolute(): + path = model_root / path + return _load_json(path, "model verification receipt") + raise RuntimeError("Model download sentinel lacks a full verification receipt.") + + +def validate_local_model_provenance( + model_id: str, resolved_revision: str | None +) -> dict[str, Any] | None: + """Validate the immutable download receipt for a local Base-only snapshot.""" + + model_root = Path(model_id) + if not model_root.exists(): + return None + if not model_root.is_dir(): + raise ValueError(f"Local H3 model path is not a directory: {model_root}") + if not isinstance(resolved_revision, str) or not _COMMIT_SHA.fullmatch( + resolved_revision + ): + raise ValueError("A local H3 snapshot requires an immutable lowercase commit SHA.") + + sentinel_path = model_root / "DOWNLOAD_SUCCESS.json" + if not sentinel_path.is_file(): + raise RuntimeError( + f"Local H3 payload lacks its verified download sentinel: {sentinel_path}." + ) + sentinel = _load_json(sentinel_path, "H3 download sentinel") + expected = { + "schema_version": 1, + "status": "success", + "repo_id": MODEL_ID, + "model_revision": resolved_revision, + "diffusers_revision": DIFFUSERS_REVISION, + "manifest_sha256": MODEL_MANIFEST_SHA256, + "payload_bytes": MODEL_PAYLOAD_BYTES, + "payload_files": MODEL_PAYLOAD_FILES, + "excluded_roots": EXCLUDED_ROOTS, + } + for name, value in expected.items(): + if sentinel.get(name) != value: + raise RuntimeError( + f"Local H3 download sentinel {name!r} mismatch: " + f"{sentinel.get(name)!r} != {value!r}." + ) + + for excluded in EXCLUDED_ROOTS: + if (model_root / excluded).exists(): + raise RuntimeError( + f"Base-only H3 model directory unexpectedly contains {excluded}." + ) + provenance_path = model_root / "download_provenance.json" + expected_provenance_sha = sentinel.get("provenance_sha256") + if ( + not provenance_path.is_file() + or provenance_path.is_symlink() + or not isinstance(expected_provenance_sha, str) + or not _SHA256.fullmatch(expected_provenance_sha) + or sha256_file(provenance_path) != expected_provenance_sha + ): + raise RuntimeError("Local H3 download provenance hash is missing or invalid.") + + verification = _verification_payload(sentinel, model_root) + verification_expected = { + "status": "verified", + "full_digest_verification": True, + "digest_files_checked": MODEL_PAYLOAD_FILES, + "expected_files": MODEL_PAYLOAD_FILES, + "expected_bytes": MODEL_PAYLOAD_BYTES, + "present_size_valid_files": MODEL_PAYLOAD_FILES, + "present_size_valid_bytes": MODEL_PAYLOAD_BYTES, + "manifest_sha256": MODEL_MANIFEST_SHA256, + } + for name, value in verification_expected.items(): + if verification.get(name) != value: + raise RuntimeError( + f"Local H3 verification receipt {name!r} mismatch: " + f"{verification.get(name)!r} != {value!r}." + ) + if verification.get("errors") != []: + raise RuntimeError("Local H3 verification receipt contains errors.") + return sentinel diff --git a/source_snapshot/minimax_h3/probe_student_lora_ab.py b/source_snapshot/minimax_h3/probe_student_lora_ab.py new file mode 100644 index 0000000000000000000000000000000000000000..12f21a3b4677dcd7d277d9abf6d4fca1ad38a2c1 --- /dev/null +++ b/source_snapshot/minimax_h3/probe_student_lora_ab.py @@ -0,0 +1,461 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Run a one-forward, same-input MiniMax-H3 Student-LoRA A/B probe. + +This is deliberately a diagnostic rather than an inference entry point. It +loads the released H3 transformer and one production Student checkpoint using +the same fail-closed loader/installer as ``infer_student``. The adapter-enabled +and adapter-disabled transformer calls receive byte-identical conditioning, +latents, layout, and timesteps. A third enabled call measures repeat noise. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping, Sequence + +import torch + +from .infer_base import _tensor_descriptor, sha256_file +from .infer_student import install_student_lora, load_student_checkpoint +from .train_dmd import validate_local_model_provenance +from .train_utils import ( + H3Condition, + STUDENT_ADAPTER, + build_row_timestep_plan, + disable_adapters_for_teacher, + h3_model_input_dtypes, + make_base_t2va_layout, +) + + +SCHEMA_VERSION = 1 +PROBE_HEIGHT = 32 +PROBE_WIDTH = 32 +PROBE_NUM_FRAMES = 124 +PROBE_TIMESTEP = 0.5 + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _atomic_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_raw = tempfile.mkstemp( + prefix=f".{path.name}.", suffix=".tmp", dir=path.parent + ) + temporary = Path(temporary_raw) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + json.dump( + payload, + handle, + ensure_ascii=False, + allow_nan=False, + indent=2, + sort_keys=True, + ) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _atomic_torch_save(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_raw = tempfile.mkstemp( + prefix=f".{path.name}.", suffix=".tmp", dir=path.parent + ) + os.close(descriptor) + temporary = Path(temporary_raw) + try: + torch.save(payload, temporary) + with temporary.open("rb") as handle: + os.fsync(handle.fileno()) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _load_one_condition(path: Path) -> tuple[dict[str, Any], H3Condition]: + payload = torch.load(path, map_location="cpu", weights_only=True) + if not isinstance(payload, Mapping) or payload.get("schema_version") != 1: + raise ValueError("Probe prompt cache is malformed or unsupported.") + for name in ("cfg_enabled", "ir", "regenerator"): + if payload.get(name) is not False: + raise ValueError(f"Probe prompt cache requires {name}=false.") + items = payload.get("conditions") + if not isinstance(items, list) or len(items) != 1: + raise ValueError("Probe prompt cache must contain exactly one condition.") + item = items[0] + if not isinstance(item, Mapping): + raise ValueError("Probe condition is not a mapping.") + condition = H3Condition( + id=item["id"], + prompt=item["prompt"], + seed=item["seed"], + prompt_embeds=item["prompt_embeds"], + text_token_tags=item["text_token_tags"], + ) + condition.validate() + metadata = {str(key): value for key, value in payload.items() if key != "conditions"} + return metadata, condition + + +def _delta_metrics( + left: Mapping[str, torch.Tensor], right: Mapping[str, torch.Tensor] +) -> dict[str, Any]: + if set(left) != {"video", "audio"} or set(right) != {"video", "audio"}: + raise ValueError("Probe outputs must contain video and audio tensors.") + rows: dict[str, Any] = {} + total_squared = 0.0 + total_absolute = 0.0 + total_count = 0 + total_nonzero = 0 + combined_dot = 0.0 + combined_left_sq = 0.0 + combined_right_sq = 0.0 + combined_max = 0.0 + for name in ("video", "audio"): + lhs = left[name].detach().float().cpu() + rhs = right[name].detach().float().cpu() + if lhs.shape != rhs.shape: + raise ValueError(f"{name} output shapes differ: {lhs.shape} != {rhs.shape}.") + delta = lhs - rhs + squared = float(torch.sum(delta.square(), dtype=torch.float64)) + absolute = float(torch.sum(delta.abs(), dtype=torch.float64)) + count = delta.numel() + nonzero = int(torch.count_nonzero(delta)) + lhs_sq = float(torch.sum(lhs.square(), dtype=torch.float64)) + rhs_sq = float(torch.sum(rhs.square(), dtype=torch.float64)) + dot = float(torch.sum(lhs * rhs, dtype=torch.float64)) + denominator = math.sqrt(lhs_sq * rhs_sq) + row = { + "elements": count, + "nonzero_elements": nonzero, + "rmse": math.sqrt(squared / count), + "mean_abs": absolute / count, + "max_abs": float(delta.abs().max()), + "cosine": dot / denominator if denominator else None, + } + if not all( + math.isfinite(float(row[key])) + for key in ("rmse", "mean_abs", "max_abs") + ): + raise FloatingPointError(f"{name} delta metrics are non-finite.") + rows[name] = row + total_squared += squared + total_absolute += absolute + total_count += count + total_nonzero += nonzero + combined_dot += dot + combined_left_sq += lhs_sq + combined_right_sq += rhs_sq + combined_max = max(combined_max, row["max_abs"]) + combined_denominator = math.sqrt(combined_left_sq * combined_right_sq) + rows["combined"] = { + "elements": total_count, + "nonzero_elements": total_nonzero, + "rmse": math.sqrt(total_squared / total_count), + "mean_abs": total_absolute / total_count, + "max_abs": combined_max, + "cosine": ( + combined_dot / combined_denominator if combined_denominator else None + ), + } + return rows + + +def _forward( + model: torch.nn.Module, + *, + video: torch.Tensor, + audio: torch.Tensor, + condition: H3Condition, + layout: Any, + timestep: torch.Tensor, + timestep_indices: torch.Tensor, +) -> dict[str, torch.Tensor]: + # ``accelerate.cpu_offload`` reconstructs arbitrary dataclasses in its + # root post-forward hook. The official two-required-field H3 ModelOutput + # cannot be reconstructed through that generic one-argument path, so ask + # the unchanged transformer for its equivalent positional tuple here. + output = model( + hidden_states=video, + audio_hidden_states=audio, + encoder_hidden_states=condition.prompt_embeds, + timestep=timestep, + timestep_indices=timestep_indices, + token_tags=layout.token_tags, + position_ids=layout.position_ids, + video_indices=layout.video_indices, + audio_indices=layout.audio_indices, + text_indices=layout.text_indices, + return_dict=False, + ) + if not isinstance(output, tuple) or len(output) != 2: + raise RuntimeError("H3 tuple forward did not return video/audio outputs.") + result = { + "video": output[0].detach().cpu().contiguous(), + "audio": output[1].detach().cpu().contiguous(), + } + if result["video"].shape != video.shape or result["audio"].shape != audio.shape: + raise RuntimeError("H3 probe output shape differs from its exact input layout.") + if not all(bool(value.isfinite().all()) for value in result.values()): + raise FloatingPointError("H3 probe produced a non-finite output.") + return result + + +def run(args: argparse.Namespace) -> dict[str, Any]: + if args.checkpoint.is_symlink() or args.prompt_cache.is_symlink(): + raise ValueError("Probe inputs must not be symlinks.") + checkpoint = load_student_checkpoint(args.checkpoint) + if checkpoint["step"] != args.expected_step: + raise ValueError( + f"Checkpoint step mismatch: {checkpoint['step']} != {args.expected_step}." + ) + if checkpoint["config"].get("resolved_revision") != args.revision: + raise ValueError("Checkpoint and requested H3 revisions differ.") + validate_local_model_provenance(str(args.model_dir), args.revision) + cache_metadata, condition_cpu = _load_one_condition(args.prompt_cache) + if cache_metadata.get("model_revision") != args.revision: + raise ValueError("Prompt cache and requested H3 revisions differ.") + + from accelerate import cpu_offload + from diffusers import MiniMaxH3Transformer3DModel + + model = MiniMaxH3Transformer3DModel.from_pretrained( + str(args.model_dir), + subfolder="transformer", + revision=args.revision, + local_files_only=True, + dtype=torch.bfloat16, + low_cpu_mem_usage=True, + ) + if model.__class__.__name__ != "MiniMaxH3Transformer3DModel": + raise TypeError("Probe loaded a non-H3 transformer.") + adapter = install_student_lora(model, checkpoint) + if not adapter.get("lora_exact_round_trip"): + raise RuntimeError("Student adapter did not pass its exact round-trip gate.") + if tuple(model.active_adapters()) != (STUDENT_ADAPTER,): + raise RuntimeError("Probe did not leave only the Student adapter active.") + model.eval() + + device = torch.device(args.device) + if device.type != "cuda" or not torch.cuda.is_available(): + raise RuntimeError("The real H3 LoRA A/B probe requires one CUDA GPU.") + cpu_offload(model, execution_device=device, offload_buffers=True) + + latent_frames = (PROBE_NUM_FRAMES - 5) // 17 * 5 + 2 + latent_height = PROBE_HEIGHT // 16 + latent_width = PROBE_WIDTH // 16 + audio_latents = int(round(PROBE_NUM_FRAMES / 24.0 * 40.0)) + patch_size = tuple(int(value) for value in model.config.patch_size) + layout = make_base_t2va_layout( + text_token_tags=condition_cpu.text_token_tags, + num_latent_frames=latent_frames, + latent_height=latent_height, + latent_width=latent_width, + num_audio_latents=audio_latents, + patch_size=patch_size, + ).to(device) + condition = condition_cpu.to(device) + video_dtype, audio_dtype = h3_model_input_dtypes(model) + video_patch_dim = int(model.config.in_channels * math.prod(patch_size)) + audio_channels = int(model.config.audio_in_channels) + generator = torch.Generator(device=device).manual_seed(args.seed) + video = torch.randn( + (1, layout.video_indices.numel(), video_patch_dim), + generator=generator, + device=device, + dtype=video_dtype, + ) + audio = torch.randn( + (1, layout.audio_indices.numel(), audio_channels), + generator=generator, + device=device, + dtype=audio_dtype, + ) + timestep, timestep_indices = build_row_timestep_plan( + layout, + video_timestep=PROBE_TIMESTEP, + audio_timestep=PROBE_TIMESTEP, + ) + + tuner_layers = [ + module + for module in model.modules() + if hasattr(module, "lora_A") and hasattr(module, "disable_adapters") + ] + if len(tuner_layers) != int(adapter["lora_target_module_count"]): + raise RuntimeError( + "LoRA tuner-layer count differs from the installed target count: " + f"{len(tuner_layers)} != {adapter['lora_target_module_count']}." + ) + + with torch.inference_mode(): + if any(bool(module.disable_adapters) for module in tuner_layers): + raise RuntimeError("A Student LoRA layer was disabled before enabled forward.") + enabled = _forward( + model, + video=video, + audio=audio, + condition=condition, + layout=layout, + timestep=timestep, + timestep_indices=timestep_indices, + ) + with disable_adapters_for_teacher(model): + if not all(bool(module.disable_adapters) for module in tuner_layers): + raise RuntimeError("Not every Student LoRA layer was disabled for Base A/B.") + disabled = _forward( + model, + video=video, + audio=audio, + condition=condition, + layout=layout, + timestep=timestep, + timestep_indices=timestep_indices, + ) + if any(bool(module.disable_adapters) for module in tuner_layers): + raise RuntimeError("Student LoRA layers stayed disabled after Base A/B.") + if tuple(model.active_adapters()) != (STUDENT_ADAPTER,): + raise RuntimeError("Student adapter was not restored after Base A/B.") + enabled_repeat = _forward( + model, + video=video, + audio=audio, + condition=condition, + layout=layout, + timestep=timestep, + timestep_indices=timestep_indices, + ) + + enabled_vs_disabled = _delta_metrics(enabled, disabled) + enabled_repeat_delta = _delta_metrics(enabled, enabled_repeat) + ab = enabled_vs_disabled["combined"] + repeat = enabled_repeat_delta["combined"] + if ab["nonzero_elements"] <= 0 or ab["rmse"] <= 0.0 or ab["max_abs"] <= 0.0: + raise RuntimeError("Student adapter enabled vs disabled produced zero delta.") + repeat_floor = max(float(repeat["rmse"]), 1e-12) + if float(ab["rmse"]) <= repeat_floor: + raise RuntimeError( + "Student adapter A/B delta does not exceed enabled-repeat numerical noise." + ) + + output_dir = args.output_dir.resolve() + output_dir.mkdir(parents=True, exist_ok=False) + tensors_path = output_dir / "probe_outputs.pt" + _atomic_torch_save( + tensors_path, + { + "schema_version": SCHEMA_VERSION, + "input": {"video": video.detach().cpu(), "audio": audio.detach().cpu()}, + "enabled": enabled, + "disabled": disabled, + "enabled_repeat": enabled_repeat, + }, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "status": "success", + "kind": "minimax_h3_student_lora_single_forward_ab", + "created_at": _utc_now(), + "checkpoint": { + "path": str(checkpoint["path"]), + "step": checkpoint["step"], + "sha256": checkpoint["sha256"], + }, + "model": { + "path": str(args.model_dir.resolve()), + "revision": args.revision, + "class": model.__class__.__name__, + "dtype": "bfloat16", + "cpu_offload": True, + }, + "condition": { + "cache": str(args.prompt_cache.resolve()), + "cache_sha256": sha256_file(args.prompt_cache), + "id": condition_cpu.id, + "prompt": condition_cpu.prompt, + "seed": condition_cpu.seed, + "prompt_embeds": _tensor_descriptor(condition_cpu.prompt_embeds), + }, + "adapter": adapter, + "adapter_runtime": { + "tuner_layer_count": len(tuner_layers), + "enabled_forward_active_adapters": [STUDENT_ADAPTER], + "disabled_forward_all_tuner_layers_disabled": True, + "enabled_repeat_active_adapters": [STUDENT_ADAPTER], + }, + "probe": { + "seed": args.seed, + "height": PROBE_HEIGHT, + "width": PROBE_WIDTH, + "num_frames": PROBE_NUM_FRAMES, + "latent_frames": latent_frames, + "latent_height": latent_height, + "latent_width": latent_width, + "num_audio_latents": audio_latents, + "timestep": PROBE_TIMESTEP, + "same_input_condition_layout_timestep": True, + "video_input": _tensor_descriptor(video), + "audio_input": _tensor_descriptor(audio), + }, + "outputs": { + "enabled": {name: _tensor_descriptor(value) for name, value in enabled.items()}, + "disabled": {name: _tensor_descriptor(value) for name, value in disabled.items()}, + "enabled_repeat": { + name: _tensor_descriptor(value) for name, value in enabled_repeat.items() + }, + }, + "delta": { + "enabled_vs_disabled": enabled_vs_disabled, + "enabled_vs_enabled_repeat": enabled_repeat_delta, + "ab_rmse_exceeds_repeat_rmse": True, + }, + "artifacts": { + "probe_outputs": { + "path": tensors_path.name, + "bytes": tensors_path.stat().st_size, + "sha256": sha256_file(tensors_path), + } + }, + } + result_path = output_dir / "LORA_AB_SUCCESS.json" + _atomic_json(result_path, payload) + print(json.dumps(payload, ensure_ascii=False, allow_nan=False, sort_keys=True)) + return payload + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--checkpoint", type=Path, required=True) + parser.add_argument("--expected-step", type=int, default=800) + parser.add_argument("--prompt-cache", type=Path, required=True) + parser.add_argument("--model-dir", type=Path, required=True) + parser.add_argument("--revision", required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--seed", type=int, default=20260812) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + run(build_parser().parse_args(argv)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/source_snapshot/minimax_h3/train_dmd.py b/source_snapshot/minimax_h3/train_dmd.py new file mode 100644 index 0000000000000000000000000000000000000000..56d73b8f6afa9cca755895fbe4d14b5aa598bf15 --- /dev/null +++ b/source_snapshot/minimax_h3/train_dmd.py @@ -0,0 +1,1317 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""MiniMax-H3 Base full-sequence, four-evaluation DMD LoRA trainer. + +This entry point intentionally lives beside (rather than inside) LongLive's +Wan-specific wrappers. It uses the official Diffusers H3 transformer/layout, +the H3 time and velocity conventions in :mod:`minimax_h3.dmd_core`, and one +shared Base checkpoint with independent ``student`` and ``critic`` adapters. +There is no unconditional prompt, CFG scale, IR module, or Regenerator path. + +The full 33B path uses classic FSDP with ``use_orig_params=True``. Loading a +full CPU checkpoint on every rank before it is sharded has a high host-memory +peak and has not been hidden behind a claim of validation: a real launch must +pass ``--acknowledge-full-rank-cpu-load`` explicitly. +""" + +from __future__ import annotations + +import argparse +import gc +import hashlib +import json +import math +import os +import re +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Sequence + +import torch + +from .infer_base import load_prompt_specs, resolve_hub_revision, sha256_file +from .model_provenance import validate_local_model_provenance +from .train_utils import ( + CRITIC_ADAPTER, + CUDA_ALLOCATOR_CONFIG, + CUDA_CACHE_POLICY, + DMD_NORMALIZATION, + NUM_MODEL_EVALUATIONS, + NUM_SIGMA_ENDPOINTS, + RELEASED_MAIN_BLOCKS, + RELEASED_REFINER_BLOCKS, + SAVED_TENSOR_OFFLOAD_PIN_MEMORY, + SAVED_TENSOR_OFFLOAD_POLICY, + STUDENT_ADAPTER, + H3CallAudit, + H3DMDTrainer, + adapter_parameters, + assert_released_h3_architecture, + h3_model_input_dtypes, + install_independent_lora_adapters, + load_prompt_cache, + make_base_t2va_layout, +) + + +MODEL_ID = "MiniMaxAI/MiniMax-H3" +WORKFLOW = "t2va" +_COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$") +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +FSDP_TOP_LEVEL_WRAP_MODULES = ( + "proj_in", + "audio_proj_in", + "context_embedder", + "time_embedder", + "token_refiner", + "norm_out", + "proj_out", + "audio_proj_out", +) +FSDP_LAYER_WRAP_CLASSES = ( + "MiniMaxH3TransformerBlock", + "MiniMaxH3TokenRefinerBlock", +) +FSDP_INNER_WRAP_MODULES = ("attn",) +FSDP_ATTENTION_PROJECTION_WRAP_MODULES = ("to_q", "to_k", "to_v", "to_out.0") +FSDP_ATTENTION_BASE_WRAP_MODULE = "base_layer" +FSDP_FFN_PROJECTION_WRAP_MODULES = ("net.0.proj", "net.2") +FSDP_FFN_BASE_WRAP_MODULE = "base_layer" +FSDP_AUTO_WRAP_POLICY_NAME = "h3_peft_attention_ffn_base_projection_v8" +FSDP_CPU_OFFLOAD_POLICY = "frozen_h3_base_handles_cpu_lora_cuda" +FSDP_CPU_OFFLOAD_WRAP_CLASSES = ( + "Linear", + "MiniMaxH3AdaLayerNormOut", + "MiniMaxH3Attention", + "MiniMaxH3TokenRefiner", + "MiniMaxH3TokenRefinerBlock", + "MiniMaxH3TransformerBlock", + "TimestepEmbedding", +) + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _atomic_torch_save(payload: Any, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=path.parent, suffix=".tmp", delete=False) as handle: + temporary = Path(handle.name) + try: + torch.save(payload, temporary) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _atomic_json(payload: Any, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + os.replace(temporary, path) + + +def _append_jsonl(payload: Any, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(payload, ensure_ascii=False, sort_keys=True) + "\n") + handle.flush() + os.fsync(handle.fileno()) + + +def _config_sha256(payload: dict[str, Any]) -> str: + raw = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(raw).hexdigest() + + +def _model_load_kwargs(args: argparse.Namespace, resolved_revision: str | None) -> dict[str, Any]: + kwargs: dict[str, Any] = { + "dtype": torch.bfloat16, + "local_files_only": args.local_files_only, + } + if resolved_revision is not None: + kwargs["revision"] = resolved_revision + return kwargs + + +def prepare_prompt_cache(args: argparse.Namespace) -> int: + """Encode exactly four positive prompts with the official standalone block.""" + + prompts = load_prompt_specs(args.prompts) + resolved_revision = resolve_hub_revision( + args.model_id, args.revision, local_files_only=args.local_files_only + ) + validate_local_model_provenance(args.model_id, resolved_revision) + try: + from diffusers.modular_pipelines.minimax_h3.before_denoise import ( # noqa: F401 + MiniMaxH3PrepareLayoutStep, + ) + from diffusers.modular_pipelines.minimax_h3.encoders import ( + MiniMaxH3TextEncoderStep, + ) + except ImportError as error: + raise RuntimeError( + "The editable diffusers_h3 checkout is required to encode MiniMax-H3 prompts." + ) from error + + block = MiniMaxH3TextEncoderStep() + pipe = block.init_pipeline(args.model_id) + pipe.load_components(**_model_load_kwargs(args, resolved_revision)) + conditions = [] + for prompt in prompts: + output = pipe( + prompt=prompt.prompt, + output=["prompt_embeds", "text_token_tags"], + ) + conditions.append( + { + "id": prompt.id, + "prompt": prompt.prompt, + "seed": prompt.seed, + "prompt_embeds": output["prompt_embeds"].detach().cpu(), + "text_token_tags": output["text_token_tags"].detach().cpu(), + } + ) + payload = { + "schema_version": 1, + "created_at": _utc_now(), + "model_id": args.model_id, + "requested_revision": args.revision, + "resolved_revision": resolved_revision, + "workflow": WORKFLOW, + "cfg_enabled": False, + "conditioner_calls": 4, + "conditions": conditions, + } + output_path = Path(args.output) + _atomic_torch_save(payload, output_path) + print( + json.dumps( + { + "prompt_cache": str(output_path.resolve()), + "sha256": sha256_file(output_path), + "resolved_revision": resolved_revision, + "conditions": len(conditions), + "cfg_enabled": False, + }, + sort_keys=True, + ) + ) + return 0 + + +def _validate_geometry(args: argparse.Namespace) -> tuple[int, int, int, int]: + if args.height <= 0 or args.width <= 0 or args.height % 32 or args.width % 32: + raise ValueError("H3 height and width must be positive multiples of 32.") + if args.num_frames < 1 or args.num_frames % 17 != 5: + raise ValueError("H3 num_frames must already have the form 17*n+5.") + if not 5.0 <= args.num_frames / 24.0 <= 15.0: + raise ValueError("H3 training clips must last between five and fifteen seconds.") + latent_frames = (args.num_frames - 5) // 17 * 5 + 2 + latent_height = args.height // 16 + latent_width = args.width // 16 + audio_latents = int(round(args.num_frames / 24.0 * 40.0)) + return latent_frames, latent_height, latent_width, audio_latents + + +def _train_config(args: argparse.Namespace, resolved_revision: str | None) -> dict[str, Any]: + config = { + "schema_version": 1, + "model_id": args.model_id, + "requested_revision": args.revision, + "resolved_revision": resolved_revision, + "workflow": WORKFLOW, + "ir": False, + "regenerator": False, + "cfg": False, + "shared_base_roles": ["student", "critic", "teacher"], + "teacher_adapter_state": "disabled", + "student_adapter": STUDENT_ADAPTER, + "critic_adapter": CRITIC_ADAPTER, + "lora_rank": args.lora_rank, + "lora_alpha": args.lora_alpha, + "lora_targets": ["to_q", "to_k", "to_v", "to_out.0"], + "rollout_mode": "full_sequence", + "attention_mode": "native_non_causal", + "attention_is_causal": False, + "temporal_ar_blocks": 0, + "model_evaluations_per_rollout": NUM_MODEL_EVALUATIONS, + "sigma_endpoints_per_rollout": NUM_SIGMA_ENDPOINTS, + "student_positive_forwards_per_rollout": NUM_MODEL_EVALUATIONS, + "transformer_block_count": RELEASED_MAIN_BLOCKS, + "token_refiner_block_count": RELEASED_REFINER_BLOCKS, + "normalization": DMD_NORMALIZATION, + "cuda_cache_policy": CUDA_CACHE_POLICY, + "cuda_empty_cache_before_generator_backward": True, + "cuda_allocator_config": CUDA_ALLOCATOR_CONFIG, + "saved_tensor_offload_policy": SAVED_TENSOR_OFFLOAD_POLICY, + "saved_tensor_offload_pin_memory": SAVED_TENSOR_OFFLOAD_PIN_MEMORY, + "video_shift": args.video_shift, + "audio_shift": args.audio_shift, + "height": args.height, + "width": args.width, + "num_frames": args.num_frames, + "strategy": args.strategy, + "gradient_checkpointing": args.gradient_checkpointing, + "seed": args.seed, + "student_lr": args.student_lr, + "critic_lr": args.critic_lr, + "max_steps": args.max_steps, + "prompt_start_index": args.prompt_start_index, + "prompt_cache": str(Path(args.prompt_cache).resolve()), + "prompt_cache_sha256": sha256_file(args.prompt_cache), + } + if args.strategy == "fsdp": + config.update( + { + "fsdp_sharding": "FULL_SHARD", + "fsdp_use_orig_params": True, + "fsdp_auto_wrap_classes": [ + "MiniMaxH3TransformerBlock", + "MiniMaxH3TokenRefinerBlock", + ], + "fsdp_auto_wrap_policy": FSDP_AUTO_WRAP_POLICY_NAME, + "fsdp_top_level_wrap_modules": list(FSDP_TOP_LEVEL_WRAP_MODULES), + "fsdp_inner_wrap_modules": list(FSDP_INNER_WRAP_MODULES), + "fsdp_attention_projection_wrap_modules": list( + FSDP_ATTENTION_PROJECTION_WRAP_MODULES + ), + "fsdp_attention_base_wrap_module": FSDP_ATTENTION_BASE_WRAP_MODULE, + "fsdp_ffn_projection_wrap_modules": list( + FSDP_FFN_PROJECTION_WRAP_MODULES + ), + "fsdp_handle_uniform_dtype_required": True, + "fsdp_root_parameter_tensors_expected": 0, + "fsdp_cpu_offload_policy": FSDP_CPU_OFFLOAD_POLICY, + "fsdp_cpu_offload_params": True, + "fsdp_limit_all_gathers": True, + "fsdp_backward_prefetch": "disabled", + "flex_attention_backend": "native", + "fsdp_sync_module_states": False, + } + ) + return config + + +def validate_train_request(args: argparse.Namespace) -> dict[str, Any]: + _validate_geometry(args) + if args.lora_rank < 1 or args.lora_alpha < 1: + raise ValueError("LoRA rank and alpha must be positive.") + if args.max_steps < 1: + raise ValueError("max_steps must be positive.") + if not args.gradient_checkpointing: + raise ValueError("Full H3 DMD training requires gradient checkpointing.") + if args.strategy not in {"fsdp", "single"}: + raise ValueError("strategy must be fsdp or single.") + if args.strategy == "single" and not args.allow_single_device_full_model: + raise ValueError( + "A full 33B single-device load is disabled by default; pass " + "--allow-single-device-full-model only on hardware that can hold it." + ) + if args.strategy == "fsdp" and not args.acknowledge_full_rank_cpu_load: + raise ValueError( + "Classic FSDP loads one full CPU checkpoint per rank before sharding. " + "Pass --acknowledge-full-rank-cpu-load after reserving sufficient host RAM." + ) + metadata, conditions = load_prompt_cache(args.prompt_cache) + if len(conditions) != 4: + raise AssertionError("Exactly four prompts are required.") + if not 0 <= args.prompt_start_index < len(conditions): + raise ValueError( + f"prompt_start_index must be in [0, {len(conditions) - 1}], " + f"got {args.prompt_start_index}." + ) + resolved_revision = resolve_hub_revision( + args.model_id, args.revision, local_files_only=args.local_files_only + ) + validate_local_model_provenance(args.model_id, resolved_revision) + resolved_cache_revision = metadata.get("resolved_revision") + model_cache_revision = metadata.get("model_revision") + if ( + resolved_cache_revision is not None + and model_cache_revision is not None + and resolved_cache_revision != model_cache_revision + ): + raise ValueError( + "Prompt cache contains conflicting resolved_revision and model_revision values." + ) + cached_revision = resolved_cache_revision or model_cache_revision + if cached_revision != resolved_revision: + raise ValueError( + "Prompt cache/checkpoint revision mismatch: " + f"cache={cached_revision!r}, requested={resolved_revision!r}." + ) + if metadata.get("model_id") not in (None, args.model_id): + raise ValueError("Prompt cache model_id does not match the training model_id.") + config = _train_config(args, resolved_revision) + config["config_sha256"] = _config_sha256(config) + return config + + +def _init_runtime(args: argparse.Namespace) -> tuple[torch.device, int, int]: + if not torch.cuda.is_available(): + raise RuntimeError("Full MiniMax-H3 training requires CUDA.") + if args.strategy == "single": + device = torch.device(args.device) + torch.cuda.set_device(device) + return device, 0, 1 + + import torch.distributed as dist + + required = ("RANK", "WORLD_SIZE", "LOCAL_RANK") + missing = [name for name in required if name not in os.environ] + if missing: + raise RuntimeError( + "FSDP training must be launched with torchrun; missing " + ", ".join(missing) + ) + if not dist.is_initialized(): + dist.init_process_group(backend="nccl") + rank = dist.get_rank() + world_size = dist.get_world_size() + if world_size < 2: + raise RuntimeError("FSDP training needs at least two ranks (eight are recommended).") + local_rank = int(os.environ["LOCAL_RANK"]) + device = torch.device("cuda", local_rank) + torch.cuda.set_device(device) + return device, rank, world_size + + +def _h3_fsdp_wrap_target_map(model: torch.nn.Module) -> dict[str, torch.nn.Module]: + """Return every nested FSDP unit needed to isolate H3 parameter dtypes. + + The block stack is wrapped one block at a time. Each attention keeps a tiny + norm-only outer handle while q/k/v/out are separate PEFT-aware projection + handles; each feed-forward uses separate input/output projection handles. + This keeps transient all-gathers below the whole-attention/FFN allocations + observed on 80-GiB H100s while preserving identical module math. Every parameter-bearing + top-level computation is also a unit, including ``token_refiner``: its + nested refiner blocks are wrapped first, so the outer refiner wrapper owns + only the leftover ``final_norm`` parameters. ``transformer_blocks`` itself + is deliberately not a unit because it is an iterated ``ModuleList`` rather + than a called module. + """ + + try: + from diffusers.models.transformers.transformer_minimax_h3 import ( + MiniMaxH3TokenRefinerBlock, + MiniMaxH3TransformerBlock, + ) + except ImportError as error: + raise RuntimeError("The editable diffusers_h3 checkout is required for H3 FSDP.") from error + + top_level = dict(model.named_children()) + missing = [name for name in FSDP_TOP_LEVEL_WRAP_MODULES if name not in top_level] + if missing: + raise RuntimeError(f"MiniMax-H3 is missing expected FSDP top-level modules: {missing}") + targets: dict[str, torch.nn.Module] = { + name: top_level[name] for name in FSDP_TOP_LEVEL_WRAP_MODULES + } + layer_types = (MiniMaxH3TransformerBlock, MiniMaxH3TokenRefinerBlock) + for name, module in model.named_modules(): + if name and isinstance(module, layer_types): + targets[name] = module + relative_modules = dict(module.named_modules()) + for relative_name in FSDP_INNER_WRAP_MODULES: + child = relative_modules.get(relative_name) + if child is None or not any(True for _ in child.parameters(recurse=True)): + raise RuntimeError( + f"H3 block {name!r} lacks parameter-bearing {relative_name!r}." + ) + targets[f"{name}.{relative_name}"] = child + for projection_name in FSDP_ATTENTION_PROJECTION_WRAP_MODULES: + relative_name = f"attn.{projection_name}" + child = relative_modules.get(relative_name) + if child is None or not any(True for _ in child.parameters(recurse=True)): + raise RuntimeError( + f"H3 block {name!r} lacks attention projection {projection_name!r}." + ) + targets[f"{name}.{relative_name}"] = child + base_relative_name = ( + f"{relative_name}.{FSDP_ATTENTION_BASE_WRAP_MODULE}" + ) + base_child = relative_modules.get(base_relative_name) + if base_child is not None: + if not any(True for _ in base_child.parameters(recurse=True)): + raise RuntimeError( + f"H3 PEFT projection {name}.{relative_name} has an empty " + f"{FSDP_ATTENTION_BASE_WRAP_MODULE!r}." + ) + targets[f"{name}.{base_relative_name}"] = base_child + for projection_name in FSDP_FFN_PROJECTION_WRAP_MODULES: + relative_name = f"ff.{projection_name}" + child = relative_modules.get(relative_name) + if child is None or not any(True for _ in child.parameters(recurse=True)): + raise RuntimeError( + f"H3 block {name!r} lacks FFN projection {projection_name!r}." + ) + targets[f"{name}.{relative_name}"] = child + # Under the LongLive-Wan-style policy, main-block FFN + # projections are PEFT Linear wrappers. Keep the trainable + # outer wrapper on CUDA and isolate only its frozen base + # Linear in a CPU-offloaded child handle. Token-refiner FFNs + # remain plain Linear modules and intentionally have no + # ``base_layer`` child. + base_relative_name = f"{relative_name}.{FSDP_FFN_BASE_WRAP_MODULE}" + base_child = relative_modules.get(base_relative_name) + if base_child is not None: + if not any(True for _ in base_child.parameters(recurse=True)): + raise RuntimeError( + f"H3 PEFT FFN projection {name}.{relative_name} has an " + f"empty {FSDP_FFN_BASE_WRAP_MODULE!r}." + ) + targets[f"{name}.{base_relative_name}"] = base_child + + # Fail when a future H3 revision adds a parameter-bearing top-level branch + # that this policy would accidentally leave in the root flat handle. + permitted_containers = {"transformer_blocks"} + uncovered = [] + for name, module in top_level.items(): + if name in targets or name in permitted_containers: + continue + if any(True for _ in module.parameters(recurse=True)): + uncovered.append(name) + if uncovered: + raise RuntimeError( + "H3 gained parameter-bearing top-level modules without an FSDP dtype unit: " + f"{uncovered}." + ) + if not any(name.startswith("transformer_blocks.") for name in targets): + raise RuntimeError("No MiniMaxH3TransformerBlock FSDP targets were found.") + if not any(name.startswith("token_refiner.refiner_blocks.") for name in targets): + raise RuntimeError("No MiniMaxH3TokenRefinerBlock FSDP targets were found.") + return targets + + +def validate_h3_fsdp_wrap_plan(model: torch.nn.Module) -> dict[str, Any]: + """Prove every planned flat handle is single-dtype and root owns no params. + + A parameter belongs to its deepest selected ancestor, matching recursive + FSDP wrapping. The report is weight-level: it inspects actual parameter + objects after LoRA injection, not model-config dtype labels. + """ + + targets = _h3_fsdp_wrap_target_map(model) + handles: dict[str, list[tuple[str, torch.nn.Parameter]]] = { + name: [] for name in targets + } + root_parameters: list[tuple[str, torch.nn.Parameter]] = [] + for parameter_name, parameter in model.named_parameters(): + candidates = [ + target_name + for target_name in targets + if parameter_name == target_name or parameter_name.startswith(target_name + ".") + ] + if not candidates: + root_parameters.append((parameter_name, parameter)) + continue + owner = max(candidates, key=lambda name: (name.count("."), len(name))) + handles[owner].append((parameter_name, parameter)) + + if root_parameters: + preview = [name for name, _ in root_parameters[:8]] + raise RuntimeError( + "H3 FSDP root would retain parameters outside dtype-isolated handles: " + f"{preview}." + ) + cpu_offload_modules = { + name + for name, module in targets.items() + if name in FSDP_TOP_LEVEL_WRAP_MODULES + or module.__class__.__name__ in FSDP_LAYER_WRAP_CLASSES + or name.endswith(".attn") + or any( + name.endswith(f".attn.{projection}") + and not hasattr(module, FSDP_ATTENTION_BASE_WRAP_MODULE) + for projection in FSDP_ATTENTION_PROJECTION_WRAP_MODULES + ) + or any( + name.endswith( + f".attn.{projection}.{FSDP_ATTENTION_BASE_WRAP_MODULE}" + ) + for projection in FSDP_ATTENTION_PROJECTION_WRAP_MODULES + ) + or any( + name.endswith(f".ff.{projection}") + and not hasattr(module, FSDP_FFN_BASE_WRAP_MODULE) + for projection in FSDP_FFN_PROJECTION_WRAP_MODULES + ) + or any( + name.endswith(f".ff.{projection}.{FSDP_FFN_BASE_WRAP_MODULE}") + for projection in FSDP_FFN_PROJECTION_WRAP_MODULES + ) + } + report_handles = [] + for name, parameters in handles.items(): + if not parameters: + raise RuntimeError( + f"H3 FSDP target {name!r} owns no parameters after nested wrapping; " + "the pinned CustomPolicy must be updated explicitly." + ) + dtypes = sorted({str(parameter.dtype).removeprefix("torch.") for _, parameter in parameters}) + if len(dtypes) != 1: + preview = [(param_name, str(param.dtype)) for param_name, param in parameters[:8]] + raise RuntimeError( + f"H3 FSDP handle {name!r} mixes parameter dtypes {dtypes}: {preview}" + ) + trainable_parameter_tensors = sum(parameter.requires_grad for _, parameter in parameters) + lora_parameter_tensors = sum("lora_" in parameter_name for parameter_name, _ in parameters) + cpu_offload = name in cpu_offload_modules + if cpu_offload and (trainable_parameter_tensors or lora_parameter_tensors): + raise RuntimeError( + f"H3 CPU-offloaded remainder {name!r} unexpectedly owns trainable/LoRA " + "parameters." + ) + report_handles.append( + { + "module": name, + "dtype": dtypes[0], + "parameter_tensors": len(parameters), + "parameter_numel": sum(parameter.numel() for _, parameter in parameters), + "trainable_parameter_tensors": trainable_parameter_tensors, + "lora_parameter_tensors": lora_parameter_tensors, + "cpu_offload": cpu_offload, + } + ) + offloaded_handles = [row for row in report_handles if row["cpu_offload"]] + return { + "policy": FSDP_AUTO_WRAP_POLICY_NAME, + "top_level_modules": list(FSDP_TOP_LEVEL_WRAP_MODULES), + "layer_classes": list(FSDP_LAYER_WRAP_CLASSES), + "inner_modules": list(FSDP_INNER_WRAP_MODULES), + "attention_projection_modules": list(FSDP_ATTENTION_PROJECTION_WRAP_MODULES), + "attention_base_module": FSDP_ATTENTION_BASE_WRAP_MODULE, + "ffn_projection_modules": list(FSDP_FFN_PROJECTION_WRAP_MODULES), + "ffn_base_module": FSDP_FFN_BASE_WRAP_MODULE, + "cpu_offload_policy": FSDP_CPU_OFFLOAD_POLICY, + "cpu_offload_params": True, + "cpu_offload_modules": [row["module"] for row in offloaded_handles], + "cpu_offload_handle_count": len(offloaded_handles), + "cpu_offload_parameter_numel": sum( + row["parameter_numel"] for row in offloaded_handles + ), + "root_parameter_tensors": 0, + "handles": report_handles, + } + + +def build_h3_fsdp_auto_wrap_policy(model: torch.nn.Module): + """Build the projection policy whose frozen top/block/FFN handles use CPU shards.""" + + from torch.distributed.fsdp import CPUOffload + from torch.distributed.fsdp.wrap import CustomPolicy + + targets = _h3_fsdp_wrap_target_map(model) + target_modules = set(targets.values()) + cpu_offload_modules = { + module + for name, module in targets.items() + if name in FSDP_TOP_LEVEL_WRAP_MODULES + or module.__class__.__name__ in FSDP_LAYER_WRAP_CLASSES + or name.endswith(".attn") + or any( + name.endswith(f".attn.{projection}") + and not hasattr(module, FSDP_ATTENTION_BASE_WRAP_MODULE) + for projection in FSDP_ATTENTION_PROJECTION_WRAP_MODULES + ) + or any( + name.endswith( + f".attn.{projection}.{FSDP_ATTENTION_BASE_WRAP_MODULE}" + ) + for projection in FSDP_ATTENTION_PROJECTION_WRAP_MODULES + ) + or any( + name.endswith(f".ff.{projection}") + and not hasattr(module, FSDP_FFN_BASE_WRAP_MODULE) + for projection in FSDP_FFN_PROJECTION_WRAP_MODULES + ) + or any( + name.endswith(f".ff.{projection}.{FSDP_FFN_BASE_WRAP_MODULE}") + for projection in FSDP_FFN_PROJECTION_WRAP_MODULES + ) + } + cpu_offload = CPUOffload(offload_params=True) + + def policy(module: torch.nn.Module) -> bool | dict[str, Any]: + if module not in target_modules: + return False + if module in cpu_offload_modules: + return {"cpu_offload": cpu_offload} + return True + + return CustomPolicy(policy) + + +def _wrap_fsdp(model: torch.nn.Module, device: torch.device) -> torch.nn.Module: + from torch.distributed.fsdp import FullyShardedDataParallel as FSDP, ShardingStrategy + + wrap_plan = validate_h3_fsdp_wrap_plan(model) + model._h3_fsdp_wrap_plan_report = wrap_plan + policy = build_h3_fsdp_auto_wrap_policy(model) + return FSDP( + model, + auto_wrap_policy=policy, + use_orig_params=True, + sharding_strategy=ShardingStrategy.FULL_SHARD, + # This correctness-first smoke performs 16 forwards through the same + # 33B stack before one backward. BACKWARD_PRE overlaps another handle + # all-gather with the current one; disable it to minimize peak. + backward_prefetch=None, + device_id=device, + sync_module_states=False, + limit_all_gathers=True, + ) + + +def validate_h3_fsdp_runtime_wrappers( + forward_model: torch.nn.Module, role_model: torch.nn.Module +) -> dict[str, Any]: + """Verify CPU offload stayed on frozen top/block/FFN projection handles.""" + + from torch.distributed.fsdp import FullyShardedDataParallel as FSDP + + wrappers = [module for module in forward_model.modules() if isinstance(module, FSDP)] + handle_rows: list[dict[str, Any]] = [] + for wrapper in wrappers: + handle = getattr(wrapper, "_handle", None) + cpu_offload = bool(wrapper.cpu_offload.offload_params) + wrapped_class = wrapper._fsdp_wrapped_module.__class__.__name__ + if handle is None: + if cpu_offload: + raise RuntimeError("An empty FSDP wrapper unexpectedly enables CPU offload.") + continue + flat_param = handle.flat_param + row = { + "wrapped_class": wrapped_class, + "cpu_offload": cpu_offload, + "flat_param_requires_grad": bool(flat_param.requires_grad), + "flat_param_device": flat_param.device.type, + "local_shard_numel": int(flat_param.numel()), + } + if cpu_offload: + if wrapped_class not in FSDP_CPU_OFFLOAD_WRAP_CLASSES: + raise RuntimeError( + f"CPU offload reached unapproved wrapper class {wrapped_class!r}." + ) + if flat_param.requires_grad: + raise RuntimeError("A CPU-offloaded FSDP flat parameter is trainable.") + if flat_param.device.type != "cpu": + raise RuntimeError("A CPU-offloaded FSDP flat parameter is not on CPU.") + elif flat_param.device.type != "cuda": + raise RuntimeError("A non-offloaded FSDP flat parameter is not on CUDA.") + handle_rows.append(row) + + cpu_offloaded_rows = [row for row in handle_rows if row["cpu_offload"]] + lora_parameters = [ + (name, parameter) + for name, parameter in role_model.named_parameters() + if "lora_" in name + ] + if not lora_parameters: + raise RuntimeError("FSDP runtime audit found no LoRA parameters.") + lora_devices = sorted({parameter.device.type for _, parameter in lora_parameters}) + if lora_devices != ["cuda"]: + raise RuntimeError(f"LoRA parameters must remain on CUDA, got {lora_devices}.") + return { + "policy": FSDP_CPU_OFFLOAD_POLICY, + "wrapper_count": len(wrappers), + "flat_handle_count": len(handle_rows), + "cpu_offload_handle_count": len(cpu_offloaded_rows), + "cpu_offloaded_classes": sorted( + {row["wrapped_class"] for row in cpu_offloaded_rows} + ), + "cpu_offloaded_flat_param_devices": sorted( + {row["flat_param_device"] for row in cpu_offloaded_rows} + ), + "cpu_offloaded_flat_params_frozen": all( + not row["flat_param_requires_grad"] for row in cpu_offloaded_rows + ), + "lora_parameter_tensors": len(lora_parameters), + "lora_parameter_devices": lora_devices, + } + + +def _load_transformer( + args: argparse.Namespace, resolved_revision: str | None, device: torch.device +) -> tuple[torch.nn.Module, torch.nn.Module]: + # Repeat the local sentinel check immediately before opening weight shards, + # rather than relying only on an earlier CLI preflight. + validate_local_model_provenance(args.model_id, resolved_revision) + try: + from diffusers import MiniMaxH3Transformer3DModel + except ImportError as error: + raise RuntimeError("Install the editable diffusers_h3 checkout first.") from error + kwargs = _model_load_kwargs(args, resolved_revision) + kwargs["low_cpu_mem_usage"] = True + model = MiniMaxH3Transformer3DModel.from_pretrained( + args.model_id, + subfolder="transformer", + **kwargs, + ) + if model.__class__.__name__ != "MiniMaxH3Transformer3DModel": + raise TypeError("Training loaded a non-H3 transformer.") + architecture = assert_released_h3_architecture(model) + if architecture["attention_is_causal"] is not False: + raise AssertionError("Released MiniMax-H3 attention must remain non-causal.") + model._h3_released_architecture = architecture + install_independent_lora_adapters( + model, + rank=args.lora_rank, + alpha=args.lora_alpha, + dropout=args.lora_dropout, + ) + model.enable_gradient_checkpointing() + model.train() + if args.strategy == "fsdp": + forward_model = _wrap_fsdp(model, device) + local_runtime_report = validate_h3_fsdp_runtime_wrappers(forward_model, model) + import torch.distributed as dist + + rank_reports: list[dict[str, Any] | None] = [None] * dist.get_world_size() + dist.all_gather_object(rank_reports, local_runtime_report) + if any(report != local_runtime_report for report in rank_reports): + raise RuntimeError("FSDP CPU-offload runtime audit differs across ranks.") + local_runtime_report["all_ranks_verified"] = True + model._h3_fsdp_runtime_report = local_runtime_report + else: + model.to(device) + forward_model = model + return forward_model, model + + +def _save_lora_checkpoint( + *, + forward_model: torch.nn.Module, + role_model: torch.nn.Module, + output_dir: Path, + step: int, + config: dict[str, Any], + rank: int, +) -> Path | None: + full_state = None + if forward_model is not role_model: + from torch.distributed.fsdp import ( + FullStateDictConfig, + FullyShardedDataParallel as FSDP, + StateDictType, + ) + + state_config = FullStateDictConfig(offload_to_cpu=True, rank0_only=True) + with FSDP.state_dict_type( + forward_model, StateDictType.FULL_STATE_DICT, state_config + ): + full_state = forward_model.state_dict() + else: + full_state = role_model.state_dict() + if rank != 0: + return None + + from peft import get_peft_model_state_dict + + checkpoint = { + "schema_version": 1, + "step": step, + "config": config, + "student_lora": get_peft_model_state_dict( + role_model, + state_dict=full_state, + adapter_name=STUDENT_ADAPTER, + ), + "critic_lora": get_peft_model_state_dict( + role_model, + state_dict=full_state, + adapter_name=CRITIC_ADAPTER, + ), + } + path = output_dir / f"checkpoint-{step:06d}" / "lora.pt" + _atomic_torch_save(checkpoint, path) + _atomic_json( + { + "step": step, + "lora_sha256": sha256_file(path), + "student_tensors": len(checkpoint["student_lora"]), + "critic_tensors": len(checkpoint["critic_lora"]), + "base_weights_included": False, + }, + path.parent / "manifest.json", + ) + return path + + +def train(args: argparse.Namespace) -> int: + config = validate_train_request(args) + if args.validate_only: + print(json.dumps(config, indent=2, sort_keys=True)) + return 0 + allocator_config = os.environ.get("PYTORCH_CUDA_ALLOC_CONF") + if args.strategy == "fsdp" and allocator_config != CUDA_ALLOCATOR_CONFIG: + raise RuntimeError( + "Full H3 FSDP requires the audited CUDA allocator configuration: " + f"actual={allocator_config!r}, expected={CUDA_ALLOCATOR_CONFIG!r}." + ) + device, rank, world_size = _init_runtime(args) + allocator_backend = ( + torch.cuda.memory.get_allocator_backend() if device.type == "cuda" else None + ) + if args.strategy == "fsdp" and allocator_backend != "native": + raise RuntimeError( + "The audited H3 memory policy requires PyTorch's native CUDA allocator; " + f"observed {allocator_backend!r}." + ) + # FSDP does not broadcast module state in this loader. Seed every rank + # identically before model construction so PEFT LoRA A initialization is + # byte-identical; rank-specific sample RNGs are created later. + torch.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) + flex_attention_compile = { + "enabled": False, + "reason": "native_non_causal_attention", + } + resolved_revision = config["resolved_revision"] + metadata, cached_conditions = load_prompt_cache(args.prompt_cache) + del metadata + forward_model, role_model = _load_transformer(args, resolved_revision, device) + video_input_dtype, audio_input_dtype = h3_model_input_dtypes(role_model) + + student_optimizer = torch.optim.AdamW( + adapter_parameters(role_model, STUDENT_ADAPTER), + lr=args.student_lr, + betas=(0.9, 0.999), + weight_decay=0.0, + ) + critic_optimizer = torch.optim.AdamW( + adapter_parameters(role_model, CRITIC_ADAPTER), + lr=args.critic_lr, + betas=(0.9, 0.999), + weight_decay=0.0, + ) + trainer = H3DMDTrainer( + forward_model=forward_model, + role_model=role_model, + student_optimizer=student_optimizer, + critic_optimizer=critic_optimizer, + video_shift=args.video_shift, + audio_shift=args.audio_shift, + max_grad_norm=args.max_grad_norm, + ) + output_dir = Path(args.output_dir) + if rank == 0: + runtime = { + **config, + "started_at": _utc_now(), + "world_size": world_size, + "torch_version": torch.__version__, + "cuda_version": torch.version.cuda, + "full_weight_run_validated_before_launch": False, + "load_risk": "one full CPU checkpoint is loaded per rank before FSDP sharding", + "transformer_load_dtype": "bfloat16", + "video_latent_dtype": str(video_input_dtype).removeprefix("torch."), + "audio_latent_dtype": str(audio_input_dtype).removeprefix("torch."), + "fsdp_wrap_plan": getattr(role_model, "_h3_fsdp_wrap_plan_report", None), + "fsdp_runtime_audit": getattr( + role_model, "_h3_fsdp_runtime_report", None + ), + "released_architecture": getattr( + role_model, "_h3_released_architecture", None + ), + "flex_attention_compile": flex_attention_compile, + "cuda_allocator_config_observed": allocator_config, + "cuda_allocator_backend_observed": allocator_backend, + } + _atomic_json(runtime, output_dir / "runtime.json") + + latent_frames, latent_height, latent_width, audio_latents = _validate_geometry(args) + patch_size = tuple(role_model.config.patch_size) + video_patch_dim = int(role_model.config.in_channels * math.prod(patch_size)) + audio_channels = int(role_model.config.audio_in_channels) + layout_cache: dict[str, Any] = {} + final_checkpoint_path: Path | None = None + + for step in range(1, args.max_steps + 1): + condition = cached_conditions[ + (args.prompt_start_index + step - 1) % len(cached_conditions) + ].to(device) + if condition.id not in layout_cache: + layout = make_base_t2va_layout( + text_token_tags=condition.text_token_tags, + num_latent_frames=latent_frames, + latent_height=latent_height, + latent_width=latent_width, + num_audio_latents=audio_latents, + patch_size=patch_size, + ).to(device) + layout_cache[condition.id] = layout + layout = layout_cache[condition.id] + + rng = torch.Generator(device=device).manual_seed( + int(args.seed + condition.seed + step * 1_000_003 + rank * 10_000_019) + ) + video_noise = torch.randn( + (1, layout.video_indices.numel(), video_patch_dim), + device=device, + dtype=video_input_dtype, + generator=rng, + ) + audio_noise = torch.randn( + (1, layout.audio_indices.numel(), audio_channels), + device=device, + dtype=audio_input_dtype, + generator=rng, + ) + + # PEFT initializes critic == Base. A critic update therefore precedes + # every generator update, including step one, so the DMD direction is + # not identically zero. + with torch.no_grad(): + critic_clean_video, critic_clean_audio = trainer.student_rollout( + video_noise=video_noise, + audio_noise=audio_noise, + condition=condition, + layout=layout, + ) + critic_sigma = float( + torch.empty((), device=device).uniform_(0.02, 0.98, generator=rng) + ) + critic = trainer.critic_step( + clean_video=critic_clean_video, + clean_audio=critic_clean_audio, + condition=condition, + layout=layout, + score_base_sigma=critic_sigma, + generator=rng, + ) + generator_sigma = float( + torch.empty((), device=device).uniform_(0.02, 0.98, generator=rng) + ) + cache_clears_before = trainer.cuda_cache_clears_before_generator_backward + offload_contexts_before = trainer.saved_tensor_offload_contexts + offload_packs_before = trainer.saved_tensor_offload_pack_count + offload_unpacks_before = trainer.saved_tensor_offload_unpack_count + offload_bytes_before = trainer.saved_tensor_offload_packed_bytes + generated = trainer.generator_step( + video_noise=video_noise, + audio_noise=audio_noise, + condition=condition, + layout=layout, + score_base_sigma=generator_sigma, + generator=rng, + ) + + cache_clears = ( + trainer.cuda_cache_clears_before_generator_backward + - cache_clears_before + ) + offload_contexts = ( + trainer.saved_tensor_offload_contexts - offload_contexts_before + ) + offload_packs = trainer.saved_tensor_offload_pack_count - offload_packs_before + offload_unpacks = ( + trainer.saved_tensor_offload_unpack_count - offload_unpacks_before + ) + offload_bytes = ( + trainer.saved_tensor_offload_packed_bytes - offload_bytes_before + ) + local_memory_policy_valid = ( + cache_clears == 1 + and offload_contexts == 1 + and offload_packs > 0 + and offload_unpacks > 0 + and offload_bytes > 0 + ) + memory_policy_all_ranks_verified = local_memory_policy_valid + if args.strategy == "fsdp": + import torch.distributed as dist + + consensus = torch.tensor( + int(local_memory_policy_valid), device=device, dtype=torch.int32 + ) + dist.all_reduce(consensus, op=dist.ReduceOp.MIN) + memory_policy_all_ranks_verified = bool(consensus.item()) + if not memory_policy_all_ranks_verified: + raise RuntimeError( + "At least one rank did not execute the audited saved-tensor/cache policy." + ) + + roles = trainer.audit.records + record = { + "step": step, + "prompt_id": condition.id, + "generator_loss": float(generated.loss), + "generator_video_loss": float(generated.video_loss), + "generator_audio_loss": float(generated.audio_loss), + "generator_grad_norm": float(generated.grad_norm), + "critic_loss": float(critic.loss), + "critic_video_loss": float(critic.video_loss), + "critic_audio_loss": float(critic.audio_loss), + "critic_grad_norm": float(critic.grad_norm), + "student_positive_forwards": sum(row.role == "student" for row in roles), + "student_rollouts": 2, + "student_positive_forwards_per_rollout": NUM_MODEL_EVALUATIONS, + "rollout_mode": "full_sequence", + "attention_is_causal": False, + "temporal_ar_blocks": 0, + "teacher_positive_forwards": sum(row.role == "teacher" for row in roles), + "unconditional_forwards": 0, + "cuda_cache_clears_before_generator_backward": cache_clears, + "saved_tensor_offload_contexts": offload_contexts, + "saved_tensor_offload_pack_count": offload_packs, + "saved_tensor_offload_unpack_count": offload_unpacks, + "saved_tensor_offload_packed_bytes": offload_bytes, + "memory_policy_all_ranks_verified": memory_policy_all_ranks_verified, + } + if rank == 0: + _append_jsonl(record, output_dir / "metrics.jsonl") + print(json.dumps(record, sort_keys=True), flush=True) + trainer.audit = H3CallAudit() + + if step % args.checkpoint_every == 0 or step == args.max_steps: + checkpoint_path = _save_lora_checkpoint( + forward_model=forward_model, + role_model=role_model, + output_dir=output_dir, + step=step, + config=config, + rank=rank, + ) + if checkpoint_path is not None: + final_checkpoint_path = checkpoint_path + if args.strategy == "fsdp": + import torch.distributed as dist + + dist.barrier() + + if rank == 0: + if trainer.cuda_cache_clears_before_generator_backward != args.max_steps: + raise AssertionError("CUDA cache-clear total differs from completed steps.") + if trainer.saved_tensor_offload_contexts != args.max_steps: + raise AssertionError("Saved-tensor offload total differs from completed steps.") + if min( + trainer.saved_tensor_offload_pack_count, + trainer.saved_tensor_offload_unpack_count, + trainer.saved_tensor_offload_packed_bytes, + ) <= 0: + raise AssertionError("Saved-tensor offload counters were not populated.") + cuda_memory_policy = { + "cuda_allocator_config": CUDA_ALLOCATOR_CONFIG, + "cuda_allocator_config_observed": allocator_config, + "cuda_allocator_backend_observed": allocator_backend, + "cuda_cache_policy": CUDA_CACHE_POLICY, + "cuda_empty_cache_before_generator_backward": True, + "cuda_cache_clears_before_generator_backward_total": ( + trainer.cuda_cache_clears_before_generator_backward + ), + "saved_tensor_offload_policy": SAVED_TENSOR_OFFLOAD_POLICY, + "saved_tensor_offload_pin_memory": SAVED_TENSOR_OFFLOAD_PIN_MEMORY, + "saved_tensor_offload_contexts_total": trainer.saved_tensor_offload_contexts, + "saved_tensor_offload_pack_count_total": trainer.saved_tensor_offload_pack_count, + "saved_tensor_offload_unpack_count_total": trainer.saved_tensor_offload_unpack_count, + "saved_tensor_offload_packed_bytes_total": trainer.saved_tensor_offload_packed_bytes, + "all_ranks_verified": True, + } + runtime_path = output_dir / "runtime.json" + runtime = json.loads(runtime_path.read_text(encoding="utf-8")) + runtime["finished_at"] = _utc_now() + runtime["completed_steps"] = args.max_steps + runtime["cuda_cache_clears_before_generator_backward_total"] = ( + trainer.cuda_cache_clears_before_generator_backward + ) + runtime["saved_tensor_offload_contexts_total"] = ( + trainer.saved_tensor_offload_contexts + ) + runtime["saved_tensor_offload_pack_count_total"] = ( + trainer.saved_tensor_offload_pack_count + ) + runtime["saved_tensor_offload_unpack_count_total"] = ( + trainer.saved_tensor_offload_unpack_count + ) + runtime["saved_tensor_offload_packed_bytes_total"] = ( + trainer.saved_tensor_offload_packed_bytes + ) + runtime["cuda_memory_policy"] = cuda_memory_policy + _atomic_json(runtime, runtime_path) + if final_checkpoint_path is None: + raise AssertionError("Rank zero completed without a LoRA checkpoint.") + run_manifest = { + "schema_version": 1, + "status": "completed", + "completed_at": _utc_now(), + "config_sha256": config["config_sha256"], + "model_id": config["model_id"], + "resolved_revision": config["resolved_revision"], + "prompt_cache": config["prompt_cache"], + "prompt_cache_sha256": config["prompt_cache_sha256"], + "normalization": DMD_NORMALIZATION, + "cuda_memory_policy": cuda_memory_policy, + "contract": { + "workflow": WORKFLOW, + "ir": False, + "regenerator": False, + "cfg": False, + "unconditional_forwards": 0, + "teacher_positive_forwards_per_generator_update": 1, + "teacher_adapters_disabled": True, + "rollout_mode": "full_sequence", + "attention_mode": "native_non_causal", + "attention_is_causal": False, + "temporal_ar_blocks": 0, + "model_evaluations_per_rollout": NUM_MODEL_EVALUATIONS, + "sigma_endpoints_per_rollout": NUM_SIGMA_ENDPOINTS, + "student_positive_forwards_per_rollout": NUM_MODEL_EVALUATIONS, + "transformer_block_count": RELEASED_MAIN_BLOCKS, + "token_refiner_block_count": RELEASED_REFINER_BLOCKS, + "student_adapter": STUDENT_ADAPTER, + "critic_adapter": CRITIC_ADAPTER, + "base_weights_shared": True, + "video_and_audio_joint": True, + "normalization": DMD_NORMALIZATION, + "cuda_cache_policy": CUDA_CACHE_POLICY, + "cuda_empty_cache_before_generator_backward": True, + "cuda_allocator_config": CUDA_ALLOCATOR_CONFIG, + "saved_tensor_offload_policy": SAVED_TENSOR_OFFLOAD_POLICY, + "saved_tensor_offload_pin_memory": SAVED_TENSOR_OFFLOAD_PIN_MEMORY, + "transformer_load_dtype": "bfloat16", + "video_latent_dtype": str(video_input_dtype).removeprefix("torch."), + "audio_latent_dtype": str(audio_input_dtype).removeprefix("torch."), + "fsdp_sharding": config.get("fsdp_sharding"), + "fsdp_use_orig_params": config.get("fsdp_use_orig_params"), + "fsdp_auto_wrap_classes": config.get("fsdp_auto_wrap_classes"), + "fsdp_auto_wrap_policy": config.get("fsdp_auto_wrap_policy"), + "fsdp_top_level_wrap_modules": config.get( + "fsdp_top_level_wrap_modules" + ), + "fsdp_inner_wrap_modules": config.get("fsdp_inner_wrap_modules"), + "fsdp_attention_projection_wrap_modules": config.get( + "fsdp_attention_projection_wrap_modules" + ), + "fsdp_attention_base_wrap_module": config.get( + "fsdp_attention_base_wrap_module" + ), + "fsdp_ffn_projection_wrap_modules": config.get( + "fsdp_ffn_projection_wrap_modules" + ), + "fsdp_handle_uniform_dtype_required": config.get( + "fsdp_handle_uniform_dtype_required" + ), + "fsdp_root_parameter_tensors_expected": config.get( + "fsdp_root_parameter_tensors_expected" + ), + "fsdp_cpu_offload_policy": config.get("fsdp_cpu_offload_policy"), + "fsdp_cpu_offload_params": config.get("fsdp_cpu_offload_params"), + "fsdp_cpu_offload_handle_count": runtime["fsdp_wrap_plan"][ + "cpu_offload_handle_count" + ], + "fsdp_cpu_offload_parameter_numel": runtime["fsdp_wrap_plan"][ + "cpu_offload_parameter_numel" + ], + "fsdp_limit_all_gathers": config.get("fsdp_limit_all_gathers"), + "fsdp_backward_prefetch": config.get("fsdp_backward_prefetch"), + "flex_attention_backend": config.get("flex_attention_backend"), + "fsdp_wrap_plan": getattr( + role_model, "_h3_fsdp_wrap_plan_report", None + ), + "fsdp_runtime_audit": runtime["fsdp_runtime_audit"], + }, + "completed_steps": args.max_steps, + "artifacts": { + "runtime": "runtime.json", + "metrics": "metrics.jsonl", + "final_lora": str(final_checkpoint_path.relative_to(output_dir)), + "final_lora_sha256": sha256_file(final_checkpoint_path), + }, + } + manifest_path = output_dir / "run_manifest.json" + _atomic_json(run_manifest, manifest_path) + success = { + "schema_version": 1, + "status": "success", + "manifest": "run_manifest.json", + "manifest_sha256": sha256_file(manifest_path), + "config_sha256": config["config_sha256"], + } + _atomic_json(success, output_dir / "SUCCESS.json") + print( + json.dumps( + {"run_manifest": str(manifest_path), **success}, sort_keys=True + ) + ) + if args.strategy == "fsdp": + import torch.distributed as dist + + dist.barrier() + dist.destroy_process_group() + del forward_model, role_model + gc.collect() + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + cache = subparsers.add_parser( + "prepare-prompt-cache", help="Encode four positive prompts with official H3." + ) + cache.add_argument("--model-id", default=MODEL_ID) + cache.add_argument("--revision", default="main") + cache.add_argument("--prompts", default="minimax_h3/prompts.json") + cache.add_argument("--output", required=True) + cache.add_argument("--local-files-only", action="store_true") + cache.set_defaults(handler=prepare_prompt_cache) + + training = subparsers.add_parser( + "train", help="Run Base H3 full-sequence few-step DMD LoRA training." + ) + training.add_argument("--model-id", "--model-dir", dest="model_id", default=MODEL_ID) + training.add_argument("--revision", default="main") + training.add_argument("--prompt-cache", required=True) + training.add_argument("--output-dir", required=True) + training.add_argument("--strategy", choices=("fsdp", "single"), default="fsdp") + training.add_argument("--fsdp", action="store_const", const="fsdp", dest="strategy") + training.add_argument("--device", default="cuda:0") + training.add_argument("--height", type=int, default=768) + training.add_argument("--width", type=int, default=1344) + training.add_argument("--num-frames", type=int, default=124) + training.add_argument("--max-steps", type=int, default=1) + training.add_argument( + "--prompt-start-index", + type=int, + default=0, + help="Rotate the audited four-prompt cache from this index before training.", + ) + training.add_argument("--seed", type=int, default=0) + training.add_argument("--checkpoint-every", type=int, default=1) + training.add_argument("--lora-rank", type=int, default=16) + training.add_argument("--lora-alpha", type=int, default=16) + training.add_argument("--lora-dropout", type=float, default=0.0) + training.add_argument("--student-lr", type=float, default=1e-5) + training.add_argument("--critic-lr", type=float, default=1e-5) + training.add_argument("--max-grad-norm", type=float, default=1.0) + training.add_argument("--video-shift", type=float, default=12.0) + training.add_argument("--audio-shift", type=float, default=3.0) + training.add_argument("--local-files-only", action="store_true") + training.add_argument( + "--gradient-checkpointing", + action=argparse.BooleanOptionalAction, + default=True, + ) + training.add_argument("--validate-only", action="store_true") + training.add_argument("--acknowledge-full-rank-cpu-load", action="store_true") + training.add_argument("--allow-single-device-full-model", action="store_true") + training.set_defaults(handler=train) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if getattr(args, "checkpoint_every", 1) < 1: + raise ValueError("checkpoint_every must be positive.") + return int(args.handler(args)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/source_snapshot/minimax_h3/train_dmd_longrun.py b/source_snapshot/minimax_h3/train_dmd_longrun.py new file mode 100644 index 0000000000000000000000000000000000000000..d88695e097b050840517828526d3983a9affae31 --- /dev/null +++ b/source_snapshot/minimax_h3/train_dmd_longrun.py @@ -0,0 +1,1102 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Restart-safe 1,000-update MiniMax-H3 DMD LoRA training. + +This entry point deliberately leaves :mod:`minimax_h3.train_dmd` unchanged. +It reuses that module's validated H3/FSDP/model-loading helpers while adding +the state needed to continue one logical 1,000-update experiment across +multiple Slurm allocations. + +The immutable LongLive-style optimization contract is: + +* critic AdamW update at every global step; +* student AdamW update when ``(global_step - 1) % 5 == 0``; +* student/critic learning rates ``1e-5``/``2e-6``; +* AdamW ``betas=(0, 0.999)`` and ``weight_decay=0``; +* LoRA rank/alpha 8, dropout 0, and gradient clipping at 1; +* 1,000 total updates and a restart checkpoint every 50 global steps. + +Each restart checkpoint keeps the schema-1 ``lora.pt`` consumed by the +existing student inference entry point and adds ``training_state.pt`` plus a +last-written ``CHECKPOINT_SUCCESS.json`` integrity sentinel. Segment metrics +and completion sentinels live in a separate ``--output-dir`` while all +restart checkpoints live below the shared ``--checkpoint-root``. +""" + +from __future__ import annotations + +import argparse +import gc +import json +import math +import os +import re +from pathlib import Path +from typing import Any, Mapping, Sequence + +import torch + +from . import train_dmd as base +from .infer_base import sha256_file +from .train_utils import ( + CRITIC_ADAPTER, + CUDA_ALLOCATOR_CONFIG, + CUDA_CACHE_POLICY, + NUM_MODEL_EVALUATIONS, + NUM_SIGMA_ENDPOINTS, + SAVED_TENSOR_OFFLOAD_PIN_MEMORY, + SAVED_TENSOR_OFFLOAD_POLICY, + STUDENT_ADAPTER, + H3CallAudit, + H3DMDTrainer, + adapter_parameters, + assert_released_h3_architecture, + h3_model_input_dtypes, + install_independent_lora_adapters, + load_prompt_cache, + make_base_t2va_layout, +) + + +TOTAL_STEPS = 1_000 +CHECKPOINT_EVERY = 50 +STUDENT_UPDATE_RATIO = 5 +STUDENT_LR = 1.0e-5 +CRITIC_LR = 2.0e-6 +ADAM_BETAS = (0.0, 0.999) +WEIGHT_DECAY = 0.0 +LORA_RANK = 8 +LORA_ALPHA = 8 +LORA_DROPOUT = 0.0 +MAX_GRAD_NORM = 1.0 + +TRAINING_STATE_KIND = "minimax_h3_dmd_longrun_training_state" +CHECKPOINT_KIND = "minimax_h3_dmd_longrun_checkpoint" +SEGMENT_KIND = "minimax_h3_dmd_longrun_segment" +_CHECKPOINT_DIR = re.compile(r"^checkpoint-(\d{6})$") + + +def student_update_due(global_step: int) -> bool: + """Return the fixed LongLive 5:1 student-update decision.""" + + if isinstance(global_step, bool) or not isinstance(global_step, int): + raise TypeError("global_step must be an integer.") + if global_step < 1: + raise ValueError("global_step must be positive.") + return (global_step - 1) % STUDENT_UPDATE_RATIO == 0 + + +def student_updates_through(global_step: int) -> int: + """Number of scheduled student updates in inclusive steps ``1..step``.""" + + if isinstance(global_step, bool) or not isinstance(global_step, int): + raise TypeError("global_step must be an integer.") + if global_step < 0: + raise ValueError("global_step cannot be negative.") + return (global_step + STUDENT_UPDATE_RATIO - 1) // STUDENT_UPDATE_RATIO + + +def student_updates_in_segment(start_step: int, stop_step: int) -> int: + if start_step < 0 or stop_step <= start_step: + raise ValueError("A segment requires 0 <= start_step < stop_step.") + return student_updates_through(stop_step) - student_updates_through(start_step) + + +def _stable_config_hash(config: Mapping[str, Any]) -> str: + payload = dict(config) + payload.pop("config_sha256", None) + return base._config_sha256(payload) + + +def _validate_fixed_hyperparameters(args: argparse.Namespace) -> None: + expected = { + "max_steps": TOTAL_STEPS, + "checkpoint_every": CHECKPOINT_EVERY, + "lora_rank": LORA_RANK, + "lora_alpha": LORA_ALPHA, + "lora_dropout": LORA_DROPOUT, + "student_lr": STUDENT_LR, + "critic_lr": CRITIC_LR, + "max_grad_norm": MAX_GRAD_NORM, + } + mismatches = { + name: {"actual": getattr(args, name), "expected": value} + for name, value in expected.items() + if getattr(args, name) != value + } + if mismatches: + raise ValueError( + "The 1,000-step LongLive optimizer contract is fixed: " + + json.dumps(mismatches, sort_keys=True) + ) + if args.stop_step < CHECKPOINT_EVERY or args.stop_step > TOTAL_STEPS: + raise ValueError(f"stop_step must lie in [{CHECKPOINT_EVERY}, {TOTAL_STEPS}].") + if args.stop_step % CHECKPOINT_EVERY: + raise ValueError(f"stop_step must be divisible by {CHECKPOINT_EVERY}.") + + +def validate_longrun_request(args: argparse.Namespace) -> dict[str, Any]: + """Validate the original H3 contract and add immutable long-run fields.""" + + _validate_fixed_hyperparameters(args) + config = base.validate_train_request(args) + config.pop("config_sha256", None) + config.update( + { + "longrun_schema_version": 1, + "total_optimizer_steps": TOTAL_STEPS, + "checkpoint_every": CHECKPOINT_EVERY, + "critic_update_every": 1, + "student_update_ratio": STUDENT_UPDATE_RATIO, + "student_update_rule": "(global_step-1)%5==0", + "student_optimizer": "AdamW", + "critic_optimizer": "AdamW", + "student_lr": STUDENT_LR, + "critic_lr": CRITIC_LR, + "adam_betas": list(ADAM_BETAS), + "student_betas": list(ADAM_BETAS), + "critic_betas": list(ADAM_BETAS), + "weight_decay": WEIGHT_DECAY, + "lora_rank": LORA_RANK, + "lora_alpha": LORA_ALPHA, + "lora_dropout": LORA_DROPOUT, + "max_grad_norm": MAX_GRAD_NORM, + "sample_rng_scheme": "seed+condition_seed+global_step+rank-v1", + } + ) + config["config_sha256"] = _stable_config_hash(config) + return config + + +def _regular_file(path: Path, *, label: str) -> Path: + if not path.is_file() or path.is_symlink(): + raise RuntimeError(f"{label} is missing or is not a regular file: {path}") + return path + + +def _load_weights_only(path: Path, *, label: str) -> dict[str, Any]: + try: + payload = torch.load(path, map_location="cpu", weights_only=True) + except Exception as error: + raise RuntimeError(f"Unable to load {label}: {path}") from error + if not isinstance(payload, dict): + raise RuntimeError(f"{label} must contain a dictionary: {path}") + return payload + + +def _validate_optimizer_state( + state_dict: Mapping[str, Any], *, expected_updates: int, label: str +) -> None: + states = state_dict.get("state") + groups = state_dict.get("param_groups") + if not isinstance(states, Mapping) or not states: + raise RuntimeError(f"{label} optimizer state is empty.") + if not isinstance(groups, list) or not groups: + raise RuntimeError(f"{label} optimizer parameter groups are missing.") + expected_lr = STUDENT_LR if label == "student" else CRITIC_LR + for group in groups: + if not isinstance(group, Mapping): + raise RuntimeError(f"{label} optimizer parameter group is invalid.") + hyperparameters = { + "lr": expected_lr, + "betas": ADAM_BETAS, + "weight_decay": WEIGHT_DECAY, + } + mismatches = { + name: {"actual": group.get(name), "expected": expected} + for name, expected in hyperparameters.items() + if group.get(name) != expected + } + if mismatches: + raise RuntimeError( + f"{label} optimizer hyperparameter mismatch: " + + json.dumps(mismatches, sort_keys=True) + ) + observed: set[int] = set() + for row in states.values(): + if not isinstance(row, Mapping) or "step" not in row: + raise RuntimeError(f"{label} optimizer has a state row without step.") + value = row["step"] + if isinstance(value, torch.Tensor): + if value.numel() != 1: + raise RuntimeError(f"{label} optimizer step must be scalar.") + value = value.item() + observed.add(int(value)) + if observed != {expected_updates}: + raise RuntimeError( + f"{label} optimizer step mismatch: {sorted(observed)} != " + f"[{expected_updates}]." + ) + + +def _validate_lora_payload( + payload: Mapping[str, Any], *, step: int, config_sha256: str +) -> None: + if payload.get("schema_version") != 1 or payload.get("step") != step: + raise RuntimeError("LoRA checkpoint schema/step mismatch.") + config = payload.get("config") + if not isinstance(config, Mapping): + raise RuntimeError("LoRA checkpoint is missing its training config.") + if config.get("config_sha256") != config_sha256: + raise RuntimeError("LoRA checkpoint config hash mismatch.") + if _stable_config_hash(config) != config_sha256: + raise RuntimeError("LoRA checkpoint embedded config is not self-consistent.") + for role in ("student_lora", "critic_lora"): + state = payload.get(role) + if not isinstance(state, Mapping) or not state: + raise RuntimeError(f"LoRA checkpoint is missing {role}.") + if any( + not isinstance(name, str) or not isinstance(value, torch.Tensor) + for name, value in state.items() + ): + raise RuntimeError(f"LoRA checkpoint {role} contains a non-tensor entry.") + + +def load_longrun_checkpoint( + checkpoint_dir: str | Path, + *, + expected_config_sha256: str, + expected_world_size: int, + checkpoint_root: str | Path | None = None, +) -> dict[str, Any]: + """Fail closed unless a complete restart checkpoint validates end to end.""" + + directory = Path(checkpoint_dir).resolve() + if not directory.is_dir() or directory.is_symlink(): + raise RuntimeError(f"Resume checkpoint directory is invalid: {directory}") + match = _CHECKPOINT_DIR.fullmatch(directory.name) + if match is None: + raise RuntimeError(f"Unexpected checkpoint directory name: {directory.name}") + directory_step = int(match.group(1)) + if checkpoint_root is not None and directory.parent != Path(checkpoint_root).resolve(): + raise RuntimeError("resume_from must be a direct child of checkpoint_root.") + + success_path = _regular_file( + directory / "CHECKPOINT_SUCCESS.json", label="checkpoint success sentinel" + ) + try: + success = json.loads(success_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise RuntimeError(f"Invalid checkpoint success sentinel: {success_path}") from error + expected_success = { + "schema_version": 1, + "status": "success", + "kind": CHECKPOINT_KIND, + "global_step": directory_step, + "world_size": expected_world_size, + "config_sha256": expected_config_sha256, + "student_update_count": student_updates_through(directory_step), + "critic_update_count": directory_step, + "base_weights_included": False, + } + mismatches = { + name: {"actual": success.get(name), "expected": value} + for name, value in expected_success.items() + if success.get(name) != value + } + if mismatches: + raise RuntimeError( + "Checkpoint success sentinel mismatch: " + json.dumps(mismatches, sort_keys=True) + ) + if directory_step < CHECKPOINT_EVERY or directory_step % CHECKPOINT_EVERY: + raise RuntimeError("Resume checkpoint step is not on the 50-step grid.") + + artifacts = success.get("artifacts") + expected_names = {"lora.pt", "manifest.json", "training_state.pt"} + if not isinstance(artifacts, Mapping) or set(artifacts) != expected_names: + raise RuntimeError("Checkpoint success sentinel artifact set is invalid.") + paths: dict[str, Path] = {} + for name in sorted(expected_names): + metadata = artifacts.get(name) + if not isinstance(metadata, Mapping): + raise RuntimeError(f"Checkpoint artifact metadata is invalid for {name}.") + path = _regular_file(directory / name, label=f"checkpoint artifact {name}") + digest = sha256_file(path) + if metadata.get("sha256") != digest or metadata.get("bytes") != path.stat().st_size: + raise RuntimeError(f"Checkpoint artifact integrity mismatch for {name}.") + paths[name] = path + + lora = _load_weights_only(paths["lora.pt"], label="LoRA checkpoint") + training = _load_weights_only(paths["training_state.pt"], label="training state") + _validate_lora_payload(lora, step=directory_step, config_sha256=expected_config_sha256) + expected_training = { + "schema_version": 1, + "kind": TRAINING_STATE_KIND, + "global_step": directory_step, + "world_size": expected_world_size, + "config_sha256": expected_config_sha256, + "student_update_count": student_updates_through(directory_step), + "critic_update_count": directory_step, + } + training_mismatches = { + name: {"actual": training.get(name), "expected": value} + for name, value in expected_training.items() + if training.get(name) != value + } + if training_mismatches: + raise RuntimeError( + "Training-state metadata mismatch: " + + json.dumps(training_mismatches, sort_keys=True) + ) + _validate_optimizer_state( + training.get("student_optimizer", {}), + expected_updates=student_updates_through(directory_step), + label="student", + ) + _validate_optimizer_state( + training.get("critic_optimizer", {}), + expected_updates=directory_step, + label="critic", + ) + rng_states = training.get("rng_states") + if not isinstance(rng_states, list) or len(rng_states) != expected_world_size: + raise RuntimeError("Training-state RNG list does not match world_size.") + if sorted(row.get("rank") for row in rng_states if isinstance(row, Mapping)) != list( + range(expected_world_size) + ): + raise RuntimeError("Training-state RNG ranks are incomplete or duplicated.") + return { + "directory": directory, + "global_step": directory_step, + "success": success, + "success_sha256": sha256_file(success_path), + "lora": lora, + "training": training, + } + + +def _capture_local_rng(device: torch.device, rank: int) -> dict[str, Any]: + cuda_state = None + if device.type == "cuda": + cuda_state = torch.cuda.get_rng_state(device).cpu() + return { + "rank": rank, + "torch_cpu": torch.get_rng_state().cpu(), + "torch_cuda": cuda_state, + } + + +def _gather_rng_states( + *, device: torch.device, rank: int, world_size: int, strategy: str +) -> list[dict[str, Any]] | None: + local = _capture_local_rng(device, rank) + if strategy == "single": + return [local] + import torch.distributed as dist + + gathered: list[dict[str, Any] | None] = [None] * world_size + dist.all_gather_object(gathered, local) + if rank != 0: + return None + if any(row is None for row in gathered): + raise RuntimeError("Failed to gather every rank RNG state.") + return [dict(row) for row in gathered if row is not None] + + +def _restore_local_rng( + rng_states: Sequence[Mapping[str, Any]], *, device: torch.device, rank: int +) -> None: + by_rank = {int(row["rank"]): row for row in rng_states} + if rank not in by_rank or len(by_rank) != len(rng_states): + raise RuntimeError("Checkpoint RNG states do not contain one unique row per rank.") + row = by_rank[rank] + cpu_state = row.get("torch_cpu") + cuda_state = row.get("torch_cuda") + if not isinstance(cpu_state, torch.Tensor) or cpu_state.dtype != torch.uint8: + raise RuntimeError("Checkpoint CPU RNG state is invalid.") + torch.set_rng_state(cpu_state.cpu()) + if device.type == "cuda": + if not isinstance(cuda_state, torch.Tensor) or cuda_state.dtype != torch.uint8: + raise RuntimeError("Checkpoint CUDA RNG state is invalid.") + torch.cuda.set_rng_state(cuda_state.cpu(), device) + elif cuda_state is not None: + raise RuntimeError("A CPU resume unexpectedly received CUDA RNG state.") + + +def _gather_full_optimizer_states( + *, + forward_model: torch.nn.Module, + role_model: torch.nn.Module, + student_optimizer: torch.optim.Optimizer, + critic_optimizer: torch.optim.Optimizer, +) -> tuple[dict[str, Any], dict[str, Any]]: + if forward_model is role_model: + return student_optimizer.state_dict(), critic_optimizer.state_dict() + + from torch.distributed.fsdp import ( + FullOptimStateDictConfig, + FullStateDictConfig, + FullyShardedDataParallel as FSDP, + StateDictType, + ) + + with FSDP.state_dict_type( + forward_model, + StateDictType.FULL_STATE_DICT, + FullStateDictConfig(offload_to_cpu=True, rank0_only=True), + FullOptimStateDictConfig(offload_to_cpu=True, rank0_only=True), + ): + student_state = FSDP.optim_state_dict(forward_model, student_optimizer) + critic_state = FSDP.optim_state_dict(forward_model, critic_optimizer) + return student_state, critic_state + + +def _checkpoint_target_is_available( + checkpoint_dir: Path, *, device: torch.device, strategy: str +) -> None: + exists = checkpoint_dir.exists() + if strategy == "fsdp": + import torch.distributed as dist + + flag = torch.tensor(int(exists), device=device, dtype=torch.int32) + dist.all_reduce(flag, op=dist.ReduceOp.MAX) + exists = bool(flag.item()) + if exists: + raise FileExistsError(f"Refusing to overwrite checkpoint directory: {checkpoint_dir}") + + +def save_longrun_checkpoint( + *, + forward_model: torch.nn.Module, + role_model: torch.nn.Module, + student_optimizer: torch.optim.Optimizer, + critic_optimizer: torch.optim.Optimizer, + checkpoint_root: Path, + step: int, + config: dict[str, Any], + device: torch.device, + rank: int, + world_size: int, + strategy: str, +) -> Path: + """Collectively save an inference artifact and exact restart state.""" + + if step % CHECKPOINT_EVERY or step < CHECKPOINT_EVERY: + raise ValueError("Long-run checkpoints must lie on the 50-step grid.") + checkpoint_dir = checkpoint_root / f"checkpoint-{step:06d}" + _checkpoint_target_is_available(checkpoint_dir, device=device, strategy=strategy) + + # Capture RNG before checkpoint collectives and filesystem operations so a + # restart restores the post-update stochastic state exactly. + rng_states = _gather_rng_states( + device=device, rank=rank, world_size=world_size, strategy=strategy + ) + lora_path = base._save_lora_checkpoint( + forward_model=forward_model, + role_model=role_model, + output_dir=checkpoint_root, + step=step, + config=config, + rank=rank, + ) + student_state, critic_state = _gather_full_optimizer_states( + forward_model=forward_model, + role_model=role_model, + student_optimizer=student_optimizer, + critic_optimizer=critic_optimizer, + ) + if rank == 0: + if lora_path is None or rng_states is None: + raise AssertionError("Rank zero did not receive gathered checkpoint state.") + _validate_optimizer_state( + student_state, + expected_updates=student_updates_through(step), + label="student", + ) + _validate_optimizer_state( + critic_state, expected_updates=step, label="critic" + ) + training_state = { + "schema_version": 1, + "kind": TRAINING_STATE_KIND, + "global_step": step, + "world_size": world_size, + "config_sha256": config["config_sha256"], + "student_update_count": student_updates_through(step), + "critic_update_count": step, + "student_optimizer": student_state, + "critic_optimizer": critic_state, + "rng_states": rng_states, + } + training_path = checkpoint_dir / "training_state.pt" + base._atomic_torch_save(training_state, training_path) + artifact_paths = { + "lora.pt": lora_path, + "manifest.json": checkpoint_dir / "manifest.json", + "training_state.pt": training_path, + } + success = { + "schema_version": 1, + "status": "success", + "kind": CHECKPOINT_KIND, + "global_step": step, + "world_size": world_size, + "config_sha256": config["config_sha256"], + "student_update_count": student_updates_through(step), + "critic_update_count": step, + "base_weights_included": False, + "created_at": base._utc_now(), + "artifacts": { + name: {"sha256": sha256_file(path), "bytes": path.stat().st_size} + for name, path in artifact_paths.items() + }, + } + base._atomic_json(success, checkpoint_dir / "CHECKPOINT_SUCCESS.json") + if strategy == "fsdp": + import torch.distributed as dist + + dist.barrier() + return checkpoint_dir + + +def _install_checkpoint_loras( + model: torch.nn.Module, checkpoint: Mapping[str, Any] +) -> None: + from peft import get_peft_model_state_dict, set_peft_model_state_dict + + for adapter_name, key in ( + (STUDENT_ADAPTER, "student_lora"), + (CRITIC_ADAPTER, "critic_lora"), + ): + expected = checkpoint[key] + set_peft_model_state_dict(model, expected, adapter_name=adapter_name) + observed = get_peft_model_state_dict(model, adapter_name=adapter_name) + if set(observed) != set(expected): + raise RuntimeError(f"{adapter_name} LoRA key round trip failed during resume.") + mismatches = [ + name + for name, value in expected.items() + if not torch.equal(observed[name].detach().cpu(), value.detach().cpu()) + ] + if mismatches: + raise RuntimeError( + f"{adapter_name} LoRA value round trip failed: {mismatches[:5]}." + ) + model.set_adapter(STUDENT_ADAPTER) + + +def _load_transformer( + args: argparse.Namespace, + resolved_revision: str | None, + device: torch.device, + resume_lora: Mapping[str, Any] | None, +) -> tuple[torch.nn.Module, torch.nn.Module]: + """Load/install LoRA, restore it before wrapping, then reuse audited FSDP.""" + + base.validate_local_model_provenance(args.model_id, resolved_revision) + try: + from diffusers import MiniMaxH3Transformer3DModel + except ImportError as error: + raise RuntimeError("Install the editable diffusers_h3 checkout first.") from error + kwargs = base._model_load_kwargs(args, resolved_revision) + kwargs["low_cpu_mem_usage"] = True + model = MiniMaxH3Transformer3DModel.from_pretrained( + args.model_id, subfolder="transformer", **kwargs + ) + if model.__class__.__name__ != "MiniMaxH3Transformer3DModel": + raise TypeError("Training loaded a non-H3 transformer.") + architecture = assert_released_h3_architecture(model) + if architecture["attention_is_causal"] is not False: + raise AssertionError("Released MiniMax-H3 attention must remain non-causal.") + model._h3_released_architecture = architecture + install_independent_lora_adapters( + model, + rank=LORA_RANK, + alpha=LORA_ALPHA, + dropout=LORA_DROPOUT, + ) + if resume_lora is not None: + _install_checkpoint_loras(model, resume_lora) + model.enable_gradient_checkpointing() + model.train() + if args.strategy == "fsdp": + forward_model = base._wrap_fsdp(model, device) + runtime_report = base.validate_h3_fsdp_runtime_wrappers(forward_model, model) + import torch.distributed as dist + + reports: list[dict[str, Any] | None] = [None] * dist.get_world_size() + dist.all_gather_object(reports, runtime_report) + if any(report != runtime_report for report in reports): + raise RuntimeError("FSDP CPU-offload runtime audit differs across ranks.") + runtime_report["all_ranks_verified"] = True + model._h3_fsdp_runtime_report = runtime_report + else: + model.to(device) + forward_model = model + return forward_model, model + + +def restore_longrun_state( + *, + bundle: Mapping[str, Any], + forward_model: torch.nn.Module, + role_model: torch.nn.Module, + student_optimizer: torch.optim.Optimizer, + critic_optimizer: torch.optim.Optimizer, + device: torch.device, + rank: int, + strategy: str, +) -> None: + """Collectively restore both optimizer shards, then restore per-rank RNG.""" + + del role_model # LoRA values were restored before FSDP wrapping. + training = bundle["training"] + step = int(bundle["global_step"]) + _validate_optimizer_state( + training["student_optimizer"], + expected_updates=student_updates_through(step), + label="student", + ) + _validate_optimizer_state( + training["critic_optimizer"], expected_updates=step, label="critic" + ) + if strategy == "single": + student_local = training["student_optimizer"] + critic_local = training["critic_optimizer"] + else: + from torch.distributed.fsdp import FullyShardedDataParallel as FSDP + + # The saved optimizer states are rank-0-only FULL state dicts. Every + # rank must enter both scatters in the same order; only rank zero passes + # the full object. This is the PyTorch 2.8 exact-resume path. + student_full = training["student_optimizer"] if rank == 0 else None + student_local = FSDP.scatter_full_optim_state_dict( + student_full, forward_model, optim=student_optimizer + ) + critic_full = training["critic_optimizer"] if rank == 0 else None + critic_local = FSDP.scatter_full_optim_state_dict( + critic_full, forward_model, optim=critic_optimizer + ) + student_optimizer.load_state_dict(student_local) + critic_optimizer.load_state_dict(critic_local) + _restore_local_rng(training["rng_states"], device=device, rank=rank) + if strategy == "fsdp": + import torch.distributed as dist + + dist.barrier() + + +def _metric_record( + *, + step: int, + prompt_id: str, + critic: Any, + generated: Any | None, + roles: Sequence[Any], + cache_clears: int, + offload_contexts: int, + offload_packs: int, + offload_unpacks: int, + offload_bytes: int, + memory_policy_verified: bool, +) -> dict[str, Any]: + student_updated = generated is not None + return { + "step": step, + "prompt_id": prompt_id, + "critic_updated": True, + "student_updated": student_updated, + "student_optimizer_updates_completed": student_updates_through(step), + "critic_optimizer_updates_completed": step, + "generator_loss": float(generated.loss) if student_updated else None, + "generator_video_loss": float(generated.video_loss) if student_updated else None, + "generator_audio_loss": float(generated.audio_loss) if student_updated else None, + "generator_grad_norm": float(generated.grad_norm) if student_updated else None, + "critic_loss": float(critic.loss), + "critic_video_loss": float(critic.video_loss), + "critic_audio_loss": float(critic.audio_loss), + "critic_grad_norm": float(critic.grad_norm), + "student_positive_forwards": sum(row.role == "student" for row in roles), + "student_rollouts": 2 if student_updated else 1, + "student_positive_forwards_per_rollout": NUM_MODEL_EVALUATIONS, + "teacher_positive_forwards": sum(row.role == "teacher" for row in roles), + "unconditional_forwards": 0, + "rollout_mode": "full_sequence", + "attention_is_causal": False, + "temporal_ar_blocks": 0, + "cuda_cache_clears_before_generator_backward": cache_clears, + "saved_tensor_offload_contexts": offload_contexts, + "saved_tensor_offload_pack_count": offload_packs, + "saved_tensor_offload_unpack_count": offload_unpacks, + "saved_tensor_offload_packed_bytes": offload_bytes, + "memory_policy_all_ranks_verified": memory_policy_verified, + } + + +def _fresh_segment_output(output_dir: Path) -> None: + conflicts = [ + output_dir / name + for name in ("runtime.json", "metrics.jsonl", "SEGMENT_SUCCESS.json") + if (output_dir / name).exists() + ] + if conflicts: + raise FileExistsError( + "Refusing to append to a non-fresh segment output: " + + ", ".join(str(path) for path in conflicts) + ) + + +def train(args: argparse.Namespace) -> int: + config = validate_longrun_request(args) + if args.validate_only: + print(json.dumps(config, indent=2, sort_keys=True)) + return 0 + + allocator_config = os.environ.get("PYTORCH_CUDA_ALLOC_CONF") + if args.strategy == "fsdp" and allocator_config != CUDA_ALLOCATOR_CONFIG: + raise RuntimeError( + "Full H3 FSDP requires the audited CUDA allocator configuration: " + f"actual={allocator_config!r}, expected={CUDA_ALLOCATOR_CONFIG!r}." + ) + device, rank, world_size = base._init_runtime(args) + allocator_backend = torch.cuda.memory.get_allocator_backend() + if args.strategy == "fsdp" and allocator_backend != "native": + raise RuntimeError( + "The audited H3 memory policy requires PyTorch's native CUDA allocator; " + f"observed {allocator_backend!r}." + ) + + checkpoint_root = Path(args.checkpoint_root).resolve() + output_dir = Path(args.output_dir).resolve() + _fresh_segment_output(output_dir) + if rank == 0: + checkpoint_root.mkdir(parents=True, exist_ok=True) + output_dir.mkdir(parents=True, exist_ok=True) + if args.strategy == "fsdp": + import torch.distributed as dist + + dist.barrier() + + bundle = None + start_step = 0 + if args.resume_from is not None: + bundle = load_longrun_checkpoint( + args.resume_from, + expected_config_sha256=config["config_sha256"], + expected_world_size=world_size, + checkpoint_root=checkpoint_root, + ) + start_step = int(bundle["global_step"]) + if start_step >= args.stop_step: + raise ValueError( + f"Resume step {start_step} must precede stop_step {args.stop_step}." + ) + + # Identical construction seed is required because FSDP does not broadcast + # model state in this audited loader. Resume LoRAs replace their initialized + # values before wrapping; rank-specific RNG is restored only after optimizer + # collectives are complete. + torch.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) + resolved_revision = config["resolved_revision"] + _, cached_conditions = load_prompt_cache(args.prompt_cache) + forward_model, role_model = _load_transformer( + args, + resolved_revision, + device, + bundle["lora"] if bundle is not None else None, + ) + video_input_dtype, audio_input_dtype = h3_model_input_dtypes(role_model) + student_optimizer = torch.optim.AdamW( + adapter_parameters(role_model, STUDENT_ADAPTER), + lr=STUDENT_LR, + betas=ADAM_BETAS, + weight_decay=WEIGHT_DECAY, + ) + critic_optimizer = torch.optim.AdamW( + adapter_parameters(role_model, CRITIC_ADAPTER), + lr=CRITIC_LR, + betas=ADAM_BETAS, + weight_decay=WEIGHT_DECAY, + ) + if bundle is not None: + restore_longrun_state( + bundle=bundle, + forward_model=forward_model, + role_model=role_model, + student_optimizer=student_optimizer, + critic_optimizer=critic_optimizer, + device=device, + rank=rank, + strategy=args.strategy, + ) + trainer = H3DMDTrainer( + forward_model=forward_model, + role_model=role_model, + student_optimizer=student_optimizer, + critic_optimizer=critic_optimizer, + video_shift=args.video_shift, + audio_shift=args.audio_shift, + max_grad_norm=MAX_GRAD_NORM, + ) + + if rank == 0: + runtime = { + **config, + "segment_kind": SEGMENT_KIND, + "started_at": base._utc_now(), + "resume_step": start_step, + "start_step": start_step + 1, + "stop_step": args.stop_step, + "steps_this_segment": args.stop_step - start_step, + "world_size": world_size, + "resume_from": str(bundle["directory"]) if bundle is not None else None, + "resume_checkpoint_success_sha256": ( + bundle["success_sha256"] if bundle is not None else None + ), + "checkpoint_root": str(checkpoint_root), + "torch_version": torch.__version__, + "cuda_version": torch.version.cuda, + "cuda_allocator_config_observed": allocator_config, + "cuda_allocator_backend_observed": allocator_backend, + "video_latent_dtype": str(video_input_dtype).removeprefix("torch."), + "audio_latent_dtype": str(audio_input_dtype).removeprefix("torch."), + "fsdp_wrap_plan": getattr(role_model, "_h3_fsdp_wrap_plan_report", None), + "fsdp_runtime_audit": getattr(role_model, "_h3_fsdp_runtime_report", None), + "released_architecture": getattr(role_model, "_h3_released_architecture", None), + } + base._atomic_json(runtime, output_dir / "runtime.json") + + latent_frames, latent_height, latent_width, audio_latents = base._validate_geometry(args) + patch_size = tuple(role_model.config.patch_size) + video_patch_dim = int(role_model.config.in_channels * math.prod(patch_size)) + audio_channels = int(role_model.config.audio_in_channels) + layout_cache: dict[str, Any] = {} + final_checkpoint_dir: Path | None = None + + for step in range(start_step + 1, args.stop_step + 1): + condition = cached_conditions[ + (args.prompt_start_index + step - 1) % len(cached_conditions) + ].to(device) + if condition.id not in layout_cache: + layout_cache[condition.id] = make_base_t2va_layout( + text_token_tags=condition.text_token_tags, + num_latent_frames=latent_frames, + latent_height=latent_height, + latent_width=latent_width, + num_audio_latents=audio_latents, + patch_size=patch_size, + ).to(device) + layout = layout_cache[condition.id] + rng = torch.Generator(device=device).manual_seed( + int(args.seed + condition.seed + step * 1_000_003 + rank * 10_000_019) + ) + video_noise = torch.randn( + (1, layout.video_indices.numel(), video_patch_dim), + device=device, + dtype=video_input_dtype, + generator=rng, + ) + audio_noise = torch.randn( + (1, layout.audio_indices.numel(), audio_channels), + device=device, + dtype=audio_input_dtype, + generator=rng, + ) + + with torch.no_grad(): + critic_clean_video, critic_clean_audio = trainer.student_rollout( + video_noise=video_noise, + audio_noise=audio_noise, + condition=condition, + layout=layout, + ) + critic_sigma = float( + torch.empty((), device=device).uniform_(0.02, 0.98, generator=rng) + ) + critic = trainer.critic_step( + clean_video=critic_clean_video, + clean_audio=critic_clean_audio, + condition=condition, + layout=layout, + score_base_sigma=critic_sigma, + generator=rng, + ) + + generated = None + cache_before = trainer.cuda_cache_clears_before_generator_backward + contexts_before = trainer.saved_tensor_offload_contexts + packs_before = trainer.saved_tensor_offload_pack_count + unpacks_before = trainer.saved_tensor_offload_unpack_count + bytes_before = trainer.saved_tensor_offload_packed_bytes + if student_update_due(step): + generator_sigma = float( + torch.empty((), device=device).uniform_(0.02, 0.98, generator=rng) + ) + generated = trainer.generator_step( + video_noise=video_noise, + audio_noise=audio_noise, + condition=condition, + layout=layout, + score_base_sigma=generator_sigma, + generator=rng, + ) + cache_clears = trainer.cuda_cache_clears_before_generator_backward - cache_before + offload_contexts = trainer.saved_tensor_offload_contexts - contexts_before + offload_packs = trainer.saved_tensor_offload_pack_count - packs_before + offload_unpacks = trainer.saved_tensor_offload_unpack_count - unpacks_before + offload_bytes = trainer.saved_tensor_offload_packed_bytes - bytes_before + due = generated is not None + local_memory_valid = ( + ( + cache_clears == 1 + and offload_contexts == 1 + and offload_packs > 0 + and offload_unpacks > 0 + and offload_bytes > 0 + ) + if due + else ( + cache_clears == 0 + and offload_contexts == 0 + and offload_packs == 0 + and offload_unpacks == 0 + and offload_bytes == 0 + ) + ) + memory_verified = local_memory_valid + if args.strategy == "fsdp": + import torch.distributed as dist + + consensus = torch.tensor(int(local_memory_valid), device=device, dtype=torch.int32) + dist.all_reduce(consensus, op=dist.ReduceOp.MIN) + memory_verified = bool(consensus.item()) + if not memory_verified: + raise RuntimeError("At least one rank violated the long-run memory policy.") + + record = _metric_record( + step=step, + prompt_id=condition.id, + critic=critic, + generated=generated, + roles=trainer.audit.records, + cache_clears=cache_clears, + offload_contexts=offload_contexts, + offload_packs=offload_packs, + offload_unpacks=offload_unpacks, + offload_bytes=offload_bytes, + memory_policy_verified=memory_verified, + ) + if rank == 0: + base._append_jsonl(record, output_dir / "metrics.jsonl") + print(json.dumps(record, sort_keys=True), flush=True) + trainer.audit = H3CallAudit() + + if step % CHECKPOINT_EVERY == 0: + final_checkpoint_dir = save_longrun_checkpoint( + forward_model=forward_model, + role_model=role_model, + student_optimizer=student_optimizer, + critic_optimizer=critic_optimizer, + checkpoint_root=checkpoint_root, + step=step, + config=config, + device=device, + rank=rank, + world_size=world_size, + strategy=args.strategy, + ) + + expected_student_updates = student_updates_in_segment(start_step, args.stop_step) + if trainer.cuda_cache_clears_before_generator_backward != expected_student_updates: + raise AssertionError("Student CUDA cache-clear count differs from scheduled updates.") + if trainer.saved_tensor_offload_contexts != expected_student_updates: + raise AssertionError("Saved-tensor offload count differs from scheduled student updates.") + if ( + final_checkpoint_dir is None + or final_checkpoint_dir.name != f"checkpoint-{args.stop_step:06d}" + ): + raise AssertionError("Segment ended without its stop-step checkpoint.") + + if rank == 0: + runtime_path = output_dir / "runtime.json" + runtime = json.loads(runtime_path.read_text(encoding="utf-8")) + runtime.update( + { + "finished_at": base._utc_now(), + "completed_global_step": args.stop_step, + "critic_updates_this_segment": args.stop_step - start_step, + "student_updates_this_segment": expected_student_updates, + "student_updates_completed": student_updates_through(args.stop_step), + "final_checkpoint": str(final_checkpoint_dir), + } + ) + base._atomic_json(runtime, runtime_path) + checkpoint_success = final_checkpoint_dir / "CHECKPOINT_SUCCESS.json" + metrics_path = output_dir / "metrics.jsonl" + segment_success = { + "schema_version": 1, + "status": "success", + "kind": SEGMENT_KIND, + "config_sha256": config["config_sha256"], + "resume_step": start_step, + "start_step": start_step + 1, + "stop_step": args.stop_step, + "critic_updates": args.stop_step - start_step, + "student_updates": expected_student_updates, + "critic_updates_completed": args.stop_step, + "student_updates_completed": student_updates_through(args.stop_step), + "world_size": world_size, + "final_checkpoint": str(final_checkpoint_dir), + "artifacts": { + "runtime.json": { + "sha256": sha256_file(runtime_path), + "bytes": runtime_path.stat().st_size, + }, + "metrics.jsonl": { + "sha256": sha256_file(metrics_path), + "bytes": metrics_path.stat().st_size, + }, + "CHECKPOINT_SUCCESS.json": { + "path": str(checkpoint_success), + "sha256": sha256_file(checkpoint_success), + "bytes": checkpoint_success.stat().st_size, + }, + }, + } + base._atomic_json(segment_success, output_dir / "SEGMENT_SUCCESS.json") + if args.strategy == "fsdp": + import torch.distributed as dist + + dist.barrier() + dist.destroy_process_group() + del forward_model, role_model + gc.collect() + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = base.build_parser() + subparsers = next( + action + for action in parser._actions + if isinstance(action, argparse._SubParsersAction) + ) + training = subparsers.choices["train"] + training.add_argument("--stop-step", type=int, required=True) + training.add_argument("--resume-from", type=Path) + training.add_argument("--checkpoint-root", type=Path, required=True) + training.set_defaults( + handler=train, + max_steps=TOTAL_STEPS, + checkpoint_every=CHECKPOINT_EVERY, + lora_rank=LORA_RANK, + lora_alpha=LORA_ALPHA, + lora_dropout=LORA_DROPOUT, + student_lr=STUDENT_LR, + critic_lr=CRITIC_LR, + max_grad_norm=MAX_GRAD_NORM, + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if getattr(args, "checkpoint_every", CHECKPOINT_EVERY) < 1: + raise ValueError("checkpoint_every must be positive.") + return int(args.handler(args)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/source_snapshot/minimax_h3/train_dmd_longrun_4step.py b/source_snapshot/minimax_h3/train_dmd_longrun_4step.py new file mode 100644 index 0000000000000000000000000000000000000000..b16db887d138d5d84c49a7784476d873c3bcb839 --- /dev/null +++ b/source_snapshot/minimax_h3/train_dmd_longrun_4step.py @@ -0,0 +1,1611 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Extra-guided-teacher 1,600-iteration four-step MiniMax-H3 DMD experiment. + +This is a new experiment contract. It leaves the provenance-locked v1-v5 +entry points unchanged while reusing their model loading, FSDP, distributed +LongLive prompt cache, and exact-resume checkpoint machinery. + +The semantic role names in the technical document map onto the historical +runtime names as ``generator -> student`` and ``fake_score -> critic``. Both +the generator DMD and fake-score DSM objectives weight independently reduced +video/audio losses at 1:2 while retaining the complete joint T2VA path. +""" + +from __future__ import annotations + +import argparse +import gc +import json +import math +import os +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterator, Mapping, Sequence + +import torch +from torch.utils.checkpoint import checkpoint as torch_checkpoint + +from . import train_dmd as base +from . import train_dmd_longrun as v1 +from . import train_dmd_longrun_v2 as v2 +from . import train_dmd_longrun_v3 as v3 +from . import train_dmd_longrun_v4 as v4 +from . import train_dmd_longrun_v5 as v5 +from . import train_utils +from .dmd_4step import ( + H3DMD4StepTrainer, + NUM_FRESH_NOISE_TRANSITIONS, + NUM_MODEL_EVALUATIONS, + NUM_SIGMA_ENDPOINTS, + build_4step_sigma_schedules, + sample_target_stage, +) +from .full_h3_lora import ( + H3_DMD_LORA_POLICY, + H3_DMD_TARGET_MODULE_COUNT, + install_full_h3_lora_adapters, + released_full_h3_target_modules, +) +from .train_utils import CRITIC_ADAPTER, STUDENT_ADAPTER +from .cfg_guidance import GuidedTeacherTrainer + + +TOTAL_STEPS = 1_600 +CACHED_CONDITIONS_PER_RANK = 2_000 +CHECKPOINT_EVERY = 100 +STUDENT_UPDATE_RATIO = 5 +GENERATOR_LR = 2.0e-5 +FAKE_SCORE_LR = 5.0e-5 +GENERATOR_BETAS = (0.0, 0.9) +FAKE_SCORE_BETAS = (0.9, 0.999) +GENERATOR_WEIGHT_DECAY = 0.01 +FAKE_SCORE_WEIGHT_DECAY = 0.0 +LORA_RANK = 128 +LORA_ALPHA = 128 +LORA_DROPOUT = 0.0 +MAX_GRAD_NORM = 1.0 +TRAINING_SEED = 20260812 +PYTORCH_ATTENTION_BACKEND = "_native_flash" +ATTENTION_BACKEND_POLICY = "pytorch_native_flash_sdpa_forced" +GRADIENT_CHECKPOINTING_POLICY = "main_transformer_blocks_reentrant" +FORMAL_WORLD_SIZE = 16 +FORMAL_NODES = 4 +FORMAL_GPUS_PER_NODE = 4 +FORMAL_HEIGHT = 768 +FORMAL_WIDTH = 1344 +FORMAL_NUM_FRAMES = 124 +VIDEO_LOSS_WEIGHT = 1.0 +AUDIO_LOSS_WEIGHT = 2.0 +LOSS_OBJECTIVE = "audio_weighted_joint_t2va" +LONGRUN_CONFIG_VERSION = 12 +CHECKPOINT_SCHEMA_VERSION = v2.CHECKPOINT_SCHEMA_VERSION +OPTIMIZER_LAYOUT = "joint_role_groups_per_role_adamw_v2" +ROLLOUT_MODE = "predict_x0_fresh_renoise" +TECHNICAL_DOCUMENT_SHA256 = ( + "e293cd2045f26ee6da47d32b440ee10db2a540eeaa4626e5e8b73a7f0318392b" +) + +student_update_due = v1.student_update_due +student_updates_through = v1.student_updates_through +student_updates_in_segment = v1.student_updates_in_segment +_stable_config_hash = v1._stable_config_hash +_ORIGINAL_METRIC_RECORD = v1._metric_record +_V4_VALIDATE_WITH_LOADER = v4._validate_longrun_request_with_loader +_REAL_ADAMW = torch.optim.AdamW + +_OUTER_STEP_SEED_MODULUS = (1 << 63) - 1 +_OUTER_STEP_STREAM_STRIDE = 1_000_000_007 +_OUTER_STEP_STREAM_IDS = { + "detached_initial_latents": 1, + "generator_initial_latents": 2, + "shared_stage_sigma": 3, + "detached_rollout": 4, + "fake_dsm": 5, + "generator_rollout": 6, + "generator_dmd": 7, +} + + +def reentrant_main_block_checkpoint( + module: torch.nn.Module, *args: Any +) -> torch.Tensor: + """Run one H3 main block with the document-required reentrant checkpoint. + + Reentrant checkpointing requires at least one direct tensor input that + requires gradients. The released 312-target policy normally satisfies this + through the trainable token-refiner LoRA. Keep a fail-safe for future cache + rows or target-policy changes: a gradient-free packed hidden state has no + upstream graph, so making that leaf require gradients preserves main-block + LoRA training without detaching anything. + """ + + if not args or not isinstance(args[0], torch.Tensor): + raise TypeError("H3 reentrant checkpoint requires hidden_states first.") + hidden_states = args[0] + if not hidden_states.requires_grad: + hidden_states = hidden_states.requires_grad_(True) + return torch_checkpoint( + module.__call__, + hidden_states, + *args[1:], + use_reentrant=True, + preserve_rng_state=True, + ) + + +def _technical_document_path() -> Path: + return Path(__file__).resolve().parents[3] / "MINIMAX_H3_DMD_TECHNICAL_DOCUMENT_ZH.md" + + +@contextmanager +def _temporary_attributes( + replacements: Sequence[tuple[Any, str, Any]], +) -> Iterator[None]: + originals: list[tuple[Any, str, Any]] = [] + for module, name, value in replacements: + originals.append((module, name, getattr(module, name))) + setattr(module, name, value) + try: + yield + finally: + for module, name, value in reversed(originals): + setattr(module, name, value) + + +@contextmanager +def _document_constant_contract() -> Iterator[None]: + """Make inherited validators/checkpointers observe the four-step recipe.""" + + replacements = [ + (v1, "TOTAL_STEPS", TOTAL_STEPS), + (v1, "CHECKPOINT_EVERY", CHECKPOINT_EVERY), + (v1, "STUDENT_UPDATE_RATIO", STUDENT_UPDATE_RATIO), + (v1, "STUDENT_LR", GENERATOR_LR), + (v1, "CRITIC_LR", FAKE_SCORE_LR), + (v1, "ADAM_BETAS", GENERATOR_BETAS), + (v1, "WEIGHT_DECAY", GENERATOR_WEIGHT_DECAY), + (v1, "LORA_RANK", LORA_RANK), + (v1, "LORA_ALPHA", LORA_ALPHA), + (v1, "LORA_DROPOUT", LORA_DROPOUT), + (v1, "NUM_MODEL_EVALUATIONS", NUM_MODEL_EVALUATIONS), + (v1, "NUM_SIGMA_ENDPOINTS", NUM_SIGMA_ENDPOINTS), + (v2, "TOTAL_STEPS", TOTAL_STEPS), + (v2, "CHECKPOINT_EVERY", CHECKPOINT_EVERY), + (v2, "STUDENT_UPDATE_RATIO", STUDENT_UPDATE_RATIO), + (v2, "STUDENT_LR", GENERATOR_LR), + (v2, "CRITIC_LR", FAKE_SCORE_LR), + (v2, "LORA_RANK", LORA_RANK), + (v2, "LORA_ALPHA", LORA_ALPHA), + (v2, "LORA_DROPOUT", LORA_DROPOUT), + (v2, "OPTIMIZER_LAYOUT", OPTIMIZER_LAYOUT), + (v3, "CHECKPOINT_EVERY", CHECKPOINT_EVERY), + (v3, "STUDENT_LR", GENERATOR_LR), + (v3, "CRITIC_LR", FAKE_SCORE_LR), + (v4, "TOTAL_STEPS", TOTAL_STEPS), + (v4, "CHECKPOINT_EVERY", CHECKPOINT_EVERY), + (v4, "STUDENT_LR", GENERATOR_LR), + (v4, "CRITIC_LR", FAKE_SCORE_LR), + (v4, "FORMAL_WORLD_SIZE", FORMAL_WORLD_SIZE), + (v4, "FORMAL_GLOBAL_STEPS", CACHED_CONDITIONS_PER_RANK), + (base, "NUM_MODEL_EVALUATIONS", NUM_MODEL_EVALUATIONS), + (base, "NUM_SIGMA_ENDPOINTS", NUM_SIGMA_ENDPOINTS), + (train_utils, "NUM_MODEL_EVALUATIONS", NUM_MODEL_EVALUATIONS), + (train_utils, "NUM_SIGMA_ENDPOINTS", NUM_SIGMA_ENDPOINTS), + ] + with _temporary_attributes(replacements): + yield + + +def _validate_formal_arguments(args: argparse.Namespace) -> None: + expected = { + "max_steps": TOTAL_STEPS, + "checkpoint_every": CHECKPOINT_EVERY, + "lora_rank": LORA_RANK, + "lora_alpha": LORA_ALPHA, + "lora_dropout": LORA_DROPOUT, + "student_lr": GENERATOR_LR, + "critic_lr": FAKE_SCORE_LR, + "max_grad_norm": MAX_GRAD_NORM, + "seed": TRAINING_SEED, + "height": FORMAL_HEIGHT, + "width": FORMAL_WIDTH, + "num_frames": FORMAL_NUM_FRAMES, + "video_shift": 12.0, + "audio_shift": 3.0, + "video_loss_weight": VIDEO_LOSS_WEIGHT, + "audio_loss_weight": AUDIO_LOSS_WEIGHT, + "gradient_checkpointing": True, + } + mismatches = { + name: {"actual": getattr(args, name), "expected": value} + for name, value in expected.items() + if getattr(args, name) != value + } + if mismatches: + raise ValueError( + "The formal four-step document experiment is fixed: " + + json.dumps(mismatches, sort_keys=True) + ) + + +def _document_contract_fields( + *, video_loss_weight: float, audio_loss_weight: float +) -> dict[str, Any]: + schedules = build_4step_sigma_schedules(dtype=torch.float64) + document = _technical_document_path() + if not document.is_file() or base.sha256_file(document) != TECHNICAL_DOCUMENT_SHA256: + raise RuntimeError("The pinned MiniMax-H3 DMD technical document changed.") + targets = released_full_h3_target_modules() + return { + "longrun_schema_version": LONGRUN_CONFIG_VERSION, + "longrun_config_version": LONGRUN_CONFIG_VERSION, + "checkpoint_schema_version": CHECKPOINT_SCHEMA_VERSION, + "technical_document": { + "path": str(document), + "sha256": TECHNICAL_DOCUMENT_SHA256, + "documented_steps": 4, + "requested_steps": NUM_MODEL_EVALUATIONS, + "grid_adaptation": "none_exact_document_four_step_grid", + }, + "semantic_role_map": { + "generator": STUDENT_ADAPTER, + "fake_score": CRITIC_ADAPTER, + "real_score_teacher": "adapters_disabled", + }, + "model_variant": "standard_full_adaln", + "adaln_curve_rank": None, + "optimizer_layout": OPTIMIZER_LAYOUT, + "student_lr": GENERATOR_LR, + "critic_lr": FAKE_SCORE_LR, + "student_betas": list(GENERATOR_BETAS), + "critic_betas": list(FAKE_SCORE_BETAS), + "student_weight_decay": GENERATOR_WEIGHT_DECAY, + "critic_weight_decay": FAKE_SCORE_WEIGHT_DECAY, + "lr_scheduler": "constant", + "student_update_ratio": STUDENT_UPDATE_RATIO, + "student_update_rule": "(global_step-1)%5==0", + "critic_update_every": 1, + "lora_rank": LORA_RANK, + "lora_alpha": LORA_ALPHA, + "lora_dropout": LORA_DROPOUT, + "lora_scale": 1.0, + "lora_master_dtype": "float32", + "lora_target_policy": H3_DMD_LORA_POLICY, + "lora_target_scope": "50_main_plus_2_token_refiner_blocks", + "lora_target_branches": ["attn", "ff"], + "lora_target_module_count": H3_DMD_TARGET_MODULE_COUNT, + "lora_targets": targets, + "lora_excluded_scopes": [ + "adaln_and_timestep", + "normalization", + "top_level_input_context_output_projections", + ], + "rollout_mode": ROLLOUT_MODE, + "model_evaluations_per_rollout": NUM_MODEL_EVALUATIONS, + "sigma_endpoints_per_rollout": NUM_SIGMA_ENDPOINTS, + "fresh_noise_transitions_per_rollout": NUM_FRESH_NOISE_TRANSITIONS, + "target_stage_distribution": "uniform_0_through_3", + "outer_step_shared_random_variables": ["target_stage", "score_base_sigma"], + "score_base_sigma_min": 0.02, + "score_base_sigma_max": 0.98, + "video_sigma_grid": schedules.video_sigmas.tolist(), + "audio_sigma_grid": schedules.audio_sigmas.tolist(), + "fresh_noise_draw_order": ["video", "audio"], + "video_audio_transition_noise_independent": True, + "sample_rng_scheme": "named_rank_local_outer_step_substreams_v1", + "sample_rng_substreams": list(_OUTER_STEP_STREAM_IDS), + "sampler_matches_training_and_inference": True, + "teacher_trajectory_regression": False, + "real_latent_regression": False, + "gan_loss": False, + "ema": False, + "loss_objective": LOSS_OBJECTIVE, + "video_loss_weight": float(video_loss_weight), + "audio_loss_weight": float(audio_loss_weight), + "audio_tokens_retained": True, + "audio_forward_retained": True, + "audio_raw_loss_metric_retained": True, + "audio_output_retained": True, + "rng_and_forward_contract_unchanged": False, + "student_rng_and_forward_contract_unchanged": True, + "gradient_checkpointing_scope": "main_transformer_blocks", + "gradient_checkpointing_policy": GRADIENT_CHECKPOINTING_POLICY, + "gradient_checkpointing_use_reentrant": True, + "token_refiner_gradient_checkpointing": False, + "attention_backend": ATTENTION_BACKEND_POLICY, + "attention_backend_dispatch_name": PYTORCH_ATTENTION_BACKEND, + "attention_requires_padless_training_sequence": True, + "layout_cache_policy": "per_step_rebuild_release_no_gpu_cache", + "distributed_backend": "FSDP_FULL_SHARD", + "documented_distributed_backend": "ZeRO-3", + "distributed_backend_deviation_reason": ( + "The validated local 16-rank launcher uses equivalent full parameter, " + "gradient, and optimizer sharding through FSDP FULL_SHARD." + ), + "data_deviation": ( + "User-requested LongLive 248221-row prompt source replaces the " + "document's Qwen-Image-Bench/H3-rewriter prompt set." + ), + "distributed_contract": { + "nodes": FORMAL_NODES, + "gpus_per_node": FORMAL_GPUS_PER_NODE, + "world_size": FORMAL_WORLD_SIZE, + "per_rank_batch_size": 1, + "global_batch_size": FORMAL_WORLD_SIZE, + "global_steps": TOTAL_STEPS, + "scheduled_conditions": FORMAL_WORLD_SIZE * TOTAL_STEPS, + }, + } + + +def _segment_contract_fields( + *, video_loss_weight: float, audio_loss_weight: float +) -> dict[str, Any]: + """Return the fail-closed Audio4/rollout fields persisted per segment.""" + + weights = (float(video_loss_weight), float(audio_loss_weight)) + if weights != (VIDEO_LOSS_WEIGHT, AUDIO_LOSS_WEIGHT): + raise RuntimeError("Segment loss weights differ from the formal objective.") + return { + "loss_objective": LOSS_OBJECTIVE, + "video_loss_weight": weights[0], + "audio_loss_weight": weights[1], + "audio_tokens_retained": True, + "audio_forward_retained": True, + "audio_raw_loss_metric_retained": True, + "audio_output_retained": True, + "rng_and_forward_contract_unchanged": False, + "student_rng_and_forward_contract_unchanged": True, + "model_evaluations_per_rollout": NUM_MODEL_EVALUATIONS, + "sigma_endpoints_per_rollout": NUM_SIGMA_ENDPOINTS, + "fresh_noise_transitions_per_rollout": NUM_FRESH_NOISE_TRANSITIONS, + "target_stage_distribution": "uniform_0_through_3", + } + + +def _validate_longrun_request_with_loader( + args: argparse.Namespace, + loader: v5._DistributedPromptCollectionLoader, +) -> dict[str, Any]: + _validate_formal_arguments(args) + with _document_constant_contract(), v5._v5_world_contract(): + config = _V4_VALIDATE_WITH_LOADER(args, loader) + config.pop("config_sha256", None) + config.update( + _document_contract_fields( + video_loss_weight=args.video_loss_weight, + audio_loss_weight=args.audio_loss_weight, + ) + ) + config["seed"] = TRAINING_SEED + config["height"] = FORMAL_HEIGHT + config["width"] = FORMAL_WIDTH + config["num_frames"] = FORMAL_NUM_FRAMES + runtime = config.get("prompt_runtime_selection") + if not isinstance(runtime, Mapping): + raise RuntimeError("Distributed prompt runtime metadata is missing.") + if ( + runtime.get("world_size") != FORMAL_WORLD_SIZE + or runtime.get("scheduled_conditions") != FORMAL_WORLD_SIZE * CACHED_CONDITIONS_PER_RANK + ): + raise RuntimeError("Prompt cache does not expose the world-16 schedule.") + reference_path = Path(args.teacher_reference_cache).resolve(strict=True) + for name in ("teacher_video_cfg", "teacher_audio_cfg"): + if not math.isfinite(getattr(args, name)) or getattr(args, name) < 1: + raise ValueError(f"{name} must be finite and >= 1") + config.update({ + "experiment_id": "minimax-h3-dmd4-extra-teacher-cfg-1600-audio2", + "teacher_video_cfg": args.teacher_video_cfg, + "teacher_audio_cfg": args.teacher_audio_cfg, + "teacher_reference_cache": str(reference_path), + "teacher_reference_cache_sha256": base.sha256_file(reference_path), + "teacher_reference_kind": "whitespace_conditioned_not_calibrated_unconditional", + "extra_guidance_target": "real_score_teacher_only", + "cfg_enabled": args.teacher_video_cfg > 1 or args.teacher_audio_cfg > 1, + "cfg": args.teacher_video_cfg > 1 or args.teacher_audio_cfg > 1, + "cfg_scope": "teacher_only_student_inference_is_single_pass", + "student_extra_cfg": False, + "critic_extra_cfg": False, + "used_conditions_per_rank": TOTAL_STEPS, + "used_total_conditions": FORMAL_WORLD_SIZE * TOTAL_STEPS, + "source_cache_conditions_per_rank": CACHED_CONDITIONS_PER_RANK, + "training_source_sha256": { + path.name: base.sha256_file(path) + for path in sorted(Path(__file__).parent.glob("*.py")) + }, + }) + config["config_sha256"] = _stable_config_hash(config) + return config + + +def validate_longrun_request(args: argparse.Namespace) -> dict[str, Any]: + loader = v5._DistributedPromptCollectionLoader(args) + try: + return _validate_longrun_request_with_loader(args, loader) + finally: + loader.clear() + + +@dataclass(frozen=True) +class H3OuterStepRNGStreams: + """Independent, deterministically-derived generators for one outer step.""" + + detached_initial_latents: torch.Generator + generator_initial_latents: torch.Generator + shared_stage_sigma: torch.Generator + detached_rollout: torch.Generator + fake_dsm: torch.Generator + generator_rollout: torch.Generator + generator_dmd: torch.Generator + seeds: Mapping[str, int] + + +@dataclass(frozen=True) +class H3OuterStepOutput: + """Outputs and the shared stochastic parameters of one DMD outer step.""" + + critic: Any + generated: Any | None + target_stage: int + shared_base_sigma: float + + +@dataclass(frozen=True) +class H3ObservedOptimizerCounters: + """Actual AdamW state counters after one joint role optimizer step.""" + + student_step: int + critic_step: int + cross_rank_verified: bool + + +def build_outer_step_rng_streams( + *, + training_seed: int, + condition_seed: int, + global_step: int, + rank: int, + device: torch.device | str, +) -> H3OuterStepRNGStreams: + """Derive named replay-stable RNG substreams without using global RNG state.""" + + values = { + "training_seed": training_seed, + "condition_seed": condition_seed, + "global_step": global_step, + "rank": rank, + } + for name, value in values.items(): + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{name} must be an integer.") + if global_step < 1 or rank < 0: + raise ValueError("global_step must be positive and rank must be non-negative.") + base_seed = ( + training_seed + + condition_seed + + global_step * 1_000_003 + + rank * 10_000_019 + ) % _OUTER_STEP_SEED_MODULUS + seeds = { + name: (base_seed + stream_id * _OUTER_STEP_STREAM_STRIDE) + % _OUTER_STEP_SEED_MODULUS + for name, stream_id in _OUTER_STEP_STREAM_IDS.items() + } + generators = { + name: torch.Generator(device=device).manual_seed(seed) + for name, seed in seeds.items() + } + return H3OuterStepRNGStreams( + detached_initial_latents=generators["detached_initial_latents"], + generator_initial_latents=generators["generator_initial_latents"], + shared_stage_sigma=generators["shared_stage_sigma"], + detached_rollout=generators["detached_rollout"], + fake_dsm=generators["fake_dsm"], + generator_rollout=generators["generator_rollout"], + generator_dmd=generators["generator_dmd"], + seeds=seeds, + ) + + +def _sample_shared_stage_and_sigma( + streams: H3OuterStepRNGStreams, + *, + device: torch.device, +) -> tuple[int, float]: + """Draw exactly one target stage and one score base sigma for the step.""" + + target_stage = sample_target_stage( + generator=streams.shared_stage_sigma, + device=device, + ) + base_sigma = float( + torch.empty((), device=device, dtype=torch.float32).uniform_( + 0.02, + 0.98, + generator=streams.shared_stage_sigma, + ) + ) + return target_stage, base_sigma + + +def run_document_outer_step( + *, + trainer: H3DMD4StepTrainer, + video_noise: torch.Tensor, + audio_noise: torch.Tensor, + condition: train_utils.H3Condition, + layout: train_utils.H3PackedLayout, + streams: H3OuterStepRNGStreams, + generator_update: bool, + allow_zero_generator_grad: bool, +) -> H3OuterStepOutput: + """Execute the document's shared-``(k, u)`` fake/generator orchestration.""" + + if not isinstance(generator_update, bool): + raise TypeError("generator_update must be a bool.") + if not isinstance(allow_zero_generator_grad, bool): + raise TypeError("allow_zero_generator_grad must be a bool.") + if allow_zero_generator_grad and not generator_update: + raise ValueError("Zero-gradient bootstrap requires a generator update.") + target_stage, shared_base_sigma = _sample_shared_stage_and_sigma( + streams, + device=video_noise.device, + ) + clean_video, clean_audio = trainer.detached_rollout( + video_noise=video_noise, + audio_noise=audio_noise, + condition=condition, + layout=layout, + target_stage=target_stage, + generator=streams.detached_rollout, + ) + fake_video_noise = train_utils._random_normal_like(clean_video, streams.fake_dsm) + fake_audio_noise = train_utils._random_normal_like(clean_audio, streams.fake_dsm) + critic = trainer.critic_step( + clean_video=clean_video, + clean_audio=clean_audio, + condition=condition, + layout=layout, + score_base_sigma=shared_base_sigma, + score_video_noise=fake_video_noise, + score_audio_noise=fake_audio_noise, + defer_optimizer_step=True, + ) + + generated = None + if generator_update: + generator_video_noise = train_utils._random_normal_like( + video_noise, streams.generator_initial_latents + ) + generator_audio_noise = train_utils._random_normal_like( + audio_noise, streams.generator_initial_latents + ) + # Draw the DMD score noise from its own stream. The H3 forward contract + # preserves A/V row shapes, so these are also the generated-clean shapes. + dmd_video_noise = train_utils._random_normal_like( + generator_video_noise, streams.generator_dmd + ) + dmd_audio_noise = train_utils._random_normal_like( + generator_audio_noise, streams.generator_dmd + ) + generated = trainer.generator_step( + video_noise=generator_video_noise, + audio_noise=generator_audio_noise, + condition=condition, + layout=layout, + score_base_sigma=shared_base_sigma, + score_video_noise=dmd_video_noise, + score_audio_noise=dmd_audio_noise, + target_stage=target_stage, + generator=streams.generator_rollout, + allow_zero_generator_grad=allow_zero_generator_grad, + defer_optimizer_step=True, + preserve_deferred_critic_gradients=True, + ) + if int(generated.target_stage) != target_stage: + raise RuntimeError("Generator did not reuse the outer step target stage.") + if not math.isclose( + float(generated.score_video_timestep), + float(critic.score_video_timestep), + rel_tol=0.0, + abs_tol=0.0, + ) or not math.isclose( + float(generated.score_audio_timestep), + float(critic.score_audio_timestep), + rel_tol=0.0, + abs_tol=0.0, + ): + raise RuntimeError("Generator and fake-score did not reuse shared base sigma.") + trainer.step_deferred_role_optimizer( + generator_update=generator_update, + device=video_noise.device, + ) + return H3OuterStepOutput( + critic=critic, + generated=generated, + target_stage=target_stage, + shared_base_sigma=shared_base_sigma, + ) + + +def _local_optimizer_role_counter( + optimizer: torch.optim.Optimizer, + parameters: Sequence[torch.nn.Parameter], +) -> tuple[int | None, bool]: + """Return one local-shard role counter and whether its state is complete.""" + + values: set[int] = set() + owned_parameters = 0 + missing_state = False + for parameter in parameters: + # With FSDP ``use_orig_params=True``, a rank may expose an empty view for + # an original parameter whose shard is owned entirely by other ranks. + if parameter.numel() == 0: + continue + owned_parameters += 1 + row = optimizer.state.get(parameter) + if not isinstance(row, Mapping) or "step" not in row: + missing_state = True + continue + try: + values.add(v2._step_value(row)) + except (RuntimeError, TypeError, ValueError): + missing_state = True + if owned_parameters == 0: + return None, True + if missing_state or len(values) != 1: + return None, False + return next(iter(values)), True + + +def _resolve_optimizer_counter_consensus( + payloads: Sequence[Sequence[int]], +) -> tuple[int, int]: + """Validate local state and exact cross-rank agreement from small payloads.""" + + if not payloads or any(len(payload) != 3 for payload in payloads): + raise RuntimeError("Optimizer counter consensus payloads are invalid.") + invalid = [rank for rank, payload in enumerate(payloads) if int(payload[0])] + if invalid: + raise RuntimeError( + "At least one rank has missing or internally inconsistent AdamW role " + f"state: ranks={invalid}." + ) + student_values = {int(payload[1]) for payload in payloads if int(payload[1]) >= 0} + critic_values = {int(payload[2]) for payload in payloads if int(payload[2]) >= 0} + if len(student_values) != 1 or len(critic_values) != 1: + raise RuntimeError( + "AdamW role counters differ across ranks or have no owning shard: " + f"student={sorted(student_values)}, critic={sorted(critic_values)}." + ) + return next(iter(student_values)), next(iter(critic_values)) + + +def observe_joint_optimizer_counters( + *, + optimizer: torch.optim.Optimizer, + student_parameters: Sequence[torch.nn.Parameter], + critic_parameters: Sequence[torch.nn.Parameter], + device: torch.device, + distributed: bool, +) -> H3ObservedOptimizerCounters: + """Read actual role counters and, under FSDP, prove rank consensus.""" + + student_step, student_valid = _local_optimizer_role_counter( + optimizer, student_parameters + ) + critic_step, critic_valid = _local_optimizer_role_counter( + optimizer, critic_parameters + ) + payload = ( + int(not (student_valid and critic_valid)), + -1 if student_step is None else student_step, + -1 if critic_step is None else critic_step, + ) + payloads: list[Sequence[int]] = [payload] + if distributed: + import torch.distributed as dist + + if not dist.is_initialized(): + raise RuntimeError("Distributed optimizer-counter audit lacks a process group.") + local = torch.tensor(payload, device=device, dtype=torch.int64) + gathered = [torch.empty_like(local) for _ in range(dist.get_world_size())] + dist.all_gather(gathered, local) + payloads = [row.cpu().tolist() for row in gathered] + resolved_student, resolved_critic = _resolve_optimizer_counter_consensus(payloads) + return H3ObservedOptimizerCounters( + student_step=resolved_student, + critic_step=resolved_critic, + cross_rank_verified=distributed, + ) + + +def validate_observed_optimizer_counters( + *, + global_step: int, + previous_student_step: int, + observed: H3ObservedOptimizerCounters, +) -> None: + """Fail immediately on TTUR drift, including every non-generator step.""" + + expected_student = student_updates_through(global_step) + expected_critic = global_step + if observed.student_step != expected_student or observed.critic_step != expected_critic: + raise RuntimeError( + "Observed AdamW counters violate the absolute outer-step contract: " + f"step={global_step}, student={observed.student_step}/{expected_student}, " + f"critic={observed.critic_step}/{expected_critic}." + ) + expected_delta = int(student_update_due(global_step)) + if observed.student_step != previous_student_step + expected_delta: + raise RuntimeError( + "Observed student AdamW counter violated the per-step update cadence: " + f"step={global_step}, previous={previous_student_step}, " + f"observed={observed.student_step}, expected_delta={expected_delta}." + ) + + # Keep the formal six-row audit explicit instead of relying only on the + # closed-form schedule above, so a future cadence rewrite cannot weaken it. + if global_step <= 6: + expected_first_six = (1, 1, 1, 1, 1, 2) + if observed.student_step != expected_first_six[global_step - 1]: + raise RuntimeError("The formal first-six generator counter audit failed.") + + +def _allow_zero_generator_bootstrap( + *, + start_step: int, + global_step: int, + resumed: bool, +) -> bool: + """Authorize the one documented bootstrap from absolute run state only.""" + + for name, value in (("start_step", start_step), ("global_step", global_step)): + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{name} must be an integer.") + if not isinstance(resumed, bool): + raise TypeError("resumed must be a bool.") + if start_step < 0 or global_step <= start_step: + raise ValueError("global_step must be positive and greater than start_step.") + return not resumed and start_step == 0 and global_step == 1 + + +def _install_document_lora_adapters( + model: torch.nn.Module, + *, + rank: int, + alpha: int, + dropout: float, + **_: Any, +) -> dict[str, list[torch.nn.Parameter]]: + if (rank, alpha, float(dropout)) != (LORA_RANK, LORA_ALPHA, LORA_DROPOUT): + raise ValueError("The document LoRA is fixed to rank/alpha/dropout 128/128/0.") + return install_full_h3_lora_adapters( + model, + rank=rank, + alpha=alpha, + dropout=dropout, + master_dtype=torch.float32, + ) + + +def _load_document_transformer(*args: Any, **kwargs: Any): + if args: + parsed_args, resolved_revision, device, *remaining = args + if remaining: + resume_lora = remaining[0] + else: + resume_lora = kwargs.pop("resume_lora", None) + else: + parsed_args = kwargs.pop("args") + resolved_revision = kwargs.pop("resolved_revision") + device = kwargs.pop("device") + resume_lora = kwargs.pop("resume_lora", None) + if kwargs: + raise TypeError(f"Unexpected transformer loader arguments: {sorted(kwargs)}") + + base.validate_local_model_provenance(parsed_args.model_id, resolved_revision) + try: + from diffusers import MiniMaxH3Transformer3DModel + except ImportError as error: + raise RuntimeError("Install the editable diffusers_h3 checkout first.") from error + load_kwargs = base._model_load_kwargs(parsed_args, resolved_revision) + load_kwargs["low_cpu_mem_usage"] = True + role_model = MiniMaxH3Transformer3DModel.from_pretrained( + parsed_args.model_id, + subfolder="transformer", + **load_kwargs, + ) + if role_model.__class__.__name__ != "MiniMaxH3Transformer3DModel": + raise TypeError("Training loaded a non-H3 transformer.") + architecture = v1.assert_released_h3_architecture(role_model) + if architecture["attention_is_causal"] is not False: + raise AssertionError("Released MiniMax-H3 attention must remain non-causal.") + role_model._h3_released_architecture = architecture + _install_document_lora_adapters( + role_model, + rank=LORA_RANK, + alpha=LORA_ALPHA, + dropout=LORA_DROPOUT, + ) + if resume_lora is not None: + v1._install_checkpoint_loras(role_model, resume_lora) + + # The full-resolution sequence is ~37.8K rows. Force PyTorch's native + # Flash-SDPA backend so an unsupported kernel fails closed instead of + # silently materializing an O(S^2) math-attention matrix. + role_model.set_attention_backend(PYTORCH_ATTENTION_BACKEND) + observed_attention_backends = { + getattr(getattr(processor, "_attention_backend", None), "value", None) + for processor in role_model.attn_processors.values() + } + if observed_attention_backends != {PYTORCH_ATTENTION_BACKEND}: + raise RuntimeError( + "H3 attention processors did not all pin native Flash-SDPA: " + f"{sorted(map(str, observed_attention_backends))}." + ) + + role_model.enable_gradient_checkpointing(reentrant_main_block_checkpoint) + if role_model._gradient_checkpointing_func is not reentrant_main_block_checkpoint: + raise RuntimeError("H3 main blocks did not retain reentrant checkpointing.") + raw_token_refiner = role_model.token_refiner + raw_token_refiner.gradient_checkpointing = False + if bool(raw_token_refiner.gradient_checkpointing): + raise RuntimeError("Token-refiner gradient checkpointing remained enabled pre-FSDP.") + role_model.train() + if parsed_args.strategy == "fsdp": + forward_model = base._wrap_fsdp(role_model, device) + wrapped_refiner = role_model.token_refiner + underlying_refiner = getattr( + wrapped_refiner, + "_fsdp_wrapped_module", + None, + ) + if underlying_refiner is not raw_token_refiner: + raise RuntimeError("FSDP did not retain the audited token-refiner module.") + if bool(underlying_refiner.gradient_checkpointing): + raise RuntimeError( + "The underlying FSDP token refiner re-enabled gradient checkpointing." + ) + runtime_report = base.validate_h3_fsdp_runtime_wrappers( + forward_model, role_model + ) + import torch.distributed as dist + + reports: list[dict[str, Any] | None] = [None] * dist.get_world_size() + dist.all_gather_object(reports, runtime_report) + if any(report != runtime_report for report in reports): + raise RuntimeError("FSDP CPU-offload runtime audit differs across ranks.") + runtime_report["all_ranks_verified"] = True + runtime_report["token_refiner_gradient_checkpointing"] = False + runtime_report["gradient_checkpointing_policy"] = ( + GRADIENT_CHECKPOINTING_POLICY + ) + runtime_report["attention_backend"] = ATTENTION_BACKEND_POLICY + runtime_report["attention_backend_dispatch_name"] = ( + PYTORCH_ATTENTION_BACKEND + ) + role_model._h3_fsdp_runtime_report = runtime_report + else: + role_model.to(device) + forward_model = role_model + + if getattr(role_model.config, "adaln_curve_rank", None) is not None: + raise RuntimeError("The four-step experiment requires standard full-AdaLN.") + lora_dtypes = { + parameter.dtype + for name, parameter in role_model.named_parameters() + if "lora_" in name + } + if lora_dtypes != {torch.float32}: + raise RuntimeError(f"LoRA masters are not FP32: {sorted(map(str, lora_dtypes))}.") + return forward_model, role_model + + +class _DocumentAdamWFactory: + """Build one FSDP-compatible optimizer with role-specific AdamW groups.""" + + def __init__(self) -> None: + self.optimizer: torch.optim.AdamW | None = None + self.parameter_ids: set[int] = set() + self.calls = 0 + + def __call__( + self, + params: Any, + *, + lr: float, + betas: tuple[float, float], + weight_decay: float, + **kwargs: Any, + ) -> torch.optim.AdamW: + del betas, weight_decay + parameters = list(params) + role = STUDENT_ADAPTER if self.calls == 0 else CRITIC_ADAPTER + expected_lr = GENERATOR_LR if role == STUDENT_ADAPTER else FAKE_SCORE_LR + role_betas = GENERATOR_BETAS if role == STUDENT_ADAPTER else FAKE_SCORE_BETAS + role_weight_decay = ( + GENERATOR_WEIGHT_DECAY + if role == STUDENT_ADAPTER + else FAKE_SCORE_WEIGHT_DECAY + ) + if self.calls >= 2 or lr != expected_lr: + raise RuntimeError("Unexpected AdamW construction in the four-step trainer.") + ids = {id(parameter) for parameter in parameters} + if not parameters or ids & self.parameter_ids: + raise RuntimeError("Four-step optimizer groups are empty or overlapping.") + self.parameter_ids.update(ids) + group = { + "params": parameters, + "role": role, + "lr": expected_lr, + "betas": role_betas, + "weight_decay": role_weight_decay, + } + if self.optimizer is None: + self.optimizer = _REAL_ADAMW([group], **kwargs) + else: + self.optimizer.add_param_group(group) + self.calls += 1 + return self.optimizer + + +def validate_joint_optimizer_state( + state_dict: Mapping[str, Any], + *, + expected_student_updates: int, + expected_critic_updates: int, +) -> None: + states = state_dict.get("state") + groups = state_dict.get("param_groups") + if not isinstance(states, Mapping) or not states: + raise RuntimeError("Joint optimizer state is empty.") + if not isinstance(groups, list) or len(groups) != 2: + raise RuntimeError("Joint optimizer must contain two role groups.") + expected = { + STUDENT_ADAPTER: { + "lr": GENERATOR_LR, + "betas": GENERATOR_BETAS, + "weight_decay": GENERATOR_WEIGHT_DECAY, + "updates": expected_student_updates, + }, + CRITIC_ADAPTER: { + "lr": FAKE_SCORE_LR, + "betas": FAKE_SCORE_BETAS, + "weight_decay": FAKE_SCORE_WEIGHT_DECAY, + "updates": expected_critic_updates, + }, + } + parameter_roles: dict[Any, str] = {} + seen_roles: set[str] = set() + for group in groups: + if not isinstance(group, Mapping): + raise RuntimeError("Optimizer parameter group is invalid.") + role = group.get("role") + if role not in expected or role in seen_roles: + raise RuntimeError("Optimizer roles must be exactly student and critic.") + seen_roles.add(role) + parameters = group.get("params") + if not isinstance(parameters, list) or not parameters: + raise RuntimeError(f"{role} optimizer group is empty.") + for parameter in parameters: + if parameter in parameter_roles: + raise RuntimeError("Optimizer parameter groups overlap.") + parameter_roles[parameter] = role + wanted = expected[role] + mismatches = { + name: {"actual": group.get(name), "expected": wanted[name]} + for name in ("lr", "betas", "weight_decay") + if group.get(name) != wanted[name] + } + if mismatches: + raise RuntimeError( + f"{role} optimizer hyperparameter mismatch: " + + json.dumps(mismatches, sort_keys=True) + ) + if seen_roles != set(expected) or set(states) != set(parameter_roles): + raise RuntimeError("Optimizer state does not cover both complete role groups.") + observed = {STUDENT_ADAPTER: set(), CRITIC_ADAPTER: set()} + for parameter, row in states.items(): + if not isinstance(row, Mapping): + raise RuntimeError("Optimizer state row is invalid.") + observed[parameter_roles[parameter]].add(v2._step_value(row)) + for role, values in observed.items(): + wanted_updates = int(expected[role]["updates"]) + if values != {wanted_updates}: + raise RuntimeError( + f"{role} optimizer step mismatch: {sorted(values)} != [{wanted_updates}]." + ) + + +def save_longrun_checkpoint(**kwargs: Any) -> Path: + with _document_constant_contract(), _temporary_attributes( + [(v2, "validate_joint_optimizer_state", validate_joint_optimizer_state)] + ): + return v2.save_longrun_checkpoint(**kwargs) + + +def load_longrun_checkpoint(*args: Any, **kwargs: Any) -> dict[str, Any]: + with _document_constant_contract(), _temporary_attributes( + [(v2, "validate_joint_optimizer_state", validate_joint_optimizer_state)] + ): + return v2.load_longrun_checkpoint(*args, **kwargs) + + +def restore_longrun_state(**kwargs: Any) -> None: + with _document_constant_contract(), _temporary_attributes( + [(v2, "validate_joint_optimizer_state", validate_joint_optimizer_state)] + ): + v2.restore_longrun_state(**kwargs) + + +def _metric_record(**kwargs: Any) -> dict[str, Any]: + teacher_video_cfg = float(kwargs.pop("teacher_video_cfg")) + teacher_audio_cfg = float(kwargs.pop("teacher_audio_cfg")) + target_stage = int(kwargs.pop("target_stage")) + shared_base_sigma = float(kwargs.pop("shared_base_sigma")) + rng_seeds = dict(kwargs.pop("rng_seeds")) + optimizer_counters = kwargs.pop("optimizer_counters") + video_loss_weight = float(kwargs.pop("video_loss_weight")) + audio_loss_weight = float(kwargs.pop("audio_loss_weight")) + if not isinstance(optimizer_counters, H3ObservedOptimizerCounters): + raise TypeError("optimizer_counters must be an observed AdamW counter record.") + if (video_loss_weight, audio_loss_weight) != ( + VIDEO_LOSS_WEIGHT, + AUDIO_LOSS_WEIGHT, + ): + raise RuntimeError("Metric loss weights differ from the formal objective.") + record = _ORIGINAL_METRIC_RECORD(**kwargs) + generated = kwargs["generated"] + critic = kwargs["critic"] + roles = kwargs["roles"] + critic_video_weighted = float(critic.video_loss) * video_loss_weight + critic_audio_weighted = float(critic.audio_loss) * audio_loss_weight + if not math.isfinite(float(critic.audio_loss)): + raise RuntimeError("Raw fake-score audio loss is non-finite.") + if not math.isclose( + float(critic.loss), + critic_video_weighted + critic_audio_weighted, + rel_tol=1e-6, + abs_tol=1e-7, + ): + raise RuntimeError("Fake-score total does not match the formal loss weights.") + generator_video_weighted = None + generator_audio_weighted = None + if generated is not None: + if not math.isfinite(float(generated.audio_loss)): + raise RuntimeError("Raw generator audio loss is non-finite.") + generator_video_weighted = float(generated.video_loss) * video_loss_weight + generator_audio_weighted = float(generated.audio_loss) * audio_loss_weight + if not math.isclose( + float(generated.loss), + generator_video_weighted + generator_audio_weighted, + rel_tol=1e-6, + abs_tol=1e-7, + ): + raise RuntimeError("Generator total does not match the formal loss weights.") + student_roles = { + "student_detached_stage", + "student_stage_cache", + "student_target_stage", + } + record.update( + { + "semantic_generator_updated": generated is not None, + "semantic_fake_score_updated": True, + "loss_objective": LOSS_OBJECTIVE, + "video_loss_weight": video_loss_weight, + "audio_loss_weight": audio_loss_weight, + "audio_tokens_retained": True, + "audio_forward_retained": True, + "audio_raw_loss_metric_retained": True, + "audio_output_retained": True, + "rng_and_forward_contract_unchanged": False, + "student_rng_and_forward_contract_unchanged": True, + "critic_video_weighted_loss": critic_video_weighted, + "critic_audio_weighted_loss": critic_audio_weighted, + "generator_video_weighted_loss": generator_video_weighted, + "generator_audio_weighted_loss": generator_audio_weighted, + "outer_target_stage": target_stage, + "outer_shared_score_base_sigma": shared_base_sigma, + "outer_rng_stream_seeds": rng_seeds, + "student_optimizer_step_observed": optimizer_counters.student_step, + "critic_optimizer_step_observed": optimizer_counters.critic_step, + "optimizer_counter_cross_rank_verified": ( + optimizer_counters.cross_rank_verified + ), + "generator_target_stage": ( + int(generated.target_stage) if generated is not None else None + ), + "generator_zero_grad_bootstrap_accepted": ( + bool(generated.zero_grad_accepted) if generated is not None else False + ), + "critic_score_video_timestep": float(critic.score_video_timestep), + "critic_score_audio_timestep": float(critic.score_audio_timestep), + "generator_score_video_timestep": ( + float(generated.score_video_timestep) if generated is not None else None + ), + "generator_score_audio_timestep": ( + float(generated.score_audio_timestep) if generated is not None else None + ), + "student_positive_forwards": sum( + row.role in student_roles for row in roles + ), + "student_positive_forwards_per_rollout": NUM_MODEL_EVALUATIONS, + "sigma_endpoints_per_rollout": NUM_SIGMA_ENDPOINTS, + "fresh_noise_transitions_per_rollout": NUM_FRESH_NOISE_TRANSITIONS, + "rollout_mode": ROLLOUT_MODE, + } + ) + expected_calls = ( + 2 * NUM_MODEL_EVALUATIONS + 3 + if generated is not None + else NUM_MODEL_EVALUATIONS + 1 + ) + # One extra frozen reference prediction on each generator update. Student + # and fake-score forward counts retain the original four-step contract. + reference_calls = sum(row.role == "teacher_reference" for row in roles) + reference_expected = int(generated is not None and (teacher_video_cfg > 1 or teacher_audio_cfg > 1)) + if reference_calls != reference_expected: + raise RuntimeError("Unexpected teacher-reference call count") + expected_calls += reference_calls + record["teacher_reference_calls"] = reference_calls + record["teacher_video_cfg"] = teacher_video_cfg + record["teacher_audio_cfg"] = teacher_audio_cfg + record["reference_kind"] = "whitespace_conditioned_not_calibrated_unconditional" + observed_calls = len(roles) + record["transformer_calls_expected"] = expected_calls + record["transformer_calls_observed"] = observed_calls + record["transformer_call_contract_verified"] = observed_calls == expected_calls + if observed_calls != expected_calls: + raise RuntimeError( + f"Four-step call contract changed: observed={observed_calls}, " + f"expected={expected_calls}." + ) + return record + + +def _train_document_loop( + args: argparse.Namespace, + *, + prompt_loader: v5._DistributedPromptCollectionLoader, + factory: _DocumentAdamWFactory, +) -> int: + """Own the four-step outer loop instead of adapting the v1 sampler loop.""" + + config = _validate_longrun_request_with_loader(args, prompt_loader) + if args.validate_only: + print(json.dumps(config, indent=2, sort_keys=True)) + return 0 + + allocator_config = os.environ.get("PYTORCH_CUDA_ALLOC_CONF") + if args.strategy == "fsdp" and allocator_config != v1.CUDA_ALLOCATOR_CONFIG: + raise RuntimeError( + "Full H3 FSDP requires the audited CUDA allocator configuration: " + f"actual={allocator_config!r}, expected={v1.CUDA_ALLOCATOR_CONFIG!r}." + ) + runtime_initializer = v5._formal_runtime_initializer(base._init_runtime) + device, rank, world_size = runtime_initializer(args) + allocator_backend = torch.cuda.memory.get_allocator_backend() + if args.strategy == "fsdp" and allocator_backend != "native": + raise RuntimeError( + "The audited H3 memory policy requires PyTorch's native CUDA allocator; " + f"observed {allocator_backend!r}." + ) + + checkpoint_root = Path(args.checkpoint_root).resolve() + output_dir = Path(args.output_dir).resolve() + v1._fresh_segment_output(output_dir) + if rank == 0: + checkpoint_root.mkdir(parents=True, exist_ok=True) + output_dir.mkdir(parents=True, exist_ok=True) + if args.strategy == "fsdp": + import torch.distributed as dist + + dist.barrier() + + bundle = None + start_step = 0 + if args.resume_from is not None: + bundle = load_longrun_checkpoint( + args.resume_from, + expected_config_sha256=config["config_sha256"], + expected_world_size=world_size, + checkpoint_root=checkpoint_root, + ) + start_step = int(bundle["global_step"]) + if start_step >= args.stop_step: + raise ValueError( + f"Resume step {start_step} must precede stop_step {args.stop_step}." + ) + + torch.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) + resolved_revision = config["resolved_revision"] + _, cached_conditions = prompt_loader(args.prompt_cache) + forward_model, role_model = _load_document_transformer( + args, + resolved_revision, + device, + bundle["lora"] if bundle is not None else None, + ) + video_input_dtype, audio_input_dtype = v1.h3_model_input_dtypes(role_model) + student_optimizer = factory( + v1.adapter_parameters(role_model, STUDENT_ADAPTER), + lr=GENERATOR_LR, + betas=GENERATOR_BETAS, + weight_decay=GENERATOR_WEIGHT_DECAY, + ) + critic_optimizer = factory( + v1.adapter_parameters(role_model, CRITIC_ADAPTER), + lr=FAKE_SCORE_LR, + betas=FAKE_SCORE_BETAS, + weight_decay=FAKE_SCORE_WEIGHT_DECAY, + ) + if student_optimizer is not critic_optimizer: + raise RuntimeError("The document recipe requires one joint role optimizer.") + if bundle is not None: + restore_longrun_state( + bundle=bundle, + forward_model=forward_model, + role_model=role_model, + student_optimizer=student_optimizer, + critic_optimizer=critic_optimizer, + device=device, + rank=rank, + strategy=args.strategy, + ) + reference_payload = torch.load(args.teacher_reference_cache, map_location="cpu", weights_only=True) + if reference_payload.get("model_revision") != resolved_revision or reference_payload.get("prompt") != " ": + raise ValueError("Teacher reference must be the pinned whitespace Qwen encoding") + reference_condition = train_utils.H3Condition( + id="teacher-whitespace-reference", prompt=" ", seed=0, + prompt_embeds=reference_payload["prompt_embeds"], + text_token_tags=reference_payload["text_token_tags"], + ) + trainer = GuidedTeacherTrainer( + reference_condition=reference_condition, + teacher_video_scale=args.teacher_video_cfg, + teacher_audio_scale=args.teacher_audio_cfg, + forward_model=forward_model, + role_model=role_model, + student_optimizer=student_optimizer, + critic_optimizer=critic_optimizer, + video_shift=args.video_shift, + audio_shift=args.audio_shift, + video_loss_weight=args.video_loss_weight, + audio_loss_weight=args.audio_loss_weight, + max_grad_norm=MAX_GRAD_NORM, + ) + + if rank == 0: + runtime = { + **config, + "segment_kind": v1.SEGMENT_KIND, + "started_at": base._utc_now(), + "resume_step": start_step, + "start_step": start_step + 1, + "stop_step": args.stop_step, + "steps_this_segment": args.stop_step - start_step, + "world_size": world_size, + "resume_from": str(bundle["directory"]) if bundle is not None else None, + "resume_checkpoint_success_sha256": ( + bundle["success_sha256"] if bundle is not None else None + ), + "checkpoint_root": str(checkpoint_root), + "torch_version": torch.__version__, + "cuda_version": torch.version.cuda, + "cuda_allocator_config_observed": allocator_config, + "cuda_allocator_backend_observed": allocator_backend, + "video_latent_dtype": str(video_input_dtype).removeprefix("torch."), + "audio_latent_dtype": str(audio_input_dtype).removeprefix("torch."), + "fsdp_wrap_plan": getattr( + role_model, "_h3_fsdp_wrap_plan_report", None + ), + "fsdp_runtime_audit": getattr( + role_model, "_h3_fsdp_runtime_report", None + ), + "released_architecture": getattr( + role_model, "_h3_released_architecture", None + ), + } + base._atomic_json(runtime, output_dir / "runtime.json") + + latent_frames, latent_height, latent_width, audio_latents = ( + base._validate_geometry(args) + ) + patch_size = tuple(role_model.config.patch_size) + video_patch_dim = int(role_model.config.in_channels * math.prod(patch_size)) + audio_channels = int(role_model.config.audio_in_channels) + final_checkpoint_dir: Path | None = None + previous_student_counter = student_updates_through(start_step) + + for step in range(start_step + 1, args.stop_step + 1): + condition = cached_conditions[ + (args.prompt_start_index + step - 1) % len(cached_conditions) + ].to(device) + # Prompt text lengths can vary. Rebuilding this small index structure is + # preferable to retaining up to 1,000 full-resolution CUDA layouts. + layout = v1.make_base_t2va_layout( + text_token_tags=condition.text_token_tags, + num_latent_frames=latent_frames, + latent_height=latent_height, + latent_width=latent_width, + num_audio_latents=audio_latents, + patch_size=patch_size, + ).to(device) + if bool((layout.token_tags < 0).any()): + raise RuntimeError( + "Formal H3 training must remain padless for forced native Flash-SDPA." + ) + streams = build_outer_step_rng_streams( + training_seed=args.seed, + condition_seed=condition.seed, + global_step=step, + rank=rank, + device=device, + ) + video_noise = torch.randn( + (1, layout.video_indices.numel(), video_patch_dim), + device=device, + dtype=video_input_dtype, + generator=streams.detached_initial_latents, + ) + audio_noise = torch.randn( + (1, layout.audio_indices.numel(), audio_channels), + device=device, + dtype=audio_input_dtype, + generator=streams.detached_initial_latents, + ) + + cache_before = trainer.cuda_cache_clears_before_generator_backward + contexts_before = trainer.saved_tensor_offload_contexts + packs_before = trainer.saved_tensor_offload_pack_count + unpacks_before = trainer.saved_tensor_offload_unpack_count + bytes_before = trainer.saved_tensor_offload_packed_bytes + due = student_update_due(step) + outer = run_document_outer_step( + trainer=trainer, + video_noise=video_noise, + audio_noise=audio_noise, + condition=condition, + layout=layout, + streams=streams, + generator_update=due, + allow_zero_generator_grad=_allow_zero_generator_bootstrap( + start_step=start_step, + global_step=step, + resumed=bundle is not None, + ), + ) + optimizer_counters = observe_joint_optimizer_counters( + optimizer=student_optimizer, + student_parameters=trainer.student_parameters, + critic_parameters=trainer.critic_parameters, + device=device, + distributed=args.strategy == "fsdp", + ) + validate_observed_optimizer_counters( + global_step=step, + previous_student_step=previous_student_counter, + observed=optimizer_counters, + ) + previous_student_counter = optimizer_counters.student_step + cache_clears = ( + trainer.cuda_cache_clears_before_generator_backward - cache_before + ) + offload_contexts = trainer.saved_tensor_offload_contexts - contexts_before + offload_packs = trainer.saved_tensor_offload_pack_count - packs_before + offload_unpacks = trainer.saved_tensor_offload_unpack_count - unpacks_before + offload_bytes = trainer.saved_tensor_offload_packed_bytes - bytes_before + local_memory_valid = ( + cache_clears == 1 + and offload_contexts == 1 + and offload_packs > 0 + and offload_unpacks > 0 + and offload_bytes > 0 + if due + else cache_clears == 0 + and offload_contexts == 0 + and offload_packs == 0 + and offload_unpacks == 0 + and offload_bytes == 0 + ) + memory_verified = local_memory_valid + if args.strategy == "fsdp": + import torch.distributed as dist + + consensus = torch.tensor( + int(local_memory_valid), device=device, dtype=torch.int32 + ) + dist.all_reduce(consensus, op=dist.ReduceOp.MIN) + memory_verified = bool(consensus.item()) + if not memory_verified: + raise RuntimeError("At least one rank violated the long-run memory policy.") + + record = _metric_record( + step=step, + prompt_id=condition.id, + critic=outer.critic, + generated=outer.generated, + roles=trainer.audit.records, + cache_clears=cache_clears, + offload_contexts=offload_contexts, + offload_packs=offload_packs, + offload_unpacks=offload_unpacks, + offload_bytes=offload_bytes, + memory_policy_verified=memory_verified, + target_stage=outer.target_stage, + shared_base_sigma=outer.shared_base_sigma, + rng_seeds=streams.seeds, + optimizer_counters=optimizer_counters, + video_loss_weight=trainer.video_loss_weight, + audio_loss_weight=trainer.audio_loss_weight, + teacher_video_cfg=trainer.teacher_video_scale, + teacher_audio_cfg=trainer.teacher_audio_scale, + ) + if outer.generated is not None: + record.update(trainer.guidance_metrics) + if rank == 0: + base._append_jsonl(record, output_dir / "metrics.jsonl") + print(json.dumps(record, sort_keys=True), flush=True) + trainer.audit = train_utils.H3CallAudit() + + if step % CHECKPOINT_EVERY == 0: + final_checkpoint_dir = save_longrun_checkpoint( + forward_model=forward_model, + role_model=role_model, + student_optimizer=student_optimizer, + critic_optimizer=critic_optimizer, + checkpoint_root=checkpoint_root, + step=step, + config=config, + device=device, + rank=rank, + world_size=world_size, + strategy=args.strategy, + ) + del outer, video_noise, audio_noise, layout, condition, streams + + expected_student_updates = student_updates_in_segment(start_step, args.stop_step) + if trainer.cuda_cache_clears_before_generator_backward != expected_student_updates: + raise AssertionError("Student CUDA cache-clear count differs from scheduled updates.") + if trainer.saved_tensor_offload_contexts != expected_student_updates: + raise AssertionError("Saved-tensor offload count differs from scheduled student updates.") + if ( + final_checkpoint_dir is None + or final_checkpoint_dir.name != f"checkpoint-{args.stop_step:06d}" + ): + raise AssertionError("Segment ended without its stop-step checkpoint.") + if factory.calls != 2 or factory.optimizer is None: + raise RuntimeError("Both four-step optimizer groups were not built.") + if len(factory.optimizer.param_groups) != 2: + raise RuntimeError("Four-step optimizer group count changed.") + prompt_loader.assert_complete() + + if rank == 0: + runtime_path = output_dir / "runtime.json" + runtime = json.loads(runtime_path.read_text(encoding="utf-8")) + runtime.update( + { + "finished_at": base._utc_now(), + "completed_global_step": args.stop_step, + "critic_updates_this_segment": args.stop_step - start_step, + "student_updates_this_segment": expected_student_updates, + "student_updates_completed": student_updates_through(args.stop_step), + "final_checkpoint": str(final_checkpoint_dir), + } + ) + base._atomic_json(runtime, runtime_path) + checkpoint_success = final_checkpoint_dir / "CHECKPOINT_SUCCESS.json" + metrics_path = output_dir / "metrics.jsonl" + segment_success = { + "schema_version": 1, + "status": "success", + "kind": v1.SEGMENT_KIND, + "config_sha256": config["config_sha256"], + "resume_step": start_step, + "start_step": start_step + 1, + "stop_step": args.stop_step, + "critic_updates": args.stop_step - start_step, + "student_updates": expected_student_updates, + "critic_updates_completed": args.stop_step, + "student_updates_completed": student_updates_through(args.stop_step), + **_segment_contract_fields( + video_loss_weight=trainer.video_loss_weight, + audio_loss_weight=trainer.audio_loss_weight, + ), + "world_size": world_size, + "final_checkpoint": str(final_checkpoint_dir), + "artifacts": { + "runtime.json": { + "sha256": base.sha256_file(runtime_path), + "bytes": runtime_path.stat().st_size, + }, + "metrics.jsonl": { + "sha256": base.sha256_file(metrics_path), + "bytes": metrics_path.stat().st_size, + }, + "CHECKPOINT_SUCCESS.json": { + "path": str(checkpoint_success), + "sha256": base.sha256_file(checkpoint_success), + "bytes": checkpoint_success.stat().st_size, + }, + }, + } + base._atomic_json(segment_success, output_dir / "SEGMENT_SUCCESS.json") + if args.strategy == "fsdp": + import torch.distributed as dist + + dist.barrier() + dist.destroy_process_group() + del forward_model, role_model + gc.collect() + return 0 + + +def train(args: argparse.Namespace) -> int: + factory = _DocumentAdamWFactory() + prompt_loader = v5._DistributedPromptCollectionLoader(args) + try: + with _document_constant_contract(), v5._v5_world_contract(): + return _train_document_loop( + args, + prompt_loader=prompt_loader, + factory=factory, + ) + finally: + prompt_loader.clear() + + +def build_parser() -> argparse.ArgumentParser: + parser = v1.build_parser() + subparsers = next( + action + for action in parser._actions + if isinstance(action, argparse._SubParsersAction) + ) + training = subparsers.choices["train"] + for action in training._actions: + if action.dest == "prompt_cache": + action.help = "Path to the verified world-16 LongLive collection.json." + elif action.dest == "prompt_start_index": + action.help = "Must remain zero; rank caches use absolute global step." + training.add_argument( + "--video-loss-weight", type=float, default=VIDEO_LOSS_WEIGHT + ) + training.add_argument( + "--audio-loss-weight", type=float, default=AUDIO_LOSS_WEIGHT + ) + training.add_argument("--teacher-reference-cache", type=Path, required=True) + training.add_argument("--teacher-video-cfg", type=float, default=1.0) + training.add_argument("--teacher-audio-cfg", type=float, default=1.5) + training.set_defaults( + handler=train, + max_steps=TOTAL_STEPS, + checkpoint_every=CHECKPOINT_EVERY, + lora_rank=LORA_RANK, + lora_alpha=LORA_ALPHA, + lora_dropout=LORA_DROPOUT, + student_lr=GENERATOR_LR, + critic_lr=FAKE_SCORE_LR, + max_grad_norm=MAX_GRAD_NORM, + seed=TRAINING_SEED, + height=FORMAL_HEIGHT, + width=FORMAL_WIDTH, + num_frames=FORMAL_NUM_FRAMES, + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + return int(args.handler(args)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/source_snapshot/minimax_h3/train_dmd_longrun_4step_autoresume.py b/source_snapshot/minimax_h3/train_dmd_longrun_4step_autoresume.py new file mode 100644 index 0000000000000000000000000000000000000000..f75aba196719ba2c223188a9a264bc529eb82bf4 --- /dev/null +++ b/source_snapshot/minimax_h3/train_dmd_longrun_4step_autoresume.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Operational 50-step recovery wrapper for the 2,000-step H3 DMD4 Audio4 run. + +The formal experiment contract, including ``checkpoint_every=100`` in the +hashed configuration, is left unchanged. This wrapper only shortens the +allocation boundary to 50 global steps so a Slurm array task stays well below +the site's roughly four-hour interruption window. The original validator is +run against the next formal 100-step boundary, which preserves the exact +configuration hash used by existing checkpoints. During execution the +checkpoint grid is temporarily narrowed to 50 so the original save/load code +can create and consume an operational recovery checkpoint. +""" + +from __future__ import annotations + +import argparse +import copy +from contextlib import contextmanager +from typing import Any, Iterator, Sequence + +from . import train_dmd_longrun_4step as formal + + +FORMAL_CHECKPOINT_EVERY = 100 +RECOVERY_CHECKPOINT_EVERY = 50 +_ORIGINAL_VALIDATE_WITH_LOADER = formal._validate_longrun_request_with_loader + + +def _formal_validation_stop(stop_step: int) -> int: + if isinstance(stop_step, bool) or not isinstance(stop_step, int): + raise TypeError("stop_step must be an integer.") + if not RECOVERY_CHECKPOINT_EVERY <= stop_step <= formal.TOTAL_STEPS: + raise ValueError( + "stop_step is outside the " + f"{RECOVERY_CHECKPOINT_EVERY}..{formal.TOTAL_STEPS} recovery range." + ) + if stop_step % RECOVERY_CHECKPOINT_EVERY: + raise ValueError("stop_step must lie on the 50-step recovery grid.") + return min( + formal.TOTAL_STEPS, + ((stop_step + FORMAL_CHECKPOINT_EVERY - 1) // FORMAL_CHECKPOINT_EVERY) + * FORMAL_CHECKPOINT_EVERY, + ) + + +def _compatible_validate_with_loader( + args: argparse.Namespace, loader: Any +) -> dict[str, Any]: + """Validate the unchanged formal contract for a 50-step allocation end.""" + + if args.checkpoint_every != FORMAL_CHECKPOINT_EVERY: + raise ValueError( + "The hashed formal checkpoint_every value must remain exactly 100." + ) + validation_args = copy.copy(args) + validation_args.stop_step = _formal_validation_stop(int(args.stop_step)) + active_grid = formal.CHECKPOINT_EVERY + formal.CHECKPOINT_EVERY = FORMAL_CHECKPOINT_EVERY + try: + return _ORIGINAL_VALIDATE_WITH_LOADER(validation_args, loader) + finally: + formal.CHECKPOINT_EVERY = active_grid + + +@contextmanager +def _operational_recovery_grid() -> Iterator[None]: + if formal.CHECKPOINT_EVERY != FORMAL_CHECKPOINT_EVERY: + raise RuntimeError("The formal H3 checkpoint grid changed before wrapping.") + original_validator = formal._validate_longrun_request_with_loader + formal.CHECKPOINT_EVERY = RECOVERY_CHECKPOINT_EVERY + formal._validate_longrun_request_with_loader = _compatible_validate_with_loader + try: + yield + finally: + formal._validate_longrun_request_with_loader = original_validator + formal.CHECKPOINT_EVERY = FORMAL_CHECKPOINT_EVERY + + +def build_parser() -> argparse.ArgumentParser: + # Build before installing the operational grid so the CLI default remains + # the formal, config-hashed checkpoint_every=100. + return formal.build_parser() + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if not hasattr(args, "stop_step"): + raise ValueError("The autoresume wrapper only supports the train command.") + _formal_validation_stop(int(args.stop_step)) + if int(args.checkpoint_every) != FORMAL_CHECKPOINT_EVERY: + raise ValueError("--checkpoint-every must remain 100.") + with _operational_recovery_grid(): + return int(args.handler(args)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/source_snapshot/minimax_h3/train_dmd_longrun_8step.py b/source_snapshot/minimax_h3/train_dmd_longrun_8step.py new file mode 100644 index 0000000000000000000000000000000000000000..97c3c508cee1b62715ede8cced9353113016dc65 --- /dev/null +++ b/source_snapshot/minimax_h3/train_dmd_longrun_8step.py @@ -0,0 +1,1443 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Document-aligned eight-step MiniMax-H3 DMD long-run experiment. + +This is a new experiment contract. It leaves the provenance-locked v1-v5 +entry points unchanged while reusing their model loading, FSDP, distributed +LongLive prompt cache, and exact-resume checkpoint machinery. + +The semantic role names in the technical document map onto the historical +runtime names as ``generator -> student`` and ``fake_score -> critic``. +""" + +from __future__ import annotations + +import argparse +import gc +import json +import math +import os +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterator, Mapping, Sequence + +import torch +from torch.utils.checkpoint import checkpoint as torch_checkpoint + +from . import train_dmd as base +from . import train_dmd_longrun as v1 +from . import train_dmd_longrun_v2 as v2 +from . import train_dmd_longrun_v3 as v3 +from . import train_dmd_longrun_v4 as v4 +from . import train_dmd_longrun_v5 as v5 +from . import train_utils +from .dmd_8step import ( + H3DMD8StepTrainer, + NUM_FRESH_NOISE_TRANSITIONS, + NUM_MODEL_EVALUATIONS, + NUM_SIGMA_ENDPOINTS, + build_8step_sigma_schedules, + sample_target_stage, +) +from .full_h3_lora import ( + H3_DMD_LORA_POLICY, + H3_DMD_TARGET_MODULE_COUNT, + install_full_h3_lora_adapters, + released_full_h3_target_modules, +) +from .train_utils import CRITIC_ADAPTER, STUDENT_ADAPTER + + +TOTAL_STEPS = 1_000 +CHECKPOINT_EVERY = 100 +STUDENT_UPDATE_RATIO = 5 +GENERATOR_LR = 2.0e-5 +FAKE_SCORE_LR = 5.0e-5 +GENERATOR_BETAS = (0.0, 0.9) +FAKE_SCORE_BETAS = (0.9, 0.999) +GENERATOR_WEIGHT_DECAY = 0.01 +FAKE_SCORE_WEIGHT_DECAY = 0.0 +LORA_RANK = 128 +LORA_ALPHA = 128 +LORA_DROPOUT = 0.0 +MAX_GRAD_NORM = 1.0 +TRAINING_SEED = 20260812 +PYTORCH_ATTENTION_BACKEND = "_native_flash" +ATTENTION_BACKEND_POLICY = "pytorch_native_flash_sdpa_forced" +GRADIENT_CHECKPOINTING_POLICY = "main_transformer_blocks_reentrant" +FORMAL_WORLD_SIZE = 16 +FORMAL_NODES = 4 +FORMAL_GPUS_PER_NODE = 4 +FORMAL_HEIGHT = 768 +FORMAL_WIDTH = 1344 +FORMAL_NUM_FRAMES = 124 +LONGRUN_CONFIG_VERSION = 8 +CHECKPOINT_SCHEMA_VERSION = v2.CHECKPOINT_SCHEMA_VERSION +OPTIMIZER_LAYOUT = "joint_role_groups_per_role_adamw_v2" +ROLLOUT_MODE = "predict_x0_fresh_renoise" +TECHNICAL_DOCUMENT_SHA256 = ( + "e293cd2045f26ee6da47d32b440ee10db2a540eeaa4626e5e8b73a7f0318392b" +) + +student_update_due = v1.student_update_due +student_updates_through = v1.student_updates_through +student_updates_in_segment = v1.student_updates_in_segment +_stable_config_hash = v1._stable_config_hash +_ORIGINAL_METRIC_RECORD = v1._metric_record +_V4_VALIDATE_WITH_LOADER = v4._validate_longrun_request_with_loader +_REAL_ADAMW = torch.optim.AdamW + +_OUTER_STEP_SEED_MODULUS = (1 << 63) - 1 +_OUTER_STEP_STREAM_STRIDE = 1_000_000_007 +_OUTER_STEP_STREAM_IDS = { + "detached_initial_latents": 1, + "generator_initial_latents": 2, + "shared_stage_sigma": 3, + "detached_rollout": 4, + "fake_dsm": 5, + "generator_rollout": 6, + "generator_dmd": 7, +} + + +def reentrant_main_block_checkpoint( + module: torch.nn.Module, *args: Any +) -> torch.Tensor: + """Run one H3 main block with the document-required reentrant checkpoint. + + Reentrant checkpointing requires at least one direct tensor input that + requires gradients. The released 312-target policy normally satisfies this + through the trainable token-refiner LoRA. Keep a fail-safe for future cache + rows or target-policy changes: a gradient-free packed hidden state has no + upstream graph, so making that leaf require gradients preserves main-block + LoRA training without detaching anything. + """ + + if not args or not isinstance(args[0], torch.Tensor): + raise TypeError("H3 reentrant checkpoint requires hidden_states first.") + hidden_states = args[0] + if not hidden_states.requires_grad: + hidden_states = hidden_states.requires_grad_(True) + return torch_checkpoint( + module.__call__, + hidden_states, + *args[1:], + use_reentrant=True, + preserve_rng_state=True, + ) + + +def _technical_document_path() -> Path: + return Path(__file__).resolve().parents[3] / "MINIMAX_H3_DMD_TECHNICAL_DOCUMENT_ZH.md" + + +@contextmanager +def _temporary_attributes( + replacements: Sequence[tuple[Any, str, Any]], +) -> Iterator[None]: + originals: list[tuple[Any, str, Any]] = [] + for module, name, value in replacements: + originals.append((module, name, getattr(module, name))) + setattr(module, name, value) + try: + yield + finally: + for module, name, value in reversed(originals): + setattr(module, name, value) + + +@contextmanager +def _document_constant_contract() -> Iterator[None]: + """Make inherited validators/checkpointers observe the eight-step recipe.""" + + replacements = [ + (v1, "TOTAL_STEPS", TOTAL_STEPS), + (v1, "CHECKPOINT_EVERY", CHECKPOINT_EVERY), + (v1, "STUDENT_UPDATE_RATIO", STUDENT_UPDATE_RATIO), + (v1, "STUDENT_LR", GENERATOR_LR), + (v1, "CRITIC_LR", FAKE_SCORE_LR), + (v1, "ADAM_BETAS", GENERATOR_BETAS), + (v1, "WEIGHT_DECAY", GENERATOR_WEIGHT_DECAY), + (v1, "LORA_RANK", LORA_RANK), + (v1, "LORA_ALPHA", LORA_ALPHA), + (v1, "LORA_DROPOUT", LORA_DROPOUT), + (v1, "NUM_MODEL_EVALUATIONS", NUM_MODEL_EVALUATIONS), + (v1, "NUM_SIGMA_ENDPOINTS", NUM_SIGMA_ENDPOINTS), + (v2, "TOTAL_STEPS", TOTAL_STEPS), + (v2, "CHECKPOINT_EVERY", CHECKPOINT_EVERY), + (v2, "STUDENT_UPDATE_RATIO", STUDENT_UPDATE_RATIO), + (v2, "STUDENT_LR", GENERATOR_LR), + (v2, "CRITIC_LR", FAKE_SCORE_LR), + (v2, "LORA_RANK", LORA_RANK), + (v2, "LORA_ALPHA", LORA_ALPHA), + (v2, "LORA_DROPOUT", LORA_DROPOUT), + (v2, "OPTIMIZER_LAYOUT", OPTIMIZER_LAYOUT), + (v3, "CHECKPOINT_EVERY", CHECKPOINT_EVERY), + (v3, "STUDENT_LR", GENERATOR_LR), + (v3, "CRITIC_LR", FAKE_SCORE_LR), + (v4, "TOTAL_STEPS", TOTAL_STEPS), + (v4, "CHECKPOINT_EVERY", CHECKPOINT_EVERY), + (v4, "STUDENT_LR", GENERATOR_LR), + (v4, "CRITIC_LR", FAKE_SCORE_LR), + (v4, "FORMAL_WORLD_SIZE", FORMAL_WORLD_SIZE), + (base, "NUM_MODEL_EVALUATIONS", NUM_MODEL_EVALUATIONS), + (base, "NUM_SIGMA_ENDPOINTS", NUM_SIGMA_ENDPOINTS), + (train_utils, "NUM_MODEL_EVALUATIONS", NUM_MODEL_EVALUATIONS), + (train_utils, "NUM_SIGMA_ENDPOINTS", NUM_SIGMA_ENDPOINTS), + ] + with _temporary_attributes(replacements): + yield + + +def _validate_formal_arguments(args: argparse.Namespace) -> None: + expected = { + "max_steps": TOTAL_STEPS, + "checkpoint_every": CHECKPOINT_EVERY, + "lora_rank": LORA_RANK, + "lora_alpha": LORA_ALPHA, + "lora_dropout": LORA_DROPOUT, + "student_lr": GENERATOR_LR, + "critic_lr": FAKE_SCORE_LR, + "max_grad_norm": MAX_GRAD_NORM, + "seed": TRAINING_SEED, + "height": FORMAL_HEIGHT, + "width": FORMAL_WIDTH, + "num_frames": FORMAL_NUM_FRAMES, + "video_shift": 12.0, + "audio_shift": 3.0, + "gradient_checkpointing": True, + } + mismatches = { + name: {"actual": getattr(args, name), "expected": value} + for name, value in expected.items() + if getattr(args, name) != value + } + if mismatches: + raise ValueError( + "The formal eight-step document experiment is fixed: " + + json.dumps(mismatches, sort_keys=True) + ) + + +def _document_contract_fields() -> dict[str, Any]: + schedules = build_8step_sigma_schedules(dtype=torch.float64) + document = _technical_document_path() + if not document.is_file() or base.sha256_file(document) != TECHNICAL_DOCUMENT_SHA256: + raise RuntimeError("The pinned MiniMax-H3 DMD technical document changed.") + targets = released_full_h3_target_modules() + return { + "longrun_schema_version": LONGRUN_CONFIG_VERSION, + "longrun_config_version": LONGRUN_CONFIG_VERSION, + "checkpoint_schema_version": CHECKPOINT_SCHEMA_VERSION, + "technical_document": { + "path": str(document), + "sha256": TECHNICAL_DOCUMENT_SHA256, + "documented_steps": 4, + "requested_steps": NUM_MODEL_EVALUATIONS, + "grid_adaptation": "linear_base_sigma_refinement", + }, + "semantic_role_map": { + "generator": STUDENT_ADAPTER, + "fake_score": CRITIC_ADAPTER, + "real_score_teacher": "adapters_disabled", + }, + "model_variant": "standard_full_adaln", + "adaln_curve_rank": None, + "optimizer_layout": OPTIMIZER_LAYOUT, + "student_lr": GENERATOR_LR, + "critic_lr": FAKE_SCORE_LR, + "student_betas": list(GENERATOR_BETAS), + "critic_betas": list(FAKE_SCORE_BETAS), + "student_weight_decay": GENERATOR_WEIGHT_DECAY, + "critic_weight_decay": FAKE_SCORE_WEIGHT_DECAY, + "lr_scheduler": "constant", + "student_update_ratio": STUDENT_UPDATE_RATIO, + "student_update_rule": "(global_step-1)%5==0", + "critic_update_every": 1, + "lora_rank": LORA_RANK, + "lora_alpha": LORA_ALPHA, + "lora_dropout": LORA_DROPOUT, + "lora_scale": 1.0, + "lora_master_dtype": "float32", + "lora_target_policy": H3_DMD_LORA_POLICY, + "lora_target_scope": "50_main_plus_2_token_refiner_blocks", + "lora_target_branches": ["attn", "ff"], + "lora_target_module_count": H3_DMD_TARGET_MODULE_COUNT, + "lora_targets": targets, + "lora_excluded_scopes": [ + "adaln_and_timestep", + "normalization", + "top_level_input_context_output_projections", + ], + "rollout_mode": ROLLOUT_MODE, + "model_evaluations_per_rollout": NUM_MODEL_EVALUATIONS, + "sigma_endpoints_per_rollout": NUM_SIGMA_ENDPOINTS, + "fresh_noise_transitions_per_rollout": NUM_FRESH_NOISE_TRANSITIONS, + "target_stage_distribution": "uniform_0_through_7", + "outer_step_shared_random_variables": ["target_stage", "score_base_sigma"], + "score_base_sigma_min": 0.02, + "score_base_sigma_max": 0.98, + "video_sigma_grid": schedules.video_sigmas.tolist(), + "audio_sigma_grid": schedules.audio_sigmas.tolist(), + "fresh_noise_draw_order": ["video", "audio"], + "video_audio_transition_noise_independent": True, + "sample_rng_scheme": "named_rank_local_outer_step_substreams_v1", + "sample_rng_substreams": list(_OUTER_STEP_STREAM_IDS), + "sampler_matches_training_and_inference": True, + "teacher_trajectory_regression": False, + "real_latent_regression": False, + "gan_loss": False, + "ema": False, + "video_loss_weight": 1.0, + "audio_loss_weight": 1.0, + "gradient_checkpointing_scope": "main_transformer_blocks", + "gradient_checkpointing_policy": GRADIENT_CHECKPOINTING_POLICY, + "gradient_checkpointing_use_reentrant": True, + "token_refiner_gradient_checkpointing": False, + "attention_backend": ATTENTION_BACKEND_POLICY, + "attention_backend_dispatch_name": PYTORCH_ATTENTION_BACKEND, + "attention_requires_padless_training_sequence": True, + "layout_cache_policy": "per_step_rebuild_release_no_gpu_cache", + "distributed_backend": "FSDP_FULL_SHARD", + "documented_distributed_backend": "ZeRO-3", + "distributed_backend_deviation_reason": ( + "The validated local 16-rank launcher uses equivalent full parameter, " + "gradient, and optimizer sharding through FSDP FULL_SHARD." + ), + "data_deviation": ( + "User-requested LongLive 248221-row prompt source replaces the " + "document's Qwen-Image-Bench/H3-rewriter prompt set." + ), + "distributed_contract": { + "nodes": FORMAL_NODES, + "gpus_per_node": FORMAL_GPUS_PER_NODE, + "world_size": FORMAL_WORLD_SIZE, + "per_rank_batch_size": 1, + "global_batch_size": FORMAL_WORLD_SIZE, + "global_steps": TOTAL_STEPS, + "scheduled_conditions": FORMAL_WORLD_SIZE * TOTAL_STEPS, + }, + } + + +def _validate_longrun_request_with_loader( + args: argparse.Namespace, + loader: v5._DistributedPromptCollectionLoader, +) -> dict[str, Any]: + _validate_formal_arguments(args) + with _document_constant_contract(), v5._v5_world_contract(): + config = _V4_VALIDATE_WITH_LOADER(args, loader) + config.pop("config_sha256", None) + config.update(_document_contract_fields()) + config["seed"] = TRAINING_SEED + config["height"] = FORMAL_HEIGHT + config["width"] = FORMAL_WIDTH + config["num_frames"] = FORMAL_NUM_FRAMES + runtime = config.get("prompt_runtime_selection") + if not isinstance(runtime, Mapping): + raise RuntimeError("Distributed prompt runtime metadata is missing.") + if ( + runtime.get("world_size") != FORMAL_WORLD_SIZE + or runtime.get("scheduled_conditions") != FORMAL_WORLD_SIZE * TOTAL_STEPS + ): + raise RuntimeError("Prompt cache does not expose the world-16 schedule.") + config["config_sha256"] = _stable_config_hash(config) + return config + + +def validate_longrun_request(args: argparse.Namespace) -> dict[str, Any]: + loader = v5._DistributedPromptCollectionLoader(args) + try: + return _validate_longrun_request_with_loader(args, loader) + finally: + loader.clear() + + +@dataclass(frozen=True) +class H3OuterStepRNGStreams: + """Independent, deterministically-derived generators for one outer step.""" + + detached_initial_latents: torch.Generator + generator_initial_latents: torch.Generator + shared_stage_sigma: torch.Generator + detached_rollout: torch.Generator + fake_dsm: torch.Generator + generator_rollout: torch.Generator + generator_dmd: torch.Generator + seeds: Mapping[str, int] + + +@dataclass(frozen=True) +class H3OuterStepOutput: + """Outputs and the shared stochastic parameters of one DMD outer step.""" + + critic: Any + generated: Any | None + target_stage: int + shared_base_sigma: float + + +@dataclass(frozen=True) +class H3ObservedOptimizerCounters: + """Actual AdamW state counters after one joint role optimizer step.""" + + student_step: int + critic_step: int + cross_rank_verified: bool + + +def build_outer_step_rng_streams( + *, + training_seed: int, + condition_seed: int, + global_step: int, + rank: int, + device: torch.device | str, +) -> H3OuterStepRNGStreams: + """Derive named replay-stable RNG substreams without using global RNG state.""" + + values = { + "training_seed": training_seed, + "condition_seed": condition_seed, + "global_step": global_step, + "rank": rank, + } + for name, value in values.items(): + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{name} must be an integer.") + if global_step < 1 or rank < 0: + raise ValueError("global_step must be positive and rank must be non-negative.") + base_seed = ( + training_seed + + condition_seed + + global_step * 1_000_003 + + rank * 10_000_019 + ) % _OUTER_STEP_SEED_MODULUS + seeds = { + name: (base_seed + stream_id * _OUTER_STEP_STREAM_STRIDE) + % _OUTER_STEP_SEED_MODULUS + for name, stream_id in _OUTER_STEP_STREAM_IDS.items() + } + generators = { + name: torch.Generator(device=device).manual_seed(seed) + for name, seed in seeds.items() + } + return H3OuterStepRNGStreams( + detached_initial_latents=generators["detached_initial_latents"], + generator_initial_latents=generators["generator_initial_latents"], + shared_stage_sigma=generators["shared_stage_sigma"], + detached_rollout=generators["detached_rollout"], + fake_dsm=generators["fake_dsm"], + generator_rollout=generators["generator_rollout"], + generator_dmd=generators["generator_dmd"], + seeds=seeds, + ) + + +def _sample_shared_stage_and_sigma( + streams: H3OuterStepRNGStreams, + *, + device: torch.device, +) -> tuple[int, float]: + """Draw exactly one target stage and one score base sigma for the step.""" + + target_stage = sample_target_stage( + generator=streams.shared_stage_sigma, + device=device, + ) + base_sigma = float( + torch.empty((), device=device, dtype=torch.float32).uniform_( + 0.02, + 0.98, + generator=streams.shared_stage_sigma, + ) + ) + return target_stage, base_sigma + + +def run_document_outer_step( + *, + trainer: H3DMD8StepTrainer, + video_noise: torch.Tensor, + audio_noise: torch.Tensor, + condition: train_utils.H3Condition, + layout: train_utils.H3PackedLayout, + streams: H3OuterStepRNGStreams, + generator_update: bool, + allow_zero_generator_grad: bool, +) -> H3OuterStepOutput: + """Execute the document's shared-``(k, u)`` fake/generator orchestration.""" + + if not isinstance(generator_update, bool): + raise TypeError("generator_update must be a bool.") + if not isinstance(allow_zero_generator_grad, bool): + raise TypeError("allow_zero_generator_grad must be a bool.") + if allow_zero_generator_grad and not generator_update: + raise ValueError("Zero-gradient bootstrap requires a generator update.") + target_stage, shared_base_sigma = _sample_shared_stage_and_sigma( + streams, + device=video_noise.device, + ) + clean_video, clean_audio = trainer.detached_rollout( + video_noise=video_noise, + audio_noise=audio_noise, + condition=condition, + layout=layout, + target_stage=target_stage, + generator=streams.detached_rollout, + ) + fake_video_noise = train_utils._random_normal_like(clean_video, streams.fake_dsm) + fake_audio_noise = train_utils._random_normal_like(clean_audio, streams.fake_dsm) + critic = trainer.critic_step( + clean_video=clean_video, + clean_audio=clean_audio, + condition=condition, + layout=layout, + score_base_sigma=shared_base_sigma, + score_video_noise=fake_video_noise, + score_audio_noise=fake_audio_noise, + defer_optimizer_step=True, + ) + + generated = None + if generator_update: + generator_video_noise = train_utils._random_normal_like( + video_noise, streams.generator_initial_latents + ) + generator_audio_noise = train_utils._random_normal_like( + audio_noise, streams.generator_initial_latents + ) + # Draw the DMD score noise from its own stream. The H3 forward contract + # preserves A/V row shapes, so these are also the generated-clean shapes. + dmd_video_noise = train_utils._random_normal_like( + generator_video_noise, streams.generator_dmd + ) + dmd_audio_noise = train_utils._random_normal_like( + generator_audio_noise, streams.generator_dmd + ) + generated = trainer.generator_step( + video_noise=generator_video_noise, + audio_noise=generator_audio_noise, + condition=condition, + layout=layout, + score_base_sigma=shared_base_sigma, + score_video_noise=dmd_video_noise, + score_audio_noise=dmd_audio_noise, + target_stage=target_stage, + generator=streams.generator_rollout, + allow_zero_generator_grad=allow_zero_generator_grad, + defer_optimizer_step=True, + preserve_deferred_critic_gradients=True, + ) + if int(generated.target_stage) != target_stage: + raise RuntimeError("Generator did not reuse the outer step target stage.") + if not math.isclose( + float(generated.score_video_timestep), + float(critic.score_video_timestep), + rel_tol=0.0, + abs_tol=0.0, + ) or not math.isclose( + float(generated.score_audio_timestep), + float(critic.score_audio_timestep), + rel_tol=0.0, + abs_tol=0.0, + ): + raise RuntimeError("Generator and fake-score did not reuse shared base sigma.") + trainer.step_deferred_role_optimizer( + generator_update=generator_update, + device=video_noise.device, + ) + return H3OuterStepOutput( + critic=critic, + generated=generated, + target_stage=target_stage, + shared_base_sigma=shared_base_sigma, + ) + + +def _local_optimizer_role_counter( + optimizer: torch.optim.Optimizer, + parameters: Sequence[torch.nn.Parameter], +) -> tuple[int | None, bool]: + """Return one local-shard role counter and whether its state is complete.""" + + values: set[int] = set() + owned_parameters = 0 + missing_state = False + for parameter in parameters: + # With FSDP ``use_orig_params=True``, a rank may expose an empty view for + # an original parameter whose shard is owned entirely by other ranks. + if parameter.numel() == 0: + continue + owned_parameters += 1 + row = optimizer.state.get(parameter) + if not isinstance(row, Mapping) or "step" not in row: + missing_state = True + continue + try: + values.add(v2._step_value(row)) + except (RuntimeError, TypeError, ValueError): + missing_state = True + if owned_parameters == 0: + return None, True + if missing_state or len(values) != 1: + return None, False + return next(iter(values)), True + + +def _resolve_optimizer_counter_consensus( + payloads: Sequence[Sequence[int]], +) -> tuple[int, int]: + """Validate local state and exact cross-rank agreement from small payloads.""" + + if not payloads or any(len(payload) != 3 for payload in payloads): + raise RuntimeError("Optimizer counter consensus payloads are invalid.") + invalid = [rank for rank, payload in enumerate(payloads) if int(payload[0])] + if invalid: + raise RuntimeError( + "At least one rank has missing or internally inconsistent AdamW role " + f"state: ranks={invalid}." + ) + student_values = {int(payload[1]) for payload in payloads if int(payload[1]) >= 0} + critic_values = {int(payload[2]) for payload in payloads if int(payload[2]) >= 0} + if len(student_values) != 1 or len(critic_values) != 1: + raise RuntimeError( + "AdamW role counters differ across ranks or have no owning shard: " + f"student={sorted(student_values)}, critic={sorted(critic_values)}." + ) + return next(iter(student_values)), next(iter(critic_values)) + + +def observe_joint_optimizer_counters( + *, + optimizer: torch.optim.Optimizer, + student_parameters: Sequence[torch.nn.Parameter], + critic_parameters: Sequence[torch.nn.Parameter], + device: torch.device, + distributed: bool, +) -> H3ObservedOptimizerCounters: + """Read actual role counters and, under FSDP, prove rank consensus.""" + + student_step, student_valid = _local_optimizer_role_counter( + optimizer, student_parameters + ) + critic_step, critic_valid = _local_optimizer_role_counter( + optimizer, critic_parameters + ) + payload = ( + int(not (student_valid and critic_valid)), + -1 if student_step is None else student_step, + -1 if critic_step is None else critic_step, + ) + payloads: list[Sequence[int]] = [payload] + if distributed: + import torch.distributed as dist + + if not dist.is_initialized(): + raise RuntimeError("Distributed optimizer-counter audit lacks a process group.") + local = torch.tensor(payload, device=device, dtype=torch.int64) + gathered = [torch.empty_like(local) for _ in range(dist.get_world_size())] + dist.all_gather(gathered, local) + payloads = [row.cpu().tolist() for row in gathered] + resolved_student, resolved_critic = _resolve_optimizer_counter_consensus(payloads) + return H3ObservedOptimizerCounters( + student_step=resolved_student, + critic_step=resolved_critic, + cross_rank_verified=distributed, + ) + + +def validate_observed_optimizer_counters( + *, + global_step: int, + previous_student_step: int, + observed: H3ObservedOptimizerCounters, +) -> None: + """Fail immediately on TTUR drift, including every non-generator step.""" + + expected_student = student_updates_through(global_step) + expected_critic = global_step + if observed.student_step != expected_student or observed.critic_step != expected_critic: + raise RuntimeError( + "Observed AdamW counters violate the absolute outer-step contract: " + f"step={global_step}, student={observed.student_step}/{expected_student}, " + f"critic={observed.critic_step}/{expected_critic}." + ) + expected_delta = int(student_update_due(global_step)) + if observed.student_step != previous_student_step + expected_delta: + raise RuntimeError( + "Observed student AdamW counter violated the per-step update cadence: " + f"step={global_step}, previous={previous_student_step}, " + f"observed={observed.student_step}, expected_delta={expected_delta}." + ) + + # Keep the formal six-row audit explicit instead of relying only on the + # closed-form schedule above, so a future cadence rewrite cannot weaken it. + if global_step <= 6: + expected_first_six = (1, 1, 1, 1, 1, 2) + if observed.student_step != expected_first_six[global_step - 1]: + raise RuntimeError("The formal first-six generator counter audit failed.") + + +def _allow_zero_generator_bootstrap( + *, + start_step: int, + global_step: int, + resumed: bool, +) -> bool: + """Authorize the one documented bootstrap from absolute run state only.""" + + for name, value in (("start_step", start_step), ("global_step", global_step)): + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{name} must be an integer.") + if not isinstance(resumed, bool): + raise TypeError("resumed must be a bool.") + if start_step < 0 or global_step <= start_step: + raise ValueError("global_step must be positive and greater than start_step.") + return not resumed and start_step == 0 and global_step == 1 + + +def _install_document_lora_adapters( + model: torch.nn.Module, + *, + rank: int, + alpha: int, + dropout: float, + **_: Any, +) -> dict[str, list[torch.nn.Parameter]]: + if (rank, alpha, float(dropout)) != (LORA_RANK, LORA_ALPHA, LORA_DROPOUT): + raise ValueError("The document LoRA is fixed to rank/alpha/dropout 128/128/0.") + return install_full_h3_lora_adapters( + model, + rank=rank, + alpha=alpha, + dropout=dropout, + master_dtype=torch.float32, + ) + + +def _load_document_transformer(*args: Any, **kwargs: Any): + if args: + parsed_args, resolved_revision, device, *remaining = args + if remaining: + resume_lora = remaining[0] + else: + resume_lora = kwargs.pop("resume_lora", None) + else: + parsed_args = kwargs.pop("args") + resolved_revision = kwargs.pop("resolved_revision") + device = kwargs.pop("device") + resume_lora = kwargs.pop("resume_lora", None) + if kwargs: + raise TypeError(f"Unexpected transformer loader arguments: {sorted(kwargs)}") + + base.validate_local_model_provenance(parsed_args.model_id, resolved_revision) + try: + from diffusers import MiniMaxH3Transformer3DModel + except ImportError as error: + raise RuntimeError("Install the editable diffusers_h3 checkout first.") from error + load_kwargs = base._model_load_kwargs(parsed_args, resolved_revision) + load_kwargs["low_cpu_mem_usage"] = True + role_model = MiniMaxH3Transformer3DModel.from_pretrained( + parsed_args.model_id, + subfolder="transformer", + **load_kwargs, + ) + if role_model.__class__.__name__ != "MiniMaxH3Transformer3DModel": + raise TypeError("Training loaded a non-H3 transformer.") + architecture = v1.assert_released_h3_architecture(role_model) + if architecture["attention_is_causal"] is not False: + raise AssertionError("Released MiniMax-H3 attention must remain non-causal.") + role_model._h3_released_architecture = architecture + _install_document_lora_adapters( + role_model, + rank=LORA_RANK, + alpha=LORA_ALPHA, + dropout=LORA_DROPOUT, + ) + if resume_lora is not None: + v1._install_checkpoint_loras(role_model, resume_lora) + + # The full-resolution sequence is ~37.8K rows. Force PyTorch's native + # Flash-SDPA backend so an unsupported kernel fails closed instead of + # silently materializing an O(S^2) math-attention matrix. + role_model.set_attention_backend(PYTORCH_ATTENTION_BACKEND) + observed_attention_backends = { + getattr(getattr(processor, "_attention_backend", None), "value", None) + for processor in role_model.attn_processors.values() + } + if observed_attention_backends != {PYTORCH_ATTENTION_BACKEND}: + raise RuntimeError( + "H3 attention processors did not all pin native Flash-SDPA: " + f"{sorted(map(str, observed_attention_backends))}." + ) + + role_model.enable_gradient_checkpointing(reentrant_main_block_checkpoint) + if role_model._gradient_checkpointing_func is not reentrant_main_block_checkpoint: + raise RuntimeError("H3 main blocks did not retain reentrant checkpointing.") + raw_token_refiner = role_model.token_refiner + raw_token_refiner.gradient_checkpointing = False + if bool(raw_token_refiner.gradient_checkpointing): + raise RuntimeError("Token-refiner gradient checkpointing remained enabled pre-FSDP.") + role_model.train() + if parsed_args.strategy == "fsdp": + forward_model = base._wrap_fsdp(role_model, device) + wrapped_refiner = role_model.token_refiner + underlying_refiner = getattr( + wrapped_refiner, + "_fsdp_wrapped_module", + None, + ) + if underlying_refiner is not raw_token_refiner: + raise RuntimeError("FSDP did not retain the audited token-refiner module.") + if bool(underlying_refiner.gradient_checkpointing): + raise RuntimeError( + "The underlying FSDP token refiner re-enabled gradient checkpointing." + ) + runtime_report = base.validate_h3_fsdp_runtime_wrappers( + forward_model, role_model + ) + import torch.distributed as dist + + reports: list[dict[str, Any] | None] = [None] * dist.get_world_size() + dist.all_gather_object(reports, runtime_report) + if any(report != runtime_report for report in reports): + raise RuntimeError("FSDP CPU-offload runtime audit differs across ranks.") + runtime_report["all_ranks_verified"] = True + runtime_report["token_refiner_gradient_checkpointing"] = False + runtime_report["gradient_checkpointing_policy"] = ( + GRADIENT_CHECKPOINTING_POLICY + ) + runtime_report["attention_backend"] = ATTENTION_BACKEND_POLICY + runtime_report["attention_backend_dispatch_name"] = ( + PYTORCH_ATTENTION_BACKEND + ) + role_model._h3_fsdp_runtime_report = runtime_report + else: + role_model.to(device) + forward_model = role_model + + if getattr(role_model.config, "adaln_curve_rank", None) is not None: + raise RuntimeError("The eight-step experiment requires standard full-AdaLN.") + lora_dtypes = { + parameter.dtype + for name, parameter in role_model.named_parameters() + if "lora_" in name + } + if lora_dtypes != {torch.float32}: + raise RuntimeError(f"LoRA masters are not FP32: {sorted(map(str, lora_dtypes))}.") + return forward_model, role_model + + +class _DocumentAdamWFactory: + """Build one FSDP-compatible optimizer with role-specific AdamW groups.""" + + def __init__(self) -> None: + self.optimizer: torch.optim.AdamW | None = None + self.parameter_ids: set[int] = set() + self.calls = 0 + + def __call__( + self, + params: Any, + *, + lr: float, + betas: tuple[float, float], + weight_decay: float, + **kwargs: Any, + ) -> torch.optim.AdamW: + del betas, weight_decay + parameters = list(params) + role = STUDENT_ADAPTER if self.calls == 0 else CRITIC_ADAPTER + expected_lr = GENERATOR_LR if role == STUDENT_ADAPTER else FAKE_SCORE_LR + role_betas = GENERATOR_BETAS if role == STUDENT_ADAPTER else FAKE_SCORE_BETAS + role_weight_decay = ( + GENERATOR_WEIGHT_DECAY + if role == STUDENT_ADAPTER + else FAKE_SCORE_WEIGHT_DECAY + ) + if self.calls >= 2 or lr != expected_lr: + raise RuntimeError("Unexpected AdamW construction in the eight-step trainer.") + ids = {id(parameter) for parameter in parameters} + if not parameters or ids & self.parameter_ids: + raise RuntimeError("Eight-step optimizer groups are empty or overlapping.") + self.parameter_ids.update(ids) + group = { + "params": parameters, + "role": role, + "lr": expected_lr, + "betas": role_betas, + "weight_decay": role_weight_decay, + } + if self.optimizer is None: + self.optimizer = _REAL_ADAMW([group], **kwargs) + else: + self.optimizer.add_param_group(group) + self.calls += 1 + return self.optimizer + + +def validate_joint_optimizer_state( + state_dict: Mapping[str, Any], + *, + expected_student_updates: int, + expected_critic_updates: int, +) -> None: + states = state_dict.get("state") + groups = state_dict.get("param_groups") + if not isinstance(states, Mapping) or not states: + raise RuntimeError("Joint optimizer state is empty.") + if not isinstance(groups, list) or len(groups) != 2: + raise RuntimeError("Joint optimizer must contain two role groups.") + expected = { + STUDENT_ADAPTER: { + "lr": GENERATOR_LR, + "betas": GENERATOR_BETAS, + "weight_decay": GENERATOR_WEIGHT_DECAY, + "updates": expected_student_updates, + }, + CRITIC_ADAPTER: { + "lr": FAKE_SCORE_LR, + "betas": FAKE_SCORE_BETAS, + "weight_decay": FAKE_SCORE_WEIGHT_DECAY, + "updates": expected_critic_updates, + }, + } + parameter_roles: dict[Any, str] = {} + seen_roles: set[str] = set() + for group in groups: + if not isinstance(group, Mapping): + raise RuntimeError("Optimizer parameter group is invalid.") + role = group.get("role") + if role not in expected or role in seen_roles: + raise RuntimeError("Optimizer roles must be exactly student and critic.") + seen_roles.add(role) + parameters = group.get("params") + if not isinstance(parameters, list) or not parameters: + raise RuntimeError(f"{role} optimizer group is empty.") + for parameter in parameters: + if parameter in parameter_roles: + raise RuntimeError("Optimizer parameter groups overlap.") + parameter_roles[parameter] = role + wanted = expected[role] + mismatches = { + name: {"actual": group.get(name), "expected": wanted[name]} + for name in ("lr", "betas", "weight_decay") + if group.get(name) != wanted[name] + } + if mismatches: + raise RuntimeError( + f"{role} optimizer hyperparameter mismatch: " + + json.dumps(mismatches, sort_keys=True) + ) + if seen_roles != set(expected) or set(states) != set(parameter_roles): + raise RuntimeError("Optimizer state does not cover both complete role groups.") + observed = {STUDENT_ADAPTER: set(), CRITIC_ADAPTER: set()} + for parameter, row in states.items(): + if not isinstance(row, Mapping): + raise RuntimeError("Optimizer state row is invalid.") + observed[parameter_roles[parameter]].add(v2._step_value(row)) + for role, values in observed.items(): + wanted_updates = int(expected[role]["updates"]) + if values != {wanted_updates}: + raise RuntimeError( + f"{role} optimizer step mismatch: {sorted(values)} != [{wanted_updates}]." + ) + + +def save_longrun_checkpoint(**kwargs: Any) -> Path: + with _document_constant_contract(), _temporary_attributes( + [(v2, "validate_joint_optimizer_state", validate_joint_optimizer_state)] + ): + return v2.save_longrun_checkpoint(**kwargs) + + +def load_longrun_checkpoint(*args: Any, **kwargs: Any) -> dict[str, Any]: + with _document_constant_contract(), _temporary_attributes( + [(v2, "validate_joint_optimizer_state", validate_joint_optimizer_state)] + ): + return v2.load_longrun_checkpoint(*args, **kwargs) + + +def restore_longrun_state(**kwargs: Any) -> None: + with _document_constant_contract(), _temporary_attributes( + [(v2, "validate_joint_optimizer_state", validate_joint_optimizer_state)] + ): + v2.restore_longrun_state(**kwargs) + + +def _metric_record(**kwargs: Any) -> dict[str, Any]: + target_stage = int(kwargs.pop("target_stage")) + shared_base_sigma = float(kwargs.pop("shared_base_sigma")) + rng_seeds = dict(kwargs.pop("rng_seeds")) + optimizer_counters = kwargs.pop("optimizer_counters") + if not isinstance(optimizer_counters, H3ObservedOptimizerCounters): + raise TypeError("optimizer_counters must be an observed AdamW counter record.") + record = _ORIGINAL_METRIC_RECORD(**kwargs) + generated = kwargs["generated"] + critic = kwargs["critic"] + roles = kwargs["roles"] + student_roles = { + "student_detached_stage", + "student_stage_cache", + "student_target_stage", + } + record.update( + { + "semantic_generator_updated": generated is not None, + "semantic_fake_score_updated": True, + "outer_target_stage": target_stage, + "outer_shared_score_base_sigma": shared_base_sigma, + "outer_rng_stream_seeds": rng_seeds, + "student_optimizer_step_observed": optimizer_counters.student_step, + "critic_optimizer_step_observed": optimizer_counters.critic_step, + "optimizer_counter_cross_rank_verified": ( + optimizer_counters.cross_rank_verified + ), + "generator_target_stage": ( + int(generated.target_stage) if generated is not None else None + ), + "generator_zero_grad_bootstrap_accepted": ( + bool(generated.zero_grad_accepted) if generated is not None else False + ), + "critic_score_video_timestep": float(critic.score_video_timestep), + "critic_score_audio_timestep": float(critic.score_audio_timestep), + "generator_score_video_timestep": ( + float(generated.score_video_timestep) if generated is not None else None + ), + "generator_score_audio_timestep": ( + float(generated.score_audio_timestep) if generated is not None else None + ), + "student_positive_forwards": sum( + row.role in student_roles for row in roles + ), + "student_positive_forwards_per_rollout": NUM_MODEL_EVALUATIONS, + "sigma_endpoints_per_rollout": NUM_SIGMA_ENDPOINTS, + "fresh_noise_transitions_per_rollout": NUM_FRESH_NOISE_TRANSITIONS, + "rollout_mode": ROLLOUT_MODE, + } + ) + expected_calls = 19 if generated is not None else 9 + observed_calls = len(roles) + record["transformer_calls_expected"] = expected_calls + record["transformer_calls_observed"] = observed_calls + record["transformer_call_contract_verified"] = observed_calls == expected_calls + if observed_calls != expected_calls: + raise RuntimeError( + f"Eight-step call contract changed: observed={observed_calls}, " + f"expected={expected_calls}." + ) + return record + + +def _train_document_loop( + args: argparse.Namespace, + *, + prompt_loader: v5._DistributedPromptCollectionLoader, + factory: _DocumentAdamWFactory, +) -> int: + """Own the eight-step outer loop instead of adapting the v1 sampler loop.""" + + config = _validate_longrun_request_with_loader(args, prompt_loader) + if args.validate_only: + print(json.dumps(config, indent=2, sort_keys=True)) + return 0 + + allocator_config = os.environ.get("PYTORCH_CUDA_ALLOC_CONF") + if args.strategy == "fsdp" and allocator_config != v1.CUDA_ALLOCATOR_CONFIG: + raise RuntimeError( + "Full H3 FSDP requires the audited CUDA allocator configuration: " + f"actual={allocator_config!r}, expected={v1.CUDA_ALLOCATOR_CONFIG!r}." + ) + runtime_initializer = v5._formal_runtime_initializer(base._init_runtime) + device, rank, world_size = runtime_initializer(args) + allocator_backend = torch.cuda.memory.get_allocator_backend() + if args.strategy == "fsdp" and allocator_backend != "native": + raise RuntimeError( + "The audited H3 memory policy requires PyTorch's native CUDA allocator; " + f"observed {allocator_backend!r}." + ) + + checkpoint_root = Path(args.checkpoint_root).resolve() + output_dir = Path(args.output_dir).resolve() + v1._fresh_segment_output(output_dir) + if rank == 0: + checkpoint_root.mkdir(parents=True, exist_ok=True) + output_dir.mkdir(parents=True, exist_ok=True) + if args.strategy == "fsdp": + import torch.distributed as dist + + dist.barrier() + + bundle = None + start_step = 0 + if args.resume_from is not None: + bundle = load_longrun_checkpoint( + args.resume_from, + expected_config_sha256=config["config_sha256"], + expected_world_size=world_size, + checkpoint_root=checkpoint_root, + ) + start_step = int(bundle["global_step"]) + if start_step >= args.stop_step: + raise ValueError( + f"Resume step {start_step} must precede stop_step {args.stop_step}." + ) + + torch.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) + resolved_revision = config["resolved_revision"] + _, cached_conditions = prompt_loader(args.prompt_cache) + forward_model, role_model = _load_document_transformer( + args, + resolved_revision, + device, + bundle["lora"] if bundle is not None else None, + ) + video_input_dtype, audio_input_dtype = v1.h3_model_input_dtypes(role_model) + student_optimizer = factory( + v1.adapter_parameters(role_model, STUDENT_ADAPTER), + lr=GENERATOR_LR, + betas=GENERATOR_BETAS, + weight_decay=GENERATOR_WEIGHT_DECAY, + ) + critic_optimizer = factory( + v1.adapter_parameters(role_model, CRITIC_ADAPTER), + lr=FAKE_SCORE_LR, + betas=FAKE_SCORE_BETAS, + weight_decay=FAKE_SCORE_WEIGHT_DECAY, + ) + if student_optimizer is not critic_optimizer: + raise RuntimeError("The document recipe requires one joint role optimizer.") + if bundle is not None: + restore_longrun_state( + bundle=bundle, + forward_model=forward_model, + role_model=role_model, + student_optimizer=student_optimizer, + critic_optimizer=critic_optimizer, + device=device, + rank=rank, + strategy=args.strategy, + ) + trainer = H3DMD8StepTrainer( + forward_model=forward_model, + role_model=role_model, + student_optimizer=student_optimizer, + critic_optimizer=critic_optimizer, + video_shift=args.video_shift, + audio_shift=args.audio_shift, + max_grad_norm=MAX_GRAD_NORM, + ) + + if rank == 0: + runtime = { + **config, + "segment_kind": v1.SEGMENT_KIND, + "started_at": base._utc_now(), + "resume_step": start_step, + "start_step": start_step + 1, + "stop_step": args.stop_step, + "steps_this_segment": args.stop_step - start_step, + "world_size": world_size, + "resume_from": str(bundle["directory"]) if bundle is not None else None, + "resume_checkpoint_success_sha256": ( + bundle["success_sha256"] if bundle is not None else None + ), + "checkpoint_root": str(checkpoint_root), + "torch_version": torch.__version__, + "cuda_version": torch.version.cuda, + "cuda_allocator_config_observed": allocator_config, + "cuda_allocator_backend_observed": allocator_backend, + "video_latent_dtype": str(video_input_dtype).removeprefix("torch."), + "audio_latent_dtype": str(audio_input_dtype).removeprefix("torch."), + "fsdp_wrap_plan": getattr( + role_model, "_h3_fsdp_wrap_plan_report", None + ), + "fsdp_runtime_audit": getattr( + role_model, "_h3_fsdp_runtime_report", None + ), + "released_architecture": getattr( + role_model, "_h3_released_architecture", None + ), + } + base._atomic_json(runtime, output_dir / "runtime.json") + + latent_frames, latent_height, latent_width, audio_latents = ( + base._validate_geometry(args) + ) + patch_size = tuple(role_model.config.patch_size) + video_patch_dim = int(role_model.config.in_channels * math.prod(patch_size)) + audio_channels = int(role_model.config.audio_in_channels) + final_checkpoint_dir: Path | None = None + previous_student_counter = student_updates_through(start_step) + + for step in range(start_step + 1, args.stop_step + 1): + condition = cached_conditions[ + (args.prompt_start_index + step - 1) % len(cached_conditions) + ].to(device) + # Prompt text lengths can vary. Rebuilding this small index structure is + # preferable to retaining up to 1,000 full-resolution CUDA layouts. + layout = v1.make_base_t2va_layout( + text_token_tags=condition.text_token_tags, + num_latent_frames=latent_frames, + latent_height=latent_height, + latent_width=latent_width, + num_audio_latents=audio_latents, + patch_size=patch_size, + ).to(device) + if bool((layout.token_tags < 0).any()): + raise RuntimeError( + "Formal H3 training must remain padless for forced native Flash-SDPA." + ) + streams = build_outer_step_rng_streams( + training_seed=args.seed, + condition_seed=condition.seed, + global_step=step, + rank=rank, + device=device, + ) + video_noise = torch.randn( + (1, layout.video_indices.numel(), video_patch_dim), + device=device, + dtype=video_input_dtype, + generator=streams.detached_initial_latents, + ) + audio_noise = torch.randn( + (1, layout.audio_indices.numel(), audio_channels), + device=device, + dtype=audio_input_dtype, + generator=streams.detached_initial_latents, + ) + + cache_before = trainer.cuda_cache_clears_before_generator_backward + contexts_before = trainer.saved_tensor_offload_contexts + packs_before = trainer.saved_tensor_offload_pack_count + unpacks_before = trainer.saved_tensor_offload_unpack_count + bytes_before = trainer.saved_tensor_offload_packed_bytes + due = student_update_due(step) + outer = run_document_outer_step( + trainer=trainer, + video_noise=video_noise, + audio_noise=audio_noise, + condition=condition, + layout=layout, + streams=streams, + generator_update=due, + allow_zero_generator_grad=_allow_zero_generator_bootstrap( + start_step=start_step, + global_step=step, + resumed=bundle is not None, + ), + ) + optimizer_counters = observe_joint_optimizer_counters( + optimizer=student_optimizer, + student_parameters=trainer.student_parameters, + critic_parameters=trainer.critic_parameters, + device=device, + distributed=args.strategy == "fsdp", + ) + validate_observed_optimizer_counters( + global_step=step, + previous_student_step=previous_student_counter, + observed=optimizer_counters, + ) + previous_student_counter = optimizer_counters.student_step + cache_clears = ( + trainer.cuda_cache_clears_before_generator_backward - cache_before + ) + offload_contexts = trainer.saved_tensor_offload_contexts - contexts_before + offload_packs = trainer.saved_tensor_offload_pack_count - packs_before + offload_unpacks = trainer.saved_tensor_offload_unpack_count - unpacks_before + offload_bytes = trainer.saved_tensor_offload_packed_bytes - bytes_before + local_memory_valid = ( + cache_clears == 1 + and offload_contexts == 1 + and offload_packs > 0 + and offload_unpacks > 0 + and offload_bytes > 0 + if due + else cache_clears == 0 + and offload_contexts == 0 + and offload_packs == 0 + and offload_unpacks == 0 + and offload_bytes == 0 + ) + memory_verified = local_memory_valid + if args.strategy == "fsdp": + import torch.distributed as dist + + consensus = torch.tensor( + int(local_memory_valid), device=device, dtype=torch.int32 + ) + dist.all_reduce(consensus, op=dist.ReduceOp.MIN) + memory_verified = bool(consensus.item()) + if not memory_verified: + raise RuntimeError("At least one rank violated the long-run memory policy.") + + record = _metric_record( + step=step, + prompt_id=condition.id, + critic=outer.critic, + generated=outer.generated, + roles=trainer.audit.records, + cache_clears=cache_clears, + offload_contexts=offload_contexts, + offload_packs=offload_packs, + offload_unpacks=offload_unpacks, + offload_bytes=offload_bytes, + memory_policy_verified=memory_verified, + target_stage=outer.target_stage, + shared_base_sigma=outer.shared_base_sigma, + rng_seeds=streams.seeds, + optimizer_counters=optimizer_counters, + ) + if rank == 0: + base._append_jsonl(record, output_dir / "metrics.jsonl") + print(json.dumps(record, sort_keys=True), flush=True) + trainer.audit = train_utils.H3CallAudit() + + if step % CHECKPOINT_EVERY == 0: + final_checkpoint_dir = save_longrun_checkpoint( + forward_model=forward_model, + role_model=role_model, + student_optimizer=student_optimizer, + critic_optimizer=critic_optimizer, + checkpoint_root=checkpoint_root, + step=step, + config=config, + device=device, + rank=rank, + world_size=world_size, + strategy=args.strategy, + ) + del outer, video_noise, audio_noise, layout, condition, streams + + expected_student_updates = student_updates_in_segment(start_step, args.stop_step) + if trainer.cuda_cache_clears_before_generator_backward != expected_student_updates: + raise AssertionError("Student CUDA cache-clear count differs from scheduled updates.") + if trainer.saved_tensor_offload_contexts != expected_student_updates: + raise AssertionError("Saved-tensor offload count differs from scheduled student updates.") + if ( + final_checkpoint_dir is None + or final_checkpoint_dir.name != f"checkpoint-{args.stop_step:06d}" + ): + raise AssertionError("Segment ended without its stop-step checkpoint.") + if factory.calls != 2 or factory.optimizer is None: + raise RuntimeError("Both eight-step optimizer groups were not built.") + if len(factory.optimizer.param_groups) != 2: + raise RuntimeError("Eight-step optimizer group count changed.") + prompt_loader.assert_complete() + + if rank == 0: + runtime_path = output_dir / "runtime.json" + runtime = json.loads(runtime_path.read_text(encoding="utf-8")) + runtime.update( + { + "finished_at": base._utc_now(), + "completed_global_step": args.stop_step, + "critic_updates_this_segment": args.stop_step - start_step, + "student_updates_this_segment": expected_student_updates, + "student_updates_completed": student_updates_through(args.stop_step), + "final_checkpoint": str(final_checkpoint_dir), + } + ) + base._atomic_json(runtime, runtime_path) + checkpoint_success = final_checkpoint_dir / "CHECKPOINT_SUCCESS.json" + metrics_path = output_dir / "metrics.jsonl" + segment_success = { + "schema_version": 1, + "status": "success", + "kind": v1.SEGMENT_KIND, + "config_sha256": config["config_sha256"], + "resume_step": start_step, + "start_step": start_step + 1, + "stop_step": args.stop_step, + "critic_updates": args.stop_step - start_step, + "student_updates": expected_student_updates, + "critic_updates_completed": args.stop_step, + "student_updates_completed": student_updates_through(args.stop_step), + "world_size": world_size, + "final_checkpoint": str(final_checkpoint_dir), + "artifacts": { + "runtime.json": { + "sha256": base.sha256_file(runtime_path), + "bytes": runtime_path.stat().st_size, + }, + "metrics.jsonl": { + "sha256": base.sha256_file(metrics_path), + "bytes": metrics_path.stat().st_size, + }, + "CHECKPOINT_SUCCESS.json": { + "path": str(checkpoint_success), + "sha256": base.sha256_file(checkpoint_success), + "bytes": checkpoint_success.stat().st_size, + }, + }, + } + base._atomic_json(segment_success, output_dir / "SEGMENT_SUCCESS.json") + if args.strategy == "fsdp": + import torch.distributed as dist + + dist.barrier() + dist.destroy_process_group() + del forward_model, role_model + gc.collect() + return 0 + + +def train(args: argparse.Namespace) -> int: + factory = _DocumentAdamWFactory() + prompt_loader = v5._DistributedPromptCollectionLoader(args) + try: + with _document_constant_contract(), v5._v5_world_contract(): + return _train_document_loop( + args, + prompt_loader=prompt_loader, + factory=factory, + ) + finally: + prompt_loader.clear() + + +def build_parser() -> argparse.ArgumentParser: + parser = v1.build_parser() + subparsers = next( + action + for action in parser._actions + if isinstance(action, argparse._SubParsersAction) + ) + training = subparsers.choices["train"] + for action in training._actions: + if action.dest == "prompt_cache": + action.help = "Path to the verified world-16 LongLive collection.json." + elif action.dest == "prompt_start_index": + action.help = "Must remain zero; rank caches use absolute global step." + training.set_defaults( + handler=train, + max_steps=TOTAL_STEPS, + checkpoint_every=CHECKPOINT_EVERY, + lora_rank=LORA_RANK, + lora_alpha=LORA_ALPHA, + lora_dropout=LORA_DROPOUT, + student_lr=GENERATOR_LR, + critic_lr=FAKE_SCORE_LR, + max_grad_norm=MAX_GRAD_NORM, + seed=TRAINING_SEED, + height=FORMAL_HEIGHT, + width=FORMAL_WIDTH, + num_frames=FORMAL_NUM_FRAMES, + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + return int(args.handler(args)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/source_snapshot/minimax_h3/train_dmd_longrun_8step_autoresume.py b/source_snapshot/minimax_h3/train_dmd_longrun_8step_autoresume.py new file mode 100644 index 0000000000000000000000000000000000000000..24a08c6aca1d56f334e0d2c8b8f93e41c2d9084c --- /dev/null +++ b/source_snapshot/minimax_h3/train_dmd_longrun_8step_autoresume.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Operational 50-step recovery wrapper for the formal H3 DMD8 run. + +The formal experiment contract, including ``checkpoint_every=100`` in the +hashed configuration, is left unchanged. This wrapper only shortens the +allocation boundary to 50 global steps so a Slurm array task stays well below +the site's roughly four-hour interruption window. The original validator is +run against the next formal 100-step boundary, which preserves the exact +configuration hash used by existing checkpoints. During execution the +checkpoint grid is temporarily narrowed to 50 so the original save/load code +can create and consume an operational recovery checkpoint. +""" + +from __future__ import annotations + +import argparse +import copy +from contextlib import contextmanager +from typing import Any, Iterator, Sequence + +from . import train_dmd_longrun_8step as formal + + +FORMAL_CHECKPOINT_EVERY = 100 +RECOVERY_CHECKPOINT_EVERY = 50 +_ORIGINAL_VALIDATE_WITH_LOADER = formal._validate_longrun_request_with_loader + + +def _formal_validation_stop(stop_step: int) -> int: + if isinstance(stop_step, bool) or not isinstance(stop_step, int): + raise TypeError("stop_step must be an integer.") + if not RECOVERY_CHECKPOINT_EVERY <= stop_step <= formal.TOTAL_STEPS: + raise ValueError("stop_step is outside the 50..1000 recovery range.") + if stop_step % RECOVERY_CHECKPOINT_EVERY: + raise ValueError("stop_step must lie on the 50-step recovery grid.") + return min( + formal.TOTAL_STEPS, + ((stop_step + FORMAL_CHECKPOINT_EVERY - 1) // FORMAL_CHECKPOINT_EVERY) + * FORMAL_CHECKPOINT_EVERY, + ) + + +def _compatible_validate_with_loader( + args: argparse.Namespace, loader: Any +) -> dict[str, Any]: + """Validate the unchanged formal contract for a 50-step allocation end.""" + + if args.checkpoint_every != FORMAL_CHECKPOINT_EVERY: + raise ValueError( + "The hashed formal checkpoint_every value must remain exactly 100." + ) + validation_args = copy.copy(args) + validation_args.stop_step = _formal_validation_stop(int(args.stop_step)) + active_grid = formal.CHECKPOINT_EVERY + formal.CHECKPOINT_EVERY = FORMAL_CHECKPOINT_EVERY + try: + return _ORIGINAL_VALIDATE_WITH_LOADER(validation_args, loader) + finally: + formal.CHECKPOINT_EVERY = active_grid + + +@contextmanager +def _operational_recovery_grid() -> Iterator[None]: + if formal.CHECKPOINT_EVERY != FORMAL_CHECKPOINT_EVERY: + raise RuntimeError("The formal H3 checkpoint grid changed before wrapping.") + original_validator = formal._validate_longrun_request_with_loader + formal.CHECKPOINT_EVERY = RECOVERY_CHECKPOINT_EVERY + formal._validate_longrun_request_with_loader = _compatible_validate_with_loader + try: + yield + finally: + formal._validate_longrun_request_with_loader = original_validator + formal.CHECKPOINT_EVERY = FORMAL_CHECKPOINT_EVERY + + +def build_parser() -> argparse.ArgumentParser: + # Build before installing the operational grid so the CLI default remains + # the formal, config-hashed checkpoint_every=100. + return formal.build_parser() + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if not hasattr(args, "stop_step"): + raise ValueError("The autoresume wrapper only supports the train command.") + _formal_validation_stop(int(args.stop_step)) + if int(args.checkpoint_every) != FORMAL_CHECKPOINT_EVERY: + raise ValueError("--checkpoint-every must remain 100.") + with _operational_recovery_grid(): + return int(args.handler(args)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/source_snapshot/minimax_h3/train_dmd_longrun_v2.py b/source_snapshot/minimax_h3/train_dmd_longrun_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..c1c5acda6c8397aa11f9f550070185bfa01a1bf5 --- /dev/null +++ b/source_snapshot/minimax_h3/train_dmd_longrun_v2.py @@ -0,0 +1,562 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Schema-v2 restart-safe MiniMax-H3 long-run DMD training. + +Version 1 is provenance-locked by the first production launch. This module +keeps its validated training loop and public CLI contract, but fixes FSDP +optimizer checkpointing by using one AdamW with two disjoint, role-labelled +parameter groups. Passing that same optimizer through both trainer roles +preserves the existing update schedule because AdamW advances only parameters +whose gradients are present on a given critic or student step. +""" + +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +from typing import Any, Mapping, Sequence + +import torch + +from . import train_dmd_longrun as v1 +from .infer_base import sha256_file +from .train_utils import CRITIC_ADAPTER, STUDENT_ADAPTER, adapter_parameters + + +TOTAL_STEPS = v1.TOTAL_STEPS +CHECKPOINT_EVERY = v1.CHECKPOINT_EVERY +STUDENT_UPDATE_RATIO = v1.STUDENT_UPDATE_RATIO +STUDENT_LR = v1.STUDENT_LR +CRITIC_LR = v1.CRITIC_LR +ADAM_BETAS = v1.ADAM_BETAS +WEIGHT_DECAY = v1.WEIGHT_DECAY +LORA_RANK = v1.LORA_RANK +LORA_ALPHA = v1.LORA_ALPHA +LORA_DROPOUT = v1.LORA_DROPOUT +MAX_GRAD_NORM = v1.MAX_GRAD_NORM +TRAINING_STATE_KIND = v1.TRAINING_STATE_KIND +CHECKPOINT_KIND = v1.CHECKPOINT_KIND +SEGMENT_KIND = v1.SEGMENT_KIND +CHECKPOINT_SCHEMA_VERSION = 2 +OPTIMIZER_LAYOUT = "joint_role_groups_v1" + +student_update_due = v1.student_update_due +student_updates_through = v1.student_updates_through +student_updates_in_segment = v1.student_updates_in_segment +_stable_config_hash = v1._stable_config_hash +_validate_fixed_hyperparameters = v1._validate_fixed_hyperparameters +_metric_record = v1._metric_record +_fresh_segment_output = v1._fresh_segment_output +_install_checkpoint_loras = v1._install_checkpoint_loras + +_V1_VALIDATE_LONGRUN_REQUEST = v1.validate_longrun_request +_V1_TRAIN = v1.train +_REAL_ADAMW = torch.optim.AdamW + + +def validate_longrun_request(args: argparse.Namespace) -> dict[str, Any]: + """Return the v1 training contract with an explicit joint optimizer.""" + + config = _V1_VALIDATE_LONGRUN_REQUEST(args) + config.pop("config_sha256", None) + config.pop("student_optimizer", None) + config.pop("critic_optimizer", None) + config.update( + { + "longrun_schema_version": CHECKPOINT_SCHEMA_VERSION, + "joint_optimizer": "AdamW", + "optimizer_layout": OPTIMIZER_LAYOUT, + "optimizer_param_group_roles": [STUDENT_ADAPTER, CRITIC_ADAPTER], + } + ) + config["config_sha256"] = _stable_config_hash(config) + return config + + +def _optimizer_groups( + role_model: torch.nn.Module, +) -> tuple[list[torch.nn.Parameter], list[torch.nn.Parameter]]: + student_parameters = adapter_parameters(role_model, STUDENT_ADAPTER) + critic_parameters = adapter_parameters(role_model, CRITIC_ADAPTER) + if not student_parameters or not critic_parameters: + raise RuntimeError("Both joint optimizer role groups must be non-empty.") + student_ids = {id(parameter) for parameter in student_parameters} + critic_ids = {id(parameter) for parameter in critic_parameters} + if student_ids & critic_ids: + raise RuntimeError("Joint optimizer student/critic groups overlap.") + return student_parameters, critic_parameters + + +def make_joint_optimizer(role_model: torch.nn.Module) -> torch.optim.AdamW: + """Create exactly two disjoint role-labelled AdamW parameter groups.""" + + student_parameters, critic_parameters = _optimizer_groups(role_model) + return _REAL_ADAMW( + [ + { + "params": student_parameters, + "role": STUDENT_ADAPTER, + "lr": STUDENT_LR, + }, + { + "params": critic_parameters, + "role": CRITIC_ADAPTER, + "lr": CRITIC_LR, + }, + ], + betas=ADAM_BETAS, + weight_decay=WEIGHT_DECAY, + ) + + +def _step_value(row: Mapping[str, Any]) -> int: + if "step" not in row: + raise RuntimeError("Joint optimizer has a state row without step.") + value = row["step"] + if isinstance(value, torch.Tensor): + if value.numel() != 1: + raise RuntimeError("Joint optimizer step must be scalar.") + value = value.item() + numeric = float(value) + if not math.isfinite(numeric) or not numeric.is_integer(): + raise RuntimeError("Joint optimizer step must be a finite integer.") + return int(numeric) + + +def validate_joint_optimizer_state( + state_dict: Mapping[str, Any], + *, + expected_student_updates: int, + expected_critic_updates: int, +) -> None: + """Validate group identity, coverage, hyperparameters, and role counters.""" + + states = state_dict.get("state") + groups = state_dict.get("param_groups") + if not isinstance(states, Mapping) or not states: + raise RuntimeError("Joint optimizer state is empty.") + if not isinstance(groups, list) or len(groups) != 2: + raise RuntimeError("Joint optimizer must contain exactly two parameter groups.") + expected = { + STUDENT_ADAPTER: (STUDENT_LR, expected_student_updates), + CRITIC_ADAPTER: (CRITIC_LR, expected_critic_updates), + } + parameter_roles: dict[Any, str] = {} + seen_roles: set[str] = set() + for group in groups: + if not isinstance(group, Mapping): + raise RuntimeError("Joint optimizer parameter group is invalid.") + role = group.get("role") + if role not in expected or role in seen_roles: + raise RuntimeError("Joint optimizer role labels must be exactly student/critic.") + seen_roles.add(role) + parameters = group.get("params") + if not isinstance(parameters, list) or not parameters: + raise RuntimeError(f"{role} optimizer parameter group is empty.") + for parameter in parameters: + if parameter in parameter_roles: + raise RuntimeError("Joint optimizer parameter groups overlap.") + parameter_roles[parameter] = role + expected_lr, _ = expected[role] + hyperparameters = { + "lr": expected_lr, + "betas": ADAM_BETAS, + "weight_decay": WEIGHT_DECAY, + } + mismatches = { + name: {"actual": group.get(name), "expected": value} + for name, value in hyperparameters.items() + if group.get(name) != value + } + if mismatches: + raise RuntimeError( + f"{role} optimizer hyperparameter mismatch: " + + json.dumps(mismatches, sort_keys=True) + ) + if seen_roles != set(expected): + raise RuntimeError("Joint optimizer role groups are incomplete.") + if set(states) != set(parameter_roles): + raise RuntimeError("Joint optimizer state does not cover every grouped parameter.") + observed = {STUDENT_ADAPTER: set(), CRITIC_ADAPTER: set()} + for parameter, row in states.items(): + if not isinstance(row, Mapping): + raise RuntimeError("Joint optimizer state row is invalid.") + observed[parameter_roles[parameter]].add(_step_value(row)) + for role, (_, expected_updates) in expected.items(): + if observed[role] != {expected_updates}: + raise RuntimeError( + f"{role} optimizer step mismatch: {sorted(observed[role])} != " + f"[{expected_updates}]." + ) + + +def _gather_full_optimizer_state( + *, + forward_model: torch.nn.Module, + role_model: torch.nn.Module, + optimizer: torch.optim.Optimizer, +) -> dict[str, Any]: + if forward_model is role_model: + return optimizer.state_dict() + from torch.distributed.fsdp import ( + FullOptimStateDictConfig, + FullStateDictConfig, + FullyShardedDataParallel as FSDP, + StateDictType, + ) + + with FSDP.state_dict_type( + forward_model, + StateDictType.FULL_STATE_DICT, + FullStateDictConfig(offload_to_cpu=True, rank0_only=True), + FullOptimStateDictConfig(offload_to_cpu=True, rank0_only=True), + ): + return FSDP.optim_state_dict(forward_model, optimizer) + + +def save_longrun_checkpoint( + *, + forward_model: torch.nn.Module, + role_model: torch.nn.Module, + student_optimizer: torch.optim.Optimizer, + critic_optimizer: torch.optim.Optimizer, + checkpoint_root: Path, + step: int, + config: dict[str, Any], + device: torch.device, + rank: int, + world_size: int, + strategy: str, +) -> Path: + """Save one full joint optimizer state and a schema-v2 success sentinel.""" + + if student_optimizer is not critic_optimizer: + raise RuntimeError("Schema-v2 requires one shared student/critic optimizer.") + if step % CHECKPOINT_EVERY or step < CHECKPOINT_EVERY: + raise ValueError("Long-run checkpoints must lie on the 50-step grid.") + checkpoint_dir = checkpoint_root / f"checkpoint-{step:06d}" + v1._checkpoint_target_is_available( + checkpoint_dir, device=device, strategy=strategy + ) + rng_states = v1._gather_rng_states( + device=device, rank=rank, world_size=world_size, strategy=strategy + ) + lora_path = v1.base._save_lora_checkpoint( + forward_model=forward_model, + role_model=role_model, + output_dir=checkpoint_root, + step=step, + config=config, + rank=rank, + ) + optimizer_state = _gather_full_optimizer_state( + forward_model=forward_model, + role_model=role_model, + optimizer=student_optimizer, + ) + if rank == 0: + if lora_path is None or rng_states is None: + raise AssertionError("Rank zero did not receive gathered checkpoint state.") + validate_joint_optimizer_state( + optimizer_state, + expected_student_updates=student_updates_through(step), + expected_critic_updates=step, + ) + training_state = { + "schema_version": CHECKPOINT_SCHEMA_VERSION, + "kind": TRAINING_STATE_KIND, + "optimizer_layout": OPTIMIZER_LAYOUT, + "global_step": step, + "world_size": world_size, + "config_sha256": config["config_sha256"], + "student_update_count": student_updates_through(step), + "critic_update_count": step, + "joint_optimizer": optimizer_state, + "rng_states": rng_states, + } + training_path = checkpoint_dir / "training_state.pt" + v1.base._atomic_torch_save(training_state, training_path) + artifact_paths = { + "lora.pt": lora_path, + "manifest.json": checkpoint_dir / "manifest.json", + "training_state.pt": training_path, + } + success = { + "schema_version": CHECKPOINT_SCHEMA_VERSION, + "status": "success", + "kind": CHECKPOINT_KIND, + "optimizer_layout": OPTIMIZER_LAYOUT, + "global_step": step, + "world_size": world_size, + "config_sha256": config["config_sha256"], + "student_update_count": student_updates_through(step), + "critic_update_count": step, + "base_weights_included": False, + "created_at": v1.base._utc_now(), + "artifacts": { + name: {"sha256": sha256_file(path), "bytes": path.stat().st_size} + for name, path in artifact_paths.items() + }, + } + v1.base._atomic_json(success, checkpoint_dir / "CHECKPOINT_SUCCESS.json") + if strategy == "fsdp": + import torch.distributed as dist + + dist.barrier() + return checkpoint_dir + + +def load_longrun_checkpoint( + checkpoint_dir: str | Path, + *, + expected_config_sha256: str, + expected_world_size: int, + checkpoint_root: str | Path | None = None, +) -> dict[str, Any]: + """Fail closed unless a complete schema-v2 checkpoint validates.""" + + directory = Path(checkpoint_dir).resolve() + if not directory.is_dir() or directory.is_symlink(): + raise RuntimeError(f"Resume checkpoint directory is invalid: {directory}") + match = v1._CHECKPOINT_DIR.fullmatch(directory.name) + if match is None: + raise RuntimeError(f"Unexpected checkpoint directory name: {directory.name}") + step = int(match.group(1)) + if checkpoint_root is not None and directory.parent != Path(checkpoint_root).resolve(): + raise RuntimeError("resume_from must be a direct child of checkpoint_root.") + success_path = v1._regular_file( + directory / "CHECKPOINT_SUCCESS.json", label="checkpoint success sentinel" + ) + try: + success = json.loads(success_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise RuntimeError(f"Invalid checkpoint success sentinel: {success_path}") from error + expected_success = { + "schema_version": CHECKPOINT_SCHEMA_VERSION, + "status": "success", + "kind": CHECKPOINT_KIND, + "optimizer_layout": OPTIMIZER_LAYOUT, + "global_step": step, + "world_size": expected_world_size, + "config_sha256": expected_config_sha256, + "student_update_count": student_updates_through(step), + "critic_update_count": step, + "base_weights_included": False, + } + mismatches = { + name: {"actual": success.get(name), "expected": value} + for name, value in expected_success.items() + if success.get(name) != value + } + if mismatches: + raise RuntimeError( + "Checkpoint success sentinel mismatch: " + + json.dumps(mismatches, sort_keys=True) + ) + if step < CHECKPOINT_EVERY or step % CHECKPOINT_EVERY: + raise RuntimeError("Resume checkpoint step is not on the 50-step grid.") + artifacts = success.get("artifacts") + expected_names = {"lora.pt", "manifest.json", "training_state.pt"} + if not isinstance(artifacts, Mapping) or set(artifacts) != expected_names: + raise RuntimeError("Checkpoint success sentinel artifact set is invalid.") + paths: dict[str, Path] = {} + for name in sorted(expected_names): + metadata = artifacts.get(name) + if not isinstance(metadata, Mapping): + raise RuntimeError(f"Checkpoint artifact metadata is invalid for {name}.") + path = v1._regular_file(directory / name, label=f"checkpoint artifact {name}") + if ( + metadata.get("sha256") != sha256_file(path) + or metadata.get("bytes") != path.stat().st_size + ): + raise RuntimeError(f"Checkpoint artifact integrity mismatch for {name}.") + paths[name] = path + lora = v1._load_weights_only(paths["lora.pt"], label="LoRA checkpoint") + training = v1._load_weights_only(paths["training_state.pt"], label="training state") + v1._validate_lora_payload( + lora, step=step, config_sha256=expected_config_sha256 + ) + expected_training = { + "schema_version": CHECKPOINT_SCHEMA_VERSION, + "kind": TRAINING_STATE_KIND, + "optimizer_layout": OPTIMIZER_LAYOUT, + "global_step": step, + "world_size": expected_world_size, + "config_sha256": expected_config_sha256, + "student_update_count": student_updates_through(step), + "critic_update_count": step, + } + training_mismatches = { + name: {"actual": training.get(name), "expected": value} + for name, value in expected_training.items() + if training.get(name) != value + } + if training_mismatches: + raise RuntimeError( + "Training-state metadata mismatch: " + + json.dumps(training_mismatches, sort_keys=True) + ) + validate_joint_optimizer_state( + training.get("joint_optimizer", {}), + expected_student_updates=student_updates_through(step), + expected_critic_updates=step, + ) + rng_states = training.get("rng_states") + if not isinstance(rng_states, list) or len(rng_states) != expected_world_size: + raise RuntimeError("Training-state RNG list does not match world_size.") + if sorted(row.get("rank") for row in rng_states if isinstance(row, Mapping)) != list( + range(expected_world_size) + ): + raise RuntimeError("Training-state RNG ranks are incomplete or duplicated.") + return { + "directory": directory, + "global_step": step, + "success": success, + "success_sha256": sha256_file(success_path), + "lora": lora, + "training": training, + } + + +def restore_longrun_state( + *, + bundle: Mapping[str, Any], + forward_model: torch.nn.Module, + role_model: torch.nn.Module, + student_optimizer: torch.optim.Optimizer, + critic_optimizer: torch.optim.Optimizer, + device: torch.device, + rank: int, + strategy: str, +) -> None: + """Restore one local joint optimizer shard, then the per-rank RNG.""" + + del role_model + if student_optimizer is not critic_optimizer: + raise RuntimeError("Schema-v2 requires one shared student/critic optimizer.") + training = bundle["training"] + step = int(bundle["global_step"]) + validate_joint_optimizer_state( + training["joint_optimizer"], + expected_student_updates=student_updates_through(step), + expected_critic_updates=step, + ) + if strategy == "single": + local_state = training["joint_optimizer"] + else: + from torch.distributed.fsdp import FullyShardedDataParallel as FSDP + + full_state = training["joint_optimizer"] if rank == 0 else None + local_state = FSDP.scatter_full_optim_state_dict( + full_state, forward_model, optim=student_optimizer + ) + student_optimizer.load_state_dict(local_state) + v1._restore_local_rng(training["rng_states"], device=device, rank=rank) + if strategy == "fsdp": + import torch.distributed as dist + + dist.barrier() + + +class _TwoCallJointAdamWFactory: + """Adapt v1's two construction sites to one optimizer without editing v1.""" + + def __init__(self) -> None: + self.optimizer: torch.optim.AdamW | None = None + self.parameter_ids: set[int] = set() + self.calls = 0 + + def __call__( + self, + params: Any, + *, + lr: float, + betas: tuple[float, float], + weight_decay: float, + **kwargs: Any, + ) -> torch.optim.AdamW: + parameters = list(params) + role = STUDENT_ADAPTER if self.calls == 0 else CRITIC_ADAPTER + expected_lr = STUDENT_LR if role == STUDENT_ADAPTER else CRITIC_LR + if self.calls >= 2 or lr != expected_lr: + raise RuntimeError("Unexpected AdamW construction in schema-v2 training.") + ids = {id(parameter) for parameter in parameters} + if not parameters or ids & self.parameter_ids: + raise RuntimeError("Schema-v2 optimizer groups are empty or overlapping.") + self.parameter_ids.update(ids) + if self.optimizer is None: + self.optimizer = _REAL_ADAMW( + [{"params": parameters, "role": role, "lr": lr}], + betas=betas, + weight_decay=weight_decay, + **kwargs, + ) + else: + self.optimizer.add_param_group( + { + "params": parameters, + "role": role, + "lr": lr, + "betas": betas, + "weight_decay": weight_decay, + } + ) + self.calls += 1 + return self.optimizer + + +def train(args: argparse.Namespace) -> int: + """Execute the provenance-locked loop with schema-v2 optimizer hooks.""" + + factory = _TwoCallJointAdamWFactory() + originals = { + "validate_longrun_request": v1.validate_longrun_request, + "load_longrun_checkpoint": v1.load_longrun_checkpoint, + "save_longrun_checkpoint": v1.save_longrun_checkpoint, + "restore_longrun_state": v1.restore_longrun_state, + "adamw": v1.torch.optim.AdamW, + } + v1.validate_longrun_request = validate_longrun_request + v1.load_longrun_checkpoint = load_longrun_checkpoint + v1.save_longrun_checkpoint = save_longrun_checkpoint + v1.restore_longrun_state = restore_longrun_state + v1.torch.optim.AdamW = factory + try: + result = _V1_TRAIN(args) + if not args.validate_only: + if factory.calls != 2 or factory.optimizer is None: + raise RuntimeError("Schema-v2 did not construct both optimizer role groups.") + if len(factory.optimizer.param_groups) != 2: + raise RuntimeError("Schema-v2 joint optimizer group count changed.") + return result + finally: + v1.validate_longrun_request = originals["validate_longrun_request"] + v1.load_longrun_checkpoint = originals["load_longrun_checkpoint"] + v1.save_longrun_checkpoint = originals["save_longrun_checkpoint"] + v1.restore_longrun_state = originals["restore_longrun_state"] + v1.torch.optim.AdamW = originals["adamw"] + + +def build_parser() -> argparse.ArgumentParser: + parser = v1.build_parser() + subparsers = next( + action + for action in parser._actions + if isinstance(action, argparse._SubParsersAction) + ) + subparsers.choices["train"].set_defaults(handler=train) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if getattr(args, "checkpoint_every", CHECKPOINT_EVERY) < 1: + raise ValueError("checkpoint_every must be positive.") + return int(args.handler(args)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/source_snapshot/minimax_h3/train_dmd_longrun_v3.py b/source_snapshot/minimax_h3/train_dmd_longrun_v3.py new file mode 100644 index 0000000000000000000000000000000000000000..5526f47ed1e72ffa7c8b34ad302a40d5e4ebfe4d --- /dev/null +++ b/source_snapshot/minimax_h3/train_dmd_longrun_v3.py @@ -0,0 +1,473 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Rank-128 LongLive-Wan-style MiniMax-H3 DMD training contract. + +The completed schema-v2/rank-8 experiment is provenance-locked. This entry +point creates a new contract without changing its historical defaults: + +* rank/alpha 128, dropout 0; +* every attention and FFN Linear in the 50 main H3 transformer blocks; +* no token-refiner, AdaLN/timestep, norm, or top-level projection LoRA; +* the schema-v2 joint role-labelled optimizer/checkpoint implementation. +""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import re +from collections.abc import Mapping +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Iterator, Sequence + +import torch + +from . import train_dmd_longrun as v1 +from . import train_dmd_longrun_v2 as v2 +from .longlive_lora import ( + LONGLIVE_H3_LORA_POLICY, + LONGLIVE_H3_MAIN_BLOCK_COUNT, + LONGLIVE_H3_TARGET_MODULE_COUNT, + install_longlive_h3_lora_adapters, + released_longlive_h3_target_modules, +) +from .train_utils import H3Condition + + +TOTAL_STEPS = v1.TOTAL_STEPS +CHECKPOINT_EVERY = v1.CHECKPOINT_EVERY +STUDENT_UPDATE_RATIO = v1.STUDENT_UPDATE_RATIO +STUDENT_LR = v1.STUDENT_LR +CRITIC_LR = v1.CRITIC_LR +ADAM_BETAS = v1.ADAM_BETAS +WEIGHT_DECAY = v1.WEIGHT_DECAY +LORA_RANK = 128 +LORA_ALPHA = 128 +LORA_DROPOUT = 0.0 +MAX_GRAD_NORM = v1.MAX_GRAD_NORM +LONGRUN_SCHEMA_VERSION = 3 +OPTIMIZER_LAYOUT = v2.OPTIMIZER_LAYOUT +PROMPT_CACHE_LOADER = "minimax_h3.longlive_prompt_set.load_prompt_cache" +PROMPT_RUNTIME_SELECTION_POLICY = "ordered_cache_global_step_cycle_v1" +PROMPT_MANIFEST_SELECTION = { + "trim": "python_str_strip", + "deduplication": "exact_keep_first_source_line", + "ordering": "sha256_utf8_ascending_then_utf8_bytes", + "selection": "first_1000", + "id": "vidprom-{source_line:06d}-{prompt_sha256[:12]}", + "seed": "source_line_minus_one", +} +_SHA256 = re.compile(r"^[0-9a-f]{64}$") + +student_update_due = v2.student_update_due +student_updates_through = v2.student_updates_through +student_updates_in_segment = v2.student_updates_in_segment +make_joint_optimizer = v2.make_joint_optimizer +validate_joint_optimizer_state = v2.validate_joint_optimizer_state +save_longrun_checkpoint = v2.save_longrun_checkpoint +load_longrun_checkpoint = v2.load_longrun_checkpoint +restore_longrun_state = v2.restore_longrun_state +_stable_config_hash = v2._stable_config_hash + + +def load_longlive_prompt_cache( + path: str | Path, +) -> tuple[dict[str, Any], list[H3Condition]]: + """Load the schema-v3 prompt set without making it a v1/v2 dependency. + + The import is intentionally lazy: schema-v1/v2 remain usable in checkouts + that do not contain the schema-v3 prompt-set implementation. + """ + + try: + from .longlive_prompt_set import load_prompt_cache as loader + except ImportError as error: + raise RuntimeError( + "Schema-v3 training requires minimax_h3.longlive_prompt_set." + ) from error + metadata, rows = loader(path) + # Keep the strict prompt-set module independent of trainer internals while + # supplying the ``.to(device)`` contract expected by the frozen v1 loop. + # These are lightweight wrappers around the same tensor objects. + conditions = [ + H3Condition( + id=row.id, + prompt=row.prompt, + seed=row.seed, + prompt_embeds=row.prompt_embeds, + text_token_tags=row.text_token_tags, + ) + for row in rows + ] + return metadata, conditions + + +class _MemoizedLongLivePromptLoader: + """Hold one validated cache payload for one schema-v3 train invocation.""" + + def __init__(self) -> None: + self._path: Path | None = None + self._value: tuple[dict[str, Any], list[Any]] | None = None + + def __call__(self, path: str | Path) -> tuple[dict[str, Any], list[Any]]: + candidate = Path(path) + resolved = candidate.resolve(strict=True) + if self._path is not None and resolved != self._path: + raise RuntimeError( + "One schema-v3 invocation cannot switch prompt caches after validation." + ) + if self._value is None: + # Preserve the caller's path for the strict loader so its explicit + # symlink rejection cannot be bypassed by pre-resolving the path. + loaded = load_longlive_prompt_cache(candidate) + if not isinstance(loaded, tuple) or len(loaded) != 2: + raise TypeError( + "load_longlive_prompt_cache must return (metadata, conditions)." + ) + metadata, conditions = loaded + if not isinstance(metadata, dict) or not isinstance(conditions, list): + raise TypeError( + "LongLive prompt metadata/conditions must be a dict and list." + ) + self._path = resolved + self._value = (metadata, conditions) + return self._value + + def legacy_validation_view( + self, path: str | Path + ) -> tuple[dict[str, Any], list[Any]]: + """Give the frozen base validator four references, never the train loop.""" + + metadata, conditions = self(path) + if len(conditions) < 4: + raise ValueError( + "Schema-v3 needs at least four prompts for the legacy validation bridge." + ) + return metadata, conditions[:4] + + def clear(self) -> None: + """Release the sole references to the potentially large cached tensors.""" + + self._value = None + self._path = None + + +@contextmanager +def _v3_prompt_loader_aliases( + loader: _MemoizedLongLivePromptLoader, +) -> Iterator[None]: + """Patch both loader names captured by the frozen v1 implementation.""" + + original_loop_loader = v1.load_prompt_cache + original_base_loader = v1.base.load_prompt_cache + v1.load_prompt_cache = loader + v1.base.load_prompt_cache = loader.legacy_validation_view + try: + yield + finally: + v1.load_prompt_cache = original_loop_loader + v1.base.load_prompt_cache = original_base_loader + + +def _canonical_provenance_mapping(value: object, label: str) -> dict[str, Any]: + if not isinstance(value, Mapping) or not value: + raise ValueError(f"{label} must be a non-empty mapping.") + if not all(isinstance(key, str) for key in value): + raise ValueError(f"{label} keys must all be strings.") + try: + encoded = json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ) + except (TypeError, ValueError) as error: + raise ValueError(f"{label} must contain canonical JSON values.") from error + decoded = json.loads(encoded) + if not isinstance(decoded, dict): + raise AssertionError(f"Canonical {label} unexpectedly stopped being a mapping.") + return decoded + + +def _ordered_prompt_ids_sha256(prompt_ids: Sequence[str]) -> str: + encoded = json.dumps( + list(prompt_ids), ensure_ascii=False, separators=(",", ":") + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _prompt_set_contract( + *, + args: argparse.Namespace, + metadata: Mapping[str, Any], + conditions: Sequence[Any], +) -> dict[str, Any]: + """Validate and normalize provenance consumed by the schema-v3 config.""" + + for name in ("cfg_enabled", "ir", "regenerator"): + if metadata.get(name) is not False: + raise ValueError(f"Schema-v3 prompt metadata requires {name}=false.") + recorded_count = len(conditions) + if recorded_count != TOTAL_STEPS: + raise ValueError( + "Schema-v3 requires one ordered prompt for each of its 1,000 global steps: " + f"conditions={recorded_count}." + ) + prompt_ids = [getattr(condition, "id", None) for condition in conditions] + if not all(isinstance(prompt_id, str) and prompt_id for prompt_id in prompt_ids): + raise ValueError("Every schema-v3 condition must have a non-empty string id.") + if len(set(prompt_ids)) != recorded_count: + raise ValueError("Schema-v3 prompt condition ids must be unique.") + start_index = getattr(args, "prompt_start_index", None) + if ( + isinstance(start_index, bool) + or not isinstance(start_index, int) + or not 0 <= start_index < recorded_count + ): + raise ValueError( + f"prompt_start_index must be an integer in [0, {recorded_count - 1}]." + ) + manifest_sha256 = metadata.get("prompt_manifest_sha256") + if not isinstance(manifest_sha256, str) or not _SHA256.fullmatch( + manifest_sha256 + ): + raise ValueError("prompt_manifest_sha256 must be a lowercase SHA-256.") + cache_path = Path(args.prompt_cache).resolve(strict=True) + return { + "prompt_count": recorded_count, + "prompt_ids": prompt_ids, + "prompt_ids_sha256": _ordered_prompt_ids_sha256(prompt_ids), + "prompt_manifest_sha256": manifest_sha256, + "prompt_source": _canonical_provenance_mapping( + metadata.get("source"), "source" + ), + "prompt_selection": _canonical_provenance_mapping( + PROMPT_MANIFEST_SELECTION, "prompt_selection" + ), + "prompt_cache_path": str(cache_path), + "prompt_cache_bytes": cache_path.stat().st_size, + "prompt_cache_sha256": v1.base.sha256_file(cache_path), + "prompt_cache_schema_version": metadata.get("schema_version"), + "prompt_cache_artifact_type": metadata.get("artifact_type"), + "prompt_cache_model_revision": metadata.get("model_revision"), + } + + +@contextmanager +def _rank128_v1_contract() -> Iterator[None]: + """Temporarily supply rank-128 constants to the frozen v1 validator.""" + + originals = (v1.LORA_RANK, v1.LORA_ALPHA, v1.LORA_DROPOUT) + v1.LORA_RANK = LORA_RANK + v1.LORA_ALPHA = LORA_ALPHA + v1.LORA_DROPOUT = LORA_DROPOUT + try: + yield + finally: + v1.LORA_RANK, v1.LORA_ALPHA, v1.LORA_DROPOUT = originals + + +def _validate_longrun_request_with_loader( + args: argparse.Namespace, loader: _MemoizedLongLivePromptLoader +) -> dict[str, Any]: + """Validate schema-v3 while isolating the frozen four-prompt validator.""" + + metadata, conditions = loader(args.prompt_cache) + prompt_contract = _prompt_set_contract( + args=args, metadata=metadata, conditions=conditions + ) + # The historical base validator checks ``len(conditions) == 4`` and bounds + # prompt_start_index against that legacy view. It never returns or trains + # on those conditions. Validate the real index above, then rotate only the + # shallow Namespace copy used by that frozen compatibility check. + legacy_args = copy.copy(args) + legacy_args.prompt_start_index = args.prompt_start_index % 4 + with _rank128_v1_contract(): + config = v2.validate_longrun_request(legacy_args) + config.pop("config_sha256", None) + targets = released_longlive_h3_target_modules() + cache_sha256 = config.get("prompt_cache_sha256") + if cache_sha256 != prompt_contract["prompt_cache_sha256"]: + raise RuntimeError( + "The legacy validator and schema-v3 loader disagree on prompt-cache SHA-256." + ) + cache_path = config.get("prompt_cache") + if cache_path != prompt_contract["prompt_cache_path"]: + raise RuntimeError( + "The legacy validator and schema-v3 loader disagree on prompt-cache path." + ) + config.update( + { + "longrun_schema_version": LONGRUN_SCHEMA_VERSION, + "lora_rank": LORA_RANK, + "lora_alpha": LORA_ALPHA, + "lora_dropout": LORA_DROPOUT, + "lora_target_policy": LONGLIVE_H3_LORA_POLICY, + "lora_target_scope": "main_transformer_blocks_only", + "lora_target_branches": ["attn", "ff"], + "lora_target_module_count": LONGLIVE_H3_TARGET_MODULE_COUNT, + "lora_targets": targets, + "lora_excluded_scopes": [ + "token_refiner", + "adaln_and_timestep", + "normalization", + "top_level_input_context_output_projections", + ], + "prompt_start_index": args.prompt_start_index, + "prompt_count": prompt_contract["prompt_count"], + "prompt_ids": prompt_contract["prompt_ids"], + "prompt_ids_sha256": prompt_contract["prompt_ids_sha256"], + "prompt_manifest_sha256": prompt_contract[ + "prompt_manifest_sha256" + ], + "prompt_source": prompt_contract["prompt_source"], + "prompt_selection": prompt_contract["prompt_selection"], + "prompt_cache_provenance": { + "loader": PROMPT_CACHE_LOADER, + "path": prompt_contract["prompt_cache_path"], + "bytes": prompt_contract["prompt_cache_bytes"], + "sha256": prompt_contract["prompt_cache_sha256"], + "payload_schema_version": prompt_contract[ + "prompt_cache_schema_version" + ], + "artifact_type": prompt_contract["prompt_cache_artifact_type"], + "model_revision": prompt_contract["prompt_cache_model_revision"], + }, + "prompt_runtime_selection": { + "policy": PROMPT_RUNTIME_SELECTION_POLICY, + "index_formula": "(prompt_start_index+global_step-1)%prompt_count", + "shared_prompt_across_ranks": True, + "rank_distinct_noise": True, + "critic_conditions_per_1000_steps": TOTAL_STEPS, + "student_update_rule": "(global_step-1)%5==0", + "student_conditions_per_1000_steps": student_updates_through( + TOTAL_STEPS + ), + "student_uses_same_condition_as_critic": True, + }, + } + ) + config["config_sha256"] = _stable_config_hash(config) + return config + + +def validate_longrun_request(args: argparse.Namespace) -> dict[str, Any]: + """Validate and record the independent schema-v3 LoRA/prompt contract.""" + + loader = _MemoizedLongLivePromptLoader() + try: + with _v3_prompt_loader_aliases(loader): + return _validate_longrun_request_with_loader(args, loader) + finally: + loader.clear() + + +def _install_rank128_blockwide_adapters( + model: torch.nn.Module, + *, + rank: int, + alpha: int, + dropout: float, + **_: Any, +) -> dict[str, list[torch.nn.Parameter]]: + if (rank, alpha, float(dropout)) != (LORA_RANK, LORA_ALPHA, LORA_DROPOUT): + raise ValueError( + "Schema-v3 LoRA is fixed to rank/alpha/dropout " + f"{LORA_RANK}/{LORA_ALPHA}/{LORA_DROPOUT}." + ) + return install_longlive_h3_lora_adapters( + model, + rank=rank, + alpha=alpha, + dropout=dropout, + expected_main_blocks=LONGLIVE_H3_MAIN_BLOCK_COUNT, + ) + + +def train(args: argparse.Namespace) -> int: + """Run the proven joint-optimizer loop with the schema-v3 LoRA policy.""" + + factory = v2._TwoCallJointAdamWFactory() + prompt_loader = _MemoizedLongLivePromptLoader() + originals = { + "validate_longrun_request": v1.validate_longrun_request, + "load_longrun_checkpoint": v1.load_longrun_checkpoint, + "save_longrun_checkpoint": v1.save_longrun_checkpoint, + "restore_longrun_state": v1.restore_longrun_state, + "install_independent_lora_adapters": v1.install_independent_lora_adapters, + "adamw": v1.torch.optim.AdamW, + "lora_rank": v1.LORA_RANK, + "lora_alpha": v1.LORA_ALPHA, + "lora_dropout": v1.LORA_DROPOUT, + } + try: + with _v3_prompt_loader_aliases(prompt_loader): + v1.validate_longrun_request = lambda parsed_args: ( + _validate_longrun_request_with_loader(parsed_args, prompt_loader) + ) + v1.load_longrun_checkpoint = load_longrun_checkpoint + v1.save_longrun_checkpoint = save_longrun_checkpoint + v1.restore_longrun_state = restore_longrun_state + v1.install_independent_lora_adapters = ( + _install_rank128_blockwide_adapters + ) + v1.torch.optim.AdamW = factory + v1.LORA_RANK = LORA_RANK + v1.LORA_ALPHA = LORA_ALPHA + v1.LORA_DROPOUT = LORA_DROPOUT + result = v2._V1_TRAIN(args) + if not args.validate_only: + if factory.calls != 2 or factory.optimizer is None: + raise RuntimeError( + "Schema-v3 did not construct both optimizer groups." + ) + if len(factory.optimizer.param_groups) != 2: + raise RuntimeError( + "Schema-v3 joint optimizer group count changed." + ) + return result + finally: + v1.validate_longrun_request = originals["validate_longrun_request"] + v1.load_longrun_checkpoint = originals["load_longrun_checkpoint"] + v1.save_longrun_checkpoint = originals["save_longrun_checkpoint"] + v1.restore_longrun_state = originals["restore_longrun_state"] + v1.install_independent_lora_adapters = originals[ + "install_independent_lora_adapters" + ] + v1.torch.optim.AdamW = originals["adamw"] + v1.LORA_RANK = originals["lora_rank"] + v1.LORA_ALPHA = originals["lora_alpha"] + v1.LORA_DROPOUT = originals["lora_dropout"] + prompt_loader.clear() + + +def build_parser() -> argparse.ArgumentParser: + parser = v1.build_parser() + subparsers = next( + action + for action in parser._actions + if isinstance(action, argparse._SubParsersAction) + ) + subparsers.choices["train"].set_defaults( + handler=train, + lora_rank=LORA_RANK, + lora_alpha=LORA_ALPHA, + lora_dropout=LORA_DROPOUT, + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if getattr(args, "checkpoint_every", CHECKPOINT_EVERY) < 1: + raise ValueError("checkpoint_every must be positive.") + return int(args.handler(args)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/source_snapshot/minimax_h3/train_dmd_longrun_v4.py b/source_snapshot/minimax_h3/train_dmd_longrun_v4.py new file mode 100644 index 0000000000000000000000000000000000000000..c20326ba8b3820e577907a31ae8bf3f7a46004c7 --- /dev/null +++ b/source_snapshot/minimax_h3/train_dmd_longrun_v4.py @@ -0,0 +1,625 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Distributed-prompt MiniMax-H3 long-run DMD training contract. + +Version 4 changes only the prompt data plane of the proven long-run trainer: + +* the complete pinned 248,221-row LongLive dataset is the sampler population; +* PyTorch ``DistributedSampler`` assigns a distinct stream to every one of the + 32 ranks with one shared seed and epoch zero; +* each rank loads only its own 1,000-condition cache shard; and +* absolute-global-step indexing is guarded so segmented resume cannot restart + the prompt stream at condition zero. + +The training loop, schema-v2 joint optimizer/checkpoint implementation, and +schema-v3 rank-128 block-wide LoRA policy are intentionally reused unchanged. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import os +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Iterator, Mapping, Sequence + +import torch + +from . import train_dmd_longrun as v1 +from . import train_dmd_longrun_v2 as v2 +from . import train_dmd_longrun_v3 as v3 +from .longlive_lora import ( + LONGLIVE_H3_LORA_POLICY, + LONGLIVE_H3_MAIN_BLOCK_COUNT, + LONGLIVE_H3_TARGET_MODULE_COUNT, + released_longlive_h3_target_modules, +) +from .longlive_prompt_distributed import ( + FORMAL_EPOCH, + FORMAL_GLOBAL_STEPS, + FORMAL_SEED, + FORMAL_WORLD_SIZE, + SOURCE_LINE_COUNT, + RankPromptCondition, + load_prompt_cache_collection, + load_rank_prompt_cache, + sha256_file, +) +from .train_utils import H3Condition + + +TOTAL_STEPS = v1.TOTAL_STEPS +CHECKPOINT_EVERY = v1.CHECKPOINT_EVERY +STUDENT_UPDATE_RATIO = v1.STUDENT_UPDATE_RATIO +STUDENT_LR = v1.STUDENT_LR +CRITIC_LR = v1.CRITIC_LR +ADAM_BETAS = v1.ADAM_BETAS +WEIGHT_DECAY = v1.WEIGHT_DECAY +LORA_RANK = v3.LORA_RANK +LORA_ALPHA = v3.LORA_ALPHA +LORA_DROPOUT = v3.LORA_DROPOUT +MAX_GRAD_NORM = v1.MAX_GRAD_NORM + +# Checkpoint payloads remain schema v2. This value identifies the immutable +# config/sampling contract and therefore deliberately changes its SHA-256. +LONGRUN_CONFIG_VERSION = 4 +CHECKPOINT_SCHEMA_VERSION = v2.CHECKPOINT_SCHEMA_VERSION +OPTIMIZER_LAYOUT = v2.OPTIMIZER_LAYOUT +PROMPT_COLLECTION_LOADER = ( + "minimax_h3.longlive_prompt_distributed.load_prompt_cache_collection" +) +PROMPT_RANK_CACHE_LOADER = ( + "minimax_h3.longlive_prompt_distributed.load_rank_prompt_cache" +) +PROMPT_RUNTIME_SELECTION_POLICY = "rank_local_absolute_global_step_v1" + +student_update_due = v2.student_update_due +student_updates_through = v2.student_updates_through +student_updates_in_segment = v2.student_updates_in_segment +make_joint_optimizer = v2.make_joint_optimizer +validate_joint_optimizer_state = v2.validate_joint_optimizer_state +save_longrun_checkpoint = v2.save_longrun_checkpoint +load_longrun_checkpoint = v2.load_longrun_checkpoint +restore_longrun_state = v2.restore_longrun_state +_stable_config_hash = v2._stable_config_hash + + +def _canonical_mapping(value: object, label: str) -> dict[str, Any]: + """Return a detached JSON-canonical mapping for the config hash.""" + + if not isinstance(value, Mapping): + raise ValueError(f"{label} must be a mapping.") + try: + encoded = json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ) + except (TypeError, ValueError) as error: + raise ValueError(f"{label} must contain canonical JSON values.") from error + decoded = json.loads(encoded) + if not isinstance(decoded, dict): + raise AssertionError(f"Canonical {label} unexpectedly stopped being a mapping.") + return decoded + + +def _legacy_validation_conditions() -> list[H3Condition]: + """Supply the frozen four-prompt validator with tensor-light sentinels. + + These rows are never returned to the training loop. Cache integrity is + established by the collection validator and the strict rank-cache loader. + """ + + return [ + H3Condition( + id=f"v4-manifest-validation-{index}", + prompt=f"v4 manifest validation row {index}", + seed=index, + prompt_embeds=torch.zeros((1, 1, 1), dtype=torch.bfloat16), + text_token_tags=torch.ones(1, dtype=torch.long), + ) + for index in range(4) + ] + + +def _resume_step(args: argparse.Namespace) -> int: + resume_from = getattr(args, "resume_from", None) + if resume_from is None: + return 0 + match = v1._CHECKPOINT_DIR.fullmatch(Path(resume_from).name) + if match is None: + raise ValueError( + "resume_from must name checkpoint-NNNNNN for absolute prompt indexing." + ) + return int(match.group(1)) + + +class _ResumeSafeRankConditions(list[H3Condition]): + """Fail closed if the frozen loop stops using absolute global-step indices.""" + + def __init__( + self, + conditions: Sequence[H3Condition], + *, + resume_step: int, + stop_step: int, + ) -> None: + super().__init__(conditions) + if len(self) != FORMAL_GLOBAL_STEPS: + raise ValueError( + f"Rank schedule must contain {FORMAL_GLOBAL_STEPS} conditions." + ) + if not 0 <= resume_step < stop_step <= len(self): + raise ValueError("Invalid resume/stop range for rank prompt schedule.") + self._next_index = resume_step + self._stop_index = stop_step + + def __getitem__(self, index: int | slice) -> H3Condition | list[H3Condition]: + if isinstance(index, slice): + return super().__getitem__(index) + if isinstance(index, bool) or not isinstance(index, int): + raise TypeError("Rank prompt schedule indices must be integers.") + if index != self._next_index or index >= self._stop_index: + raise RuntimeError( + "Prompt selection is not the expected absolute global-step sequence: " + f"requested={index}, expected={self._next_index}." + ) + condition = super().__getitem__(index) + self._next_index += 1 + return condition + + def assert_complete(self) -> None: + if self._next_index != self._stop_index: + raise RuntimeError( + "Training returned before consuming its complete rank prompt segment: " + f"next={self._next_index}, stop={self._stop_index}." + ) + + +def _adapt_rank_conditions( + rows: Sequence[RankPromptCondition], +) -> list[H3Condition]: + conditions: list[H3Condition] = [] + for position, row in enumerate(rows, start=1): + if row.global_step != position: + raise ValueError("Rank prompt cache global_step ordering changed.") + # v3 used the source index as the per-prompt RNG component. Preserve + # that deterministic meaning while each rank now sees a different row. + conditions.append( + H3Condition( + id=row.id, + prompt=row.prompt, + seed=row.source_index, + prompt_embeds=row.prompt_embeds, + text_token_tags=row.text_token_tags, + ) + ) + return conditions + + +class _DistributedPromptCollectionLoader: + """Validate one collection and lazily load only the current rank shard.""" + + def __init__(self, args: argparse.Namespace) -> None: + self._args = args + self._path: Path | None = None + self._manifest: dict[str, Any] | None = None + self._shards_sha_verified = False + self._runtime_metadata: dict[str, Any] | None = None + self._runtime_schedule: _ResumeSafeRankConditions | None = None + + def load_manifest( + self, path: str | Path, *, verify_shards: bool + ) -> dict[str, Any]: + candidate = Path(path) + resolved = candidate.resolve(strict=True) + if self._path is not None and resolved != self._path: + raise RuntimeError( + "One schema-v4 invocation cannot switch prompt-cache collections." + ) + if self._manifest is None or (verify_shards and not self._shards_sha_verified): + # Pass the original path so the strict loader can reject symlinks. + manifest = load_prompt_cache_collection( + candidate, + verify_shards=verify_shards, + expected_world_size=FORMAL_WORLD_SIZE, + expected_seed=FORMAL_SEED, + expected_epoch=FORMAL_EPOCH, + expected_global_steps=FORMAL_GLOBAL_STEPS, + ) + self._path = resolved + self._manifest = manifest + self._shards_sha_verified = verify_shards + return self._manifest + + def legacy_validation_view( + self, path: str | Path + ) -> tuple[dict[str, Any], list[H3Condition]]: + manifest = self.load_manifest( + path, verify_shards=bool(getattr(self._args, "validate_only", False)) + ) + metadata = { + "model_revision": manifest["model_revision"], + "workflow": manifest["workflow"], + "cfg_enabled": manifest["cfg_enabled"], + "ir": manifest["ir"], + "regenerator": manifest["regenerator"], + } + return metadata, _legacy_validation_conditions() + + def __call__( + self, path: str | Path + ) -> tuple[dict[str, Any], _ResumeSafeRankConditions]: + if self._runtime_schedule is not None: + if Path(path).resolve(strict=True) != self._path: + raise RuntimeError("Runtime prompt-cache collection path changed.") + assert self._runtime_metadata is not None + return self._runtime_metadata, self._runtime_schedule + + manifest = self.load_manifest(path, verify_shards=False) + import torch.distributed as dist + + if not dist.is_initialized(): + raise RuntimeError( + "Schema-v4 rank cache loading requires an initialized process group." + ) + rank = dist.get_rank() + world_size = dist.get_world_size() + if world_size != FORMAL_WORLD_SIZE or not 0 <= rank < FORMAL_WORLD_SIZE: + raise RuntimeError( + "Schema-v4 production requires exactly 32 distributed ranks: " + f"rank={rank}, world_size={world_size}." + ) + + shard = manifest["shards"][rank] + assert self._path is not None + shard_path = self._path.parent / shard["path"] + observed_sha = sha256_file(shard_path) + if observed_sha != shard["sha256"]: + raise RuntimeError( + f"Rank {rank} prompt-cache SHA-256 mismatch: {shard_path}." + ) + metadata, raw_conditions = load_rank_prompt_cache( + shard_path, + expected_rank=rank, + expected_world_size=world_size, + expected_seed=FORMAL_SEED, + expected_epoch=FORMAL_EPOCH, + expected_global_steps=FORMAL_GLOBAL_STEPS, + ) + if metadata.get("model_revision") != manifest["model_revision"]: + raise RuntimeError("Rank cache and collection model revisions disagree.") + conditions = _adapt_rank_conditions(raw_conditions) + schedule = _ResumeSafeRankConditions( + conditions, + resume_step=_resume_step(self._args), + stop_step=self._args.stop_step, + ) + runtime_metadata = { + **metadata, + "collection_path": str(self._path), + "collection_sha256": v1.base.sha256_file(self._path), + "rank_cache_path": str(shard_path.resolve()), + "rank_cache_sha256": observed_sha, + "runtime_rank": rank, + "runtime_world_size": world_size, + } + self._runtime_metadata = runtime_metadata + self._runtime_schedule = schedule + print( + json.dumps( + { + "event": "v4_rank_prompt_cache_loaded", + "rank": rank, + "world_size": world_size, + "conditions": len(schedule), + "cache": str(shard_path.resolve()), + "cache_sha256": observed_sha, + "resume_step": _resume_step(self._args), + "stop_step": self._args.stop_step, + }, + sort_keys=True, + ), + flush=True, + ) + return runtime_metadata, schedule + + def assert_complete(self) -> None: + if self._runtime_schedule is None: + raise RuntimeError("Training never loaded its rank prompt cache.") + self._runtime_schedule.assert_complete() + + def clear(self) -> None: + self._runtime_metadata = None + self._runtime_schedule = None + self._manifest = None + self._path = None + self._shards_sha_verified = False + + +@contextmanager +def _v4_prompt_loader_aliases( + loader: _DistributedPromptCollectionLoader, +) -> Iterator[None]: + original_loop_loader = v1.load_prompt_cache + original_base_loader = v1.base.load_prompt_cache + v1.load_prompt_cache = loader + v1.base.load_prompt_cache = loader.legacy_validation_view + try: + yield + finally: + v1.load_prompt_cache = original_loop_loader + v1.base.load_prompt_cache = original_base_loader + + +def _prompt_collection_contract( + args: argparse.Namespace, manifest: Mapping[str, Any] +) -> dict[str, Any]: + if getattr(args, "prompt_start_index", None) != 0: + raise ValueError( + "Schema-v4 fixes prompt_start_index=0 so cache position equals global_step-1." + ) + dataset = _canonical_mapping(manifest.get("dataset"), "dataset") + sampler = _canonical_mapping(manifest.get("sampler"), "sampler") + audit = _canonical_mapping(manifest.get("schedule_audit"), "schedule_audit") + if dataset.get("line_count") != SOURCE_LINE_COUNT: + raise ValueError("Schema-v4 requires the complete 248,221-row dataset.") + if audit.get("scheduled_rows") != FORMAL_WORLD_SIZE * FORMAL_GLOBAL_STEPS: + raise ValueError("Schema-v4 collection does not schedule 32,000 rows.") + if audit.get("unique_source_indices") != audit["scheduled_rows"]: + raise ValueError("Schema-v4 scheduled prefix is not globally rank-distinct.") + collection_path = Path(args.prompt_cache).resolve(strict=True) + shards = manifest.get("shards") + if not isinstance(shards, list): + raise ValueError("Schema-v4 prompt-cache collection shards must be a list.") + canonical_shards = [ + _canonical_mapping(shard, f"shards[{index}]") + for index, shard in enumerate(shards) + ] + return { + "dataset": dataset, + "sampler": sampler, + "schedule_audit": audit, + "shards": canonical_shards, + "collection_path": str(collection_path), + "collection_bytes": collection_path.stat().st_size, + "collection_sha256": v1.base.sha256_file(collection_path), + "collection_schema_version": manifest.get("schema_version"), + "collection_artifact_type": manifest.get("artifact_type"), + "collection_model_revision": manifest.get("model_revision"), + } + + +def _validate_longrun_request_with_loader( + args: argparse.Namespace, loader: _DistributedPromptCollectionLoader +) -> dict[str, Any]: + """Validate the v4 collection offline and build a v4-hashed config.""" + + verify_all_shards = bool(getattr(args, "validate_only", False)) + manifest = loader.load_manifest(args.prompt_cache, verify_shards=verify_all_shards) + prompt_contract = _prompt_collection_contract(args, manifest) + legacy_args = copy.copy(args) + legacy_args.prompt_start_index = 0 + with _v4_prompt_loader_aliases(loader), v3._rank128_v1_contract(): + config = v2.validate_longrun_request(legacy_args) + config.pop("config_sha256", None) + + if config.get("prompt_cache") != prompt_contract["collection_path"]: + raise RuntimeError( + "The legacy validator and v4 loader disagree on the collection path." + ) + if config.get("prompt_cache_sha256") != prompt_contract["collection_sha256"]: + raise RuntimeError( + "The legacy validator and v4 loader disagree on the collection SHA-256." + ) + if config.get("resolved_revision") != prompt_contract["collection_model_revision"]: + raise ValueError( + "Prompt-cache collection/checkpoint revision mismatch: " + f"cache={prompt_contract['collection_model_revision']!r}, " + f"requested={config.get('resolved_revision')!r}." + ) + + targets = released_longlive_h3_target_modules() + rank_epoch_rows = SOURCE_LINE_COUNT // FORMAL_WORLD_SIZE + config.update( + { + "longrun_schema_version": LONGRUN_CONFIG_VERSION, + "longrun_config_version": LONGRUN_CONFIG_VERSION, + "checkpoint_schema_version": CHECKPOINT_SCHEMA_VERSION, + "optimizer_layout": OPTIMIZER_LAYOUT, + "lora_rank": LORA_RANK, + "lora_alpha": LORA_ALPHA, + "lora_dropout": LORA_DROPOUT, + "lora_target_policy": LONGLIVE_H3_LORA_POLICY, + "lora_target_scope": "main_transformer_blocks_only", + "lora_target_branches": ["attn", "ff"], + "lora_target_module_count": LONGLIVE_H3_TARGET_MODULE_COUNT, + "lora_targets": targets, + "lora_excluded_scopes": [ + "token_refiner", + "adaln_and_timestep", + "normalization", + "top_level_input_context_output_projections", + ], + "prompt_start_index": 0, + "prompt_dataset": prompt_contract["dataset"], + "prompt_dataset_row_count": SOURCE_LINE_COUNT, + "prompt_sampler": prompt_contract["sampler"], + "prompt_schedule_audit": prompt_contract["schedule_audit"], + "prompt_collection_provenance": { + "loader": PROMPT_COLLECTION_LOADER, + "path": prompt_contract["collection_path"], + "bytes": prompt_contract["collection_bytes"], + "sha256": prompt_contract["collection_sha256"], + "payload_schema_version": prompt_contract[ + "collection_schema_version" + ], + "artifact_type": prompt_contract["collection_artifact_type"], + "model_revision": prompt_contract["collection_model_revision"], + "shards": prompt_contract["shards"], + # These describe the immutable execution contract, not whether + # this invocation is the offline validator or a train rank. + # Consequently --validate-only cannot change config_sha256. + "validate_only_full_shard_sha256_readback": True, + "production_manifest_verifies_all_shard_sizes": True, + "production_every_rank_verifies_own_shard_sha256": True, + }, + "prompt_runtime_selection": { + "policy": PROMPT_RUNTIME_SELECTION_POLICY, + "rank_cache_loader": PROMPT_RANK_CACHE_LOADER, + "index_formula": "global_step-1", + "absolute_global_step_resume_guard": True, + "dataset_rows": SOURCE_LINE_COUNT, + "sampler_seed_shared_across_ranks": FORMAL_SEED, + "sampler_epoch": FORMAL_EPOCH, + "sampler_shuffle": True, + "sampler_drop_last": True, + "sampler_rank_epoch_rows": rank_epoch_rows, + "sampler_epoch_dropped_rows": ( + SOURCE_LINE_COUNT - rank_epoch_rows * FORMAL_WORLD_SIZE + ), + "world_size": FORMAL_WORLD_SIZE, + "conditions_per_rank": FORMAL_GLOBAL_STEPS, + "scheduled_conditions": FORMAL_WORLD_SIZE * FORMAL_GLOBAL_STEPS, + "shared_prompt_across_ranks": False, + "rank_distinct_prompt": True, + "rank_distinct_noise": True, + "critic_conditions_per_rank": TOTAL_STEPS, + "student_update_rule": "(global_step-1)%5==0", + "student_conditions_per_rank": student_updates_through(TOTAL_STEPS), + "student_uses_same_rank_condition_as_critic": True, + }, + "sample_rng_scheme": "seed+source_index+global_step+rank-v4", + } + ) + config["config_sha256"] = _stable_config_hash(config) + return config + + +def validate_longrun_request(args: argparse.Namespace) -> dict[str, Any]: + """Validate the complete v4 manifest without initializing distributed.""" + + loader = _DistributedPromptCollectionLoader(args) + try: + return _validate_longrun_request_with_loader(args, loader) + finally: + loader.clear() + + +def _formal_runtime_initializer(original: Any) -> Any: + def initialize(args: argparse.Namespace) -> tuple[torch.device, int, int]: + device, rank, world_size = original(args) + if world_size != FORMAL_WORLD_SIZE or not 0 <= rank < FORMAL_WORLD_SIZE: + raise RuntimeError( + "Schema-v4 production requires exactly 32 ranks: " + f"rank={rank}, world_size={world_size}." + ) + if args.strategy != "fsdp": + raise RuntimeError("Schema-v4 production must use the 32-rank FSDP path.") + environment_rank = int(os.environ.get("RANK", "-1")) + environment_world = int(os.environ.get("WORLD_SIZE", "-1")) + if (environment_rank, environment_world) != (rank, world_size): + raise RuntimeError( + "Distributed rank/world environment disagrees with the process group." + ) + return device, rank, world_size + + return initialize + + +def train(args: argparse.Namespace) -> int: + """Run v1's loop with v2 checkpointing, v3 LoRA, and v4 rank prompts.""" + + factory = v2._TwoCallJointAdamWFactory() + prompt_loader = _DistributedPromptCollectionLoader(args) + originals = { + "validate_longrun_request": v1.validate_longrun_request, + "load_longrun_checkpoint": v1.load_longrun_checkpoint, + "save_longrun_checkpoint": v1.save_longrun_checkpoint, + "restore_longrun_state": v1.restore_longrun_state, + "install_independent_lora_adapters": v1.install_independent_lora_adapters, + "init_runtime": v1.base._init_runtime, + "adamw": v1.torch.optim.AdamW, + "lora_rank": v1.LORA_RANK, + "lora_alpha": v1.LORA_ALPHA, + "lora_dropout": v1.LORA_DROPOUT, + } + try: + with _v4_prompt_loader_aliases(prompt_loader): + v1.validate_longrun_request = lambda parsed_args: ( + _validate_longrun_request_with_loader(parsed_args, prompt_loader) + ) + v1.load_longrun_checkpoint = load_longrun_checkpoint + v1.save_longrun_checkpoint = save_longrun_checkpoint + v1.restore_longrun_state = restore_longrun_state + v1.install_independent_lora_adapters = v3._install_rank128_blockwide_adapters + v1.base._init_runtime = _formal_runtime_initializer(originals["init_runtime"]) + v1.torch.optim.AdamW = factory + v1.LORA_RANK = LORA_RANK + v1.LORA_ALPHA = LORA_ALPHA + v1.LORA_DROPOUT = LORA_DROPOUT + result = v2._V1_TRAIN(args) + if not args.validate_only: + if factory.calls != 2 or factory.optimizer is None: + raise RuntimeError( + "Schema-v4 did not construct both optimizer groups." + ) + if len(factory.optimizer.param_groups) != 2: + raise RuntimeError( + "Schema-v4 joint optimizer group count changed." + ) + prompt_loader.assert_complete() + return result + finally: + v1.validate_longrun_request = originals["validate_longrun_request"] + v1.load_longrun_checkpoint = originals["load_longrun_checkpoint"] + v1.save_longrun_checkpoint = originals["save_longrun_checkpoint"] + v1.restore_longrun_state = originals["restore_longrun_state"] + v1.install_independent_lora_adapters = originals[ + "install_independent_lora_adapters" + ] + v1.base._init_runtime = originals["init_runtime"] + v1.torch.optim.AdamW = originals["adamw"] + v1.LORA_RANK = originals["lora_rank"] + v1.LORA_ALPHA = originals["lora_alpha"] + v1.LORA_DROPOUT = originals["lora_dropout"] + prompt_loader.clear() + + +def build_parser() -> argparse.ArgumentParser: + parser = v1.build_parser() + subparsers = next( + action + for action in parser._actions + if isinstance(action, argparse._SubParsersAction) + ) + training = subparsers.choices["train"] + for action in training._actions: + if action.dest == "prompt_cache": + action.help = "Path to the v4 rank-local prompt-cache collection.json." + elif action.dest == "prompt_start_index": + action.help = "Must remain zero; v4 indexes rank caches by global_step-1." + training.set_defaults( + handler=train, + lora_rank=LORA_RANK, + lora_alpha=LORA_ALPHA, + lora_dropout=LORA_DROPOUT, + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if getattr(args, "checkpoint_every", CHECKPOINT_EVERY) < 1: + raise ValueError("checkpoint_every must be positive.") + return int(args.handler(args)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/source_snapshot/minimax_h3/train_dmd_longrun_v5.py b/source_snapshot/minimax_h3/train_dmd_longrun_v5.py new file mode 100644 index 0000000000000000000000000000000000000000..b16ccde88c903df7c3d8fe089847330f01080787 --- /dev/null +++ b/source_snapshot/minimax_h3/train_dmd_longrun_v5.py @@ -0,0 +1,406 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""World-16, 10x-learning-rate MiniMax-H3 long-run DMD contract. + +Version 5 is an independent experiment derived from the v4 rank-sharded +prompt/data contract. It deliberately leaves the provenance-locked v1-v4 +entry points unchanged while fixing the following experiment parameters: + +* 4 nodes x 4 GPUs, for exactly 16 distributed ranks; +* 1,000 global steps and 16,000 rank-distinct prompt conditions; +* rank/alpha 128 block-wide student and critic LoRAs; and +* student/critic learning rates 1e-4 and 2e-5 (10x the v4 rates). + +The inherited v1 loop and schema-v2 checkpoint implementation read learning +rates from module globals, including during optimizer construction and resume +validation. Merely changing parser defaults or the JSON recipe would therefore +be unsafe. ``_v5_learning_rate_contract`` temporarily updates both v1 and v2 +constants for the complete operation and restores them on every exit path. +""" + +from __future__ import annotations + +import argparse +import os +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Iterator, Mapping, Sequence + +import torch + +from . import train_dmd_longrun as v1 +from . import train_dmd_longrun_v2 as v2 +from . import train_dmd_longrun_v3 as v3 +from . import train_dmd_longrun_v4 as v4 + + +TOTAL_STEPS = 1_000 +CHECKPOINT_EVERY = v1.CHECKPOINT_EVERY +STUDENT_UPDATE_RATIO = v1.STUDENT_UPDATE_RATIO +STUDENT_LR = 1.0e-4 +CRITIC_LR = 2.0e-5 +BASELINE_STUDENT_LR = 1.0e-5 +BASELINE_CRITIC_LR = 2.0e-6 +LEARNING_RATE_MULTIPLIER = 10 +ADAM_BETAS = v1.ADAM_BETAS +WEIGHT_DECAY = v1.WEIGHT_DECAY +LORA_RANK = v3.LORA_RANK +LORA_ALPHA = v3.LORA_ALPHA +LORA_DROPOUT = v3.LORA_DROPOUT +MAX_GRAD_NORM = v1.MAX_GRAD_NORM + +FORMAL_WORLD_SIZE = 16 +FORMAL_GLOBAL_STEPS = TOTAL_STEPS +SCHEDULED_CONDITIONS = FORMAL_WORLD_SIZE * FORMAL_GLOBAL_STEPS +LONGRUN_CONFIG_VERSION = 5 +CHECKPOINT_SCHEMA_VERSION = v2.CHECKPOINT_SCHEMA_VERSION +OPTIMIZER_LAYOUT = v2.OPTIMIZER_LAYOUT + +student_update_due = v2.student_update_due +student_updates_through = v2.student_updates_through +student_updates_in_segment = v2.student_updates_in_segment +_stable_config_hash = v2._stable_config_hash + +# Capture the implementation before train() temporarily installs the v5 +# validator into the v4 compatibility layer. +_V4_VALIDATE_WITH_LOADER = v4._validate_longrun_request_with_loader + + +@contextmanager +def _v5_learning_rate_contract() -> Iterator[None]: + """Make every inherited LR consumer observe the v5 values, then restore it.""" + + modules = (v1, v2, v4) + originals = [ + (module, module.STUDENT_LR, module.CRITIC_LR) for module in modules + ] + for module in modules: + module.STUDENT_LR = STUDENT_LR + module.CRITIC_LR = CRITIC_LR + try: + if (v1.STUDENT_LR, v1.CRITIC_LR) != (STUDENT_LR, CRITIC_LR): + raise AssertionError("v1 did not enter the v5 learning-rate contract.") + if (v2.STUDENT_LR, v2.CRITIC_LR) != (STUDENT_LR, CRITIC_LR): + raise AssertionError("v2 did not enter the v5 learning-rate contract.") + yield + finally: + for module, student_lr, critic_lr in reversed(originals): + module.STUDENT_LR = student_lr + module.CRITIC_LR = critic_lr + + +@contextmanager +def _v5_world_contract() -> Iterator[None]: + """Parameterize the reused v4 prompt implementation for exactly 16 ranks.""" + + original = v4.FORMAL_WORLD_SIZE + v4.FORMAL_WORLD_SIZE = FORMAL_WORLD_SIZE + try: + yield + finally: + v4.FORMAL_WORLD_SIZE = original + + +class _DistributedPromptCollectionLoader(v4._DistributedPromptCollectionLoader): + """The v4 rank-local loader bound to the v5 world-size contract.""" + + def load_manifest( + self, path: str | Path, *, verify_shards: bool + ) -> dict[str, Any]: + with _v5_world_contract(): + return super().load_manifest(path, verify_shards=verify_shards) + + def __call__( + self, path: str | Path + ) -> tuple[dict[str, Any], v4._ResumeSafeRankConditions]: + import torch.distributed as dist + + if not dist.is_initialized(): + raise RuntimeError( + "Schema-v5 rank cache loading requires an initialized process group." + ) + rank = dist.get_rank() + world_size = dist.get_world_size() + if world_size != FORMAL_WORLD_SIZE or not 0 <= rank < FORMAL_WORLD_SIZE: + raise RuntimeError( + "Schema-v5 production requires exactly 16 distributed ranks: " + f"rank={rank}, world_size={world_size}." + ) + with _v5_world_contract(): + return super().__call__(path) + + +@contextmanager +def _v5_prompt_loader_aliases( + loader: _DistributedPromptCollectionLoader, +) -> Iterator[None]: + original_loop_loader = v1.load_prompt_cache + original_base_loader = v1.base.load_prompt_cache + v1.load_prompt_cache = loader + v1.base.load_prompt_cache = loader.legacy_validation_view + try: + yield + finally: + v1.load_prompt_cache = original_loop_loader + v1.base.load_prompt_cache = original_base_loader + + +def _validate_longrun_request_with_loader( + args: argparse.Namespace, loader: _DistributedPromptCollectionLoader +) -> dict[str, Any]: + """Build and hash the independent v5 world/LR experiment contract.""" + + with _v5_world_contract(), _v5_learning_rate_contract(): + config = _V4_VALIDATE_WITH_LOADER(args, loader) + config.pop("config_sha256", None) + + if (config.get("student_lr"), config.get("critic_lr")) != ( + STUDENT_LR, + CRITIC_LR, + ): + raise RuntimeError( + "Inherited validation did not record the v5 learning-rate contract." + ) + runtime = config.get("prompt_runtime_selection") + if not isinstance(runtime, Mapping): + raise RuntimeError("Inherited validation omitted prompt runtime metadata.") + if ( + runtime.get("world_size") != FORMAL_WORLD_SIZE + or runtime.get("scheduled_conditions") != SCHEDULED_CONDITIONS + ): + raise RuntimeError("Inherited validation did not record the world-16 schedule.") + + config.update( + { + "longrun_schema_version": LONGRUN_CONFIG_VERSION, + "longrun_config_version": LONGRUN_CONFIG_VERSION, + "checkpoint_schema_version": CHECKPOINT_SCHEMA_VERSION, + "optimizer_layout": OPTIMIZER_LAYOUT, + "student_lr": STUDENT_LR, + "critic_lr": CRITIC_LR, + "learning_rate_contract": { + "baseline": "v4", + "multiplier": LEARNING_RATE_MULTIPLIER, + "baseline_student_lr": BASELINE_STUDENT_LR, + "baseline_critic_lr": BASELINE_CRITIC_LR, + "student_lr": STUDENT_LR, + "critic_lr": CRITIC_LR, + "applies_to": [ + "v1_training_loop_optimizer_construction", + "v1_optimizer_state_validation", + "v2_joint_optimizer_factory", + "v2_checkpoint_load_and_restore_validation", + ], + }, + "distributed_contract": { + "nodes": 4, + "gpus_per_node": 4, + "world_size": FORMAL_WORLD_SIZE, + "per_rank_batch_size": 1, + "global_batch_size": FORMAL_WORLD_SIZE, + "global_steps": FORMAL_GLOBAL_STEPS, + "scheduled_conditions": SCHEDULED_CONDITIONS, + }, + "sample_rng_scheme": "seed+source_index+global_step+rank-v5", + } + ) + config["config_sha256"] = _stable_config_hash(config) + return config + + +def validate_longrun_request(args: argparse.Namespace) -> dict[str, Any]: + """Validate the complete v5 manifest without initializing distributed.""" + + loader = _DistributedPromptCollectionLoader(args) + try: + return _validate_longrun_request_with_loader(args, loader) + finally: + loader.clear() + + +def make_joint_optimizer(role_model: torch.nn.Module) -> torch.optim.AdamW: + """Create the schema-v2 joint optimizer with the v5 learning rates.""" + + with _v5_learning_rate_contract(): + return v2.make_joint_optimizer(role_model) + + +def validate_joint_optimizer_state( + state_dict: Mapping[str, Any], + *, + expected_student_updates: int, + expected_critic_updates: int, +) -> None: + """Validate optimizer groups and counters against the v5 rates.""" + + with _v5_learning_rate_contract(): + v2.validate_joint_optimizer_state( + state_dict, + expected_student_updates=expected_student_updates, + expected_critic_updates=expected_critic_updates, + ) + + +def save_longrun_checkpoint(**kwargs: Any) -> Path: + """Save only after schema-v2 validates optimizer groups at v5 rates.""" + + with _v5_learning_rate_contract(): + return v2.save_longrun_checkpoint(**kwargs) + + +def load_longrun_checkpoint(*args: Any, **kwargs: Any) -> dict[str, Any]: + """Load only checkpoints whose joint optimizer carries the v5 rates.""" + + with _v5_learning_rate_contract(): + return v2.load_longrun_checkpoint(*args, **kwargs) + + +def restore_longrun_state(**kwargs: Any) -> None: + """Restore only after validating the persisted v5 optimizer contract.""" + + with _v5_learning_rate_contract(): + v2.restore_longrun_state(**kwargs) + + +def _formal_runtime_initializer(original: Any) -> Any: + def initialize(args: argparse.Namespace) -> tuple[torch.device, int, int]: + device, rank, world_size = original(args) + if world_size != FORMAL_WORLD_SIZE or not 0 <= rank < FORMAL_WORLD_SIZE: + raise RuntimeError( + "Schema-v5 production requires exactly 16 ranks: " + f"rank={rank}, world_size={world_size}." + ) + if args.strategy != "fsdp": + raise RuntimeError("Schema-v5 production must use the 16-rank FSDP path.") + environment_rank = int(os.environ.get("RANK", "-1")) + environment_world = int(os.environ.get("WORLD_SIZE", "-1")) + if (environment_rank, environment_world) != (rank, world_size): + raise RuntimeError( + "Distributed rank/world environment disagrees with the process group." + ) + local_world_size = int(os.environ.get("LOCAL_WORLD_SIZE", "-1")) + local_rank = int(os.environ.get("LOCAL_RANK", "-1")) + group_world_size = int(os.environ.get("GROUP_WORLD_SIZE", "-1")) + if ( + local_world_size != 4 + or group_world_size != 4 + or not 0 <= local_rank < 4 + ): + raise RuntimeError( + "Schema-v5 production requires four nodes with four local ranks " + "per node: " + f"LOCAL_WORLD_SIZE={local_world_size}, " + f"GROUP_WORLD_SIZE={group_world_size}, LOCAL_RANK={local_rank}." + ) + return device, rank, world_size + + return initialize + + +def train(args: argparse.Namespace) -> int: + """Run the audited loop with v5 world, LoRA, optimizer, and resume contracts.""" + + factory = v2._TwoCallJointAdamWFactory() + prompt_loader = _DistributedPromptCollectionLoader(args) + originals = { + "validate_longrun_request": v1.validate_longrun_request, + "load_longrun_checkpoint": v1.load_longrun_checkpoint, + "save_longrun_checkpoint": v1.save_longrun_checkpoint, + "restore_longrun_state": v1.restore_longrun_state, + "install_independent_lora_adapters": v1.install_independent_lora_adapters, + "init_runtime": v1.base._init_runtime, + "adamw": v1.torch.optim.AdamW, + "lora_rank": v1.LORA_RANK, + "lora_alpha": v1.LORA_ALPHA, + "lora_dropout": v1.LORA_DROPOUT, + } + try: + with ( + _v5_world_contract(), + _v5_learning_rate_contract(), + _v5_prompt_loader_aliases(prompt_loader), + ): + v1.validate_longrun_request = lambda parsed_args: ( + _validate_longrun_request_with_loader(parsed_args, prompt_loader) + ) + v1.load_longrun_checkpoint = load_longrun_checkpoint + v1.save_longrun_checkpoint = save_longrun_checkpoint + v1.restore_longrun_state = restore_longrun_state + v1.install_independent_lora_adapters = ( + v3._install_rank128_blockwide_adapters + ) + v1.base._init_runtime = _formal_runtime_initializer( + originals["init_runtime"] + ) + v1.torch.optim.AdamW = factory + v1.LORA_RANK = LORA_RANK + v1.LORA_ALPHA = LORA_ALPHA + v1.LORA_DROPOUT = LORA_DROPOUT + result = v2._V1_TRAIN(args) + if not args.validate_only: + if factory.calls != 2 or factory.optimizer is None: + raise RuntimeError( + "Schema-v5 did not construct both optimizer groups." + ) + groups = factory.optimizer.param_groups + if len(groups) != 2 or [group["lr"] for group in groups] != [ + STUDENT_LR, + CRITIC_LR, + ]: + raise RuntimeError( + "Schema-v5 joint optimizer learning-rate groups changed." + ) + prompt_loader.assert_complete() + return result + finally: + v1.validate_longrun_request = originals["validate_longrun_request"] + v1.load_longrun_checkpoint = originals["load_longrun_checkpoint"] + v1.save_longrun_checkpoint = originals["save_longrun_checkpoint"] + v1.restore_longrun_state = originals["restore_longrun_state"] + v1.install_independent_lora_adapters = originals[ + "install_independent_lora_adapters" + ] + v1.base._init_runtime = originals["init_runtime"] + v1.torch.optim.AdamW = originals["adamw"] + v1.LORA_RANK = originals["lora_rank"] + v1.LORA_ALPHA = originals["lora_alpha"] + v1.LORA_DROPOUT = originals["lora_dropout"] + prompt_loader.clear() + + +def build_parser() -> argparse.ArgumentParser: + parser = v1.build_parser() + subparsers = next( + action + for action in parser._actions + if isinstance(action, argparse._SubParsersAction) + ) + training = subparsers.choices["train"] + for action in training._actions: + if action.dest == "prompt_cache": + action.help = "Path to the v5 world-16 rank-local collection.json." + elif action.dest == "prompt_start_index": + action.help = "Must remain zero; v5 indexes rank caches by global_step-1." + training.set_defaults( + handler=train, + max_steps=TOTAL_STEPS, + lora_rank=LORA_RANK, + lora_alpha=LORA_ALPHA, + lora_dropout=LORA_DROPOUT, + student_lr=STUDENT_LR, + critic_lr=CRITIC_LR, + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if getattr(args, "checkpoint_every", CHECKPOINT_EVERY) < 1: + raise ValueError("checkpoint_every must be positive.") + return int(args.handler(args)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/source_snapshot/minimax_h3/train_utils.py b/source_snapshot/minimax_h3/train_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1bb006b301403212b987839523e43c5994b97745 --- /dev/null +++ b/source_snapshot/minimax_h3/train_utils.py @@ -0,0 +1,1048 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Executable MiniMax-H3 full-sequence few-step DMD primitives. + +The released H3 checkpoint is guidance-distilled. This module consequently +has no negative-prompt or CFG API: every transformer call is conditional and +the frozen teacher is the shared Base transformer with all adapters disabled. + +One H3 transformer owns two independent PEFT adapters. ``student`` is used for +one four-evaluation rollout over the complete packed A/V sequence; ``critic`` +is the trainable fake score; disabling both adapters exposes the frozen Base +teacher. The stock Diffusers attention processor remains untouched and uses +``is_causal=False``. Sharing the Base weights is important for a 33B checkpoint +and still gives the three DMD roles distinct parameter states. +""" + +from __future__ import annotations + +import contextlib +import copy +import inspect +import math +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Iterator, Mapping, Sequence + +import torch + +from .dmd_core import ( + DMD_NORMALIZATION, + H3_AUDIO_TAG, + H3_TEXT_TAG, + H3_VIDEO_TAG, + PositiveOnlyCallCounter, + build_h3_sigma_schedules, + critic_flow_matching_loss, + dmd_generator_pseudo_loss, + h3_add_noise, + h3_data_velocity_to_x0, +) + + +STUDENT_ADAPTER = "student" +CRITIC_ADAPTER = "critic" +NUM_MODEL_EVALUATIONS = 4 +NUM_SIGMA_ENDPOINTS = NUM_MODEL_EVALUATIONS + 1 +RELEASED_MAIN_BLOCKS = 50 +RELEASED_REFINER_BLOCKS = 2 +CUDA_ALLOCATOR_CONFIG = "expandable_segments:True" +CUDA_CACHE_POLICY = "empty_cache_before_generator_backward" +SAVED_TENSOR_OFFLOAD_POLICY = "student_rollout_save_on_cpu" +SAVED_TENSOR_OFFLOAD_PIN_MEMORY = False + + +@dataclass(frozen=True) +class H3PackedLayout: + """Static row layout consumed by ``MiniMaxH3Transformer3DModel``.""" + + position_ids: torch.Tensor + token_tags: torch.Tensor + video_indices: torch.Tensor + audio_indices: torch.Tensor + text_indices: torch.Tensor + + def validate(self) -> None: + if self.position_ids.ndim != 2 or self.position_ids.shape[-1] != 3: + raise ValueError("position_ids must have shape [sequence, 3].") + sequence_length = self.position_ids.shape[0] + if self.token_tags.shape != (sequence_length,): + raise ValueError("token_tags must have shape [sequence].") + for name, indices in ( + ("video_indices", self.video_indices), + ("audio_indices", self.audio_indices), + ("text_indices", self.text_indices), + ): + if indices.ndim != 1 or indices.dtype != torch.long: + raise ValueError(f"{name} must be a one-dimensional torch.long tensor.") + if indices.numel() and ( + int(indices.min()) < 0 or int(indices.max()) >= sequence_length + ): + raise ValueError(f"{name} contains an out-of-range row index.") + all_indices = torch.cat( + (self.video_indices, self.audio_indices, self.text_indices), dim=0 + ) + if all_indices.numel() != sequence_length: + raise ValueError( + "A Base T2VA layout must account for every row exactly once; " + f"got {all_indices.numel()} indices for {sequence_length} rows." + ) + if torch.unique(all_indices).numel() != sequence_length: + raise ValueError("video/audio/text row indices must be disjoint.") + expected_tags = torch.empty_like(self.token_tags) + expected_tags[self.video_indices] = H3_VIDEO_TAG + expected_tags[self.audio_indices] = H3_AUDIO_TAG + expected_tags[self.text_indices] = H3_TEXT_TAG + if not torch.equal(self.token_tags, expected_tags): + raise ValueError( + "token_tags disagree with the supplied video/audio/text indices; " + "training supports Base T2VA only (no IR/reference rows)." + ) + devices = { + tensor.device + for tensor in ( + self.position_ids, + self.token_tags, + self.video_indices, + self.audio_indices, + self.text_indices, + ) + } + if len(devices) != 1: + raise ValueError("Every layout tensor must be on the same device.") + + @property + def sequence_length(self) -> int: + return int(self.token_tags.numel()) + + def to(self, device: torch.device | str) -> "H3PackedLayout": + return H3PackedLayout( + position_ids=self.position_ids.to(device), + token_tags=self.token_tags.to(device), + video_indices=self.video_indices.to(device), + audio_indices=self.audio_indices.to(device), + text_indices=self.text_indices.to(device), + ) + + +@dataclass(frozen=True) +class H3Condition: + """One cached positive prompt presentation.""" + + id: str + prompt: str + seed: int + prompt_embeds: torch.Tensor + text_token_tags: torch.Tensor + + def validate(self) -> None: + if not self.id or not self.prompt: + raise ValueError("A cached condition needs non-empty id and prompt strings.") + if isinstance(self.seed, bool) or not isinstance(self.seed, int): + raise TypeError("A cached prompt seed must be an integer.") + if self.prompt_embeds.ndim != 3 or self.prompt_embeds.shape[0] != 1: + raise ValueError("prompt_embeds must have shape [1, text_rows, text_dim].") + if self.text_token_tags.shape != (self.prompt_embeds.shape[1],): + raise ValueError("text_token_tags must match the text-row dimension.") + if not bool((self.text_token_tags == H3_TEXT_TAG).all()): + raise ValueError( + "Base T2VA prompt caches may contain text rows only; IR/vision rows are not supported." + ) + if not bool(torch.isfinite(self.prompt_embeds).all()): + raise ValueError("prompt_embeds contain non-finite values.") + + def to(self, device: torch.device | str) -> "H3Condition": + return H3Condition( + id=self.id, + prompt=self.prompt, + seed=self.seed, + prompt_embeds=self.prompt_embeds.to(device), + text_token_tags=self.text_token_tags.to(device), + ) + + +@dataclass(frozen=True) +class RoleCall: + """One auditable positive transformer call.""" + + role: str + active_adapters: tuple[str, ...] + adapters_enabled: bool + block_causal: bool + current_block: int | None + + +class H3CallAudit: + """In-memory call trace used by tests and emitted in training metrics.""" + + def __init__(self) -> None: + self.records: list[RoleCall] = [] + + def record( + self, + *, + role: str, + active_adapters: Sequence[str], + adapters_enabled: bool, + block_causal: bool, + current_block: int | None = None, + ) -> None: + self.records.append( + RoleCall( + role=role, + active_adapters=tuple(active_adapters), + adapters_enabled=adapters_enabled, + block_causal=block_causal, + current_block=current_block, + ) + ) + + def count(self, role: str) -> int: + return sum(record.role == role for record in self.records) + + +@dataclass(frozen=True) +class GeneratorStepOutput: + generated_video: torch.Tensor + generated_audio: torch.Tensor + loss: torch.Tensor + video_loss: torch.Tensor + audio_loss: torch.Tensor + grad_norm: torch.Tensor + score_video_timestep: float + score_audio_timestep: float + + +@dataclass(frozen=True) +class CriticStepOutput: + loss: torch.Tensor + video_loss: torch.Tensor + audio_loss: torch.Tensor + grad_norm: torch.Tensor + score_video_timestep: float + score_audio_timestep: float + + +def _adapter_marker(adapter_name: str) -> str: + if not adapter_name or "." in adapter_name: + raise ValueError("Adapter names must be non-empty and cannot contain dots.") + return f".{adapter_name}." + + +def adapter_named_parameters( + model: torch.nn.Module, adapter_name: str +) -> list[tuple[str, torch.nn.Parameter]]: + """Return one named PEFT adapter without accidentally selecting the other.""" + + marker = _adapter_marker(adapter_name) + rows = [ + (name, parameter) + for name, parameter in model.named_parameters() + if "lora_" in name and marker in name + ] + if not rows: + raise RuntimeError(f"No LoRA parameters were found for adapter {adapter_name!r}.") + return rows + + +def adapter_parameters(model: torch.nn.Module, adapter_name: str) -> list[torch.nn.Parameter]: + return [parameter for _, parameter in adapter_named_parameters(model, adapter_name)] + + +def assert_base_frozen(model: torch.nn.Module) -> None: + trainable_base = [ + name + for name, parameter in model.named_parameters() + if parameter.requires_grad and "lora_" not in name + ] + if trainable_base: + preview = ", ".join(trainable_base[:5]) + raise AssertionError(f"Base H3 parameters must be frozen; trainable: {preview}") + + +def h3_model_input_dtypes(model: torch.nn.Module) -> tuple[torch.dtype, torch.dtype]: + """Return the exact dtypes expected by H3's video and audio input projections.""" + + video_projection = getattr(model, "proj_in", None) + audio_projection = getattr(model, "audio_proj_in", None) + if video_projection is None or audio_projection is None: + raise TypeError("Expected MiniMax-H3 proj_in and audio_proj_in modules.") + video_dtype = video_projection.weight.dtype + audio_dtype = audio_projection.weight.dtype + if not video_dtype.is_floating_point or not audio_dtype.is_floating_point: + raise TypeError("H3 input projection weights must use floating-point dtypes.") + return video_dtype, audio_dtype + + +def assert_released_h3_architecture(model: torch.nn.Module) -> dict[str, Any]: + """Fail closed unless ``model`` keeps the released 50+2 H3 topology. + + Few-step distillation changes only the denoising schedule and LoRA weights. + It must not replace, truncate, or reinterpret MiniMax-H3's 50 main blocks + and two text-refiner blocks. + """ + + config = getattr(model, "config", None) + main_blocks = getattr(model, "transformer_blocks", None) + token_refiner = getattr(model, "token_refiner", None) + refiner_blocks = getattr(token_refiner, "refiner_blocks", None) + observed = { + "config_num_layers": getattr(config, "num_layers", None), + "config_num_refiner_layers": getattr(config, "num_refiner_layers", None), + "main_transformer_blocks": len(main_blocks) if main_blocks is not None else None, + "token_refiner_blocks": len(refiner_blocks) if refiner_blocks is not None else None, + } + expected = { + "config_num_layers": RELEASED_MAIN_BLOCKS, + "config_num_refiner_layers": RELEASED_REFINER_BLOCKS, + "main_transformer_blocks": RELEASED_MAIN_BLOCKS, + "token_refiner_blocks": RELEASED_REFINER_BLOCKS, + } + if observed != expected: + raise RuntimeError( + "MiniMax-H3 architecture differs from the released 50-main/2-refiner " + f"setting: observed={observed}, expected={expected}." + ) + attention_blocks = [*main_blocks, *refiner_blocks] + processors = [ + getattr(getattr(block, "attn", None), "processor", None) + for block in attention_blocks + ] + processor_classes = { + processor.__class__.__name__ + for processor in processors + if processor is not None + } + if ( + len(processors) != RELEASED_MAIN_BLOCKS + RELEASED_REFINER_BLOCKS + or None in processors + ): + raise RuntimeError("Released H3 blocks must all expose their native attention processor.") + if processor_classes != {"MiniMaxH3AttnProcessor"}: + raise RuntimeError( + "MiniMax-H3 attention processor was replaced: " + f"observed={sorted(processor_classes)}." + ) + processor_source = inspect.getsource(processors[0].__class__.__call__) + if "is_causal=False" not in processor_source: + raise RuntimeError("Pinned Diffusers H3 attention is not explicitly is_causal=False.") + return { + **observed, + "attention_implementation": "official_diffusers_native", + "attention_is_causal": False, + "attention_mask_override": False, + "stock_attention_processor_count": len(processors), + } + + +def install_independent_lora_adapters( + model: torch.nn.Module, + *, + rank: int = 16, + alpha: int | None = None, + dropout: float = 0.0, + target_modules: Sequence[str] = ("to_q", "to_k", "to_v", "to_out.0"), +) -> dict[str, list[torch.nn.Parameter]]: + """Freeze Base H3 and install disjoint student/critic PEFT LoRAs.""" + + if isinstance(rank, bool) or not isinstance(rank, int) or rank < 1: + raise ValueError("LoRA rank must be a positive integer.") + if alpha is None: + alpha = rank + if isinstance(alpha, bool) or not isinstance(alpha, int) or alpha < 1: + raise ValueError("LoRA alpha must be a positive integer.") + if not 0.0 <= float(dropout) < 1.0: + raise ValueError("LoRA dropout must lie in [0, 1).") + if not target_modules: + raise ValueError("At least one LoRA target module is required.") + if not all(hasattr(model, name) for name in ("add_adapter", "set_adapter")): + raise TypeError("The model must expose Diffusers' PeftAdapterMixin API.") + + try: + from peft import LoraConfig + except ImportError as error: + raise RuntimeError("PEFT is required for MiniMax-H3 DMD LoRA training.") from error + + model.requires_grad_(False) + config = LoraConfig( + r=rank, + lora_alpha=alpha, + lora_dropout=float(dropout), + target_modules=list(target_modules), + init_lora_weights=True, + bias="none", + ) + try: + model.add_adapter(config, adapter_name=STUDENT_ADAPTER) + model.add_adapter(copy.deepcopy(config), adapter_name=CRITIC_ADAPTER) + except ImportError as error: + raise RuntimeError( + "PEFT LoRA injection failed. Check optional acceleration packages (in particular, " + "remove an incompatible torchao build) before loading the 33B checkpoint." + ) from error + + student = adapter_parameters(model, STUDENT_ADAPTER) + critic = adapter_parameters(model, CRITIC_ADAPTER) + if {id(parameter) for parameter in student} & {id(parameter) for parameter in critic}: + raise AssertionError("Student and critic adapters unexpectedly share parameters.") + model.set_adapter(STUDENT_ADAPTER) + assert_base_frozen(model) + if not all(parameter.requires_grad for parameter in student): + raise AssertionError("Activating the student adapter did not make all of it trainable.") + if any(parameter.requires_grad for parameter in critic): + raise AssertionError("The inactive critic adapter must not be trainable.") + return {STUDENT_ADAPTER: student, CRITIC_ADAPTER: critic} + + +def _active_adapters(model: torch.nn.Module) -> tuple[str, ...]: + active = model.active_adapters() + if isinstance(active, str): + return (active,) + return tuple(active) + + +@contextlib.contextmanager +def use_adapter(model: torch.nn.Module, adapter_name: str) -> Iterator[None]: + """Activate one adapter and restore the preceding adapter after the call.""" + + previous = _active_adapters(model) + model.enable_adapters() + model.set_adapter(adapter_name) + if _active_adapters(model) != (adapter_name,): + raise AssertionError(f"Failed to activate only adapter {adapter_name!r}.") + try: + yield + finally: + model.enable_adapters() + if previous: + model.set_adapter(list(previous) if len(previous) > 1 else previous[0]) + + +@contextlib.contextmanager +def disable_adapters_for_teacher(model: torch.nn.Module) -> Iterator[None]: + """Expose the frozen Base checkpoint and reliably restore the active LoRA.""" + + previous = _active_adapters(model) + model.disable_adapters() + try: + yield + finally: + model.enable_adapters() + if previous: + model.set_adapter(list(previous) if len(previous) > 1 else previous[0]) + + +def build_row_timestep_plan( + layout: H3PackedLayout, + *, + video_timestep: float | torch.Tensor, + audio_timestep: float | torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """Build the official full-sequence H3 row-timestep plan. + + All video rows share the video clock, all audio rows share the audio clock, + and text follows video, matching the released Base T2VA convention. + """ + + video = torch.as_tensor( + video_timestep, device=layout.token_tags.device, dtype=torch.float32 + ) + audio = torch.as_tensor( + audio_timestep, device=layout.token_tags.device, dtype=torch.float32 + ) + if video.numel() != 1 or audio.numel() != 1: + raise ValueError("One H3 request uses one video and one audio timestep per call.") + if not bool(torch.isfinite(video)) or not bool(torch.isfinite(audio)): + raise ValueError("H3 timesteps must be finite.") + if not (0.0 <= float(video) <= 1.0 and 0.0 <= float(audio) <= 1.0): + raise ValueError("H3 timesteps must lie in [0, 1].") + row_timesteps = torch.full( + (layout.sequence_length,), + float(video), + dtype=torch.float32, + device=layout.token_tags.device, + ) + row_timesteps[layout.audio_indices] = audio + return torch.unique(row_timesteps, sorted=True, return_inverse=True) + + +def _shift_sigma(base_sigma: float, shift: float) -> float: + if not 0.0 < float(base_sigma) < 1.0: + raise ValueError("A DMD score sigma must lie strictly between zero and one.") + if not math.isfinite(float(shift)) or shift <= 0: + raise ValueError("A sigma shift must be positive and finite.") + return float(shift * base_sigma / (1.0 + (shift - 1.0) * base_sigma)) + + +def paired_score_timesteps( + base_sigma: float, *, video_shift: float = 12.0, audio_shift: float = 3.0 +) -> tuple[float, float]: + """Map one random base sigma onto H3's paired video/audio clocks.""" + + return ( + 1.0 - _shift_sigma(base_sigma, video_shift), + 1.0 - _shift_sigma(base_sigma, audio_shift), + ) + + +def h3_euler_step( + sample: torch.Tensor, + velocity: torch.Tensor, + *, + timestep: torch.Tensor, + sigma: torch.Tensor, + sigma_next: torch.Tensor, +) -> torch.Tensor: + """Differentiable equivalent of the official MiniMaxH3Scheduler step.""" + + if sample.shape != velocity.shape: + raise ValueError("H3 Euler sample and velocity shapes must match.") + timestep = timestep.to(device=sample.device, dtype=sample.dtype) + sigma_from_timestep = 1.0 - timestep + denoised = sample + sigma_from_timestep * velocity + compute_dtype = ( + torch.float32 + if sample.dtype in (torch.float16, torch.bfloat16) + else sample.dtype + ) + sigma = sigma.to(device=sample.device, dtype=compute_dtype) + sigma_next = sigma_next.to(device=sample.device, dtype=compute_dtype) + ratio = sigma_next / sigma + updated = ratio * sample.to(compute_dtype) + (1.0 - ratio) * denoised.to( + compute_dtype + ) + return updated.to(sample.dtype) + + +def _random_normal_like( + reference: torch.Tensor, generator: torch.Generator | None +) -> torch.Tensor: + return torch.randn( + reference.shape, + generator=generator, + device=reference.device, + dtype=reference.dtype, + ) + + +def _parameter_grad_norm(parameters: Sequence[torch.nn.Parameter]) -> torch.Tensor: + gradients = [ + parameter.grad.detach().float().norm(2) + for parameter in parameters + if parameter.grad is not None + ] + if not gradients: + return torch.tensor(0.0) + return torch.stack(gradients).norm(2) + + +class H3DMDTrainer: + """A one-base, two-adapter H3 DMD training engine. + + ``forward_model`` may be the raw transformer or an FSDP wrapper. Adapter + switching happens on ``role_model``, the underlying H3 module. The model's + native full-sequence attention processors are never replaced. + """ + + def __init__( + self, + *, + forward_model: torch.nn.Module, + role_model: torch.nn.Module, + student_optimizer: torch.optim.Optimizer, + critic_optimizer: torch.optim.Optimizer, + video_shift: float = 12.0, + audio_shift: float = 3.0, + max_grad_norm: float = 1.0, + audit: H3CallAudit | None = None, + ) -> None: + if not math.isfinite(float(max_grad_norm)) or max_grad_norm <= 0: + raise ValueError("max_grad_norm must be positive and finite.") + self.forward_model = forward_model + self.role_model = role_model + self.student_optimizer = student_optimizer + self.critic_optimizer = critic_optimizer + self.video_shift = float(video_shift) + self.audio_shift = float(audio_shift) + self.max_grad_norm = float(max_grad_norm) + self.audit = audit if audit is not None else H3CallAudit() + self.cuda_cache_clears_before_generator_backward = 0 + self.saved_tensor_offload_contexts = 0 + self.saved_tensor_offload_pack_count = 0 + self.saved_tensor_offload_unpack_count = 0 + self.saved_tensor_offload_packed_bytes = 0 + self.student_parameters = adapter_parameters(role_model, STUDENT_ADAPTER) + self.critic_parameters = adapter_parameters(role_model, CRITIC_ADAPTER) + assert_base_frozen(role_model) + + def _clip_active_gradients( + self, parameters: Sequence[torch.nn.Parameter], device: torch.device + ) -> torch.Tensor: + if self.forward_model is not self.role_model and hasattr( + self.forward_model, "clip_grad_norm_" + ): + # FSDP's implementation performs the required cross-rank norm + # reduction. Only the active adapter has gradients at this point. + norm = self.forward_model.clip_grad_norm_(self.max_grad_norm) + return torch.as_tensor(norm, device=device, dtype=torch.float32) + norm = _parameter_grad_norm(parameters).to(device) + torch.nn.utils.clip_grad_norm_(parameters, self.max_grad_norm) + return norm + + def _forward( + self, + *, + role: str, + video_rows: torch.Tensor, + audio_rows: torch.Tensor, + condition: H3Condition, + layout: H3PackedLayout, + video_timestep: float | torch.Tensor, + audio_timestep: float | torch.Tensor, + adapters_enabled: bool, + ) -> tuple[torch.Tensor, torch.Tensor]: + timestep, timestep_indices = build_row_timestep_plan( + layout, + video_timestep=video_timestep, + audio_timestep=audio_timestep, + ) + self.audit.record( + role=role, + active_adapters=_active_adapters(self.role_model), + adapters_enabled=adapters_enabled, + block_causal=False, + current_block=None, + ) + output = self.forward_model( + hidden_states=video_rows, + audio_hidden_states=audio_rows, + encoder_hidden_states=condition.prompt_embeds, + timestep=timestep, + timestep_indices=timestep_indices, + token_tags=layout.token_tags, + position_ids=layout.position_ids, + video_indices=layout.video_indices, + audio_indices=layout.audio_indices, + text_indices=layout.text_indices, + return_dict=True, + ) + video = output.sample + audio = output.audio_sample + if video.shape != video_rows.shape or audio.shape != audio_rows.shape: + raise RuntimeError( + "MiniMax-H3 changed the video/audio row shapes during a score call." + ) + if not bool(torch.isfinite(video).all()) or not bool(torch.isfinite(audio).all()): + raise FloatingPointError(f"Non-finite {role} velocity prediction.") + return video, audio + + @contextlib.contextmanager + def _offload_student_saved_tensors(self, device: torch.device) -> Iterator[None]: + """Move differentiable-rollout saved tensors to CPU with runtime counters.""" + + if device.type != "cuda": + yield + return + offloader = torch.autograd.graph.save_on_cpu( + pin_memory=SAVED_TENSOR_OFFLOAD_PIN_MEMORY, + device_type="cuda", + ) + + def pack(tensor: torch.Tensor) -> tuple[torch.device, torch.Tensor]: + packed = offloader.pack_hook(tensor) + if tensor.device.type == "cuda": + self.saved_tensor_offload_pack_count += 1 + self.saved_tensor_offload_packed_bytes += ( + tensor.numel() * tensor.element_size() + ) + return packed + + def unpack(packed: tuple[torch.device, torch.Tensor]) -> torch.Tensor: + tensor = offloader.unpack_hook(packed) + if tensor.device.type == "cuda": + self.saved_tensor_offload_unpack_count += 1 + return tensor + + self.saved_tensor_offload_contexts += 1 + with torch.autograd.graph.saved_tensors_hooks(pack, unpack): + yield + + def student_rollout( + self, + *, + video_noise: torch.Tensor, + audio_noise: torch.Tensor, + condition: H3Condition, + layout: H3PackedLayout, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Denoise the complete native H3 sequence in four evaluations. + + Each call uses the released full-sequence, non-causal attention and + advances every video and audio row together on its modality-specific + shifted schedule. Four model evaluations consume five sigma endpoints. + """ + + schedules = build_h3_sigma_schedules( + num_evaluations=NUM_MODEL_EVALUATIONS, + video_shift=self.video_shift, + audio_shift=self.audio_shift, + device=video_noise.device, + dtype=torch.float32, + ) + if schedules.num_evaluations != NUM_MODEL_EVALUATIONS: + raise AssertionError("H3 few-step rollout must contain exactly four evaluations.") + video = video_noise + audio = audio_noise + with use_adapter(self.role_model, STUDENT_ADAPTER): + with PositiveOnlyCallCounter( + expected_calls=NUM_MODEL_EVALUATIONS + ) as calls: + for index in range(NUM_MODEL_EVALUATIONS): + video_timestep = schedules.video_timesteps[index] + audio_timestep = schedules.audio_timesteps[index] + video_velocity, audio_velocity = calls.call( + self._forward, + role="student", + video_rows=video, + audio_rows=audio, + condition=condition, + layout=layout, + video_timestep=video_timestep, + audio_timestep=audio_timestep, + adapters_enabled=True, + ) + video = h3_euler_step( + video, + video_velocity, + timestep=video_timestep, + sigma=schedules.video_sigmas[index], + sigma_next=schedules.video_sigmas[index + 1], + ) + audio = h3_euler_step( + audio, + audio_velocity, + timestep=audio_timestep, + sigma=schedules.audio_sigmas[index], + sigma_next=schedules.audio_sigmas[index + 1], + ) + return video, audio + + def _score_pair( + self, + *, + noisy_video: torch.Tensor, + noisy_audio: torch.Tensor, + condition: H3Condition, + layout: H3PackedLayout, + video_timestep: float, + audio_timestep: float, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + # The teacher is one positive Base call. There is deliberately no + # parameter, state, or branch through which an unconditional call can + # enter this function. + with torch.no_grad(): + with disable_adapters_for_teacher(self.role_model): + with PositiveOnlyCallCounter(expected_calls=1) as teacher_calls: + teacher_video, teacher_audio = teacher_calls.call( + self._forward, + role="teacher", + video_rows=noisy_video, + audio_rows=noisy_audio, + condition=condition, + layout=layout, + video_timestep=video_timestep, + audio_timestep=audio_timestep, + adapters_enabled=False, + ) + with use_adapter(self.role_model, CRITIC_ADAPTER): + critic_video, critic_audio = self._forward( + role="critic_target", + video_rows=noisy_video, + audio_rows=noisy_audio, + condition=condition, + layout=layout, + video_timestep=video_timestep, + audio_timestep=audio_timestep, + adapters_enabled=True, + ) + return teacher_video, teacher_audio, critic_video, critic_audio + + def generator_step( + self, + *, + video_noise: torch.Tensor, + audio_noise: torch.Tensor, + condition: H3Condition, + layout: H3PackedLayout, + score_base_sigma: float, + score_video_noise: torch.Tensor | None = None, + score_audio_noise: torch.Tensor | None = None, + generator: torch.Generator | None = None, + ) -> GeneratorStepOutput: + """Update only the student adapter with the joint video/audio DMD loss.""" + + self.student_optimizer.zero_grad(set_to_none=True) + self.critic_optimizer.zero_grad(set_to_none=True) + with self._offload_student_saved_tensors(video_noise.device): + generated_video, generated_audio = self.student_rollout( + video_noise=video_noise, + audio_noise=audio_noise, + condition=condition, + layout=layout, + ) + video_timestep, audio_timestep = paired_score_timesteps( + score_base_sigma, + video_shift=self.video_shift, + audio_shift=self.audio_shift, + ) + if score_video_noise is None: + score_video_noise = _random_normal_like(generated_video, generator) + if score_audio_noise is None: + score_audio_noise = _random_normal_like(generated_audio, generator) + noisy_video = h3_add_noise( + generated_video.detach(), score_video_noise, video_timestep + ) + noisy_audio = h3_add_noise( + generated_audio.detach(), score_audio_noise, audio_timestep + ) + teacher_video_v, teacher_audio_v, critic_video_v, critic_audio_v = ( + self._score_pair( + noisy_video=noisy_video, + noisy_audio=noisy_audio, + condition=condition, + layout=layout, + video_timestep=video_timestep, + audio_timestep=audio_timestep, + ) + ) + teacher_video_x0 = h3_data_velocity_to_x0( + noisy_video, teacher_video_v, video_timestep + ) + teacher_audio_x0 = h3_data_velocity_to_x0( + noisy_audio, teacher_audio_v, audio_timestep + ) + critic_video_x0 = h3_data_velocity_to_x0( + noisy_video, critic_video_v, video_timestep + ) + critic_audio_x0 = h3_data_velocity_to_x0( + noisy_audio, critic_audio_v, audio_timestep + ) + losses = dmd_generator_pseudo_loss( + generated_video=generated_video, + real_video_x0=teacher_video_x0, + fake_video_x0=critic_video_x0, + generated_audio=generated_audio, + real_audio_x0=teacher_audio_x0, + fake_audio_x0=critic_audio_x0, + ) + if losses.audio is None or not bool(torch.isfinite(losses.total)): + raise FloatingPointError("The joint H3 generator DMD loss is not finite.") + + # Adapter switching while constructing detached score targets leaves + # student active again before autograd traverses its rollout graph. + self.role_model.set_adapter(STUDENT_ADAPTER) + # The released 33B checkpoint runs close to the H100 memory ceiling. + # Emptying only unused cache here is a low-risk auxiliary measure; an + # allocator already retries cache release on OOM, so this is not the + # primary live-memory fix. Saved-tensor CPU offload above is audited + # separately through pack and backward-time unpack counters. + if losses.total.device.type == "cuda": + torch.cuda.empty_cache() + self.cuda_cache_clears_before_generator_backward += 1 + losses.total.backward() + grad_norm = self._clip_active_gradients( + self.student_parameters, losses.total.device + ) + if not bool(torch.isfinite(grad_norm)) or float(grad_norm) == 0.0: + raise FloatingPointError("The student LoRA gradient is zero or non-finite.") + if any(parameter.grad is not None for parameter in self.critic_parameters): + raise AssertionError("Generator DMD backward leaked into the critic adapter.") + self.student_optimizer.step() + return GeneratorStepOutput( + generated_video=generated_video.detach(), + generated_audio=generated_audio.detach(), + loss=losses.total.detach(), + video_loss=losses.video.detach(), + audio_loss=losses.audio.detach(), + grad_norm=grad_norm.detach(), + score_video_timestep=video_timestep, + score_audio_timestep=audio_timestep, + ) + + def critic_step( + self, + *, + clean_video: torch.Tensor, + clean_audio: torch.Tensor, + condition: H3Condition, + layout: H3PackedLayout, + score_base_sigma: float, + score_video_noise: torch.Tensor | None = None, + score_audio_noise: torch.Tensor | None = None, + generator: torch.Generator | None = None, + ) -> CriticStepOutput: + """Update only critic LoRA against H3's data-ward flow target.""" + + self.critic_optimizer.zero_grad(set_to_none=True) + self.student_optimizer.zero_grad(set_to_none=True) + clean_video = clean_video.detach() + clean_audio = clean_audio.detach() + video_timestep, audio_timestep = paired_score_timesteps( + score_base_sigma, + video_shift=self.video_shift, + audio_shift=self.audio_shift, + ) + if score_video_noise is None: + score_video_noise = _random_normal_like(clean_video, generator) + if score_audio_noise is None: + score_audio_noise = _random_normal_like(clean_audio, generator) + noisy_video = h3_add_noise(clean_video, score_video_noise, video_timestep) + noisy_audio = h3_add_noise(clean_audio, score_audio_noise, audio_timestep) + with use_adapter(self.role_model, CRITIC_ADAPTER): + predicted_video, predicted_audio = self._forward( + role="critic_train", + video_rows=noisy_video, + audio_rows=noisy_audio, + condition=condition, + layout=layout, + video_timestep=video_timestep, + audio_timestep=audio_timestep, + adapters_enabled=True, + ) + losses = critic_flow_matching_loss( + predicted_video_velocity=predicted_video, + clean_video=clean_video, + video_noise=score_video_noise, + predicted_audio_velocity=predicted_audio, + clean_audio=clean_audio, + audio_noise=score_audio_noise, + ) + if losses.audio is None or not bool(torch.isfinite(losses.total)): + raise FloatingPointError("The joint H3 critic loss is not finite.") + losses.total.backward() + grad_norm = self._clip_active_gradients( + self.critic_parameters, losses.total.device + ) + if not bool(torch.isfinite(grad_norm)) or float(grad_norm) == 0.0: + raise FloatingPointError("The critic LoRA gradient is zero or non-finite.") + if any(parameter.grad is not None for parameter in self.student_parameters): + raise AssertionError("Critic backward leaked into the student adapter.") + self.critic_optimizer.step() + return CriticStepOutput( + loss=losses.total.detach(), + video_loss=losses.video.detach(), + audio_loss=losses.audio.detach(), + grad_norm=grad_norm.detach(), + score_video_timestep=video_timestep, + score_audio_timestep=audio_timestep, + ) + + +def make_base_t2va_layout( + *, + text_token_tags: torch.Tensor, + num_latent_frames: int, + latent_height: int, + latent_width: int, + num_audio_latents: int, + patch_size: tuple[int, int, int], +) -> H3PackedLayout: + """Delegate Base layout construction to the official H3 Diffusers block.""" + + try: + from diffusers.modular_pipelines.minimax_h3.before_denoise import ( + MiniMaxH3PrepareLayoutStep, + ) + except ImportError as error: + raise RuntimeError( + "Install the shared diffusers_h3 checkout before constructing H3 layouts." + ) from error + ( + position_ids, + token_tags, + video_indices, + audio_indices, + text_indices, + num_condition_video_rows, + num_condition_audio_rows, + ) = MiniMaxH3PrepareLayoutStep.build_packed_sequence( + text_token_tags=text_token_tags.cpu(), + num_latent_frames=num_latent_frames, + latent_height=latent_height, + latent_width=latent_width, + num_audio_latents=num_audio_latents, + patch_size=patch_size, + keyframe_anchors=(), + ) + if num_condition_video_rows != 0 or num_condition_audio_rows != 0: + raise AssertionError("Base T2VA unexpectedly produced conditioning rows.") + layout = H3PackedLayout( + position_ids=position_ids, + token_tags=token_tags, + video_indices=video_indices, + audio_indices=audio_indices, + text_indices=text_indices, + ) + layout.validate() + return layout + + +def load_prompt_cache(path: str | Path) -> tuple[dict[str, Any], list[H3Condition]]: + """Load a tensor-only, positive-only prompt cache and validate all four prompts.""" + + payload = torch.load(Path(path), map_location="cpu", weights_only=True) + if not isinstance(payload, Mapping) or payload.get("schema_version") != 1: + raise ValueError("Unsupported or malformed MiniMax-H3 prompt cache.") + if payload.get("workflow") != "t2va" or payload.get("cfg_enabled") is not False: + raise ValueError("Prompt cache must be positive-only Base T2VA with CFG disabled.") + items = payload.get("conditions") + if not isinstance(items, list) or len(items) != 4: + raise ValueError("The training prompt cache must contain exactly four conditions.") + conditions = [] + for item in items: + if not isinstance(item, Mapping): + raise ValueError("Every prompt-cache condition must be a mapping.") + condition = H3Condition( + id=item["id"], + prompt=item["prompt"], + seed=item["seed"], + prompt_embeds=item["prompt_embeds"], + text_token_tags=item["text_token_tags"], + ) + condition.validate() + conditions.append(condition) + if len({condition.id for condition in conditions}) != 4: + raise ValueError("Prompt-cache ids must be unique.") + metadata = {key: value for key, value in payload.items() if key != "conditions"} + return metadata, conditions + + +__all__ = [ + "CRITIC_ADAPTER", + "DMD_NORMALIZATION", + "CriticStepOutput", + "GeneratorStepOutput", + "H3CallAudit", + "H3Condition", + "H3DMDTrainer", + "H3PackedLayout", + "NUM_MODEL_EVALUATIONS", + "NUM_SIGMA_ENDPOINTS", + "RELEASED_MAIN_BLOCKS", + "RELEASED_REFINER_BLOCKS", + "STUDENT_ADAPTER", + "adapter_named_parameters", + "adapter_parameters", + "assert_base_frozen", + "assert_released_h3_architecture", + "build_row_timestep_plan", + "disable_adapters_for_teacher", + "h3_euler_step", + "h3_model_input_dtypes", + "install_independent_lora_adapters", + "load_prompt_cache", + "make_base_t2va_layout", + "paired_score_timesteps", + "use_adapter", +] diff --git a/source_snapshot/minimax_h3/visualize_teacher_x0.py b/source_snapshot/minimax_h3/visualize_teacher_x0.py new file mode 100644 index 0000000000000000000000000000000000000000..e58e299de5d8e0ea7ddb21272ac4fea10ca44fc8 --- /dev/null +++ b/source_snapshot/minimax_h3/visualize_teacher_x0.py @@ -0,0 +1,674 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""Visualize intermediate MiniMax-H3 Student states and frozen-Base x0. + +The Student always follows the existing four-evaluation, five-endpoint, +full-sequence schedule. After Euler updates 1, 2, and 3 this runner captures +the current *noisy* post-Euler state ``x_t`` and decodes it as ``student_xt``. +It then disables the Student adapter, evaluates the frozen Base teacher once +on exactly that packed A/V state and endpoint, and decodes +``teacher_x0 = x_t + (1 - t) * velocity``. + +This is a diagnostic analogue of the DMD score call, not the exact training +path: the current smoke trainer first completes all four Student evaluations +and then re-noises that completed sample at a random score sigma. +""" + +from __future__ import annotations + +import argparse +import gc +import json +import os +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping, Sequence + +import torch + +from diffusers.modular_pipelines.minimax_h3.denoise import ( + MiniMaxH3DenoiseLoopWrapper, + MiniMaxH3LoopDenoiser, + MiniMaxH3LoopSchedulerStep, +) + +from .dmd_core import h3_data_velocity_to_x0 +from .infer_base import ( + EXPECTED_PROMPT_COUNT, + FPS, + MODEL_ID, + WORKFLOW, + RunConfig, + _default_video_encoder, + _prepare_video_for_encoder, + _tensor_descriptor, + assert_official_t2va_contract, + build_official_pipeline, + load_prompt_specs, + resolve_hub_revision, + sha256_file, +) +from .infer_student import install_student_lora, load_student_checkpoint +from .train_utils import ( + NUM_MODEL_EVALUATIONS, + NUM_SIGMA_ENDPOINTS, + RELEASED_MAIN_BLOCKS, + RELEASED_REFINER_BLOCKS, + STUDENT_ADAPTER, + disable_adapters_for_teacher, + use_adapter, +) + + +STOP_EVALUATIONS = (1, 2, 3) +TRAINING_SCORE_PATH = "full_rollout_then_random_renoise" +TEACHER_X0_FORMULA = "x0=x_t+(1-t)*v" +RUNNER = "LongLive/minimax_h3/visualize_teacher_x0.py" + +DIAGNOSTIC_CONTRACT: dict[str, Any] = { + "workflow": WORKFLOW, + "rollout_mode": "full_sequence", + "attention_mode": "native_non_causal", + "attention_is_causal": False, + "attention_mask_override": False, + "temporal_ar_blocks": 0, + "transformer_block_count": RELEASED_MAIN_BLOCKS, + "token_refiner_block_count": RELEASED_REFINER_BLOCKS, + "model_evaluations_per_rollout": NUM_MODEL_EVALUATIONS, + "sigma_endpoints_per_rollout": NUM_SIGMA_ENDPOINTS, + "stop_evaluations": list(STOP_EVALUATIONS), + "student_state": "post_euler_endpoint", + "teacher_positive_forwards_per_stop": 1, + "teacher_unconditional_forwards_per_stop": 0, + "teacher_adapters_disabled": True, + "teacher_x0_formula": TEACHER_X0_FORMULA, + "cfg": False, + "uses_cfg": False, + "uses_context_ir": False, + "uses_regenerator": False, + "video_and_audio_joint": True, + "diagnostic_matches_training_score_path": False, + "training_score_path": TRAINING_SCORE_PATH, +} + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _atomic_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + os.replace(temporary, path) + + +@dataclass(frozen=True) +class EndpointSnapshot: + """CPU copy of one shared Student/Teacher diagnostic endpoint.""" + + stop_eval: int + video_sigma: float + video_timestep: float + audio_sigma: float + audio_timestep: float + student_video_rows: torch.Tensor + student_audio_rows: torch.Tensor + teacher_video_x0_rows: torch.Tensor + teacher_audio_x0_rows: torch.Tensor + num_latent_frames: int + latent_height: int + latent_width: int + num_audio_latents: int + + +def validate_endpoint_schedule( + video_sigmas: torch.Tensor, + audio_sigmas: torch.Tensor, + video_timesteps: torch.Tensor, + audio_timesteps: torch.Tensor, +) -> None: + """Fail closed unless this is the fixed four-eval H3 paired schedule.""" + + expected_endpoints = NUM_SIGMA_ENDPOINTS + expected_evals = NUM_MODEL_EVALUATIONS + if tuple(video_sigmas.shape) != (expected_endpoints,) or tuple( + audio_sigmas.shape + ) != (expected_endpoints,): + raise RuntimeError("Diagnostic requires exactly five video/audio sigma endpoints.") + if tuple(video_timesteps.shape) != (expected_evals,) or tuple( + audio_timesteps.shape + ) != (expected_evals,): + raise RuntimeError("Diagnostic requires exactly four video/audio model evaluations.") + for name, sigmas, timesteps in ( + ("video", video_sigmas, video_timesteps), + ("audio", audio_sigmas, audio_timesteps), + ): + if not bool((sigmas[:-1] > sigmas[1:]).all()) or float(sigmas[-1]) != 0.0: + raise RuntimeError(f"{name} sigmas must strictly decrease to zero.") + if not torch.allclose(timesteps.float(), 1.0 - sigmas[:-1].float()): + raise RuntimeError(f"{name} timestep/sigma convention is not t=1-sigma.") + + +def unpack_generated_rows( + video_rows: torch.Tensor, + audio_rows: torch.Tensor, + *, + patch_size: tuple[int, int, int], + vae_latent_channels: int, + audio_channels: int, + num_latent_frames: int, + latent_height: int, + latent_width: int, + num_audio_latents: int, +) -> tuple[torch.Tensor, torch.Tensor]: + """Inverse of official H3 T2VA packing for generated video/audio rows.""" + + patch_t, patch_h, patch_w = patch_size + if any( + value <= 0 + for value in ( + patch_t, + patch_h, + patch_w, + vae_latent_channels, + audio_channels, + num_latent_frames, + latent_height, + latent_width, + num_audio_latents, + ) + ): + raise ValueError("H3 unpack geometry must be positive.") + if ( + num_latent_frames % patch_t + or latent_height % patch_h + or latent_width % patch_w + ): + raise ValueError("H3 latent geometry must be divisible by the patch size.") + expected_video_rows = ( + num_latent_frames // patch_t + * (latent_height // patch_h) + * (latent_width // patch_w) + ) + expected_video_features = vae_latent_channels * patch_t * patch_h * patch_w + if tuple(video_rows.shape) != (expected_video_rows, expected_video_features): + raise ValueError( + "Packed video row shape mismatch: " + f"{tuple(video_rows.shape)} != {(expected_video_rows, expected_video_features)}." + ) + if audio_rows.ndim != 2 or audio_rows.shape[0] != audio_channels * num_audio_latents: + raise ValueError("Packed audio row shape does not match channel/latent geometry.") + + rows = video_rows.reshape( + -1, + num_latent_frames // patch_t, + latent_height // patch_h, + latent_width // patch_w, + vae_latent_channels, + patch_t, + patch_h, + patch_w, + ) + rows = rows.permute(0, 4, 1, 5, 2, 6, 3, 7) + video = rows.reshape( + -1, + vae_latent_channels, + num_latent_frames, + latent_height, + latent_width, + ).contiguous() + audio = audio_rows.reshape(audio_channels, num_audio_latents, -1) + audio = audio.permute(0, 2, 1).contiguous() + return video, audio + + +class MiniMaxH3StudentTeacherX0DenoiseStep(MiniMaxH3DenoiseLoopWrapper): + """Official H3 loop plus shared-prefix endpoint capture and Base x0 calls.""" + + block_classes = [MiniMaxH3LoopDenoiser, MiniMaxH3LoopSchedulerStep] + block_names = ["denoiser", "update"] + + def __init__(self) -> None: + super().__init__() + self.snapshots: list[EndpointSnapshot] = [] + self.student_positive_calls = 0 + self.teacher_positive_calls = 0 + self.teacher_unconditional_calls = 0 + + @torch.no_grad() + def __call__(self, components: Any, state: Any) -> Any: + geometry = { + name: state.get(name) + for name in ( + "num_latent_frames", + "latent_height", + "latent_width", + "num_audio_latents", + ) + } + if any(value is None for value in geometry.values()): + raise RuntimeError("H3 diagnostic state is missing latent geometry.") + block_state = self.get_block_state(state) + validate_endpoint_schedule( + components.scheduler.sigmas, + components.audio_scheduler.sigmas, + block_state.timesteps, + block_state.audio_timesteps, + ) + if block_state.num_condition_video_rows != 0 or block_state.num_condition_audio_rows != 0: + raise RuntimeError("Teacher-x0 diagnostic supports Base T2VA only; conditioning rows are forbidden.") + if tuple(components.transformer.active_adapters()) != (STUDENT_ADAPTER,): + raise RuntimeError("Teacher-x0 diagnostic requires only the Student adapter active.") + + self.snapshots.clear() + self.student_positive_calls = 0 + self.teacher_positive_calls = 0 + self.teacher_unconditional_calls = 0 + denoiser = self.sub_blocks["denoiser"] + update = self.sub_blocks["update"] + + with use_adapter(components.transformer, STUDENT_ADAPTER): + with self.progress_bar(total=len(block_state.timesteps)) as progress_bar: + for index, timestep in enumerate(block_state.timesteps): + components, block_state = denoiser( + components, block_state, i=index, t=timestep + ) + self.student_positive_calls += 1 + components, block_state = update( + components, block_state, i=index, t=timestep + ) + stop_eval = index + 1 + if stop_eval in STOP_EVALUATIONS: + # This is the current noisy state at endpoint k, after + # the k-th Student Euler update. It is not an x0. + student_video = block_state.latents.detach().clone() + student_audio = block_state.audio_latents.detach().clone() + endpoint_video_t = block_state.timesteps[stop_eval] + endpoint_audio_t = block_state.audio_timesteps[stop_eval] + with disable_adapters_for_teacher(components.transformer): + components, block_state = denoiser( + components, + block_state, + i=stop_eval, + t=endpoint_video_t, + ) + self.teacher_positive_calls += 1 + teacher_video_x0 = h3_data_velocity_to_x0( + student_video, + block_state.noise_pred[0], + endpoint_video_t, + ) + teacher_audio_x0 = h3_data_velocity_to_x0( + student_audio, + block_state.audio_noise_pred[0], + endpoint_audio_t, + ) + self.snapshots.append( + EndpointSnapshot( + stop_eval=stop_eval, + video_sigma=float(components.scheduler.sigmas[stop_eval]), + video_timestep=float(endpoint_video_t), + audio_sigma=float(components.audio_scheduler.sigmas[stop_eval]), + audio_timestep=float(endpoint_audio_t), + student_video_rows=student_video.cpu(), + student_audio_rows=student_audio.cpu(), + teacher_video_x0_rows=teacher_video_x0.cpu(), + teacher_audio_x0_rows=teacher_audio_x0.cpu(), + num_latent_frames=int(geometry["num_latent_frames"]), + latent_height=int(geometry["latent_height"]), + latent_width=int(geometry["latent_width"]), + num_audio_latents=int(geometry["num_audio_latents"]), + ) + ) + progress_bar.update() + + if self.student_positive_calls != NUM_MODEL_EVALUATIONS: + raise AssertionError("Student diagnostic did not execute exactly four positive forwards.") + if self.teacher_positive_calls != len(STOP_EVALUATIONS): + raise AssertionError("Teacher diagnostic did not execute one positive forward per stop.") + if self.teacher_unconditional_calls != 0: + raise AssertionError("MiniMax-H3 diagnostic must not execute unconditional/CFG calls.") + if [item.stop_eval for item in self.snapshots] != list(STOP_EVALUATIONS): + raise AssertionError("Teacher-x0 diagnostic did not capture endpoints 1, 2, and 3.") + self.set_block_state(state, block_state) + return components, state + + +def install_diagnostic_block(pipe: Any) -> MiniMaxH3StudentTeacherX0DenoiseStep: + """Replace only the official loop wrapper; its native denoiser/update stay intact.""" + + # ``ModularPipeline.blocks`` returns a deepcopy in the pinned Diffusers + # implementation. The runtime executes ``_blocks``; mutating the public + # inspection copy would silently leave the official loop in place. + blocks = getattr(pipe, "_blocks", None) + sub_blocks = getattr(blocks, "sub_blocks", None) + key = "denoise.denoise" + if sub_blocks is None or key not in sub_blocks: + raise RuntimeError("Pinned H3 T2VA pipeline has no denoise.denoise block.") + original = sub_blocks[key] + if original.__class__.__name__ != "MiniMaxH3DenoiseStep": + raise RuntimeError( + f"Expected official MiniMaxH3DenoiseStep, found {original.__class__.__name__}." + ) + diagnostic = MiniMaxH3StudentTeacherX0DenoiseStep() + sub_blocks[key] = diagnostic + return diagnostic + + +@torch.no_grad() +def decode_rows(pipe: Any, snapshot: EndpointSnapshot, role: str) -> tuple[Any, torch.Tensor, int, torch.Tensor, torch.Tensor]: + """Decode one packed A/V role with the official released H3 VAEs.""" + + if role == "student_xt": + video_rows = snapshot.student_video_rows + audio_rows = snapshot.student_audio_rows + elif role == "teacher_x0": + video_rows = snapshot.teacher_video_x0_rows + audio_rows = snapshot.teacher_audio_x0_rows + else: + raise ValueError(f"Unknown diagnostic role: {role!r}.") + video_latents, audio_latents = unpack_generated_rows( + video_rows, + audio_rows, + patch_size=tuple(pipe.patch_size), + vae_latent_channels=int(pipe.vae_latent_channels), + audio_channels=int(pipe.audio_channels), + num_latent_frames=snapshot.num_latent_frames, + latent_height=snapshot.latent_height, + latent_width=snapshot.latent_width, + num_audio_latents=snapshot.num_audio_latents, + ) + device = pipe._execution_device + video_device = video_latents.to(device) + mean = torch.tensor(pipe.vae.config.latents_mean, device=device).view(1, -1, 1, 1, 1) + std = torch.tensor(pipe.vae.config.latents_std, device=device).view(1, -1, 1, 1, 1) + with torch.autocast(device_type=device.type, dtype=torch.float16, enabled=device.type == "cuda"): + decoded = pipe.vae.decode(video_device * std + mean, return_dict=False)[0] + pixel_mean = torch.tensor(pipe.pixel_mean, device=device).view(1, -1, 1, 1, 1) + pixel_std = torch.tensor(pipe.pixel_std, device=device).view(1, -1, 1, 1, 1) + decoded = (decoded.float() * pixel_std + pixel_mean).clamp(0, 1) + video = pipe.video_processor.postprocess_video(decoded, output_type="pt") + + audio_device = audio_latents.to(device) + audio_mean = torch.tensor(pipe.audio_vae.config.latents_mean, device=device).view(1, -1, 1) + audio_std = torch.tensor(pipe.audio_vae.config.latents_std, device=device).view(1, -1, 1) + audio = pipe.audio_vae.decode(audio_device * audio_std + audio_mean, return_dict=False)[0] + audio = audio.float().permute(1, 0, 2) + return video, audio, int(pipe.audio_sampling_rate), video_latents, audio_latents + + +def _write_role( + pipe: Any, + snapshot: EndpointSnapshot, + role: str, + stop_dir: Path, +) -> tuple[dict[str, Any], dict[str, Any]]: + video, audio, sampling_rate, video_latents, audio_latents = decode_rows( + pipe, snapshot, role + ) + frames = _prepare_video_for_encoder(video) + video_path = stop_dir / f"{role}.mp4" + latent_path = stop_dir / f"{role}_latents.pt" + _default_video_encoder( + video=frames, + fps=FPS, + output_path=str(video_path), + audio=audio[0].detach().cpu(), + audio_sample_rate=sampling_rate, + ) + torch.save( + { + "video_latents": video_latents.detach().cpu().contiguous(), + "audio_latents": audio_latents.detach().cpu().contiguous(), + }, + latent_path, + ) + record = { + "video": video_path.name, + "video_sha256": sha256_file(video_path), + "latents": latent_path.name, + "latents_sha256": sha256_file(latent_path), + } + metadata = { + **record, + "semantics": ( + "post_euler_noisy_state_xt_direct_vae_decode" + if role == "student_xt" + else "frozen_base_direct_x0_from_same_xt_and_endpoint" + ), + "numeric_outputs": { + "video_latents": _tensor_descriptor(video_latents), + "audio_latents": _tensor_descriptor(audio_latents), + }, + } + del video, audio, frames + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + return record, metadata + + +def run_diagnostic_batch( + pipe: Any, + diagnostic: MiniMaxH3StudentTeacherX0DenoiseStep, + prompts: Sequence[Any], + config: RunConfig, + checkpoint: Mapping[str, Any], + adapter: Mapping[str, Any], + output_dir: str | Path, +) -> dict[str, Any]: + """Generate four prompts x three stops x two A/V diagnostic roles.""" + + if len(prompts) != EXPECTED_PROMPT_COUNT: + raise ValueError(f"Expected exactly four prompts, found {len(prompts)}.") + if config.num_inference_steps != NUM_SIGMA_ENDPOINTS: + raise ValueError("Teacher-x0 diagnostic requires five endpoints/four evaluations.") + assert_official_t2va_contract(pipe, expected_revision=config.resolved_revision) + destination = Path(output_dir).resolve() + destination.mkdir(parents=True, exist_ok=True) + samples: list[dict[str, Any]] = [] + + for prompt_index, prompt in enumerate(prompts): + sample_dir = destination / f"{prompt_index:02d}_{prompt.id}" + sample_dir.mkdir(parents=True, exist_ok=True) + results = pipe( + prompt=prompt.prompt, + height=config.height, + width=config.width, + num_frames=config.num_frames, + num_inference_steps=NUM_SIGMA_ENDPOINTS, + generator=torch.Generator().manual_seed(prompt.seed), + output_type="pt", + output=["videos", "audio", "sampling_rate"], + ) + if not isinstance(results, Mapping): + raise TypeError("Official H3 pipeline did not return a mapping.") + if len(diagnostic.snapshots) != len(STOP_EVALUATIONS): + raise RuntimeError("Diagnostic loop returned an incomplete endpoint set.") + # The stock pipeline still performs its terminal decode. It is not a + # diagnostic artifact, so release that large tensor before decoding + # the six captured endpoint roles below. + del results + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + stops: list[dict[str, Any]] = [] + for snapshot in diagnostic.snapshots: + stop_dir = sample_dir / f"k{snapshot.stop_eval:02d}" + stop_dir.mkdir(parents=True, exist_ok=True) + roles: dict[str, Any] = {} + metadata_roles: dict[str, Any] = {} + for role in ("student_xt", "teacher_x0"): + roles[role], metadata_roles[role] = _write_role( + pipe, snapshot, role, stop_dir + ) + calls = { + "student_cumulative": snapshot.stop_eval, + "teacher_positive": 1, + "teacher_unconditional": 0, + } + metadata = { + "schema_version": 1, + "created_at": _utc_now(), + "sample": asdict(prompt), + "stop_eval": snapshot.stop_eval, + "endpoint_index": snapshot.stop_eval, + "video": { + "sigma": snapshot.video_sigma, + "timestep": snapshot.video_timestep, + }, + "audio": { + "sigma": snapshot.audio_sigma, + "timestep": snapshot.audio_timestep, + }, + "contract": dict(DIAGNOSTIC_CONTRACT), + "model": { + "id": config.model_id, + "resolved_revision": config.resolved_revision, + "transformer_block_count": RELEASED_MAIN_BLOCKS, + "token_refiner_block_count": RELEASED_REFINER_BLOCKS, + }, + "calls": {**calls, "teacher_adapters_disabled": True}, + "roles": metadata_roles, + } + metadata_path = stop_dir / "metadata.json" + _atomic_json(metadata_path, metadata) + stops.append( + { + "stop_eval": snapshot.stop_eval, + "endpoint_index": snapshot.stop_eval, + "video": metadata["video"], + "audio": metadata["audio"], + "calls": calls, + "roles": roles, + "metadata": str(metadata_path.relative_to(destination)), + "metadata_sha256": sha256_file(metadata_path), + } + ) + samples.append( + { + "id": prompt.id, + "prompt": prompt.prompt, + "seed": prompt.seed, + "stops": stops, + } + ) + + config_payload = asdict(config) + config_payload["dtype"] = "bfloat16" + manifest = { + "schema_version": 1, + "status": "success", + "created_at": _utc_now(), + "runner": RUNNER, + "config": config_payload, + "contract": dict(DIAGNOSTIC_CONTRACT), + "checkpoint": { + "path": str(checkpoint["path"]), + "sha256": checkpoint["sha256"], + "step": checkpoint["step"], + }, + "adapter": dict(adapter), + "samples": samples, + } + manifest_path = destination / "manifest.json" + _atomic_json(manifest_path, manifest) + _atomic_json( + destination / "SUCCESS.json", + { + "schema_version": 1, + "status": "success", + "manifest": manifest_path.name, + "manifest_sha256": sha256_file(manifest_path), + }, + ) + return manifest + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--checkpoint", type=Path, required=True) + parser.add_argument("--prompts", type=Path, default=Path(__file__).with_name("prompts.json")) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--model-id", "--model-dir", dest="model_id", default=MODEL_ID) + parser.add_argument("--revision", default="main") + parser.add_argument("--height", type=int, default=768) + parser.add_argument("--width", type=int, default=1344) + parser.add_argument("--num-frames", type=int, default=124) + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--memory-reserve-margin", default="12GB") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--validate-only", action="store_true") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + prompts = load_prompt_specs(args.prompts) + checkpoint = load_student_checkpoint(args.checkpoint) + revision = resolve_hub_revision( + args.model_id, args.revision, local_files_only=args.local_files_only + ) + if checkpoint["config"].get("resolved_revision") != revision: + raise ValueError("Student checkpoint/model revision mismatch.") + config = RunConfig( + model_id=args.model_id, + requested_revision=args.revision, + resolved_revision=revision, + height=args.height, + width=args.width, + num_frames=args.num_frames, + num_inference_steps=NUM_SIGMA_ENDPOINTS, + device=args.device, + memory_reserve_margin=args.memory_reserve_margin, + local_files_only=args.local_files_only, + ) + config.validate() + if args.validate_only: + print( + json.dumps( + { + "checkpoint": { + "path": str(checkpoint["path"]), + "sha256": checkpoint["sha256"], + "step": checkpoint["step"], + }, + "config": {**asdict(config), "dtype": "bfloat16"}, + "contract": DIAGNOSTIC_CONTRACT, + "prompts": [asdict(prompt) for prompt in prompts], + }, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + ) + return 0 + + adapter: dict[str, Any] = {} + + def setup_student(transformer: torch.nn.Module) -> None: + adapter.update(install_student_lora(transformer, checkpoint)) + + pipe = build_official_pipeline(config, transformer_setup=setup_student) + diagnostic = install_diagnostic_block(pipe) + manifest = run_diagnostic_batch( + pipe, + diagnostic, + prompts, + config, + checkpoint, + adapter, + args.output_dir, + ) + print(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/training_config.json b/training_config.json new file mode 100644 index 0000000000000000000000000000000000000000..18db4e1f73f3b4b42ed07c6a1f12396624f49490 --- /dev/null +++ b/training_config.json @@ -0,0 +1,880 @@ +{ + "schema_version": 1, + "model_id": "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/shyang/LongLive-LoRA/Minimax-H3/training/LongLive-LoRA/models/MiniMax-H3-Base-T2VA-fa9c8ab1", + "requested_revision": "fa9c8ab1eaa21c8ae25e7e40b83b2e6002f340af", + "resolved_revision": "fa9c8ab1eaa21c8ae25e7e40b83b2e6002f340af", + "workflow": "t2va", + "ir": false, + "regenerator": false, + "cfg": false, + "shared_base_roles": [ + "student", + "critic", + "teacher" + ], + "teacher_adapter_state": "disabled", + "student_adapter": "student", + "critic_adapter": "critic", + "lora_rank": 128, + "lora_alpha": 128, + "lora_targets": [ + "token_refiner.refiner_blocks.0.attn.to_k", + "token_refiner.refiner_blocks.0.attn.to_out.0", + "token_refiner.refiner_blocks.0.attn.to_q", + "token_refiner.refiner_blocks.0.attn.to_v", + "token_refiner.refiner_blocks.0.ff.net.0.proj", + "token_refiner.refiner_blocks.0.ff.net.2", + "token_refiner.refiner_blocks.1.attn.to_k", + "token_refiner.refiner_blocks.1.attn.to_out.0", + "token_refiner.refiner_blocks.1.attn.to_q", + "token_refiner.refiner_blocks.1.attn.to_v", + "token_refiner.refiner_blocks.1.ff.net.0.proj", + "token_refiner.refiner_blocks.1.ff.net.2", + "transformer_blocks.0.attn.to_k", + "transformer_blocks.0.attn.to_out.0", + "transformer_blocks.0.attn.to_q", + "transformer_blocks.0.attn.to_v", + "transformer_blocks.0.ff.net.0.proj", + "transformer_blocks.0.ff.net.2", + "transformer_blocks.1.attn.to_k", + "transformer_blocks.1.attn.to_out.0", + "transformer_blocks.1.attn.to_q", + "transformer_blocks.1.attn.to_v", + "transformer_blocks.1.ff.net.0.proj", + "transformer_blocks.1.ff.net.2", + "transformer_blocks.10.attn.to_k", + "transformer_blocks.10.attn.to_out.0", + "transformer_blocks.10.attn.to_q", + "transformer_blocks.10.attn.to_v", + "transformer_blocks.10.ff.net.0.proj", + "transformer_blocks.10.ff.net.2", + "transformer_blocks.11.attn.to_k", + "transformer_blocks.11.attn.to_out.0", + "transformer_blocks.11.attn.to_q", + "transformer_blocks.11.attn.to_v", + "transformer_blocks.11.ff.net.0.proj", + "transformer_blocks.11.ff.net.2", + "transformer_blocks.12.attn.to_k", + "transformer_blocks.12.attn.to_out.0", + "transformer_blocks.12.attn.to_q", + "transformer_blocks.12.attn.to_v", + "transformer_blocks.12.ff.net.0.proj", + "transformer_blocks.12.ff.net.2", + "transformer_blocks.13.attn.to_k", + "transformer_blocks.13.attn.to_out.0", + "transformer_blocks.13.attn.to_q", + "transformer_blocks.13.attn.to_v", + "transformer_blocks.13.ff.net.0.proj", + "transformer_blocks.13.ff.net.2", + "transformer_blocks.14.attn.to_k", + "transformer_blocks.14.attn.to_out.0", + "transformer_blocks.14.attn.to_q", + "transformer_blocks.14.attn.to_v", + "transformer_blocks.14.ff.net.0.proj", + "transformer_blocks.14.ff.net.2", + "transformer_blocks.15.attn.to_k", + "transformer_blocks.15.attn.to_out.0", + "transformer_blocks.15.attn.to_q", + "transformer_blocks.15.attn.to_v", + "transformer_blocks.15.ff.net.0.proj", + "transformer_blocks.15.ff.net.2", + "transformer_blocks.16.attn.to_k", + "transformer_blocks.16.attn.to_out.0", + "transformer_blocks.16.attn.to_q", + "transformer_blocks.16.attn.to_v", + "transformer_blocks.16.ff.net.0.proj", + "transformer_blocks.16.ff.net.2", + "transformer_blocks.17.attn.to_k", + "transformer_blocks.17.attn.to_out.0", + "transformer_blocks.17.attn.to_q", + "transformer_blocks.17.attn.to_v", + "transformer_blocks.17.ff.net.0.proj", + "transformer_blocks.17.ff.net.2", + "transformer_blocks.18.attn.to_k", + "transformer_blocks.18.attn.to_out.0", + "transformer_blocks.18.attn.to_q", + "transformer_blocks.18.attn.to_v", + "transformer_blocks.18.ff.net.0.proj", + "transformer_blocks.18.ff.net.2", + "transformer_blocks.19.attn.to_k", + "transformer_blocks.19.attn.to_out.0", + "transformer_blocks.19.attn.to_q", + "transformer_blocks.19.attn.to_v", + "transformer_blocks.19.ff.net.0.proj", + "transformer_blocks.19.ff.net.2", + "transformer_blocks.2.attn.to_k", + "transformer_blocks.2.attn.to_out.0", + "transformer_blocks.2.attn.to_q", + "transformer_blocks.2.attn.to_v", + "transformer_blocks.2.ff.net.0.proj", + "transformer_blocks.2.ff.net.2", + "transformer_blocks.20.attn.to_k", + "transformer_blocks.20.attn.to_out.0", + "transformer_blocks.20.attn.to_q", + "transformer_blocks.20.attn.to_v", + "transformer_blocks.20.ff.net.0.proj", + "transformer_blocks.20.ff.net.2", + "transformer_blocks.21.attn.to_k", + "transformer_blocks.21.attn.to_out.0", + "transformer_blocks.21.attn.to_q", + "transformer_blocks.21.attn.to_v", + "transformer_blocks.21.ff.net.0.proj", + "transformer_blocks.21.ff.net.2", + "transformer_blocks.22.attn.to_k", + "transformer_blocks.22.attn.to_out.0", + "transformer_blocks.22.attn.to_q", + "transformer_blocks.22.attn.to_v", + "transformer_blocks.22.ff.net.0.proj", + "transformer_blocks.22.ff.net.2", + "transformer_blocks.23.attn.to_k", + "transformer_blocks.23.attn.to_out.0", + "transformer_blocks.23.attn.to_q", + "transformer_blocks.23.attn.to_v", + "transformer_blocks.23.ff.net.0.proj", + "transformer_blocks.23.ff.net.2", + "transformer_blocks.24.attn.to_k", + "transformer_blocks.24.attn.to_out.0", + "transformer_blocks.24.attn.to_q", + "transformer_blocks.24.attn.to_v", + "transformer_blocks.24.ff.net.0.proj", + "transformer_blocks.24.ff.net.2", + "transformer_blocks.25.attn.to_k", + "transformer_blocks.25.attn.to_out.0", + "transformer_blocks.25.attn.to_q", + "transformer_blocks.25.attn.to_v", + "transformer_blocks.25.ff.net.0.proj", + "transformer_blocks.25.ff.net.2", + "transformer_blocks.26.attn.to_k", + "transformer_blocks.26.attn.to_out.0", + "transformer_blocks.26.attn.to_q", + "transformer_blocks.26.attn.to_v", + "transformer_blocks.26.ff.net.0.proj", + "transformer_blocks.26.ff.net.2", + "transformer_blocks.27.attn.to_k", + "transformer_blocks.27.attn.to_out.0", + "transformer_blocks.27.attn.to_q", + "transformer_blocks.27.attn.to_v", + "transformer_blocks.27.ff.net.0.proj", + "transformer_blocks.27.ff.net.2", + "transformer_blocks.28.attn.to_k", + "transformer_blocks.28.attn.to_out.0", + "transformer_blocks.28.attn.to_q", + "transformer_blocks.28.attn.to_v", + "transformer_blocks.28.ff.net.0.proj", + "transformer_blocks.28.ff.net.2", + "transformer_blocks.29.attn.to_k", + "transformer_blocks.29.attn.to_out.0", + "transformer_blocks.29.attn.to_q", + "transformer_blocks.29.attn.to_v", + "transformer_blocks.29.ff.net.0.proj", + "transformer_blocks.29.ff.net.2", + "transformer_blocks.3.attn.to_k", + "transformer_blocks.3.attn.to_out.0", + "transformer_blocks.3.attn.to_q", + "transformer_blocks.3.attn.to_v", + "transformer_blocks.3.ff.net.0.proj", + "transformer_blocks.3.ff.net.2", + "transformer_blocks.30.attn.to_k", + "transformer_blocks.30.attn.to_out.0", + "transformer_blocks.30.attn.to_q", + "transformer_blocks.30.attn.to_v", + "transformer_blocks.30.ff.net.0.proj", + "transformer_blocks.30.ff.net.2", + "transformer_blocks.31.attn.to_k", + "transformer_blocks.31.attn.to_out.0", + "transformer_blocks.31.attn.to_q", + "transformer_blocks.31.attn.to_v", + "transformer_blocks.31.ff.net.0.proj", + "transformer_blocks.31.ff.net.2", + "transformer_blocks.32.attn.to_k", + "transformer_blocks.32.attn.to_out.0", + "transformer_blocks.32.attn.to_q", + "transformer_blocks.32.attn.to_v", + "transformer_blocks.32.ff.net.0.proj", + "transformer_blocks.32.ff.net.2", + "transformer_blocks.33.attn.to_k", + "transformer_blocks.33.attn.to_out.0", + "transformer_blocks.33.attn.to_q", + "transformer_blocks.33.attn.to_v", + "transformer_blocks.33.ff.net.0.proj", + "transformer_blocks.33.ff.net.2", + "transformer_blocks.34.attn.to_k", + "transformer_blocks.34.attn.to_out.0", + "transformer_blocks.34.attn.to_q", + "transformer_blocks.34.attn.to_v", + "transformer_blocks.34.ff.net.0.proj", + "transformer_blocks.34.ff.net.2", + "transformer_blocks.35.attn.to_k", + "transformer_blocks.35.attn.to_out.0", + "transformer_blocks.35.attn.to_q", + "transformer_blocks.35.attn.to_v", + "transformer_blocks.35.ff.net.0.proj", + "transformer_blocks.35.ff.net.2", + "transformer_blocks.36.attn.to_k", + "transformer_blocks.36.attn.to_out.0", + "transformer_blocks.36.attn.to_q", + "transformer_blocks.36.attn.to_v", + "transformer_blocks.36.ff.net.0.proj", + "transformer_blocks.36.ff.net.2", + "transformer_blocks.37.attn.to_k", + "transformer_blocks.37.attn.to_out.0", + "transformer_blocks.37.attn.to_q", + "transformer_blocks.37.attn.to_v", + "transformer_blocks.37.ff.net.0.proj", + "transformer_blocks.37.ff.net.2", + "transformer_blocks.38.attn.to_k", + "transformer_blocks.38.attn.to_out.0", + "transformer_blocks.38.attn.to_q", + "transformer_blocks.38.attn.to_v", + "transformer_blocks.38.ff.net.0.proj", + "transformer_blocks.38.ff.net.2", + "transformer_blocks.39.attn.to_k", + "transformer_blocks.39.attn.to_out.0", + "transformer_blocks.39.attn.to_q", + "transformer_blocks.39.attn.to_v", + "transformer_blocks.39.ff.net.0.proj", + "transformer_blocks.39.ff.net.2", + "transformer_blocks.4.attn.to_k", + "transformer_blocks.4.attn.to_out.0", + "transformer_blocks.4.attn.to_q", + "transformer_blocks.4.attn.to_v", + "transformer_blocks.4.ff.net.0.proj", + "transformer_blocks.4.ff.net.2", + "transformer_blocks.40.attn.to_k", + "transformer_blocks.40.attn.to_out.0", + "transformer_blocks.40.attn.to_q", + "transformer_blocks.40.attn.to_v", + "transformer_blocks.40.ff.net.0.proj", + "transformer_blocks.40.ff.net.2", + "transformer_blocks.41.attn.to_k", + "transformer_blocks.41.attn.to_out.0", + "transformer_blocks.41.attn.to_q", + "transformer_blocks.41.attn.to_v", + "transformer_blocks.41.ff.net.0.proj", + "transformer_blocks.41.ff.net.2", + "transformer_blocks.42.attn.to_k", + "transformer_blocks.42.attn.to_out.0", + "transformer_blocks.42.attn.to_q", + "transformer_blocks.42.attn.to_v", + "transformer_blocks.42.ff.net.0.proj", + "transformer_blocks.42.ff.net.2", + "transformer_blocks.43.attn.to_k", + "transformer_blocks.43.attn.to_out.0", + "transformer_blocks.43.attn.to_q", + "transformer_blocks.43.attn.to_v", + "transformer_blocks.43.ff.net.0.proj", + "transformer_blocks.43.ff.net.2", + "transformer_blocks.44.attn.to_k", + "transformer_blocks.44.attn.to_out.0", + "transformer_blocks.44.attn.to_q", + "transformer_blocks.44.attn.to_v", + "transformer_blocks.44.ff.net.0.proj", + "transformer_blocks.44.ff.net.2", + "transformer_blocks.45.attn.to_k", + "transformer_blocks.45.attn.to_out.0", + "transformer_blocks.45.attn.to_q", + "transformer_blocks.45.attn.to_v", + "transformer_blocks.45.ff.net.0.proj", + "transformer_blocks.45.ff.net.2", + "transformer_blocks.46.attn.to_k", + "transformer_blocks.46.attn.to_out.0", + "transformer_blocks.46.attn.to_q", + "transformer_blocks.46.attn.to_v", + "transformer_blocks.46.ff.net.0.proj", + "transformer_blocks.46.ff.net.2", + "transformer_blocks.47.attn.to_k", + "transformer_blocks.47.attn.to_out.0", + "transformer_blocks.47.attn.to_q", + "transformer_blocks.47.attn.to_v", + "transformer_blocks.47.ff.net.0.proj", + "transformer_blocks.47.ff.net.2", + "transformer_blocks.48.attn.to_k", + "transformer_blocks.48.attn.to_out.0", + "transformer_blocks.48.attn.to_q", + "transformer_blocks.48.attn.to_v", + "transformer_blocks.48.ff.net.0.proj", + "transformer_blocks.48.ff.net.2", + "transformer_blocks.49.attn.to_k", + "transformer_blocks.49.attn.to_out.0", + "transformer_blocks.49.attn.to_q", + "transformer_blocks.49.attn.to_v", + "transformer_blocks.49.ff.net.0.proj", + "transformer_blocks.49.ff.net.2", + "transformer_blocks.5.attn.to_k", + "transformer_blocks.5.attn.to_out.0", + "transformer_blocks.5.attn.to_q", + "transformer_blocks.5.attn.to_v", + "transformer_blocks.5.ff.net.0.proj", + "transformer_blocks.5.ff.net.2", + "transformer_blocks.6.attn.to_k", + "transformer_blocks.6.attn.to_out.0", + "transformer_blocks.6.attn.to_q", + "transformer_blocks.6.attn.to_v", + "transformer_blocks.6.ff.net.0.proj", + "transformer_blocks.6.ff.net.2", + "transformer_blocks.7.attn.to_k", + "transformer_blocks.7.attn.to_out.0", + "transformer_blocks.7.attn.to_q", + "transformer_blocks.7.attn.to_v", + "transformer_blocks.7.ff.net.0.proj", + "transformer_blocks.7.ff.net.2", + "transformer_blocks.8.attn.to_k", + "transformer_blocks.8.attn.to_out.0", + "transformer_blocks.8.attn.to_q", + "transformer_blocks.8.attn.to_v", + "transformer_blocks.8.ff.net.0.proj", + "transformer_blocks.8.ff.net.2", + "transformer_blocks.9.attn.to_k", + "transformer_blocks.9.attn.to_out.0", + "transformer_blocks.9.attn.to_q", + "transformer_blocks.9.attn.to_v", + "transformer_blocks.9.ff.net.0.proj", + "transformer_blocks.9.ff.net.2" + ], + "rollout_mode": "predict_x0_fresh_renoise", + "attention_mode": "native_non_causal", + "attention_is_causal": false, + "temporal_ar_blocks": 0, + "model_evaluations_per_rollout": 4, + "sigma_endpoints_per_rollout": 5, + "student_positive_forwards_per_rollout": 4, + "transformer_block_count": 50, + "token_refiner_block_count": 2, + "normalization": "per_modality_full_sequence", + "cuda_cache_policy": "empty_cache_before_generator_backward", + "cuda_empty_cache_before_generator_backward": true, + "cuda_allocator_config": "expandable_segments:True", + "saved_tensor_offload_policy": "student_rollout_save_on_cpu", + "saved_tensor_offload_pin_memory": false, + "video_shift": 12, + "audio_shift": 3, + "height": 768, + "width": 1344, + "num_frames": 124, + "strategy": "fsdp", + "gradient_checkpointing": true, + "seed": 20260812, + "student_lr": 0.00002, + "critic_lr": 0.00005, + "max_steps": 1500, + "prompt_start_index": 0, + "prompt_cache": "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/shyang/LongLive-LoRA/Minimax-H3/training/LongLive-LoRA/artifacts/minimax_h3/prompt-cache-v5-w16-s2000/collection.json", + "prompt_cache_sha256": "590c072c951049c660ad8cbe3399746d0224029eb1ed4083119a9b50e4eb8e46", + "fsdp_sharding": "FULL_SHARD", + "fsdp_use_orig_params": true, + "fsdp_auto_wrap_classes": [ + "MiniMaxH3TransformerBlock", + "MiniMaxH3TokenRefinerBlock" + ], + "fsdp_auto_wrap_policy": "h3_peft_attention_ffn_base_projection_v8", + "fsdp_top_level_wrap_modules": [ + "proj_in", + "audio_proj_in", + "context_embedder", + "time_embedder", + "token_refiner", + "norm_out", + "proj_out", + "audio_proj_out" + ], + "fsdp_inner_wrap_modules": [ + "attn" + ], + "fsdp_attention_projection_wrap_modules": [ + "to_q", + "to_k", + "to_v", + "to_out.0" + ], + "fsdp_attention_base_wrap_module": "base_layer", + "fsdp_ffn_projection_wrap_modules": [ + "net.0.proj", + "net.2" + ], + "fsdp_handle_uniform_dtype_required": true, + "fsdp_root_parameter_tensors_expected": 0, + "fsdp_cpu_offload_policy": "frozen_h3_base_handles_cpu_lora_cuda", + "fsdp_cpu_offload_params": true, + "fsdp_limit_all_gathers": true, + "fsdp_backward_prefetch": "disabled", + "flex_attention_backend": "native", + "fsdp_sync_module_states": false, + "longrun_schema_version": 12, + "total_optimizer_steps": 1500, + "checkpoint_every": 100, + "critic_update_every": 1, + "student_update_ratio": 5, + "student_update_rule": "(global_step-1)%5==0", + "adam_betas": [ + 0, + 0.9 + ], + "student_betas": [ + 0, + 0.9 + ], + "critic_betas": [ + 0.9, + 0.999 + ], + "weight_decay": 0.01, + "lora_dropout": 0, + "max_grad_norm": 1, + "sample_rng_scheme": "named_rank_local_outer_step_substreams_v1", + "joint_optimizer": "AdamW", + "optimizer_layout": "joint_role_groups_per_role_adamw_v2", + "optimizer_param_group_roles": [ + "student", + "critic" + ], + "longrun_config_version": 12, + "checkpoint_schema_version": 2, + "lora_target_policy": "h3_full_50_main_plus_2_token_refiner_attention_ffn_v1", + "lora_target_scope": "50_main_plus_2_token_refiner_blocks", + "lora_target_branches": [ + "attn", + "ff" + ], + "lora_target_module_count": 312, + "lora_excluded_scopes": [ + "adaln_and_timestep", + "normalization", + "top_level_input_context_output_projections" + ], + "prompt_runtime_selection": { + "policy": "rank_balanced_50pct_authored_audio_v1", + "rank_cache_loader": "minimax_h3.longlive_prompt_distributed.load_rank_prompt_cache", + "index_formula": "global_step-1", + "absolute_global_step_resume_guard": true, + "dataset_rows": 248221, + "sampler_seed_shared_across_ranks": 20260806, + "sampler_epoch": 0, + "sampler_shuffle": true, + "sampler_drop_last": true, + "sampler_rank_epoch_rows": 15513, + "sampler_epoch_dropped_rows": 13, + "world_size": 16, + "conditions_per_rank": 2000, + "scheduled_conditions": 32000, + "shared_prompt_across_ranks": false, + "rank_distinct_noise": true, + "critic_conditions_per_rank": 1500, + "student_update_rule": "(global_step-1)%5==0", + "student_conditions_per_rank": 300, + "student_uses_same_rank_condition_as_critic": true, + "source_original_cache_conditions": 32000, + "used_outer_condition_slots": 24000, + "used_authored_outer_slots": 12000, + "used_authored_student_slots": 2400, + "original_condition_seed_retained": true, + "text_selection_uses_rng": false, + "distinct_prompt_text_per_rank_verified": false, + "rank_distinct_condition_ids": true + }, + "technical_document": { + "path": "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/shyang/LongLive-LoRA/Minimax-H3/MINIMAX_H3_DMD_TECHNICAL_DOCUMENT_ZH.md", + "sha256": "e293cd2045f26ee6da47d32b440ee10db2a540eeaa4626e5e8b73a7f0318392b", + "documented_steps": 4, + "requested_steps": 4, + "grid_adaptation": "none_exact_document_four_step_grid" + }, + "semantic_role_map": { + "generator": "student", + "fake_score": "critic", + "real_score_teacher": "adapters_disabled" + }, + "model_variant": "standard_full_adaln", + "adaln_curve_rank": null, + "student_weight_decay": 0.01, + "critic_weight_decay": 0, + "lr_scheduler": "constant", + "lora_scale": 1, + "lora_master_dtype": "float32", + "fresh_noise_transitions_per_rollout": 3, + "target_stage_distribution": "uniform_0_through_3", + "outer_step_shared_random_variables": [ + "target_stage", + "score_base_sigma" + ], + "score_base_sigma_min": 0.02, + "score_base_sigma_max": 0.98, + "video_sigma_grid": [ + 1, + 0.972972972972973, + 0.9230769230769231, + 0.8, + 0 + ], + "audio_sigma_grid": [ + 1, + 0.9, + 0.75, + 0.5, + 0 + ], + "fresh_noise_draw_order": [ + "video", + "audio" + ], + "video_audio_transition_noise_independent": true, + "sample_rng_substreams": [ + "detached_initial_latents", + "generator_initial_latents", + "shared_stage_sigma", + "detached_rollout", + "fake_dsm", + "generator_rollout", + "generator_dmd" + ], + "sampler_matches_training_and_inference": true, + "teacher_trajectory_regression": true, + "real_latent_regression": false, + "gan_loss": false, + "ema": false, + "loss_objective": "audio_quality_audio_mixed_endpoint_latent", + "video_loss_weight": 1, + "audio_loss_weight": 2, + "audio_tokens_retained": true, + "audio_forward_retained": true, + "audio_raw_loss_metric_retained": true, + "audio_output_retained": true, + "rng_and_forward_contract_unchanged": false, + "student_rng_and_forward_contract_unchanged": true, + "gradient_checkpointing_scope": "main_transformer_blocks", + "gradient_checkpointing_policy": "main_transformer_blocks_reentrant", + "gradient_checkpointing_use_reentrant": true, + "token_refiner_gradient_checkpointing": false, + "attention_backend": "pytorch_native_flash_sdpa_forced", + "attention_backend_dispatch_name": "_native_flash", + "attention_requires_padless_training_sequence": true, + "layout_cache_policy": "per_step_rebuild_release_no_gpu_cache", + "distributed_backend": "FSDP_FULL_SHARD", + "documented_distributed_backend": "ZeRO-3", + "distributed_backend_deviation_reason": "The validated local 16-rank launcher uses equivalent full parameter, gradient, and optimizer sharding through FSDP FULL_SHARD.", + "data_deviation": "Half of each global batch retains original LongLive text; half uses separately authored audio-conditioned text. Other endpoint-latent settings are unchanged.", + "distributed_contract": { + "nodes": 4, + "gpus_per_node": 4, + "world_size": 16, + "per_rank_batch_size": 1, + "global_batch_size": 16, + "global_steps": 1500, + "scheduled_conditions": 24000 + }, + "experiment_id": "audio-data-mixed-fresh-20260914", + "teacher_video_cfg": 1, + "teacher_audio_cfg": 1, + "teacher_reference_cache": "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/shyang/LongLive-LoRA/Minimax-H3/experiments/audio_cfg_1600_20260911/pilot_outputs/reference-rank0.pt", + "teacher_reference_cache_sha256": "902fd97f6ced7146ebaeb790bd2d5ef89173594ef1f9a650099a0bbeeca342d3", + "teacher_reference_kind": "whitespace_conditioned_not_calibrated_unconditional", + "extra_guidance_target": "real_score_teacher_only", + "cfg_enabled": false, + "cfg_scope": "teacher_only_student_inference_is_single_pass", + "student_extra_cfg": false, + "critic_extra_cfg": false, + "used_conditions_per_rank": 1500, + "used_total_conditions": 24000, + "source_cache_conditions_per_rank": 2000, + "training_source_sha256": { + "__init__.py": "7928c5364f079a90f05d14d9ef8c3949173f1e5caf9010f08779793b3fa50291", + "build_prompt_cache.py": "5ed06cb54ca28bc776d92f6da13b37aaa5baccb9e7e537fbcd37282f871cae98", + "cfg_guidance.py": "ee796ccfc5a03aef1e98054eaf30585d74b190e54308ba1ca192e2373e06df24", + "dmd_4step.py": "3d72c4dd3b3793f91fdcce3d37a809e18be805889fd3cc4bf57d96426eef8d0f", + "dmd_8step.py": "77ab7e2c7ce40cad12b141b1b24467406ff66a7fa420b5a752bdc5d5b74dc2b1", + "dmd_core.py": "4c2cc6909d41356d27c95275a0e32a5a3d96038634cb83e42a270424f6a86164", + "encode_longlive_prompt_cache.py": "7bdd4004bf4973158bac9866975c08b825579ee279c45916bd7789eacd57dc42", + "encode_longlive_prompt_cache_v4.py": "b39b37977235320b541e54d78833088cabd3199c8fe77059399ea8438cabcb29", + "extract_dmd8_inline5.py": "9f43ef1bad719a328f7ece6d14657534ec9cf76ebe91405760fed15d5bed821e", + "fresh_noise_scheduler.py": "404cbb1e6eb05e5a11222b95f86ba5eec88fb09718d32094a04dd4ac834fbcdb", + "fresh_noise_scheduler_4step.py": "ae6d4887471edb94096be0d65dfcd850131f819620d35450a26289dd90a5dc8e", + "full_h3_lora.py": "ad4b08ff2aae784452f5dd3468fc7e571cecb9665ada80ef5f60f04bf8cf4343", + "infer_base.py": "dd6a5ef8aad12ba76c65d74048726fd0f9623a116f216b4a300ef42272d09b93", + "infer_student.py": "c3dfa6456fcc1d49500c5a6445431e0254181f0d5342fa24353dc57e8c18c5d8", + "infer_student_4step.py": "3bbb180764ef744791065e172acf759b65f8ff7b8e85a3ac8cdc9ee59c738db3", + "infer_student_8step.py": "867252e18cb1c53c266d2a3a658453d63e7b9a38711a94ec36bcf88d7f944f5b", + "longlive_lora.py": "4d061332055b2b59dfa7632ca7502f98ecdace283333b28f872b1f74f2c1cb4e", + "longlive_prompt_distributed.py": "eff9f6e68cc17f4b0237990ac79478a45d5a6e846a4b9b812a508cc03faca48d", + "longlive_prompt_set.py": "ed80195faf3f1614c2ec5d3678298505eae00490456ebc365df995324cc33703", + "model_provenance.py": "8b5409504258657fdab9665dbd0a51a09e5503e468a9b0e7eb76ad0c2f55ab20", + "probe_student_lora_ab.py": "6ff009a751dc07c54cf8dedc98b4061f36080eec93c5068996c5bcec448f0150", + "train_dmd.py": "a2f2c71ead08514d96898019eee6c39536192eeb5b017bb2c7454d1cfb0e7172", + "train_dmd_longrun.py": "5843b7c57eea8c602bd06cae4580d4efae6fc15be4d370b4a2951cf60baf6ecb", + "train_dmd_longrun_4step.py": "5a751327012031d7a51e5004dc3d47094d7810e52ed72d05b04b513bedcee1bf", + "train_dmd_longrun_4step_autoresume.py": "42abbef8788b0d86ae6d2d6094bec6144d754d355748eefea6a7f189b7bd8918", + "train_dmd_longrun_8step.py": "2f0302f973ba0729040687741eed95edbf321bd4c85745ec1ca977fe91c9e7a0", + "train_dmd_longrun_8step_autoresume.py": "21b033ae059b73587257f6923db3587ce6b16c18aa4e8b6f5a91a615905d68dd", + "train_dmd_longrun_v2.py": "2ef1d89d400df60a86be10b8828ea7c4b87fb96112538bfd25e7baaf5e8bc357", + "train_dmd_longrun_v3.py": "ca1905890c1e3c849ccb7f189166d7d7ddaa22b9d6162c629beab8e0b5c65c81", + "train_dmd_longrun_v4.py": "e27328e359c6ff4fccd38623fb7eadb84c48f131fb547ca82bd7364c3f2dd44c", + "train_dmd_longrun_v5.py": "14778be13523721a38fb539d973c6d0ad7eff8054ab146a76adcb684bdd9c7cf", + "train_utils.py": "fd64c02074680470776fa9e8f58578bde7fdb90603d2cad3eab5e9216c6f8e9c", + "visualize_teacher_x0.py": "962f86b0fb699accecc36eb0b57fdc278835c22731f78f5e4d5fe36c3a33d49f" + }, + "initialization": "original_Base_fa9c8ab1_new_student_and_critic_LoRA_A_random_B_zero", + "old_lora_warm_start": false, + "operational_resume_scope": "this_new_experiment_only", + "audio_objective": { + "dmd_weight": 1, + "teacher_endpoint_nfe": 50, + "endpoint_weight": 1, + "spectral_weight": 0, + "label": "新初始化 · 50% 音频描述数据 · DMD + 教师音频目标", + "endpoint_target": "50_frozen_Base_Euler_calls_from_exact_selected_student_noisy_AV_input_to_zero_sigma", + "endpoint_is_not_cached_Base50_output": true, + "endpoint_grid": "official_shared_base_sigma_subgrid_video_shift12_audio_shift3", + "spectral_ffts": [], + "spectral_hop": "n_fft/4", + "spectral_log_floor": 0.0001, + "spectral_decoder": null, + "waveform_gain_normalization": false, + "inference_postprocessing": false + }, + "fresh_source_sha256": { + "train_fresh.py": "6b478c00a64c5bb405823960f541fa3d0faf4cc14b17db5cc762043a71ddec10", + "common.py": "60d1bd3d7774dca1ac1c54234ac05fd2fcb7a0ad1a01450d789fe882f7ad2ef7", + "audio_objectives.py": "4e22fd00a67f5e7e7ce03bec2607911079aad7129fc6b50c88cae3385bc1c605" + }, + "milestone_evaluation_every": 200, + "pilot_training_budget_is_not_goal_completion": true, + "original_prompt_component": { + "prompt_dataset": { + "bytes": 145875068, + "line_count": 248221, + "path": "prompts/vidprom_filtered_extended.txt", + "repo_id": "Efficient-Large-Model/LongLive-1.3B", + "revision": "cda9138d93872ef109d9a74c270184eb3a5acc6e", + "row_policy": "nonempty_python_str_strip_then_python_str_rstrip", + "sha256": "7896742f468bc8aef9e4547424d1ce0a951acdb2a82233790155401a99bf5aa5" + }, + "prompt_dataset_row_count": 248221, + "prompt_sampler": { + "batch_size_per_rank": 1, + "dataset_size": 248221, + "drop_last": true, + "epoch": 0, + "global_steps": 2000, + "name": "torch.utils.data.distributed.DistributedSampler", + "seed": 20260806, + "shuffle": true, + "world_size": 16 + }, + "prompt_schedule_audit": { + "dataset_size": 248221, + "global_steps": 2000, + "scheduled_rows": 32000, + "unique_source_indices": 32000, + "world_size": 16 + }, + "prompt_collection_provenance": { + "loader": "minimax_h3.longlive_prompt_distributed.load_prompt_cache_collection", + "path": "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/shyang/LongLive-LoRA/Minimax-H3/training/LongLive-LoRA/artifacts/minimax_h3/prompt-cache-v5-w16-s2000/collection.json", + "bytes": 4539, + "sha256": "590c072c951049c660ad8cbe3399746d0224029eb1ed4083119a9b50e4eb8e46", + "payload_schema_version": 1, + "artifact_type": "minimax_h3_longlive_distributed_prompt_cache_collection", + "model_revision": "fa9c8ab1eaa21c8ae25e7e40b83b2e6002f340af", + "shards": [ + { + "bytes": 2340850895, + "conditions": 2000, + "path": "rank-00000-of-00016.pt", + "rank": 0, + "sha256": "ac25de38c7aa2a1f0f8e0ccd76d662eeba514bc3e807b43877ab4b3f6c408bac" + }, + { + "bytes": 2336362127, + "conditions": 2000, + "path": "rank-00001-of-00016.pt", + "rank": 1, + "sha256": "ef800ca248af17d3d05c10354b1e1fed2431f63c7fd7b6bfeb4cb9b40ca36025" + }, + { + "bytes": 2339642127, + "conditions": 2000, + "path": "rank-00002-of-00016.pt", + "rank": 2, + "sha256": "e258544f5b2c1eb79912418445242f728d44bf0bc4026f3930e2bfa035e6b0ed" + }, + { + "bytes": 2340954703, + "conditions": 2000, + "path": "rank-00003-of-00016.pt", + "rank": 3, + "sha256": "d9ef8adb270eca7923354dc9f6df0cb606604e6beb5aa750fdc463af0a41fd4d" + }, + { + "bytes": 2333896015, + "conditions": 2000, + "path": "rank-00004-of-00016.pt", + "rank": 4, + "sha256": "12a94a253a5f9a94441e5c89c3c097cdfe665ea23200557b9caad7462b3952dd" + }, + { + "bytes": 2337396047, + "conditions": 2000, + "path": "rank-00005-of-00016.pt", + "rank": 5, + "sha256": "c0f349ca80065522c529f40581b32c928b1e04dc4245a6dd54dbc52c4b184482" + }, + { + "bytes": 2352031311, + "conditions": 2000, + "path": "rank-00006-of-00016.pt", + "rank": 6, + "sha256": "3e74eb0061eda77f3e620680fccea76cc1e596a871b27c9436a5cc964a11cfd6" + }, + { + "bytes": 2346638735, + "conditions": 2000, + "path": "rank-00007-of-00016.pt", + "rank": 7, + "sha256": "51cccd347d6055dafc27c6734d81d23f93d202681d8d857ac3d40e2de0d052fe" + }, + { + "bytes": 2336423759, + "conditions": 2000, + "path": "rank-00008-of-00016.pt", + "rank": 8, + "sha256": "b97823b091e0530b21a925f0b4103a8e022e9dd276422dd182079a2827d6a100" + }, + { + "bytes": 2327823759, + "conditions": 2000, + "path": "rank-00009-of-00016.pt", + "rank": 9, + "sha256": "d5307b0250d532ac54484726d18e0f53fee3043164c6f634b26b196dd85be4f8" + }, + { + "bytes": 2349843407, + "conditions": 2000, + "path": "rank-00010-of-00016.pt", + "rank": 10, + "sha256": "33e47a61beb520341cce9313ca278946c6d978a626186cc18a2bab4016700417" + }, + { + "bytes": 2318440719, + "conditions": 2000, + "path": "rank-00011-of-00016.pt", + "rank": 11, + "sha256": "5810fcddac96e8e8ed91a0a3c60630b97aa03ccf5f4006ff00bf166917c13046" + }, + { + "bytes": 2330704527, + "conditions": 2000, + "path": "rank-00012-of-00016.pt", + "rank": 12, + "sha256": "5815f9a772903857ead5d6d70a5c0135e73e475b545746a4a449f5e2dc2b1c4a" + }, + { + "bytes": 2326816463, + "conditions": 2000, + "path": "rank-00013-of-00016.pt", + "rank": 13, + "sha256": "894effe10ed17b74d6b5d6dbd20e11da7d49385fbe8b99bb912d0983dc1a5298" + }, + { + "bytes": 2327963919, + "conditions": 2000, + "path": "rank-00014-of-00016.pt", + "rank": 14, + "sha256": "3b6c62e0619f1e280720ea776b3b888b4b4882e7f75b032c140d160e805a3724" + }, + { + "bytes": 2350203407, + "conditions": 2000, + "path": "rank-00015-of-00016.pt", + "rank": 15, + "sha256": "6fc63df45b01b08172fecb3b9229ea338b6083af62f3fdc5cfef1aacb287af4d" + } + ], + "validate_only_full_shard_sha256_readback": true, + "production_manifest_verifies_all_shard_sizes": true, + "production_every_rank_verifies_own_shard_sha256": true + }, + "prompt_runtime_selection": { + "policy": "rank_local_absolute_global_step_v1", + "rank_cache_loader": "minimax_h3.longlive_prompt_distributed.load_rank_prompt_cache", + "index_formula": "global_step-1", + "absolute_global_step_resume_guard": true, + "dataset_rows": 248221, + "sampler_seed_shared_across_ranks": 20260806, + "sampler_epoch": 0, + "sampler_shuffle": true, + "sampler_drop_last": true, + "sampler_rank_epoch_rows": 15513, + "sampler_epoch_dropped_rows": 13, + "world_size": 16, + "conditions_per_rank": 2000, + "scheduled_conditions": 32000, + "shared_prompt_across_ranks": false, + "rank_distinct_prompt": true, + "rank_distinct_noise": true, + "critic_conditions_per_rank": 1500, + "student_update_rule": "(global_step-1)%5==0", + "student_conditions_per_rank": 300, + "student_uses_same_rank_condition_as_critic": true + } + }, + "prompt_dataset": { + "kind": "deterministic_mixture", + "original_fraction": 0.5, + "authored_audio_fraction": 0.5, + "authored_candidate_manifest": "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/shyang/LongLive-LoRA/Minimax-H3/experiments/audio_fresh_quality_20260914/audio_data_candidates.json", + "authored_candidate_manifest_sha256": "4142bb4f21ba8ec2513e9839d8d4a793f6b4cca6580cc4e8bf09397351863f0d", + "authored_embedding_collection": "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/shyang/LongLive-LoRA/Minimax-H3/experiments/audio_fresh_quality_20260914/audio_candidate_cache/COLLECTION.json", + "authored_embedding_collection_sha256": "dd3f32d488f0b7eb40d0f485c994d5c881c20e491c8e258b6c6b83fce55c90a3", + "authored_cache_contract": { + "artifact_type": "h3_authored_audio_candidate_embeddings_v1", + "schema_version": 1, + "model_revision": "fa9c8ab1eaa21c8ae25e7e40b83b2e6002f340af", + "workflow": "positive_text_encoding", + "IR": false, + "candidate_manifest_sha256": "4142bb4f21ba8ec2513e9839d8d4a793f6b4cca6580cc4e8bf09397351863f0d", + "qualified_teacher_receipt_sha256": "39eea24416458a42a59fa9cf38b3e36df1fa361058f713f55332d772fd673db3", + "qualification_scope": "Sampled transcript gate only, not perceptual or whole-pool quality.", + "sources": { + "audio_candidate_cache.py": "234d4a9cfad2952b2f98b76f70c77a46399483e9d6da5030ac80639e1fb4b616", + "encode_audio_candidates.py": "5adcb38b74cb83423ba39b6c690e4c9a9294f9a7cc0ed3796a83f7964352de18", + "audio_data_candidates.py": "28cebc27e0c8d914cd716401f9ebb9c03133c5803b362165ddb2e56aa7744274" + }, + "official_encoder_sha256": "dc37019187d287d1346bb0a0f7d080bb9d1443ce1ae4b0f65dba57b40a75d8eb", + "text_dim": 5120 + }, + "authored_prompt_strings": 256, + "authored_core_scenes": 32, + "evaluation_text_and_reference_phrases_excluded": true, + "semantic_near_duplicate_exclusion_proven": false + }, + "prompt_cache_argument_scope": "original_LongLive_component_and_preserved_noise_seed_schedule_only", + "data_trial_arm": "audio_mixed_endpoint_latent", + "matched_loss_recipe": "endpoint_latent", + "data_trial_source_sha256": { + "train_audio_mixed.py": "6193189881a1fef24144c4ddf57aa77965bf69308a1392724c826720e8d6c338", + "audio_mixed_conditions.py": "2b41ff3d1249ccdd7857d7b384476e8b65ec09593a44c810e931c6106d863dfd", + "audio_candidate_cache.py": "234d4a9cfad2952b2f98b76f70c77a46399483e9d6da5030ac80639e1fb4b616", + "audio_data_candidates.py": "28cebc27e0c8d914cd716401f9ebb9c03133c5803b362165ddb2e56aa7744274" + }, + "training_budget_extension": { + "same_run_not_foreign_LoRA_warm_start": true, + "pilot_config_sha256": "7114d47ab94108ab0e54a972e96dd536d442ec5ce5dc406b41bf999440e19ab1", + "original_planned_outer_iterations": 400, + "extended_outer_iterations": 1500, + "transition_checkpoint_step": 50, + "transition_checkpoint_success_sha256": "a702baf7e06d8c823bce49399ee85126bc68a517a1e12d2dc486e8edc98db00d", + "optimizer_and_per_rank_rng_restored": true, + "checkpoint_payloads_are_never_rewritten": true, + "evaluation_outer_iterations": [ + 200, + 400, + 600, + 800, + 1000, + 1200, + 1400, + 1500 + ], + "source_sha256": "309b8877de66a25c3ecdc85ffad46ad65aa007f978b04ed441fb4d855e37c40b", + "registry_source_sha256": "138bd3bc2a4f9fd7a386d55d1d4c6641e880d921b20182e76d8babbf6a68a66f" + }, + "config_sha256": "916da7041fe91de966917d8600ac7c1ec64a9b9d4171c457151c687efeb7bf5a" +} diff --git a/training_recipe/audio_candidate_cache.py b/training_recipe/audio_candidate_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..0d0a5004b9ce4c04463a0e10d4e6d55c2790d93b --- /dev/null +++ b/training_recipe/audio_candidate_cache.py @@ -0,0 +1,108 @@ +"""Separate, truthful cache schema for authored audio-training candidates.""" +from common import * +from audio_data_candidates import candidates,validate + +CACHE=EXP/'audio_candidate_cache' + + +def candidate_manifest(): + manifest=read(EXP/'audio_data_candidates.json') + rows=candidates() + if manifest['candidates']!=rows or manifest['checks']!=validate(rows) or manifest['source_sha256']!=sha(EXP/'audio_data_candidates.py'): + raise RuntimeError('Authored candidate source/manifest mismatch') + return manifest + + +def qualified_teacher(): + root=EXP/'audio_data_qualification' + receipt=read(root/'SUCCESS.json') + if receipt['status']!='success' or receipt['samples']!=8 or receipt['speech_samples']!=4: + raise RuntimeError('Teacher-data preflight is incomplete') + for filename,key in (('MANIFEST.json','manifest_sha256'),('AUDIO_DIAGNOSTICS.json','diagnostics_sha256'), + ('asr/TRAINING_ASR.json','asr_sha256')): + if sha(root/filename)!=receipt[key]: + raise RuntimeError('Teacher-data preflight report changed') + manifest=read(root/'MANIFEST.json') + from qualify_audio_data import plan as qualification_plan + if manifest['plan']!=qualification_plan(): + raise RuntimeError('Teacher preflight used different data or Base revision') + selected={row['id']:row for row in manifest['plan']['samples']} + if len(manifest['samples'])!=8 or {row['id'] for row in manifest['samples']}!=set(selected): + raise RuntimeError('Teacher-data media coverage mismatch') + for row in manifest['samples']: + if any(row[key]!=selected[row['id']][key] for key in ('prompt','seed')): + raise RuntimeError('Teacher-data media request mismatch') + if sha(row['video'])!=row['video_sha256'] or sha(row['native_wav'])!=row['native_wav_sha256']: + raise RuntimeError('Teacher-data media changed') + asr=read(root/'asr/TRAINING_ASR.json') + speech={key:row for key,row in selected.items() if row['phrase']} + if len(asr['results'])!=4 or {r['case'] for r in asr['results']}!=set(speech) or any( + r['edit_distance']!=0 or r['text_prompt_supplied_to_asr'] or r['reference']!=speech[r['case']]['phrase'] + or r['language']!=speech[r['case']]['language'] for r in asr['results']): + raise RuntimeError('Sampled teacher transcripts require review before encoding this trial') + if not receipt['sampled_speech_reference_gate_passed'] or receipt['speech_exact_matches']!=4: + raise RuntimeError('Sampled teacher speech-reference gate has not passed') + return sha(root/'SUCCESS.json') + + +def contract(): + candidate_manifest() + source_contract() + return {'artifact_type':'h3_authored_audio_candidate_embeddings_v1','schema_version':1, + 'model_revision':REVISION,'workflow':'positive_text_encoding','IR':False, + 'candidate_manifest_sha256':sha(EXP/'audio_data_candidates.json'), + 'qualified_teacher_receipt_sha256':qualified_teacher(), + 'qualification_scope':'Sampled transcript gate only, not perceptual or whole-pool quality.', + 'sources':{name:sha(EXP/name) for name in ('audio_candidate_cache.py','encode_audio_candidates.py','audio_data_candidates.py')}, + 'official_encoder_sha256':sha(ASSETS/'external/diffusers-minimax-h3/src/diffusers/modular_pipelines/minimax_h3/encoders.py'), + 'text_dim':read(BASE/'transformer/config.json')['text_dim']} + + +def validate_condition(condition,row): + imports() + from minimax_h3.encode_longlive_prompt_cache_v4 import _validate_encoded_condition + expected={'id':row['id'],'prompt':row['prompt'], + 'prompt_sha256':hashlib.sha256(row['prompt'].encode()).hexdigest()} + if any(condition.get(key)!=value for key,value in expected.items()): + raise RuntimeError('Encoded candidate identity mismatch') + _validate_encoded_condition(condition) + if condition['prompt_embeds'].shape[-1]!=read(BASE/'transformer/config.json')['text_dim']: + raise RuntimeError('Candidate embedding width differs from original Base') + + +def load_condition(row,expected_contract): + import torch + root=CACHE/row['id'] + receipt=read(root/'SUCCESS.json') + path=root/'condition.pt' + if receipt['status']!='success' or receipt['contract']!=expected_contract or receipt['id']!=row['id']: + raise RuntimeError('Candidate encoding receipt mismatch') + if path.stat().st_size!=receipt['bytes'] or sha(path)!=receipt['sha256']: + raise RuntimeError('Candidate tensor payload changed') + payload=torch.load(path,map_location='cpu',weights_only=True) + if payload['contract']!=expected_contract: + raise RuntimeError('Candidate tensor contract mismatch') + condition=payload['condition'] + validate_condition(condition,row) + return condition + + +def verify_collection(): + expected=contract() + manifest=read(CACHE/'COLLECTION.json') + rows=candidate_manifest()['candidates'] + if manifest['contract']!=expected or [r['id'] for r in manifest['conditions']]!=[r['id'] for r in rows]: + raise RuntimeError('Audio candidate cache coverage/contract mismatch') + result={} + for row,record in zip(rows,manifest['conditions'],strict=True): + if record['receipt_sha256']!=sha(CACHE/row['id']/'SUCCESS.json'): + raise RuntimeError('Audio candidate row receipt changed') + result[row['id']]=load_condition(row,expected) + return result,{'collection_sha256':sha(CACHE/'COLLECTION.json'),'contract':expected} + + +def load_pool(): + receipt=read(CACHE/'SUCCESS.json') + if receipt['status']!='success' or receipt['conditions']!=256 or receipt['collection_sha256']!=sha(CACHE/'COLLECTION.json'): + raise RuntimeError('Audio candidate cache is incomplete or changed') + return verify_collection() diff --git a/training_recipe/audio_data_candidates.py b/training_recipe/audio_data_candidates.py new file mode 100644 index 0000000000000000000000000000000000000000..bc55be5802e20a6eb9730faad6b6c28603fa7e89 --- /dev/null +++ b/training_recipe/audio_data_candidates.py @@ -0,0 +1,171 @@ +"""Authored audio-conditioned training candidates, not model-generated results.""" +import argparse +import unicodedata +from collections import Counter +from common import * + +GROUPS=('clean_speech','ambient_speech','discrete_effects','continuous_effects') +# Eight core scenes per group. Variants change framing, not the underlying event. +SCENES={ + 'clean_speech':[ + ('palette','An adult art instructor holds a small color palette in a quiet studio.', + 'english','The blue paint needs more water.','Only very soft indoor room tone under the voice.'), + ('fabric','An adult tailor gently lifts a folded fabric sample in a quiet fitting room.', + 'english','This fabric feels soft and light.','Only very soft indoor room tone under the voice.'), + ('chess','An adult chess tutor points to a knight on a wooden board in a quiet study.', + 'english','Move the knight beside the king.','One quiet fingertip contact on the board before the voice.'), + ('calendar','An adult office worker holds a blank calendar in a quiet meeting room.', + 'english','Our meeting starts at half past two.','Soft natural room tone; no other voices.'), + ('pencil','An adult drawing teacher holds a pencil in a quiet classroom.', + 'chinese','这支铅笔很好用。','Only very soft indoor room tone under the voice.'), + ('scarf','An adult customer holds a wool scarf in a quiet clothing shop.', + 'chinese','这条围巾很柔软。','A faint cloth rustle followed by the clear voice.'), + ('chair','An adult furniture maker stands beside a wooden chair in a quiet showroom.', + 'chinese','这把椅子很结实。','Soft natural room tone; no other voices.'), + ('notebook','An adult designer closes an unmarked notebook in a quiet workspace.', + 'chinese','请把本子放在这里。','A gentle notebook closing sound before the clear voice.'), + ], + 'ambient_speech':[ + ('aquarium','An adult caretaker stands beside an indoor aquarium with a visible small water filter.', + 'english','The water looks clear this morning.','A low continuous filter hum and gentle water trickle remain behind the foreground voice.'), + ('laundry','An adult attendant stands beside a slowly turning washing machine in a laundry room.', + 'english','The towels will be dry soon.','A steady low drum motor sound remains behind the foreground voice.'), + ('printer','An adult worker stands beside an office printer as one plain sheet comes out.', + 'english','Please leave the pages on the desk.','A short paper-feed whir and soft sheet rustle, then quiet room tone under the voice.'), + ('window_blind','An adult resident gently raises a fabric window blind in an apartment.', + 'english','The room feels brighter already.','Soft blind-cord movement and distant mild street ambience behind the voice.'), + ('ceiling_fan','An adult resident stands near a slowly rotating ceiling fan in a living room.', + 'chinese','风扇转得很慢。','A soft continuous fan hum remains behind the clear foreground voice.'), + ('water_dispenser','An adult office worker stands beside a water dispenser while filling a glass.', + 'chinese','杯子里的水够了。','A brief natural stream of water ends before the foreground voice.'), + ('elevator','An adult building attendant stands beside an opening elevator in a quiet lobby.', + 'chinese','电梯已经到了。','A soft arrival chime and a gentle door motor sound precede the clear voice.'), + ('paper_fan','An adult craft instructor slowly opens a paper fan in a workshop.', + 'chinese','我们先把扇子打开。','Light paper and bamboo movement synchronized with the opening, under the foreground voice.'), + ], + 'discrete_effects':[ + ('wood_blocks','Two adult hands tap two small wooden blocks together twice, with a clear pause between taps.', + None,None,'Exactly two dry wooden contact sounds match the two visible taps, with a natural short decay and quiet room tone between them.'), + ('ceramic_cup','An adult hand sets an empty ceramic cup on a wooden coaster and then releases the handle.', + None,None,'One gentle ceramic-on-wood contact at the visible landing, then quiet room tone.'), + ('scissors','An adult hand makes three slow cuts through one sheet of plain paper with scissors.', + None,None,'Three crisp paper-cutting sounds align with the three visible scissor closures; quiet pauses remain between cuts.'), + ('door_latch','An adult hand slowly closes a wooden cabinet door until its latch engages.', + None,None,'A soft hinge movement followed by one clear latch click at the final contact.'), + ('zipper','An adult hand smoothly closes the zipper of a canvas pouch and lets go.', + None,None,'One brief zipper rasp follows the visible zipper motion and stops immediately when the hand stops.'), + ('keys','An adult hand places a small ring of metal keys onto a felt mat and withdraws.', + None,None,'A brief light key jingle and a muffled landing sound coincide with contact; no continuing rattle after the keys settle.'), + ('velcro','Two adult hands slowly separate a short hook-and-loop fabric strap and hold it apart.', + None,None,'A short natural fabric tearing rasp tracks the separation and ends when the two pieces are still.'), + ('ping_pong','An adult hand drops one table-tennis ball onto a wooden desk; it bounces three times and is caught.', + None,None,'Three light hollow impacts become closer together, followed by a soft catch and quiet room tone.'), + ], + 'continuous_effects':[ + ('desk_fan','A small desktop fan rotates steadily beside a plain curtain in a quiet room; the curtain moves gently.', + None,None,'A soft stable rotor hum with light air movement, matching the steady fan speed. Preserve the intended hum without extra crackles or unrelated effects.'), + ('sewing_machine','An adult slowly feeds plain cotton under a domestic sewing machine needle at constant speed.', + None,None,'An even sequence of small needle clicks over a low motor tone follows the visible steady mechanism, then stops with the needle.'), + ('vinyl_brush','An adult hand gently sweeps a soft brush across a wooden tabletop in one continuous movement.', + None,None,'A soft continuous bristle-on-wood brushing sound follows the moving hand and ends when it lifts.'), + ('hand_pump','An adult steadily turns the handle of a small manual water pump in a utility room.', + None,None,'A gentle periodic handle squeak and a steady water trickle follow the visible motion, with natural room ambience.'), + ('faucet','A narrow stream of water flows from a kitchen faucet into a ceramic sink at a steady rate.', + None,None,'Continuous natural running water and small splashes correspond to the visible stream; no tonal buzzing or speech.'), + ('vent_grille','A fixed view shows a ventilation grille with a lightweight ribbon moving steadily in its gentle airflow.', + None,None,'A low stable ventilation sound with faint air movement. Retain the intended airflow and do not add intermittent alarms or clicks.'), + ('sand_tray','An adult hand slowly pours a narrow stream of dry sand into a shallow wooden tray.', + None,None,'A fine continuous granular patter tracks the falling sand and fades as the stream ends.'), + ('hand_crank','An adult smoothly rotates a simple hand-cranked wheel at an even speed on a tabletop.', + None,None,'A quiet regular mechanical rotation sound follows the wheel, with no voices or unrelated sound effects.'), + ], +} +FRAMINGS=( + 'A locked medium close view at natural eye level with soft neutral lighting.', + 'A locked medium view from slightly to the left with soft neutral lighting.', + 'A locked medium view from slightly to the right with soft natural window light.', + 'A slow, very small camera push-in with gentle diffuse indoor lighting.', + 'A steady frontal view with realistic texture and balanced exposure.', + 'A steady three-quarter view with a softly defocused background.', + 'A medium documentary view with a fixed camera and natural colors.', + 'A slightly wider fixed view that keeps the subject and relevant object clearly visible.', +) + + +def normalize(value): + return ''.join(c for c in unicodedata.normalize('NFKC',value).casefold() if c.isalnum()) + + +def candidates(): + rows=[] + for group in GROUPS: + for core,(name,scene,language,phrase,sound) in enumerate(SCENES[group]): + for variant,framing in enumerate(FRAMINGS): + prompt=f'{scene} {framing} One continuous five-second shot; keep object identity and contact motion coherent. ' + if phrase: + spoken_language='Mandarin Chinese' if language=='chinese' else 'English' + prompt+=f'The visible adult speaks once in {spoken_language}, saying exactly, "{phrase}" ' + prompt+='Use natural clear foreground speech and synchronized mouth movement. ' + prompt+='Synchronized stereo audio: '+sound+' No music, subtitles, added narration or text overlays.' + if not phrase: + prompt+=' No speech.' + rows.append({'id':f'audio_data_{group}_{name}_v{variant}', + 'group':group,'core':core,'variant':variant,'prompt':prompt, + 'language':language,'phrase':phrase}) + return rows + + +def validate(rows): + if len(rows)!=256 or len({r['id'] for r in rows})!=256 or len({r['prompt'] for r in rows})!=256: + raise ValueError('Expected 32 core scenes x eight framing variants') + if Counter(r['group'] for r in rows)!={group:64 for group in GROUPS}: + raise ValueError('Unbalanced audio curriculum') + evaluations=read(EXP/'prompts.json')['prompts'] + evaluation_text={normalize(r['prompt']) for r in evaluations} + references=read(EXP/'speech_references.json') + evaluation_phrases={normalize(phrase) for _,phrases in references.values() for phrase in phrases} + for row in rows: + text=normalize(row['prompt']) + if text in evaluation_text or any(phrase in text for phrase in evaluation_phrases): + raise ValueError('Training candidate reuses an evaluation text/utterance') + if any(term in text for term in ('orbitallaunch','roboticsprecisionassembly','stageseparation')): + raise ValueError('Highlighted evaluation scenario leaked into candidates') + if bool(row['phrase'])!=(row['group'] in GROUPS[:2]): + raise ValueError('Speech reference/category mismatch') + if row['phrase'] and row['phrase'] not in row['prompt']: + raise ValueError('Speech phrase absent from actual training prompt') + return {'candidate_prompts':256,'core_scenes':32,'framing_variants_per_scene':8, + 'explicit_speech_prompts':128,'effects_only_prompts':128, + 'evaluation_exact_text_or_reference_phrase_reused':False, + 'semantic_near_duplicate_exclusion_proven':False} + + +def preflight(rows): + # One English and one Chinese item in each speech group; two event types in + # each effects group. This is an eight-item preflight, not pool-wide proof. + selected=[r for r in rows if r['core'] in (0,4) and r['variant']==0] + if len(selected)!=8: + raise ValueError('Expected eight fixed preflight candidates') + return [dict(row,seed=2026091400+i) for i,row in enumerate(selected)] + + +def main(): + parser=argparse.ArgumentParser() + parser.add_argument('--validate-only',action='store_true') + args=parser.parse_args() + rows=candidates() + checks=validate(rows) + result={'schema_version':1,'purpose':'authored_candidate_training_text_not_model_results', + 'source_sha256':sha(__file__),'checks':checks,'candidates':rows,'preflight':preflight(rows), + 'proposed_mixture':{'original_rank_condition_fraction':0.5,'audio_candidate_fraction':0.5}, + 'training_launched':False,'teacher_preflight_completed':False,'goal_complete':False} + if not args.validate_only: + path=EXP/'audio_data_candidates.json' + if path.exists() and read(path)!=result: + raise RuntimeError('Candidate manifest already exists with different content; version it explicitly') + save(path,result) + print(json.dumps(checks,indent=2)) + + +if __name__=='__main__': + main() diff --git a/training_recipe/audio_mixed_conditions.py b/training_recipe/audio_mixed_conditions.py new file mode 100644 index 0000000000000000000000000000000000000000..bc2c8ee8e04a13e89659c507ebf6e0561100a5c3 --- /dev/null +++ b/training_recipe/audio_mixed_conditions.py @@ -0,0 +1,63 @@ +"""Deterministic 50:50 data ablation without changing the original RNG schedule.""" +from common import * +from audio_data_candidates import GROUPS,candidates + +LOOKUP={(row['group'],row['core'],row['variant']):row for row in candidates()} + + +def candidate_for(step,rank): + if isinstance(step,bool) or not isinstance(step,int) or not 1<=step<=2000: + raise ValueError('Expected absolute global step 1..2000') + if isinstance(rank,bool) or not isinstance(rank,int) or not 0<=rank<16: + raise ValueError('Expected rank 0..15') + index=step-1 + if (rank+index)%2: + return None + slot=rank//2 + group=GROUPS[slot%4] + core=(2*index+slot//4)%8 + variant=(index//4)%8 + return LOOKUP[(group,core,variant)] + + +class MixedRankConditions: + """Consume the original resume guard on every step, then replace only text. + + The original condition seed is retained even when text changes. No RNG is + used to select data. Eight ranks retain their exact original object; eight + receive an authored condition, two per audio category at every global step. + """ + def __init__(self,original,pool,rank): + if len(original)!=2000 or not 0<=rank<16: + raise ValueError('Expected the unchanged 2000-row rank schedule') + if set(pool)!={row['id'] for row in LOOKUP.values()}: + raise ValueError('Audio pool is incomplete') + self.original=original + self.pool=pool + self.rank=rank + self.last_selection=None + + def __len__(self): + return len(self.original) + + def __getitem__(self,index): + if isinstance(index,bool) or not isinstance(index,int): + raise TypeError('Mixed training schedule requires one integer index') + row=candidate_for(index+1,self.rank) + original=self.original[index] # Also advances/checks the original resume guard. + self.last_selection={'rank':self.rank,'step':index+1,'original_prompt_id':original.id, + 'rng_condition_seed_retained':original.seed,'method':'original_LongLive' if row is None else 'authored_audio', + 'audio_category':None if row is None else row['group'], + 'selected_prompt_id':original.id if row is None else row['id']} + if row is None: + return original + encoded=self.pool[row['id']] + if encoded['prompt']!=row['prompt'] or encoded['id']!=row['id']: + raise RuntimeError('Selected encoded prompt identity mismatch') + imports() + from minimax_h3.train_utils import H3Condition + return H3Condition(id=row['id'],prompt=row['prompt'],seed=original.seed, + prompt_embeds=encoded['prompt_embeds'],text_token_tags=encoded['text_token_tags']) + + def assert_complete(self): + return self.original.assert_complete() diff --git a/training_recipe/audio_objectives.py b/training_recipe/audio_objectives.py new file mode 100644 index 0000000000000000000000000000000000000000..de85cb687a92e39c7b852b63b09477edfe6aec05 --- /dev/null +++ b/training_recipe/audio_objectives.py @@ -0,0 +1,166 @@ +"""Teacher-endpoint audio regression and differentiable waveform supervision.""" +from contextlib import contextmanager +import torch +from torch.utils.checkpoint import checkpoint +from common import imports, BASE, REVISION + +imports() +from minimax_h3 import dmd_4step +from minimax_h3.cfg_guidance import GuidedTeacherTrainer +from minimax_h3.dmd_core import MultimodalLoss +from minimax_h3.train_utils import disable_adapters_for_teacher + + +def teacher_endpoint_grid(state,nfe,video_shift,audio_shift): + """Use the same shared-base clock coupling as the official Base scheduler. + + sigma = shift*u/(1+(shift-1)*u); invert the selected audio sigma to u, + subdivide u, then apply BOTH original shifts. Linear interpolation of the + two shifted sigmas separately would query a different A/V clock path. + """ + if nfe<1: + raise ValueError('Teacher endpoint requires a positive NFE') + sigma=state.audio_sigma.detach().float() + base=sigma/(audio_shift-(audio_shift-1)*sigma) + grid=torch.linspace(1,0,nfe+1,device=state.audio_rows.device)*base + video=video_shift*grid/(1+(video_shift-1)*grid) + audio=audio_shift*grid/(1+(audio_shift-1)*grid) + if not torch.allclose(video[0],state.video_sigma.float(),rtol=1e-6,atol=1e-7) or not torch.allclose(audio[0],state.audio_sigma.float(),rtol=1e-6,atol=1e-7): + raise RuntimeError('Selected student state is not on the official paired A/V clock') + return video,audio + + +def multiresolution_spectral_loss(prediction, target): + """Stereo channels evaluated independently; no waveform gain normalization. + + Mean of spectral convergence plus log-magnitude L1 over 256/1024/2048 + FFTs. Target is detached. This compares aligned teacher/student states, + not unrelated full clips or different seeds. + """ + if prediction.shape != target.shape or prediction.ndim != 2: + raise ValueError('MR-STFT expects equal [stereo_channel, waveform_sample] tensors') + if prediction.shape[-1] < 2048: + raise ValueError('Waveform too short for the declared FFTs') + target = target.detach().float() + prediction = prediction.float() + losses = [] + for fft in (256,1024,2048): + window = torch.hann_window(fft,device=prediction.device,dtype=torch.float32) + def magnitude(waveform): + return torch.stft(waveform,n_fft=fft,hop_length=fft//4,window=window, + center=False,return_complex=True).abs() + actual, wanted = magnitude(prediction), magnitude(target) + # Per channel reduction avoids one loud channel dominating the other. + convergence = torch.linalg.vector_norm(actual-wanted,dim=(-2,-1)) / torch.linalg.vector_norm(wanted,dim=(-2,-1)).clamp_min(1e-4) + log_l1 = (actual.clamp_min(1e-4).log()-wanted.clamp_min(1e-4).log()).abs().mean(dim=(-2,-1)) + losses.append((convergence+log_l1).mean()) + return torch.stack(losses).mean() + + +class FrozenAudioDecoder: + def __init__(self,device): + from diffusers import AutoencoderKLMiniMaxH3Audio + self.model = AutoencoderKLMiniMaxH3Audio.from_pretrained(str(BASE),subfolder='audio_vae', + revision=REVISION,local_files_only=True,torch_dtype=torch.float32).eval().to(device) + self.model.requires_grad_(False) + self.mean = torch.tensor(self.model.config.latents_mean,device=device,dtype=torch.float32).view(1,-1,1) + self.std = torch.tensor(self.model.config.latents_std,device=device,dtype=torch.float32).view(1,-1,1) + + def unpack(self,rows): + if rows.ndim!=3 or rows.shape[0]!=1 or rows.shape[1]%2: + raise ValueError('H3 training audio must be [1,2*T,C], channel-major') + latents = rows[0].reshape(2,-1,rows.shape[-1]).permute(0,2,1).contiguous().float() + if latents.shape[1]!=self.mean.shape[1]: + raise ValueError('Audio latent channel mismatch') + return latents*self.std+self.mean + + def __call__(self,rows,*,with_grad): + latents = self.unpack(rows) + def decode(value): + return self.model.decode(value,return_dict=False)[0][:,0,:].float() + if with_grad: + # Checkpoint only the frozen decoder; preserve gradients to latents. + with torch.autograd.graph.save_on_cpu(pin_memory=True): + waveform = checkpoint(decode,latents,use_reentrant=False) + else: + with torch.no_grad(): + waveform = decode(latents.detach()) + if not bool(torch.isfinite(waveform).all()): + raise FloatingPointError('Nonfinite decoded waveform in audio objective') + return waveform + + +class EndpointAudioTrainer(GuidedTeacherTrainer): + def __init__(self,*,endpoint_nfe,endpoint_weight,spectral_weight,**kwargs): + super().__init__(**kwargs) + if self.teacher_audio_scale!=1 or self.teacher_video_scale!=1: + raise ValueError('Fresh audio experiment uses the original positive-only Base teacher') + self.endpoint_nfe = endpoint_nfe + self.endpoint_weight = endpoint_weight + self.spectral_weight = spectral_weight + self.audio_decoder = None + self.endpoint_state = None + self.endpoint_audio_target = None + self.audio_objective_metrics = {} + + def target_stage_rollout(self,**kwargs): + output = super().target_stage_rollout(**kwargs) + self.endpoint_state = output.stage_states[output.target_stage] + return output + + def _score_pair(self,**kwargs): + scores = super()._score_pair(**kwargs) + state = self.endpoint_state + if state is None: + raise RuntimeError('Teacher endpoint requires the selected student input') + video, audio = state.video_rows.detach(), state.audio_rows.detach() + # Frozen Base Euler substeps from the EXACT selected input, + # preserving the official shared-base-sigma clock coupling and shifts. + video_sigmas,audio_sigmas=teacher_endpoint_grid(state,self.endpoint_nfe,self.video_shift,self.audio_shift) + with torch.no_grad(), disable_adapters_for_teacher(self.role_model): + for step in range(self.endpoint_nfe): + vv,av = self._forward(role='teacher_audio_endpoint',video_rows=video,audio_rows=audio, + condition=kwargs['condition'],layout=kwargs['layout'], + video_timestep=1-video_sigmas[step],audio_timestep=1-audio_sigmas[step], + adapters_enabled=False) + video = video+(video_sigmas[step]-video_sigmas[step+1])*vv + audio = audio+(audio_sigmas[step]-audio_sigmas[step+1])*av + self.endpoint_audio_target = audio.detach() + self.endpoint_state = None + return scores + + def generator_step(self,**kwargs): + original = dmd_4step.dmd_generator_pseudo_loss + self.audio_objective_metrics = {} + def objective(**loss_kwargs): + dmd = original(**loss_kwargs) + actual = loss_kwargs['generated_audio'] + target = self.endpoint_audio_target + if target is None or target.shape!=actual.shape or target.requires_grad: + raise RuntimeError('Missing or differentiable teacher endpoint target') + latent = 0.5*(actual.float()-target.float()).square().mean() + spectral = torch.zeros((),device=actual.device,dtype=torch.float32) + if self.spectral_weight: + if self.audio_decoder is None: + self.audio_decoder = FrozenAudioDecoder(actual.device) + wanted = self.audio_decoder(target,with_grad=False) + predicted = self.audio_decoder(actual,with_grad=True) + spectral = multiresolution_spectral_loss(predicted,wanted) + audio_loss = dmd.audio+self.endpoint_weight*latent+self.spectral_weight*spectral + total = loss_kwargs['video_weight']*dmd.video+loss_kwargs['audio_weight']*audio_loss + self.audio_objective_metrics = {'audio_dmd_pseudo_loss':float(dmd.audio.detach()), + 'audio_teacher_endpoint_mse':float(latent.detach()), + 'audio_mrstft_loss':float(spectral.detach()), + 'teacher_audio_endpoint_nfe':self.endpoint_nfe, + 'audio_endpoint_weight':self.endpoint_weight,'audio_spectral_weight':self.spectral_weight, + 'audio_decoder_frozen':self.audio_decoder is None or not any(p.requires_grad for p in self.audio_decoder.model.parameters())} + return MultimodalLoss(total=total,video=dmd.video,audio=audio_loss) + # One training process has one trainer; scope the shared loss hook to + # this synchronous generator call and restore it on every exit path. + dmd_4step.dmd_generator_pseudo_loss = objective + try: + return super().generator_step(**kwargs) + finally: + dmd_4step.dmd_generator_pseudo_loss = original + self.endpoint_state = None + self.endpoint_audio_target = None diff --git a/training_recipe/common.py b/training_recipe/common.py new file mode 100644 index 0000000000000000000000000000000000000000..7ca7128dce7ad5fa009fb1a9fa54b2ea82be3bd9 --- /dev/null +++ b/training_recipe/common.py @@ -0,0 +1,79 @@ +"""Independent from-scratch audio-quality experiments. No old LoRA warm starts.""" +import hashlib +import json +import os +from pathlib import Path +import sys + +EXP = Path(__file__).resolve().parent +ROOT = EXP.parents[1] +SOURCE = ROOT / "training/LongLive-LoRA-dmd4-cfg-1600" +ASSETS = ROOT / "training/LongLive-LoRA" +BASE = ASSETS / "models/MiniMax-H3-Base-T2VA-fa9c8ab1" +REVISION = "fa9c8ab1eaa21c8ae25e7e40b83b2e6002f340af" +PY = str(ASSETS / ".envs/minimax-h3/bin/python3.11") +PREVIOUS = ROOT / "experiments/audio_quality_forks_20260913" +CFG_EXP = ROOT / "experiments/audio_cfg_1600_20260911" +REFERENCE = CFG_EXP / "pilot_outputs/reference-rank0.pt" +TARGET = 400 +ARMS = { + "endpoint_spectral": {"teacher_endpoint_nfe": 50, "endpoint_weight": 1.0, + "spectral_weight": 0.02, "label": "新初始化 · DMD + 教师音频目标 + 多尺度频谱"}, + "endpoint_latent": {"teacher_endpoint_nfe": 50, "endpoint_weight": 1.0, + "spectral_weight": 0.0, "label": "新初始化 · DMD + 教师音频目标"}, + "fresh_dmd_control": {"teacher_endpoint_nfe": 0, "endpoint_weight": 0.0, + "spectral_weight": 0.0, "label": "新初始化 · 原 DMD 对照"}, +} + + +def read(path): + return json.loads(Path(path).read_text()) + + +def sha(path): + h = hashlib.sha256() + with Path(path).open('rb') as stream: + for block in iter(lambda:stream.read(16*1024*1024),b''): + h.update(block) + return h.hexdigest() + + +def save(path, value): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(path.suffix + '.tmp') + tmp.write_text(json.dumps(value, ensure_ascii=False, indent=2)+'\n') + tmp.replace(path) + + +def imports(): + for path in (SOURCE, ASSETS/'external/diffusers-minimax-h3/src'): + sys.path.insert(0, str(path)) + + +def environment(): + env = os.environ.copy() + env.update({'PYTHONPATH':f'{EXP}:{SOURCE}:{ASSETS / "external/diffusers-minimax-h3/src"}', + 'PYTHONDONTWRITEBYTECODE':'1','PYTHONUNBUFFERED':'1','PYTHONSAFEPATH':'1', + 'HF_HUB_OFFLINE':'1','TRANSFORMERS_OFFLINE':'1','DIFFUSERS_OFFLINE':'1', + 'HF_HOME':str(ASSETS/'models/.hf_home'),'HF_HUB_CACHE':str(ASSETS/'models/.hf_home/hub'), + 'TOKENIZERS_PARALLELISM':'false','OMP_NUM_THREADS':'8','MKL_NUM_THREADS':'8', + 'OPENBLAS_NUM_THREADS':'8','NUMEXPR_NUM_THREADS':'8', + 'PYTORCH_CUDA_ALLOC_CONF':'expandable_segments:True', + 'TORCH_NCCL_ASYNC_ERROR_HANDLING':'1','NCCL_DEBUG':'WARN'}) + env['PATH'] = str(ASSETS/'.envs/minimax-h3/bin')+':'+env.get('PATH',os.defpath) + return env + + +def run_dir(arm): + if arm not in ARMS: + raise ValueError(arm) + return EXP/'runs'/arm + + +def source_contract(): + config = read(SOURCE/'artifacts/audio2-vcfg1-acfg1p5/CONFIG.json') + for name,digest in config['training_source_sha256'].items(): + if sha(SOURCE/'minimax_h3'/name)!=digest: + raise RuntimeError('Frozen source changed: '+name) + return config['training_source_sha256'] diff --git a/training_recipe/train_audio_mixed.py b/training_recipe/train_audio_mixed.py new file mode 100644 index 0000000000000000000000000000000000000000..bed58eb0324578d016ea91be8d6d8b292f0d5394 --- /dev/null +++ b/training_recipe/train_audio_mixed.py @@ -0,0 +1,146 @@ +"""Independent from-Base data ablation; original training source stays untouched.""" +from contextlib import contextmanager +from collections import Counter +from common import * +from audio_candidate_cache import CACHE,load_pool +from audio_mixed_conditions import MixedRankConditions,candidate_for +from audio_data_candidates import GROUPS + +imports() +import train_fresh as fresh +from minimax_h3 import train_dmd_longrun_4step as formal +from minimax_h3 import train_dmd_longrun_4step_autoresume as operational + +ARM='audio_mixed_endpoint_latent' +TRIAL=EXP/'audio_data_trial' + + +def trial_dir(arm): + if arm!=ARM: + raise ValueError('Unexpected data-trial arm') + return TRIAL + + +def validate_selection_group(rows,step): + if len(rows)!=16 or {r['rank'] for r in rows}!=set(range(16)): + raise RuntimeError('Missing or duplicate distributed data selections') + for row in rows: + wanted=candidate_for(step,row['rank']) + if row['step']!=step or not row['rng_seeds_match_original_condition']: + raise RuntimeError('Data selection step/RNG mismatch') + if row['method']!=('authored_audio' if wanted else 'original_LongLive'): + raise RuntimeError('Distributed data mixture ratio changed') + if row['selected_prompt_id']!=(wanted['id'] if wanted else row['original_prompt_id']): + raise RuntimeError('Distributed selected prompt identity mismatch') + if row['audio_category']!=(wanted['group'] if wanted else None): + raise RuntimeError('Distributed audio category mismatch') + if Counter(row['audio_category'] for row in rows if row['method']=='authored_audio')!={key:2 for key in GROUPS}: + raise RuntimeError('Distributed audio categories are unbalanced') + + +def augment_config(parent,pool_metadata): + # Original source metadata is retained under a clearly scoped component; + # the effective training dataset is explicitly labelled as a mixture. + config=json.loads(json.dumps(parent)) + config.pop('config_sha256',None) + component={key:config.pop(key) for key in ( + 'prompt_dataset','prompt_dataset_row_count','prompt_sampler','prompt_schedule_audit', + 'prompt_collection_provenance') if key in config} + component['prompt_runtime_selection']=dict(config['prompt_runtime_selection']) + config['original_prompt_component']=component + config['prompt_dataset']={'kind':'deterministic_mixture','original_fraction':0.5,'authored_audio_fraction':0.5, + 'authored_candidate_manifest':str(EXP/'audio_data_candidates.json'), + 'authored_candidate_manifest_sha256':sha(EXP/'audio_data_candidates.json'), + 'authored_embedding_collection':str(CACHE/'COLLECTION.json'), + 'authored_embedding_collection_sha256':pool_metadata['collection_sha256'], + 'authored_cache_contract':pool_metadata['contract'], + 'authored_prompt_strings':256,'authored_core_scenes':32, + 'evaluation_text_and_reference_phrases_excluded':True,'semantic_near_duplicate_exclusion_proven':False} + config['prompt_cache_argument_scope']='original_LongLive_component_and_preserved_noise_seed_schedule_only' + config['prompt_runtime_selection'].pop('rank_distinct_prompt',None) + config['prompt_runtime_selection'].update(policy='rank_balanced_50pct_authored_audio_v1', + source_original_cache_conditions=32000,used_outer_condition_slots=16*TARGET, + used_authored_outer_slots=8*TARGET,used_authored_student_slots=8*((TARGET+4)//5), + original_condition_seed_retained=True,text_selection_uses_rng=False, + distinct_prompt_text_per_rank_verified=False, + rank_distinct_condition_ids=True) + config['data_deviation']='Half of each global batch retains original LongLive text; half uses separately authored audio-conditioned text. Other endpoint-latent settings are unchanged.' + config['experiment_id']='audio-data-mixed-fresh-20260914' + config['data_trial_arm']=ARM + config['matched_loss_recipe']='endpoint_latent' + config['data_trial_source_sha256']={name:sha(EXP/name) for name in ( + 'train_audio_mixed.py','audio_mixed_conditions.py','audio_candidate_cache.py','audio_data_candidates.py')} + config['config_sha256']=formal._stable_config_hash(config) + return config + + +@contextmanager +def mixed_contract(): + recipe=dict(ARMS['endpoint_latent']) + recipe['label']='新初始化 · 50% 音频描述数据 · DMD + 教师音频目标' + # These are process-local aliases only. Neither the shared source files nor + # the running three-arm processes are modified by this new entrypoint. + with formal._temporary_attributes([(fresh,'ARMS',{ARM:recipe}),(fresh,'run_dir',trial_dir)]): + with fresh.fresh_contract(ARM): + inherited_validate=operational._ORIGINAL_VALIDATE_WITH_LOADER + inherited_metric=formal._metric_record + parent_loader=formal.v5._DistributedPromptCollectionLoader + active={} + + def validate(args,loader): + # Preserve all original model/LoRA/own-run resume guards first. + config=inherited_validate(args,loader) + if 'pool' not in active: + active['pool'],active['pool_metadata']=load_pool() + return augment_config(config,active['pool_metadata']) + + class MixedLoader(parent_loader): + def __call__(self,path): + metadata,original=super().__call__(path) + if not hasattr(self,'mixed_schedule'): + if 'pool' not in active: + raise RuntimeError('Audio cache was not verified during request validation') + import torch.distributed as dist + self.mixed_schedule=MixedRankConditions(original,active['pool'],dist.get_rank()) + active['loader']=self + return dict(metadata,effective_text_source='half_original_half_authored_audio', + audio_collection_sha256=active['pool_metadata']['collection_sha256']),self.mixed_schedule + + def metric(**kwargs): + record=inherited_metric(**kwargs) + import torch.distributed as dist + loader=active.get('loader') + if loader is None or loader.mixed_schedule.last_selection is None: + raise RuntimeError('No actual mixed-prompt selection for this update') + selection=dict(loader.mixed_schedule.last_selection) + if selection['step']!=kwargs['step'] or selection['selected_prompt_id']!=kwargs['prompt_id']: + raise RuntimeError('Training used a different prompt from the audited selection') + expected=formal.build_outer_step_rng_streams(training_seed=formal.TRAINING_SEED, + condition_seed=selection['rng_condition_seed_retained'],global_step=kwargs['step'], + rank=selection['rank'],device='cpu').seeds + selection['rng_seeds_match_original_condition']=expected==kwargs['rng_seeds'] + if not selection['rng_seeds_match_original_condition']: + raise RuntimeError('The data ablation changed a training RNG seed') + selections=[None]*dist.get_world_size() + dist.all_gather_object(selections,selection) + validate_selection_group(selections,kwargs['step']) + record.update(data_trial_arm=ARM,data_selection_by_rank=selections, + data_selection_all_ranks_verified=True,data_rng_seeds_preserved_all_ranks=True, + audio_candidate_collection_sha256=active['pool_metadata']['collection_sha256']) + return record + + with formal._temporary_attributes([(operational,'_ORIGINAL_VALIDATE_WITH_LOADER',validate), + (formal.v5,'_DistributedPromptCollectionLoader',MixedLoader),(formal,'_metric_record',metric)]): + yield + + +def main(argv=None): + arguments=list(sys.argv[1:] if argv is None else argv) + if '--validate-only' not in arguments and not os.environ.get('SLURM_JOB_ID'): + raise RuntimeError('Actual mixed-data training requires a Slurm allocation') + with mixed_contract(): + return operational.main(arguments) + + +if __name__=='__main__': + raise SystemExit(main()) diff --git a/training_recipe/train_fresh.py b/training_recipe/train_fresh.py new file mode 100644 index 0000000000000000000000000000000000000000..bd3fcb267d1cdd44bb049044aa28e854244bdbce --- /dev/null +++ b/training_recipe/train_fresh.py @@ -0,0 +1,119 @@ +"""New Base initialization plus explicit audio-loss extensions; old runs untouched.""" +import argparse +from contextlib import contextmanager +import math +from common import * + +imports() +from minimax_h3 import train_dmd_longrun_4step as formal +from minimax_h3 import train_dmd_longrun_4step_autoresume as operational +from audio_objectives import EndpointAudioTrainer + + +@contextmanager +def fresh_contract(arm): + recipe = ARMS[arm] + source_contract() + original_validate = operational._ORIGINAL_VALIDATE_WITH_LOADER + original_trainer = formal.GuidedTeacherTrainer + original_metric = formal._metric_record + original_install = formal._install_document_lora_adapters + active = {} + + def validate(args,loader): + if Path(args.checkpoint_root).resolve()!=run_dir(arm)/'checkpoints': + raise RuntimeError('Unexpected fresh experiment root') + if args.teacher_video_cfg!=1.0 or args.teacher_audio_cfg!=1.0: + raise ValueError('No extra teacher CFG in this new experiment') + if args.resume_from is not None: + directory = Path(args.resume_from).resolve(strict=True) + if directory.parent!=run_dir(arm)/'checkpoints': + raise ValueError('Old or foreign LoRA warm starts are forbidden') + step = int(directory.name.split('-')[-1]) + if not 0