Publish SWD factor checkpoints

.gitattributes

@@ -1,35 +1 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

.pytest_cache/.gitignore

@@ -0,0 +1,2 @@
+# Created by pytest automatically.
+*

.pytest_cache/CACHEDIR.TAG

@@ -0,0 +1,4 @@
+Signature: 8a477f597d28d172789f06886806bc55
+# This file is a cache directory tag created by pytest.
+# For information about cache directory tags, see:
+#	https://bford.info/cachedir/spec.html

.pytest_cache/README.md

@@ -0,0 +1,8 @@
+# pytest cache directory #
+
+This directory contains data from the pytest's cache plugin,
+which provides the `--lf` and `--ff` options, as well as the `cache` fixture.
+
+**Do not** commit this to version control.
+
+See [the docs](https://docs.pytest.org/en/stable/how-to/cache.html) for more information.

.pytest_cache/v/cache/nodeids

@@ -0,0 +1,3 @@
+[
+  "tests/test_loader.py::test_factorized_and_folded"
+]

.pytest_cache/v/cache/stepwise

@@ -0,0 +1 @@
+[]

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

NOTICE

@@ -0,0 +1,14 @@
+Sparse Weight Decomposition Checkpoints
+Copyright 2026 SWD authors
+
+This distribution contains factor checkpoints derived from the following
+pretrained models:
+
+- GPT-2, Copyright 2019 OpenAI, distributed under the Modified MIT License.
+  See THIRD_PARTY_LICENSES/GPT2-MODIFIED-MIT.txt.
+- Qwen2.5-0.5B and Qwen2.5-1.5B, Copyright 2024 Alibaba Cloud, distributed
+  under the Apache License 2.0.
+- Qwen3.5-27B, Copyright 2026 Alibaba Cloud, distributed under the Apache
+  License 2.0.
+
+No base-model weights are redistributed in this repository.

README.md

@@ -0,0 +1,128 @@
+---
+library_name: pytorch
+license: apache-2.0
+tags:
+- mechanistic-interpretability
+- sparse-models
+- model-decomposition
+- circuit-analysis
+base_model:
+- gpt2
+- Qwen/Qwen2.5-0.5B
+- Qwen/Qwen2.5-1.5B
+- Qwen/Qwen3.5-27B
+datasets:
+- HuggingFaceFW/fineweb-edu
+---
+
+# Sparse Weight Decomposition Checkpoints
+
+This repository contains factor-only Sparse Weight Decomposition (SWD)
+checkpoints used in our replacement-fidelity and circuit-extraction
+experiments. It does **not** redistribute any base model. Load the corresponding
+base model first, then apply one checkpoint with `swd_loader.py`.
+
+Each replaced matrix is represented as
+
+```text
+output = input @ read @ write + bias
+```
+
+The intermediate coordinates are the SWD bottleneck units used for circuit
+scoring and ablation. `s=0.5` and `s=0.75` mean that 50% and 75% of all entries
+across the two factors are zero, respectively.
+
+## Included Checkpoints
+
+| Base model | Replacement | Setting | Data used | CE delta vs dense |
+|---|---|---:|---:|---:|
+| GPT-2 Small | Layer 8 `mlp.c_proj` | `s=0.5` | 16,384 tokens | 0.000889 |
+| GPT-2 Small | Layer 8 `mlp.c_proj` | `s=0.75` | 16,384 tokens | 0.008292 |
+| Qwen2.5-0.5B | Layer 12 `mlp.down_proj` | `s=0.5` | 1,024 tokens | 0.001222 |
+| Qwen2.5-0.5B | Layer 12 `mlp.down_proj` | `s=0.75` | 1,048,576 tokens | 0.005010 |
+| Qwen2.5-1.5B | Layer 14 `mlp.down_proj` | `s=0.5` | 1,024 tokens | 0.000733 |
+| Qwen2.5-1.5B | Layer 14 `mlp.down_proj` | `s=0.75` | 1,048,576 tokens | 0.001222 |
+| Qwen3.5-27B | Layer 31 `mlp.down_proj` | `s=0.5` | 2,048 tokens | -0.000427 |
+| Qwen3.5-27B | Layer 31 `mlp.down_proj` | `s=0.75` | 1,048,576 tokens | -0.000448 |
+| GPT-2 Small | Layer 8 complete MLP | `s=0.5` | 16,384 tokens | 0.004003 |
+| GPT-2 Small | Layer 8 complete MLP | `s=0.75` | 1,048,576 tokens | 0.015521 |
+| GPT-2 Small | All 48 transformer-block linear projections | fixed-support SWD-FT | 20,578,304 tokens | 0.151263* |
+
+`*` The full-model value uses its full-model stress evaluation and should not
+be numerically compared with the single-matrix unified CE rows.
+
+The Qwen2.5-3B checkpoints are distributed separately at
+[`veri-safe/SWD-Qwen2.5-3B`](https://huggingface.co/veri-safe/SWD-Qwen2.5-3B)
+because the upstream model uses the Qwen Research License.
+
+## Usage
+
+Install the lightweight loader dependencies:
+
+```bash
+pip install torch safetensors transformers huggingface_hub
+```
+
+After downloading this repository, load a base model and apply a checkpoint:
+
+```python
+from transformers import AutoModelForCausalLM
+from swd_loader import apply_swd_checkpoint
+
+model = AutoModelForCausalLM.from_pretrained("gpt2")
+apply_swd_checkpoint(
+    model,
+    "checkpoints/gpt2-small/layer8-cproj/s0p5-tokens16384",
+    mode="factorized",
+)
+```
+
+`mode="factorized"` installs `SWDLinear`, exposing
+`component_activations(inputs)`. Use `mode="folded"` to write `read @ write`
+back into the original dense module for conventional inference.
+
+The base model must be fully materialized before applying a checkpoint. For
+large models loaded with a device map, each replacement is moved to the device
+and dtype of its target module.
+
+## Format
+
+Every checkpoint directory contains:
+
+```text
+model.safetensors  # factor tensors only; no pickle
+config.json        # base model, module paths, shapes, sparsity, and token exposure
+provenance.json    # source hashes, conversion rule, and validation result
+```
+
+All public factors follow `[input, rank] @ [rank, output]`, independent of the
+source framework's dense-weight layout. Qwen2.5 source factors are transposed
+into this convention; Qwen3.5 feature shards are concatenated along the rank
+dimension without changing dtype or values. Full-model GPT-2 checkpoint biases
+are included because they belong to the fixed-support fine-tuned checkpoint.
+
+`RELEASE_MANIFEST.csv` is the machine-readable checkpoint index. Each
+checkpoint's `provenance.json` records its source hashes and conversion
+validation.
+
+## Validation
+
+Before release, every checkpoint was checked for source identity, finite
+tensors, shape compatibility, factor nonzero counts, and exact tensor equality
+after the safetensors round-trip.
+
+The release intentionally excludes base-model weights, activation Grams, dense
+target/reconstructed matrices, optimizer state, data caches, remote-transfer
+archives, credentials, and cluster-local paths.
+
+## Links
+
+- Code: https://github.com/Veri-Safe/SWD
+- Models: https://huggingface.co/veri-safe/SWD
+
+## License
+
+The SWD release code and the checkpoints in this repository are distributed
+under the Apache License 2.0. The GPT-2-derived checkpoints also retain the
+upstream Modified MIT notice in
+`THIRD_PARTY_LICENSES/GPT2-MODIFIED-MIT.txt`. See `NOTICE` for attribution.

RELEASE_MANIFEST.csv

@@ -0,0 +1,12 @@
+variant_id,base_model,replacement_scope,sparsity,token_exposure,ce_delta_vs_dense,module_count,bottleneck_units,active_factor_scalars,relative_path,weights_bytes,weights_sha256
+gpt2_small_l8_cproj_s0p5,gpt2,single_matrix,0.5,16384,0.0008888365819190724,1,768,1179646,checkpoints/gpt2-small/layer8-cproj/s0p5-tokens16384,11796752,e4dc8d3fbe3f9b491a3ccadf928de65d0ad98f6826844b3143d04e1bd06a6942
+gpt2_small_l8_cproj_s0p75,gpt2,single_matrix,0.75,16384,0.0082921466752812,1,768,589822,checkpoints/gpt2-small/layer8-cproj/s0p75-tokens16384,11796752,f6db8a41cfacc8cb6ce54429c6c39be7b74ba36a7d9a1aa9b3e55b1f0e0b097e
+qwen25_0p5b_l12_downproj_s0p5,Qwen/Qwen2.5-0.5B,single_matrix,0.5,1024,0.0012218963831869267,1,896,2179070,checkpoints/qwen2.5-0.5b/layer12-down-proj/s0p5-tokens1024,20644120,400f49a25d6862f584c8057d138b83802d47e13c31b4b58acb1b4a740805ca6e
+qwen25_0p5b_l12_downproj_s0p75,Qwen/Qwen2.5-0.5B,single_matrix,0.75,1048576,0.005009775171065822,1,896,1089534,checkpoints/qwen2.5-0.5b/layer12-down-proj/s0p75-tokens1048576,20644120,530b19e4f58da33e6f3b6a7bad749cd3e8bafa8b67ee41939075d3dcb1fb4078
+qwen25_1p5b_l14_downproj_s0p5,Qwen/Qwen2.5-1.5B,single_matrix,0.5,1024,0.0007331378299122449,1,1536,6881278,checkpoints/qwen2.5-1.5b/layer14-down-proj/s0p5-tokens1024,64487712,ad3be13722726a1a447c2ea7f41c1ee55a21099ac231ae1ea2089950d1a204dc
+qwen25_1p5b_l14_downproj_s0p75,Qwen/Qwen2.5-1.5B,single_matrix,0.75,1048576,0.0012218963831869267,1,1536,3440638,checkpoints/qwen2.5-1.5b/layer14-down-proj/s0p75-tokens1048576,64487712,aa3fa378549da027bb4bf2778fa80b158c01840aa0306025c5f828baa5bcf7dd
+qwen35_27b_l31_downproj_s0p5,Qwen/Qwen3.5-27B,single_matrix,0.5,2048,-0.00042653013883819924,1,5120,44564478,checkpoints/qwen3.5-27b/layer31-down-proj/s0p5-tokens2048,230687008,84c6673c889c229853de6cea4556f4ee1b7e054b03e191353fb96563c7d31877
+qwen35_27b_l31_downproj_s0p75,Qwen/Qwen3.5-27B,single_matrix,0.75,1048576,-0.00044806658464224824,1,5120,22282238,checkpoints/qwen3.5-27b/layer31-down-proj/s0p75-tokens1048576,230687008,92eeb9672d70cbc16c4483bfa9673593781bf73069e145aad75bf6350df86066
+gpt2_small_l8_mlp_s0p5,gpt2,mlp_replacement,0.5,16384,0.004003,2,1536,2359292,checkpoints/gpt2-small/layer8-mlp-sequential/s0p5-tokens16384,23593408,a56f5ae70bf0b3c3fa0750e53d226928a7de3e27b84458d9fcef4b159e16ea57
+gpt2_small_l8_mlp_s0p75,gpt2,mlp_replacement,0.75,1048576,0.01552128791809082,2,1536,1179644,checkpoints/gpt2-small/layer8-mlp-sequential/s0p75-tokens1048576,23593408,8852687de6d0d890e55c2df5a4e097224e6125d6bd4987391e50625b222953b3
+gpt2_small_full_model_swdft,gpt2,full_model_linear_replacement,matched_active_budget,20578304,0.151263,48,36864,26770800,checkpoints/gpt2-small/full-model/swd-ft-tokens20578304,453329504,b18359f09dcd61ccd3197ad2c5e66e47a55ab0cc3979870dc67ce667edf2f51e

THIRD_PARTY_LICENSES/GPT2-MODIFIED-MIT.txt

@@ -0,0 +1,24 @@
+Modified MIT License
+
+Software Copyright (c) 2019 OpenAI
+
+We don’t claim ownership of the content you create with GPT-2, so it is yours to do with as you please.
+We only ask that you use GPT-2 responsibly and clearly indicate your content was created using GPT-2.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+associated documentation files (the "Software"), to deal in the Software without restriction,
+including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+The above copyright notice and this permission notice need not be included
+with content created by the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
+OR OTHER DEALINGS IN THE SOFTWARE.

checkpoints/gpt2-small/full-model/swd-ft-tokens20578304/config.json

@@ -0,0 +1,832 @@
+{
+  "base_model": "gpt2",
+  "ce_delta_vs_dense": 0.151263,
+  "factorization_tokens": 4194304,
+  "fixed_support_finetuning_tokens": 16384000,
+  "method": "SWD",
+  "module_count": 48,
+  "module_replacements": [
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.000.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.0.attn.c_attn",
+      "output_dim": 2304,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.000.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.000.write",
+      "write_nnz": 410270
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.001.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.0.attn.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.001.read",
+      "read_nnz": 94370,
+      "write_dtype": "float32",
+      "write_key": "modules.001.write",
+      "write_nnz": 91537
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.002.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.0.mlp.c_fc",
+      "output_dim": 3072,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.002.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.002.write",
+      "write_nnz": 596179
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.003.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 3072,
+      "module_path": "transformer.h.0.mlp.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.003.read",
+      "read_nnz": 596179,
+      "write_dtype": "float32",
+      "write_key": "modules.003.write",
+      "write_nnz": 147455
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.004.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.1.attn.c_attn",
+      "output_dim": 2304,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.004.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.004.write",
+      "write_nnz": 410270
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.005.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.1.attn.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.005.read",
+      "read_nnz": 94370,
+      "write_dtype": "float32",
+      "write_key": "modules.005.write",
+      "write_nnz": 91537
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.006.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.1.mlp.c_fc",
+      "output_dim": 3072,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.006.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.006.write",
+      "write_nnz": 596179
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.007.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 3072,
+      "module_path": "transformer.h.1.mlp.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.007.read",
+      "read_nnz": 596179,
+      "write_dtype": "float32",
+      "write_key": "modules.007.write",
+      "write_nnz": 147455
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.008.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.2.attn.c_attn",
+      "output_dim": 2304,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.008.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.008.write",
+      "write_nnz": 410270
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.009.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.2.attn.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.009.read",
+      "read_nnz": 94370,
+      "write_dtype": "float32",
+      "write_key": "modules.009.write",
+      "write_nnz": 91537
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.010.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.2.mlp.c_fc",
+      "output_dim": 3072,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.010.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.010.write",
+      "write_nnz": 596179
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.011.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 3072,
+      "module_path": "transformer.h.2.mlp.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.011.read",
+      "read_nnz": 596179,
+      "write_dtype": "float32",
+      "write_key": "modules.011.write",
+      "write_nnz": 147455
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.012.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.3.attn.c_attn",
+      "output_dim": 2304,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.012.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.012.write",
+      "write_nnz": 410270
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.013.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.3.attn.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.013.read",
+      "read_nnz": 94370,
+      "write_dtype": "float32",
+      "write_key": "modules.013.write",
+      "write_nnz": 91537
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.014.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.3.mlp.c_fc",
+      "output_dim": 3072,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.014.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.014.write",
+      "write_nnz": 596179
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.015.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 3072,
+      "module_path": "transformer.h.3.mlp.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.015.read",
+      "read_nnz": 596179,
+      "write_dtype": "float32",
+      "write_key": "modules.015.write",
+      "write_nnz": 147455
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.016.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.4.attn.c_attn",
+      "output_dim": 2304,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.016.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.016.write",
+      "write_nnz": 410270
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.017.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.4.attn.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.017.read",
+      "read_nnz": 94370,
+      "write_dtype": "float32",
+      "write_key": "modules.017.write",
+      "write_nnz": 91537
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.018.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.4.mlp.c_fc",
+      "output_dim": 3072,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.018.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.018.write",
+      "write_nnz": 596179
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.019.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 3072,
+      "module_path": "transformer.h.4.mlp.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.019.read",
+      "read_nnz": 596179,
+      "write_dtype": "float32",
+      "write_key": "modules.019.write",
+      "write_nnz": 147455
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.020.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.5.attn.c_attn",
+      "output_dim": 2304,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.020.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.020.write",
+      "write_nnz": 410270
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.021.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.5.attn.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.021.read",
+      "read_nnz": 94370,
+      "write_dtype": "float32",
+      "write_key": "modules.021.write",
+      "write_nnz": 91537
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.022.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.5.mlp.c_fc",
+      "output_dim": 3072,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.022.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.022.write",
+      "write_nnz": 596179
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.023.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 3072,
+      "module_path": "transformer.h.5.mlp.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.023.read",
+      "read_nnz": 596179,
+      "write_dtype": "float32",
+      "write_key": "modules.023.write",
+      "write_nnz": 147455
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.024.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.6.attn.c_attn",
+      "output_dim": 2304,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.024.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.024.write",
+      "write_nnz": 410270
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.025.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.6.attn.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.025.read",
+      "read_nnz": 94370,
+      "write_dtype": "float32",
+      "write_key": "modules.025.write",
+      "write_nnz": 91537
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.026.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.6.mlp.c_fc",
+      "output_dim": 3072,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.026.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.026.write",
+      "write_nnz": 596179
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.027.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 3072,
+      "module_path": "transformer.h.6.mlp.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.027.read",
+      "read_nnz": 596179,
+      "write_dtype": "float32",
+      "write_key": "modules.027.write",
+      "write_nnz": 147455
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.028.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.7.attn.c_attn",
+      "output_dim": 2304,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.028.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.028.write",
+      "write_nnz": 410270
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.029.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.7.attn.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.029.read",
+      "read_nnz": 94370,
+      "write_dtype": "float32",
+      "write_key": "modules.029.write",
+      "write_nnz": 91537
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.030.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.7.mlp.c_fc",
+      "output_dim": 3072,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.030.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.030.write",
+      "write_nnz": 596179
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.031.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 3072,
+      "module_path": "transformer.h.7.mlp.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.031.read",
+      "read_nnz": 596179,
+      "write_dtype": "float32",
+      "write_key": "modules.031.write",
+      "write_nnz": 147455
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.032.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.8.attn.c_attn",
+      "output_dim": 2304,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.032.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.032.write",
+      "write_nnz": 410270
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.033.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.8.attn.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.033.read",
+      "read_nnz": 94370,
+      "write_dtype": "float32",
+      "write_key": "modules.033.write",
+      "write_nnz": 91537
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.034.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.8.mlp.c_fc",
+      "output_dim": 3072,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.034.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.034.write",
+      "write_nnz": 596179
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.035.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 3072,
+      "module_path": "transformer.h.8.mlp.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.035.read",
+      "read_nnz": 596179,
+      "write_dtype": "float32",
+      "write_key": "modules.035.write",
+      "write_nnz": 147455
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.036.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.9.attn.c_attn",
+      "output_dim": 2304,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.036.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.036.write",
+      "write_nnz": 410270
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.037.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.9.attn.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.037.read",
+      "read_nnz": 94370,
+      "write_dtype": "float32",
+      "write_key": "modules.037.write",
+      "write_nnz": 91537
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.038.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.9.mlp.c_fc",
+      "output_dim": 3072,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.038.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.038.write",
+      "write_nnz": 596179
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.039.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 3072,
+      "module_path": "transformer.h.9.mlp.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.039.read",
+      "read_nnz": 596179,
+      "write_dtype": "float32",
+      "write_key": "modules.039.write",
+      "write_nnz": 147455
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.040.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.10.attn.c_attn",
+      "output_dim": 2304,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.040.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.040.write",
+      "write_nnz": 410270
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.041.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.10.attn.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.041.read",
+      "read_nnz": 94370,
+      "write_dtype": "float32",
+      "write_key": "modules.041.write",
+      "write_nnz": 91537
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.042.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.10.mlp.c_fc",
+      "output_dim": 3072,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.042.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.042.write",
+      "write_nnz": 596179
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.043.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 3072,
+      "module_path": "transformer.h.10.mlp.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.043.read",
+      "read_nnz": 596179,
+      "write_dtype": "float32",
+      "write_key": "modules.043.write",
+      "write_nnz": 147455
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.044.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.11.attn.c_attn",
+      "output_dim": 2304,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.044.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.044.write",
+      "write_nnz": 410270
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.045.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.11.attn.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.045.read",
+      "read_nnz": 94370,
+      "write_dtype": "float32",
+      "write_key": "modules.045.write",
+      "write_nnz": 91537
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.046.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.11.mlp.c_fc",
+      "output_dim": 3072,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.046.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.046.write",
+      "write_nnz": 596179
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": "modules.047.bias",
+      "bias_policy": "checkpoint",
+      "conversion_rule": "read=A; write=B; bias=checkpoint bias",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 3072,
+      "module_path": "transformer.h.11.mlp.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.047.read",
+      "read_nnz": 596179,
+      "write_dtype": "float32",
+      "write_key": "modules.047.write",
+      "write_nnz": 147455
+    }
+  ],
+  "replacement_scope": "full_model_linear_replacement",
+  "schema_version": "swd_factor_checkpoint_v1",
+  "sparsity": "matched_active_budget",
+  "token_exposure": 20578304,
+  "variant_id": "gpt2_small_full_model_swdft",
+  "weights_file": "model.safetensors"
+}

checkpoints/gpt2-small/full-model/swd-ft-tokens20578304/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b18359f09dcd61ccd3197ad2c5e66e47a55ab0cc3979870dc67ce667edf2f51e
+size 453329504

checkpoints/gpt2-small/full-model/swd-ft-tokens20578304/provenance.json

@@ -0,0 +1,224 @@
+{
+  "conversion": [
+    {
+      "module_path": "transformer.h.0.attn.c_attn",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.0.attn.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.0.mlp.c_fc",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.0.mlp.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.1.attn.c_attn",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.1.attn.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.1.mlp.c_fc",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.1.mlp.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.2.attn.c_attn",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.2.attn.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.2.mlp.c_fc",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.2.mlp.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.3.attn.c_attn",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.3.attn.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.3.mlp.c_fc",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.3.mlp.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.4.attn.c_attn",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.4.attn.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.4.mlp.c_fc",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.4.mlp.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.5.attn.c_attn",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.5.attn.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.5.mlp.c_fc",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.5.mlp.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.6.attn.c_attn",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.6.attn.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.6.mlp.c_fc",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.6.mlp.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.7.attn.c_attn",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.7.attn.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.7.mlp.c_fc",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.7.mlp.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.8.attn.c_attn",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.8.attn.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.8.mlp.c_fc",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.8.mlp.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.9.attn.c_attn",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.9.attn.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.9.mlp.c_fc",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.9.mlp.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.10.attn.c_attn",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.10.attn.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.10.mlp.c_fc",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.10.mlp.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.11.attn.c_attn",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.11.attn.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.11.mlp.c_fc",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    },
+    {
+      "module_path": "transformer.h.11.mlp.c_proj",
+      "rule": "read=A; write=B; bias=checkpoint bias"
+    }
+  ],
+  "excluded": [
+    "base model weights",
+    "activation Gram matrices",
+    "dense target and reconstructed weights",
+    "optimizer state and data caches",
+    "cluster-local absolute paths"
+  ],
+  "notes": [
+    "Contains all 48 transformer-block linear projections.",
+    "The reported CE delta uses the full-model stress evaluation rather than the single-matrix unified CE surface."
+  ],
+  "source_artifact_hashes": [
+    {
+      "bundle_id": "gpt2_small_full_model_swdft",
+      "hash_semantics": "source_file",
+      "sha256": "a21298cc05d95aff90b55119248c22756e9136002f3717df5dff1a380e47538d"
+    }
+  ],
+  "source_bundle_ids": [
+    "gpt2_small_full_model_swdft"
+  ],
+  "source_format": "gpt2_full_model_fixed_support_swdft",
+  "validation": {
+    "all_tensors_finite": true,
+    "orientation_random_input_max_abs_error": 0.0,
+    "safetensors_exact_round_trip": true
+  },
+  "variant_id": "gpt2_small_full_model_swdft"
+}

checkpoints/gpt2-small/layer8-cproj/s0p5-tokens16384/config.json

@@ -0,0 +1,30 @@
+{
+  "base_model": "gpt2",
+  "ce_delta_vs_dense": 0.0008888365819190724,
+  "method": "SWD",
+  "module_replacements": [
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": null,
+      "bias_policy": "preserve_base",
+      "conversion_rule": "read=A_prop; write=B_prop",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 3072,
+      "module_path": "transformer.h.8.mlp.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.000.read",
+      "read_nnz": 1032191,
+      "write_dtype": "float32",
+      "write_key": "modules.000.write",
+      "write_nnz": 147455
+    }
+  ],
+  "replacement_scope": "single_matrix",
+  "schema_version": "swd_factor_checkpoint_v1",
+  "sparsity": 0.5,
+  "token_exposure": 16384,
+  "variant_id": "gpt2_small_l8_cproj_s0p5",
+  "weights_file": "model.safetensors"
+}

checkpoints/gpt2-small/layer8-cproj/s0p5-tokens16384/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4dc8d3fbe3f9b491a3ccadf928de65d0ad98f6826844b3143d04e1bd06a6942
+size 11796752

checkpoints/gpt2-small/layer8-cproj/s0p5-tokens16384/provenance.json

@@ -0,0 +1,33 @@
+{
+  "conversion": [
+    {
+      "module_path": "transformer.h.8.mlp.c_proj",
+      "rule": "read=A_prop; write=B_prop"
+    }
+  ],
+  "excluded": [
+    "base model weights",
+    "activation Gram matrices",
+    "dense target and reconstructed weights",
+    "optimizer state and data caches",
+    "cluster-local absolute paths"
+  ],
+  "notes": [],
+  "source_artifact_hashes": [
+    {
+      "bundle_id": "gpt2_small_l8_cproj_s0p5",
+      "hash_semantics": "source_file",
+      "sha256": "ecafa0a74c7e06824e223520abb99c47292ba7e19071e2f15066dffce6101ebe"
+    }
+  ],
+  "source_bundle_ids": [
+    "gpt2_small_l8_cproj_s0p5"
+  ],
+  "source_format": "gpt2_factor_artifact",
+  "validation": {
+    "all_tensors_finite": true,
+    "orientation_random_input_max_abs_error": 0.0,
+    "safetensors_exact_round_trip": true
+  },
+  "variant_id": "gpt2_small_l8_cproj_s0p5"
+}

checkpoints/gpt2-small/layer8-cproj/s0p75-tokens16384/config.json

@@ -0,0 +1,30 @@
+{
+  "base_model": "gpt2",
+  "ce_delta_vs_dense": 0.0082921466752812,
+  "method": "SWD",
+  "module_replacements": [
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": null,
+      "bias_policy": "preserve_base",
+      "conversion_rule": "read=A_prop; write=B_prop",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 3072,
+      "module_path": "transformer.h.8.mlp.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.000.read",
+      "read_nnz": 442367,
+      "write_dtype": "float32",
+      "write_key": "modules.000.write",
+      "write_nnz": 147455
+    }
+  ],
+  "replacement_scope": "single_matrix",
+  "schema_version": "swd_factor_checkpoint_v1",
+  "sparsity": 0.75,
+  "token_exposure": 16384,
+  "variant_id": "gpt2_small_l8_cproj_s0p75",
+  "weights_file": "model.safetensors"
+}

checkpoints/gpt2-small/layer8-cproj/s0p75-tokens16384/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6db8a41cfacc8cb6ce54429c6c39be7b74ba36a7d9a1aa9b3e55b1f0e0b097e
+size 11796752

checkpoints/gpt2-small/layer8-cproj/s0p75-tokens16384/provenance.json

@@ -0,0 +1,33 @@
+{
+  "conversion": [
+    {
+      "module_path": "transformer.h.8.mlp.c_proj",
+      "rule": "read=A_prop; write=B_prop"
+    }
+  ],
+  "excluded": [
+    "base model weights",
+    "activation Gram matrices",
+    "dense target and reconstructed weights",
+    "optimizer state and data caches",
+    "cluster-local absolute paths"
+  ],
+  "notes": [],
+  "source_artifact_hashes": [
+    {
+      "bundle_id": "gpt2_small_l8_cproj_s0p75",
+      "hash_semantics": "source_file",
+      "sha256": "a9f9798ce6c6d2ff06409b663285605f37ca51d84386549b66b9cd75016aa70d"
+    }
+  ],
+  "source_bundle_ids": [
+    "gpt2_small_l8_cproj_s0p75"
+  ],
+  "source_format": "gpt2_factor_artifact",
+  "validation": {
+    "all_tensors_finite": true,
+    "orientation_random_input_max_abs_error": 0.0,
+    "safetensors_exact_round_trip": true
+  },
+  "variant_id": "gpt2_small_l8_cproj_s0p75"
+}

checkpoints/gpt2-small/layer8-mlp-sequential/s0p5-tokens16384/config.json

@@ -0,0 +1,48 @@
+{
+  "base_model": "gpt2",
+  "ce_delta_vs_dense": 0.004003,
+  "method": "SWD",
+  "module_replacements": [
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": null,
+      "bias_policy": "preserve_base",
+      "conversion_rule": "read=A_prop; write=B_prop",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.8.mlp.c_fc",
+      "output_dim": 3072,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.000.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.000.write",
+      "write_nnz": 1032191
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": null,
+      "bias_policy": "preserve_base",
+      "conversion_rule": "read=A_prop; write=B_prop",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 3072,
+      "module_path": "transformer.h.8.mlp.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.001.read",
+      "read_nnz": 1032191,
+      "write_dtype": "float32",
+      "write_key": "modules.001.write",
+      "write_nnz": 147455
+    }
+  ],
+  "preserves_base_mlp_nonlinearity": true,
+  "replacement_scope": "mlp_replacement",
+  "schema_version": "swd_factor_checkpoint_v1",
+  "sparsity": 0.5,
+  "token_exposure": 16384,
+  "variant_id": "gpt2_small_l8_mlp_s0p5",
+  "weights_file": "model.safetensors"
+}

checkpoints/gpt2-small/layer8-mlp-sequential/s0p5-tokens16384/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a56f5ae70bf0b3c3fa0750e53d226928a7de3e27b84458d9fcef4b159e16ea57
+size 23593408

checkpoints/gpt2-small/layer8-mlp-sequential/s0p5-tokens16384/provenance.json

@@ -0,0 +1,45 @@
+{
+  "conversion": [
+    {
+      "module_path": "transformer.h.8.mlp.c_fc",
+      "rule": "read=A_prop; write=B_prop"
+    },
+    {
+      "module_path": "transformer.h.8.mlp.c_proj",
+      "rule": "read=A_prop; write=B_prop"
+    }
+  ],
+  "excluded": [
+    "base model weights",
+    "activation Gram matrices",
+    "dense target and reconstructed weights",
+    "optimizer state and data caches",
+    "cluster-local absolute paths"
+  ],
+  "notes": [
+    "The base model GELU between c_fc and c_proj is retained."
+  ],
+  "source_artifact_hashes": [
+    {
+      "bundle_id": "gpt2_small_l8_mlp_s0p5_cfc",
+      "hash_semantics": "source_file",
+      "sha256": "a1b4486e628f3a7a5010f851c70244a2a7f05391a91d63bf12d9f7bf64ba1431"
+    },
+    {
+      "bundle_id": "gpt2_small_l8_mlp_s0p5_cproj",
+      "hash_semantics": "source_file",
+      "sha256": "4a3fbe7294d367fc4820fc0014d9a6ef67511cf9c312fce94c3cacb03cb2e240"
+    }
+  ],
+  "source_bundle_ids": [
+    "gpt2_small_l8_mlp_s0p5_cfc",
+    "gpt2_small_l8_mlp_s0p5_cproj"
+  ],
+  "source_format": "gpt2_sequential_mlp_factor_artifacts",
+  "validation": {
+    "all_tensors_finite": true,
+    "orientation_random_input_max_abs_error": 0.0,
+    "safetensors_exact_round_trip": true
+  },
+  "variant_id": "gpt2_small_l8_mlp_s0p5"
+}

checkpoints/gpt2-small/layer8-mlp-sequential/s0p75-tokens1048576/config.json

@@ -0,0 +1,48 @@
+{
+  "base_model": "gpt2",
+  "ce_delta_vs_dense": 0.01552128791809082,
+  "method": "SWD",
+  "module_replacements": [
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": null,
+      "bias_policy": "preserve_base",
+      "conversion_rule": "read=A_prop; write=B_prop",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 768,
+      "module_path": "transformer.h.8.mlp.c_fc",
+      "output_dim": 3072,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.000.read",
+      "read_nnz": 147455,
+      "write_dtype": "float32",
+      "write_key": "modules.000.write",
+      "write_nnz": 442367
+    },
+    {
+      "base_weight_layout": "in_out",
+      "bias_key": null,
+      "bias_policy": "preserve_base",
+      "conversion_rule": "read=A_prop; write=B_prop",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 3072,
+      "module_path": "transformer.h.8.mlp.c_proj",
+      "output_dim": 768,
+      "rank": 768,
+      "read_dtype": "float32",
+      "read_key": "modules.001.read",
+      "read_nnz": 442367,
+      "write_dtype": "float32",
+      "write_key": "modules.001.write",
+      "write_nnz": 147455
+    }
+  ],
+  "preserves_base_mlp_nonlinearity": true,
+  "replacement_scope": "mlp_replacement",
+  "schema_version": "swd_factor_checkpoint_v1",
+  "sparsity": 0.75,
+  "token_exposure": 1048576,
+  "variant_id": "gpt2_small_l8_mlp_s0p75",
+  "weights_file": "model.safetensors"
+}

checkpoints/gpt2-small/layer8-mlp-sequential/s0p75-tokens1048576/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8852687de6d0d890e55c2df5a4e097224e6125d6bd4987391e50625b222953b3
+size 23593408

checkpoints/gpt2-small/layer8-mlp-sequential/s0p75-tokens1048576/provenance.json

@@ -0,0 +1,45 @@
+{
+  "conversion": [
+    {
+      "module_path": "transformer.h.8.mlp.c_fc",
+      "rule": "read=A_prop; write=B_prop"
+    },
+    {
+      "module_path": "transformer.h.8.mlp.c_proj",
+      "rule": "read=A_prop; write=B_prop"
+    }
+  ],
+  "excluded": [
+    "base model weights",
+    "activation Gram matrices",
+    "dense target and reconstructed weights",
+    "optimizer state and data caches",
+    "cluster-local absolute paths"
+  ],
+  "notes": [
+    "The base model GELU between c_fc and c_proj is retained."
+  ],
+  "source_artifact_hashes": [
+    {
+      "bundle_id": "gpt2_small_l8_mlp_s0p75_cfc",
+      "hash_semantics": "source_file",
+      "sha256": "8bc0ae08ecc7a56d762ae642af9641b25bf12f5e461ab97dc4a38945863df700"
+    },
+    {
+      "bundle_id": "gpt2_small_l8_mlp_s0p75_cproj",
+      "hash_semantics": "source_file",
+      "sha256": "4414abe8b83d4244b6b11c6af3e59aa234f4904400df4a038e7d77185c0d7a6f"
+    }
+  ],
+  "source_bundle_ids": [
+    "gpt2_small_l8_mlp_s0p75_cfc",
+    "gpt2_small_l8_mlp_s0p75_cproj"
+  ],
+  "source_format": "gpt2_sequential_mlp_factor_artifacts",
+  "validation": {
+    "all_tensors_finite": true,
+    "orientation_random_input_max_abs_error": 0.0,
+    "safetensors_exact_round_trip": true
+  },
+  "variant_id": "gpt2_small_l8_mlp_s0p75"
+}

checkpoints/qwen2.5-0.5b/layer12-down-proj/s0p5-tokens1024/config.json

@@ -0,0 +1,30 @@
+{
+  "base_model": "Qwen/Qwen2.5-0.5B",
+  "ce_delta_vs_dense": 0.0012218963831869267,
+  "method": "SWD",
+  "module_replacements": [
+    {
+      "base_weight_layout": "out_in",
+      "bias_key": null,
+      "bias_policy": "preserve_base",
+      "conversion_rule": "source stores Linear weight as A@B; read=B.T; write=A.T",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 4864,
+      "module_path": "model.layers.12.mlp.down_proj",
+      "output_dim": 896,
+      "rank": 896,
+      "read_dtype": "float32",
+      "read_key": "modules.000.read",
+      "read_nnz": 1978367,
+      "write_dtype": "float32",
+      "write_key": "modules.000.write",
+      "write_nnz": 200703
+    }
+  ],
+  "replacement_scope": "single_matrix",
+  "schema_version": "swd_factor_checkpoint_v1",
+  "sparsity": 0.5,
+  "token_exposure": 1024,
+  "variant_id": "qwen25_0p5b_l12_downproj_s0p5",
+  "weights_file": "model.safetensors"
+}

checkpoints/qwen2.5-0.5b/layer12-down-proj/s0p5-tokens1024/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:400f49a25d6862f584c8057d138b83802d47e13c31b4b58acb1b4a740805ca6e
+size 20644120

checkpoints/qwen2.5-0.5b/layer12-down-proj/s0p5-tokens1024/provenance.json

@@ -0,0 +1,33 @@
+{
+  "conversion": [
+    {
+      "module_path": "model.layers.12.mlp.down_proj",
+      "rule": "source stores Linear weight as A@B; read=B.T; write=A.T"
+    }
+  ],
+  "excluded": [
+    "base model weights",
+    "activation Gram matrices",
+    "dense target and reconstructed weights",
+    "optimizer state and data caches",
+    "cluster-local absolute paths"
+  ],
+  "notes": [],
+  "source_artifact_hashes": [
+    {
+      "bundle_id": "qwen25_0p5b_l12_downproj_s0p5",
+      "hash_semantics": "source_file",
+      "sha256": "3202ce0270f0f75bfb14d6bb253fbda1b4dbfee077d892a295a30fbf449e1059"
+    }
+  ],
+  "source_bundle_ids": [
+    "qwen25_0p5b_l12_downproj_s0p5"
+  ],
+  "source_format": "qwen25_factor_and_baseline_artifact",
+  "validation": {
+    "all_tensors_finite": true,
+    "orientation_random_input_max_abs_error": 1.7881393432617188e-06,
+    "safetensors_exact_round_trip": true
+  },
+  "variant_id": "qwen25_0p5b_l12_downproj_s0p5"
+}

checkpoints/qwen2.5-0.5b/layer12-down-proj/s0p75-tokens1048576/config.json

@@ -0,0 +1,30 @@
+{
+  "base_model": "Qwen/Qwen2.5-0.5B",
+  "ce_delta_vs_dense": 0.005009775171065822,
+  "method": "SWD",
+  "module_replacements": [
+    {
+      "base_weight_layout": "out_in",
+      "bias_key": null,
+      "bias_policy": "preserve_base",
+      "conversion_rule": "source stores Linear weight as A@B; read=B.T; write=A.T",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 4864,
+      "module_path": "model.layers.12.mlp.down_proj",
+      "output_dim": 896,
+      "rank": 896,
+      "read_dtype": "float32",
+      "read_key": "modules.000.read",
+      "read_nnz": 888831,
+      "write_dtype": "float32",
+      "write_key": "modules.000.write",
+      "write_nnz": 200703
+    }
+  ],
+  "replacement_scope": "single_matrix",
+  "schema_version": "swd_factor_checkpoint_v1",
+  "sparsity": 0.75,
+  "token_exposure": 1048576,
+  "variant_id": "qwen25_0p5b_l12_downproj_s0p75",
+  "weights_file": "model.safetensors"
+}

checkpoints/qwen2.5-0.5b/layer12-down-proj/s0p75-tokens1048576/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:530b19e4f58da33e6f3b6a7bad749cd3e8bafa8b67ee41939075d3dcb1fb4078
+size 20644120

checkpoints/qwen2.5-0.5b/layer12-down-proj/s0p75-tokens1048576/provenance.json

@@ -0,0 +1,33 @@
+{
+  "conversion": [
+    {
+      "module_path": "model.layers.12.mlp.down_proj",
+      "rule": "source stores Linear weight as A@B; read=B.T; write=A.T"
+    }
+  ],
+  "excluded": [
+    "base model weights",
+    "activation Gram matrices",
+    "dense target and reconstructed weights",
+    "optimizer state and data caches",
+    "cluster-local absolute paths"
+  ],
+  "notes": [],
+  "source_artifact_hashes": [
+    {
+      "bundle_id": "qwen25_0p5b_l12_downproj_s0p75",
+      "hash_semantics": "source_file",
+      "sha256": "55862fab1505f2def48246d94cc1aa23e384511deb4906b8ce1de43f0ac130ca"
+    }
+  ],
+  "source_bundle_ids": [
+    "qwen25_0p5b_l12_downproj_s0p75"
+  ],
+  "source_format": "qwen25_factor_and_baseline_artifact",
+  "validation": {
+    "all_tensors_finite": true,
+    "orientation_random_input_max_abs_error": 1.430511474609375e-06,
+    "safetensors_exact_round_trip": true
+  },
+  "variant_id": "qwen25_0p5b_l12_downproj_s0p75"
+}

checkpoints/qwen2.5-1.5b/layer14-down-proj/s0p5-tokens1024/config.json

@@ -0,0 +1,30 @@
+{
+  "base_model": "Qwen/Qwen2.5-1.5B",
+  "ce_delta_vs_dense": 0.0007331378299122449,
+  "method": "SWD",
+  "module_replacements": [
+    {
+      "base_weight_layout": "out_in",
+      "bias_key": null,
+      "bias_policy": "preserve_base",
+      "conversion_rule": "source stores Linear weight as A@B; read=B.T; write=A.T",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 8960,
+      "module_path": "model.layers.14.mlp.down_proj",
+      "output_dim": 1536,
+      "rank": 1536,
+      "read_dtype": "float32",
+      "read_key": "modules.000.read",
+      "read_nnz": 6291455,
+      "write_dtype": "float32",
+      "write_key": "modules.000.write",
+      "write_nnz": 589823
+    }
+  ],
+  "replacement_scope": "single_matrix",
+  "schema_version": "swd_factor_checkpoint_v1",
+  "sparsity": 0.5,
+  "token_exposure": 1024,
+  "variant_id": "qwen25_1p5b_l14_downproj_s0p5",
+  "weights_file": "model.safetensors"
+}

checkpoints/qwen2.5-1.5b/layer14-down-proj/s0p5-tokens1024/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad3be13722726a1a447c2ea7f41c1ee55a21099ac231ae1ea2089950d1a204dc
+size 64487712

checkpoints/qwen2.5-1.5b/layer14-down-proj/s0p5-tokens1024/provenance.json

@@ -0,0 +1,33 @@
+{
+  "conversion": [
+    {
+      "module_path": "model.layers.14.mlp.down_proj",
+      "rule": "source stores Linear weight as A@B; read=B.T; write=A.T"
+    }
+  ],
+  "excluded": [
+    "base model weights",
+    "activation Gram matrices",
+    "dense target and reconstructed weights",
+    "optimizer state and data caches",
+    "cluster-local absolute paths"
+  ],
+  "notes": [],
+  "source_artifact_hashes": [
+    {
+      "bundle_id": "qwen25_1p5b_l14_downproj_s0p5",
+      "hash_semantics": "source_file",
+      "sha256": "7f7eae2b0b8c0e309adc7f1d09667f3bc8a894abac2a99f0038729527960e0fd"
+    }
+  ],
+  "source_bundle_ids": [
+    "qwen25_1p5b_l14_downproj_s0p5"
+  ],
+  "source_format": "qwen25_factor_and_baseline_artifact",
+  "validation": {
+    "all_tensors_finite": true,
+    "orientation_random_input_max_abs_error": 4.5299530029296875e-06,
+    "safetensors_exact_round_trip": true
+  },
+  "variant_id": "qwen25_1p5b_l14_downproj_s0p5"
+}

checkpoints/qwen2.5-1.5b/layer14-down-proj/s0p75-tokens1048576/config.json

@@ -0,0 +1,30 @@
+{
+  "base_model": "Qwen/Qwen2.5-1.5B",
+  "ce_delta_vs_dense": 0.0012218963831869267,
+  "method": "SWD",
+  "module_replacements": [
+    {
+      "base_weight_layout": "out_in",
+      "bias_key": null,
+      "bias_policy": "preserve_base",
+      "conversion_rule": "source stores Linear weight as A@B; read=B.T; write=A.T",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 8960,
+      "module_path": "model.layers.14.mlp.down_proj",
+      "output_dim": 1536,
+      "rank": 1536,
+      "read_dtype": "float32",
+      "read_key": "modules.000.read",
+      "read_nnz": 2850815,
+      "write_dtype": "float32",
+      "write_key": "modules.000.write",
+      "write_nnz": 589823
+    }
+  ],
+  "replacement_scope": "single_matrix",
+  "schema_version": "swd_factor_checkpoint_v1",
+  "sparsity": 0.75,
+  "token_exposure": 1048576,
+  "variant_id": "qwen25_1p5b_l14_downproj_s0p75",
+  "weights_file": "model.safetensors"
+}

checkpoints/qwen2.5-1.5b/layer14-down-proj/s0p75-tokens1048576/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa3fa378549da027bb4bf2778fa80b158c01840aa0306025c5f828baa5bcf7dd
+size 64487712

checkpoints/qwen2.5-1.5b/layer14-down-proj/s0p75-tokens1048576/provenance.json

@@ -0,0 +1,33 @@
+{
+  "conversion": [
+    {
+      "module_path": "model.layers.14.mlp.down_proj",
+      "rule": "source stores Linear weight as A@B; read=B.T; write=A.T"
+    }
+  ],
+  "excluded": [
+    "base model weights",
+    "activation Gram matrices",
+    "dense target and reconstructed weights",
+    "optimizer state and data caches",
+    "cluster-local absolute paths"
+  ],
+  "notes": [],
+  "source_artifact_hashes": [
+    {
+      "bundle_id": "qwen25_1p5b_l14_downproj_s0p75",
+      "hash_semantics": "source_file",
+      "sha256": "993fab945123bd20ff08a5c6dc2a7c1de58b4764e3531b86b207ab0c8e887c09"
+    }
+  ],
+  "source_bundle_ids": [
+    "qwen25_1p5b_l14_downproj_s0p75"
+  ],
+  "source_format": "qwen25_factor_and_baseline_artifact",
+  "validation": {
+    "all_tensors_finite": true,
+    "orientation_random_input_max_abs_error": 3.814697265625e-06,
+    "safetensors_exact_round_trip": true
+  },
+  "variant_id": "qwen25_1p5b_l14_downproj_s0p75"
+}

checkpoints/qwen3.5-27b/layer31-down-proj/s0p5-tokens2048/config.json

@@ -0,0 +1,31 @@
+{
+  "base_model": "Qwen/Qwen3.5-27B",
+  "ce_delta_vs_dense": -0.00042653013883819924,
+  "method": "SWD",
+  "module_replacements": [
+    {
+      "base_weight_layout": "out_in",
+      "bias_key": null,
+      "bias_policy": "preserve_base",
+      "conversion_rule": "read=cat(shard.A, dim=1); write=cat(shard.B, dim=0)",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 17408,
+      "module_path": "model.layers.31.mlp.down_proj",
+      "output_dim": 5120,
+      "rank": 5120,
+      "read_dtype": "bfloat16",
+      "read_key": "modules.000.read",
+      "read_nnz": 38010879,
+      "write_dtype": "bfloat16",
+      "write_key": "modules.000.write",
+      "write_nnz": 6553599
+    }
+  ],
+  "replacement_scope": "single_matrix",
+  "schema_version": "swd_factor_checkpoint_v1",
+  "source_shard_count": 4,
+  "sparsity": 0.5,
+  "token_exposure": 2048,
+  "variant_id": "qwen35_27b_l31_downproj_s0p5",
+  "weights_file": "model.safetensors"
+}

checkpoints/qwen3.5-27b/layer31-down-proj/s0p5-tokens2048/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84c6673c889c229853de6cea4556f4ee1b7e054b03e191353fb96563c7d31877
+size 230687008

checkpoints/qwen3.5-27b/layer31-down-proj/s0p5-tokens2048/provenance.json

@@ -0,0 +1,35 @@
+{
+  "conversion": [
+    {
+      "module_path": "model.layers.31.mlp.down_proj",
+      "rule": "read=cat(shard.A, dim=1); write=cat(shard.B, dim=0)"
+    }
+  ],
+  "excluded": [
+    "base model weights",
+    "activation Gram matrices",
+    "dense target and reconstructed weights",
+    "optimizer state and data caches",
+    "cluster-local absolute paths"
+  ],
+  "notes": [
+    "Source BF16 precision is preserved."
+  ],
+  "source_artifact_hashes": [
+    {
+      "bundle_id": "qwen35_27b_l31_downproj_s0p5",
+      "hash_semantics": "source_checkpoint_aggregate",
+      "sha256": "ab3528492d78d7935cb8422b687672eb11dcad7bda77a20dc3ff0d8cfb09c18f"
+    }
+  ],
+  "source_bundle_ids": [
+    "qwen35_27b_l31_downproj_s0p5"
+  ],
+  "source_format": "qwen35_four_shard_factor_checkpoint",
+  "validation": {
+    "all_tensors_finite": true,
+    "orientation_random_input_max_abs_error": 3.814697265625e-06,
+    "safetensors_exact_round_trip": true
+  },
+  "variant_id": "qwen35_27b_l31_downproj_s0p5"
+}

checkpoints/qwen3.5-27b/layer31-down-proj/s0p75-tokens1048576/config.json

@@ -0,0 +1,31 @@
+{
+  "base_model": "Qwen/Qwen3.5-27B",
+  "ce_delta_vs_dense": -0.00044806658464224824,
+  "method": "SWD",
+  "module_replacements": [
+    {
+      "base_weight_layout": "out_in",
+      "bias_key": null,
+      "bias_policy": "preserve_base",
+      "conversion_rule": "read=cat(shard.A, dim=1); write=cat(shard.B, dim=0)",
+      "forward_equation": "output = input @ read @ write + bias",
+      "input_dim": 17408,
+      "module_path": "model.layers.31.mlp.down_proj",
+      "output_dim": 5120,
+      "rank": 5120,
+      "read_dtype": "bfloat16",
+      "read_key": "modules.000.read",
+      "read_nnz": 15728639,
+      "write_dtype": "bfloat16",
+      "write_key": "modules.000.write",
+      "write_nnz": 6553599
+    }
+  ],
+  "replacement_scope": "single_matrix",
+  "schema_version": "swd_factor_checkpoint_v1",
+  "source_shard_count": 4,
+  "sparsity": 0.75,
+  "token_exposure": 1048576,
+  "variant_id": "qwen35_27b_l31_downproj_s0p75",
+  "weights_file": "model.safetensors"
+}

checkpoints/qwen3.5-27b/layer31-down-proj/s0p75-tokens1048576/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92eeb9672d70cbc16c4483bfa9673593781bf73069e145aad75bf6350df86066
+size 230687008

checkpoints/qwen3.5-27b/layer31-down-proj/s0p75-tokens1048576/provenance.json

@@ -0,0 +1,35 @@
+{
+  "conversion": [
+    {
+      "module_path": "model.layers.31.mlp.down_proj",
+      "rule": "read=cat(shard.A, dim=1); write=cat(shard.B, dim=0)"
+    }
+  ],
+  "excluded": [
+    "base model weights",
+    "activation Gram matrices",
+    "dense target and reconstructed weights",
+    "optimizer state and data caches",
+    "cluster-local absolute paths"
+  ],
+  "notes": [
+    "Source BF16 precision is preserved."
+  ],
+  "source_artifact_hashes": [
+    {
+      "bundle_id": "qwen35_27b_l31_downproj_s0p75",
+      "hash_semantics": "source_checkpoint_aggregate",
+      "sha256": "3b94e88ab0e48e4c1a3d7977aa05a5c579d461a013dde2a4a3671a020ff3e124"
+    }
+  ],
+  "source_bundle_ids": [
+    "qwen35_27b_l31_downproj_s0p75"
+  ],
+  "source_format": "qwen35_four_shard_factor_checkpoint",
+  "validation": {
+    "all_tensors_finite": true,
+    "orientation_random_input_max_abs_error": 2.384185791015625e-06,
+    "safetensors_exact_round_trip": true
+  },
+  "variant_id": "qwen35_27b_l31_downproj_s0p75"
+}

examples/load_checkpoint.py

@@ -0,0 +1,17 @@
+"""Minimal example for applying an SWD checkpoint to a base model."""
+
+from pathlib import Path
+
+from transformers import AutoModelForCausalLM
+
+from swd_loader import apply_swd_checkpoint
+
+
+repo_root = Path(__file__).resolve().parents[1]
+checkpoint = repo_root / "checkpoints/gpt2-small/layer8-cproj/s0p5-tokens16384"
+model = AutoModelForCausalLM.from_pretrained("gpt2")
+
+# Use mode="folded" for a conventional dense module after loading.
+applied = apply_swd_checkpoint(model, checkpoint, mode="factorized")
+print(applied)
+print(model.transformer.h[8].mlp.c_proj)

requirements.txt

@@ -0,0 +1,4 @@
+torch>=2.1
+safetensors>=0.4
+huggingface_hub>=1.0
+transformers

swd_loader.py

@@ -0,0 +1,228 @@
+"""Load SWD factor checkpoints into an already constructed PyTorch model."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Literal
+
+import torch
+from safetensors.torch import load_file
+from torch import nn
+
+
+@dataclass(frozen=True)
+class AppliedReplacement:
+    module_path: str
+    input_dim: int
+    rank: int
+    output_dim: int
+    mode: str
+
+
+class SWDLinear(nn.Module):
+    """Two-factor linear map with explicit scalar bottleneck activations."""
+
+    def __init__(
+        self,
+        read: torch.Tensor,
+        write: torch.Tensor,
+        bias: torch.Tensor | None = None,
+    ) -> None:
+        super().__init__()
+        if read.ndim != 2 or write.ndim != 2 or read.shape[1] != write.shape[0]:
+            raise ValueError(
+                f"Invalid SWD shapes: read={tuple(read.shape)}, write={tuple(write.shape)}"
+            )
+        self.read = nn.Parameter(read.detach().contiguous(), requires_grad=False)
+        self.write = nn.Parameter(write.detach().contiguous(), requires_grad=False)
+        self.bias = (
+            None
+            if bias is None
+            else nn.Parameter(bias.detach().contiguous(), requires_grad=False)
+        )
+
+    @property
+    def in_features(self) -> int:
+        return int(self.read.shape[0])
+
+    @property
+    def rank(self) -> int:
+        return int(self.read.shape[1])
+
+    @property
+    def out_features(self) -> int:
+        return int(self.write.shape[1])
+
+    def component_activations(self, inputs: torch.Tensor) -> torch.Tensor:
+        return inputs.matmul(self.read)
+
+    def forward(self, inputs: torch.Tensor) -> torch.Tensor:
+        outputs = self.component_activations(inputs).matmul(self.write)
+        if self.bias is not None:
+            outputs = outputs + self.bias
+        return outputs
+
+
+def _get_child(module: Any, name: str) -> Any:
+    if name.isdigit() and isinstance(module, (nn.ModuleList, nn.Sequential)):
+        return module[int(name)]
+    return getattr(module, name)
+
+
+def _set_child(module: Any, name: str, value: nn.Module) -> None:
+    if name.isdigit() and isinstance(module, (nn.ModuleList, nn.Sequential)):
+        module[int(name)] = value
+        return
+    setattr(module, name, value)
+
+
+def _resolve_parent(model: nn.Module, module_path: str) -> tuple[Any, str, nn.Module]:
+    parts = module_path.split(".")
+    if not parts or any(not part for part in parts):
+        raise ValueError(f"Invalid module path: {module_path!r}")
+    parent: Any = model
+    for part in parts[:-1]:
+        parent = _get_child(parent, part)
+    leaf = parts[-1]
+    target = _get_child(parent, leaf)
+    if not isinstance(target, nn.Module):
+        raise TypeError(f"Target at {module_path!r} is not an nn.Module")
+    return parent, leaf, target
+
+
+def _module_device_dtype(module: nn.Module) -> tuple[torch.device, torch.dtype]:
+    weight = getattr(module, "weight", None)
+    if not torch.is_tensor(weight):
+        raise TypeError("Target module must expose a materialized weight tensor")
+    if weight.device.type == "meta":
+        raise ValueError("Load/materialize the base model before applying SWD factors")
+    if not weight.dtype.is_floating_point:
+        raise TypeError(f"Unsupported target weight dtype: {weight.dtype}")
+    return weight.device, weight.dtype
+
+
+def _validate_base_weight(
+    module: nn.Module,
+    *,
+    module_path: str,
+    input_dim: int,
+    output_dim: int,
+    layout: str,
+) -> None:
+    weight = getattr(module, "weight", None)
+    if not torch.is_tensor(weight):
+        raise TypeError(f"{module_path} does not expose a weight tensor")
+    expected = (
+        (input_dim, output_dim) if layout == "in_out" else (output_dim, input_dim)
+    )
+    if tuple(weight.shape) != expected:
+        raise ValueError(
+            f"Base weight mismatch at {module_path}: got {tuple(weight.shape)}, expected {expected}"
+        )
+
+
+def _select_bias(
+    module: nn.Module,
+    tensors: dict[str, torch.Tensor],
+    spec: dict[str, Any],
+    *,
+    device: torch.device,
+    dtype: torch.dtype,
+) -> torch.Tensor | None:
+    policy = spec["bias_policy"]
+    if policy == "checkpoint":
+        bias = tensors[spec["bias_key"]]
+    elif policy == "preserve_base":
+        bias = getattr(module, "bias", None)
+    elif policy == "none":
+        bias = None
+    else:
+        raise ValueError(f"Unknown bias policy: {policy}")
+    return None if bias is None else bias.detach().to(device=device, dtype=dtype)
+
+
+def _fold_into_module(
+    module: nn.Module,
+    read: torch.Tensor,
+    write: torch.Tensor,
+    bias: torch.Tensor | None,
+    *,
+    layout: str,
+) -> None:
+    dense_in_out = read.matmul(write)
+    dense = dense_in_out if layout == "in_out" else dense_in_out.transpose(0, 1)
+    weight = getattr(module, "weight")
+    with torch.no_grad():
+        weight.copy_(dense.to(device=weight.device, dtype=weight.dtype))
+        existing_bias = getattr(module, "bias", None)
+        if bias is not None:
+            if existing_bias is None:
+                raise ValueError("Checkpoint provides bias but base module has no bias parameter")
+            existing_bias.copy_(bias.to(existing_bias.device, existing_bias.dtype))
+
+
+def load_swd_config(checkpoint_dir: str | Path) -> dict[str, Any]:
+    path = Path(checkpoint_dir) / "config.json"
+    with path.open(encoding="utf-8") as handle:
+        config = json.load(handle)
+    if config.get("schema_version") != "swd_factor_checkpoint_v1":
+        raise ValueError(f"Unsupported SWD schema: {config.get('schema_version')!r}")
+    return config
+
+
+def apply_swd_checkpoint(
+    model: nn.Module,
+    checkpoint_dir: str | Path,
+    *,
+    mode: Literal["factorized", "folded"] = "factorized",
+) -> list[AppliedReplacement]:
+    """Apply one release checkpoint to a loaded base model.
+
+    ``factorized`` installs :class:`SWDLinear` modules and preserves explicit
+    bottleneck activations. ``folded`` writes ``read @ write`` into the existing
+    dense modules for conventional inference.
+    """
+
+    if mode not in {"factorized", "folded"}:
+        raise ValueError(f"Unknown mode: {mode}")
+    root = Path(checkpoint_dir)
+    config = load_swd_config(root)
+    tensors = load_file(root / config["weights_file"], device="cpu")
+    applied: list[AppliedReplacement] = []
+
+    for spec in config["module_replacements"]:
+        path = spec["module_path"]
+        parent, leaf, module = _resolve_parent(model, path)
+        input_dim = int(spec["input_dim"])
+        rank = int(spec["rank"])
+        output_dim = int(spec["output_dim"])
+        layout = spec["base_weight_layout"]
+        _validate_base_weight(
+            module,
+            module_path=path,
+            input_dim=input_dim,
+            output_dim=output_dim,
+            layout=layout,
+        )
+        device, dtype = _module_device_dtype(module)
+        read = tensors[spec["read_key"]]
+        write = tensors[spec["write_key"]]
+        if tuple(read.shape) != (input_dim, rank):
+            raise ValueError(f"Read tensor mismatch for {path}: {tuple(read.shape)}")
+        if tuple(write.shape) != (rank, output_dim):
+            raise ValueError(f"Write tensor mismatch for {path}: {tuple(write.shape)}")
+        read = read.to(device=device, dtype=dtype)
+        write = write.to(device=device, dtype=dtype)
+        bias = _select_bias(module, tensors, spec, device=device, dtype=dtype)
+
+        if mode == "factorized":
+            _set_child(parent, leaf, SWDLinear(read, write, bias))
+        else:
+            _fold_into_module(module, read, write, bias, layout=layout)
+
+        applied.append(
+            AppliedReplacement(path, input_dim, rank, output_dim, mode)
+        )
+    return applied