REAX Engine: final release engine tree EC 8dfb7d67 (weights unchanged)
Browse files- engine/LICENSE +21 -202
- engine/README.md +6 -4
- engine/SEALED-CONTRACT.md +7 -6
- engine/SHA256SUMS.json +17 -35
- engine/SHA256SUMS.shipped.json +42 -0
- engine/THIRD_PARTY_NOTICES.md +2 -1
- engine/autotune_guard.py +24 -1
- engine/backend.py +6 -2
- engine/benchmark.py +15 -4
- engine/fast_fp8_gemm.py +72 -0
- engine/probabilities.py +1 -1
- engine/reax13.json +2 -1
- engine/reax13_engine.py +77 -9
- engine/serve.py +12 -1
- engine/test_bundle.py +475 -0
- engine/test_fast_fp8_gemm.py +21 -0
- engine/test_probabilities.py +1 -1
- engine/test_serve.py +1 -1
engine/LICENSE
CHANGED
|
@@ -1,202 +1,21 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
-
|
| 24 |
-
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
-
exercising permissions granted by this License.
|
| 26 |
-
|
| 27 |
-
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
-
including but not limited to software source code, documentation
|
| 29 |
-
source, and configuration files.
|
| 30 |
-
|
| 31 |
-
"Object" form shall mean any form resulting from mechanical
|
| 32 |
-
transformation or translation of a Source form, including but
|
| 33 |
-
not limited to compiled object code, generated documentation,
|
| 34 |
-
and conversions to other media types.
|
| 35 |
-
|
| 36 |
-
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
-
Object form, made available under the License, as indicated by a
|
| 38 |
-
copyright notice that is included in or attached to the work
|
| 39 |
-
(an example is provided in the Appendix below).
|
| 40 |
-
|
| 41 |
-
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
-
form, that is based on (or derived from) the Work and for which the
|
| 43 |
-
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
-
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
-
of this License, Derivative Works shall not include works that remain
|
| 46 |
-
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
-
the Work and Derivative Works thereof.
|
| 48 |
-
|
| 49 |
-
"Contribution" shall mean any work of authorship, including
|
| 50 |
-
the original version of the Work and any modifications or additions
|
| 51 |
-
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
-
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
-
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
-
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
-
means any form of electronic, verbal, or written communication sent
|
| 56 |
-
to the Licensor or its representatives, including but not limited to
|
| 57 |
-
communication on electronic mailing lists, source code control systems,
|
| 58 |
-
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
-
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
-
excluding communication that is conspicuously marked or otherwise
|
| 61 |
-
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
-
|
| 63 |
-
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
-
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
-
subsequently incorporated within the Work.
|
| 66 |
-
|
| 67 |
-
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
-
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
-
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
-
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
-
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
-
Work and such Derivative Works in Source or Object form.
|
| 73 |
-
|
| 74 |
-
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
-
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
-
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
-
(except as stated in this section) patent license to make, have made,
|
| 78 |
-
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
-
where such license applies only to those patent claims licensable
|
| 80 |
-
by such Contributor that are necessarily infringed by their
|
| 81 |
-
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
-
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
-
institute patent litigation against any entity (including a
|
| 84 |
-
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
-
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
-
or contributory patent infringement, then any patent licenses
|
| 87 |
-
granted to You under this License for that Work shall terminate
|
| 88 |
-
as of the date such litigation is filed.
|
| 89 |
-
|
| 90 |
-
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
-
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
-
modifications, and in Source or Object form, provided that You
|
| 93 |
-
meet the following conditions:
|
| 94 |
-
|
| 95 |
-
(a) You must give any other recipients of the Work or
|
| 96 |
-
Derivative Works a copy of this License; and
|
| 97 |
-
|
| 98 |
-
(b) You must cause any modified files to carry prominent notices
|
| 99 |
-
stating that You changed the files; and
|
| 100 |
-
|
| 101 |
-
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
-
that You distribute, all copyright, patent, trademark, and
|
| 103 |
-
attribution notices from the Source form of the Work,
|
| 104 |
-
excluding those notices that do not pertain to any part of
|
| 105 |
-
the Derivative Works; and
|
| 106 |
-
|
| 107 |
-
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
-
distribution, then any Derivative Works that You distribute must
|
| 109 |
-
include a readable copy of the attribution notices contained
|
| 110 |
-
within such NOTICE file, excluding those notices that do not
|
| 111 |
-
pertain to any part of the Derivative Works, in at least one
|
| 112 |
-
of the following places: within a NOTICE text file distributed
|
| 113 |
-
as part of the Derivative Works; within the Source form or
|
| 114 |
-
documentation, if provided along with the Derivative Works; or,
|
| 115 |
-
within a display generated by the Derivative Works, if and
|
| 116 |
-
wherever such third-party notices normally appear. The contents
|
| 117 |
-
of the NOTICE file are for informational purposes only and
|
| 118 |
-
do not modify the License. You may add Your own attribution
|
| 119 |
-
notices within Derivative Works that You distribute, alongside
|
| 120 |
-
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
-
that such additional attribution notices cannot be construed
|
| 122 |
-
as modifying the License.
|
| 123 |
-
|
| 124 |
-
You may add Your own copyright statement to Your modifications and
|
| 125 |
-
may provide additional or different license terms and conditions
|
| 126 |
-
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
-
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
-
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
-
the conditions stated in this License.
|
| 130 |
-
|
| 131 |
-
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
-
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
-
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
-
this License, without any additional terms or conditions.
|
| 135 |
-
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
-
the terms of any separate license agreement you may have executed
|
| 137 |
-
with Licensor regarding such Contributions.
|
| 138 |
-
|
| 139 |
-
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
-
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
-
except as required for reasonable and customary use in describing the
|
| 142 |
-
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
-
|
| 144 |
-
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
-
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
-
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
-
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
-
implied, including, without limitation, any warranties or conditions
|
| 149 |
-
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
-
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
-
appropriateness of using or redistributing the Work and assume any
|
| 152 |
-
risks associated with Your exercise of permissions under this License.
|
| 153 |
-
|
| 154 |
-
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
-
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
-
unless required by applicable law (such as deliberate and grossly
|
| 157 |
-
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
-
liable to You for damages, including any direct, indirect, special,
|
| 159 |
-
incidental, or consequential damages of any character arising as a
|
| 160 |
-
result of this License or out of the use or inability to use the
|
| 161 |
-
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
-
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
-
other commercial damages or losses), even if such Contributor
|
| 164 |
-
has been advised of the possibility of such damages.
|
| 165 |
-
|
| 166 |
-
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
-
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
-
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
-
or other liability obligations and/or rights consistent with this
|
| 170 |
-
License. However, in accepting such obligations, You may act only
|
| 171 |
-
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
-
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
-
defend, and hold each Contributor harmless for any liability
|
| 174 |
-
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
-
of your accepting any such warranty or additional liability.
|
| 176 |
-
|
| 177 |
-
END OF TERMS AND CONDITIONS
|
| 178 |
-
|
| 179 |
-
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
-
|
| 181 |
-
To apply the Apache License to your work, attach the following
|
| 182 |
-
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
-
replaced with your own identifying information. (Don't include
|
| 184 |
-
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
-
comment syntax for the file format. We also recommend that a
|
| 186 |
-
file or class name and description of purpose be included on the
|
| 187 |
-
same "printed page" as the copyright notice for easier
|
| 188 |
-
identification within third-party archives.
|
| 189 |
-
|
| 190 |
-
Copyright 2026 The REAX Engine authors
|
| 191 |
-
|
| 192 |
-
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
-
you may not use this file except in compliance with the License.
|
| 194 |
-
You may obtain a copy of the License at
|
| 195 |
-
|
| 196 |
-
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
-
|
| 198 |
-
Unless required by applicable law or agreed to in writing, software
|
| 199 |
-
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
-
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
-
See the License for the specific language governing permissions and
|
| 202 |
-
limitations under the License.
|
|
|
|
| 1 |
+
MIT License
|
| 2 |
+
|
| 3 |
+
Copyright (c) 2026 productivity-boost.com Betriebs UG (haftungsbeschränkt) & Co. KG
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
SOFTWARE.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
engine/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
| 2 |
|
| 3 |
`Reax13Engine` is a custom engine for the public [Decision Index kit](https://github.com/apolinario/decision-index/blob/25c6ec7b6b5f44a216c63c3a700de3a9c6d6699d/docs/engines.md). It loads both parts of one model repository on **one NVIDIA RTX PRO 6000 Blackwell 96 GB**. The language-model linear weights use dynamic fine-grained FP8, block128. Heads, non-language towers and small gated-delta projections remain BF16. Both parts must fit together; this is not two-GPU inference.
|
| 4 |
|
| 5 |
-
For every request, two host workers launch the native Gemma/KnowLine and Qwen/d3 paths on separate CUDA streams. Every question goes to both parts. Their per-option probabilities are averaged with weights 0.5/0.5, checked for completeness and normalized for rounding error. The result contains every supplied option. External requests are serialized. A host lock protects Triton’s shared mutable autotuner during tuning and dispatch; GPU kernels remain asynchronous after launch. Prompts, native temperature policies and option readouts are retained; no truncation or benchmark-specific rules.
|
| 6 |
|
| 7 |
The default `reax13` mode enables the official causal-conv1d kernel and within-request Qwen prefix/state reuse for multi-question prefixes up to24576 tokens. Prefixes above16384 wait for the Gemma forward before the cached Qwen pass to bound peak memory. An allocation failure retries the native uncached Qwen path. The cache is cleared between requests; no answers are cached. The conservative mode does not require miner projection kernels, CUDA graphs or RMSNorm compilation.
|
| 8 |
|
|
@@ -67,7 +67,7 @@ docker run --rm --network none --gpus device=0 --read-only \
|
|
| 67 |
--rows /inputs/sample.jsonl --out /out/run --fresh
|
| 68 |
```
|
| 69 |
|
| 70 |
-
For index-style latency, pass `--option warmup_rows=/inputs/sample.jsonl`. This adds three longest rows and the first ten sample rows to the kit's two warmups, then measures each request once, sequentially. The official kit supports choice/noul. For a mixed sample containing native score questions, use the included compatibility driver with the same Engine API and timing boundaries: replace the final command above with `python /engine/benchmark.py --config /engine/reax13.json --rows /inputs/sample.jsonl --out /out/latency`. It adds score-distribution and expected-value validation without changing inputs. Startup/loading/warmup are excluded. The measured wall time includes IPC, both prompt renderers/tokenizers, all native model work, synchronization and probability combination. Engine raw output includes internal times and peak allocated/reserved GPU bytes. Peak GPU process memory is also sampled externally in the validation run.
|
| 71 |
|
| 72 |
A Python caller with the pinned kit available can construct `Reax13Engine(config=..., model=...)`, then call `engine(state, questions)` to get `(response, diagnostics)`. Use `engine.close()` when finished. The supplied adapter requires a container isolation marker. The experimental `optimization=plain` configuration is a sequential FP8 reference for parity diagnostics, not the fast mode. The public kit at this pin validates choice and noul; the backend also handles native score questions.
|
| 73 |
|
|
@@ -78,9 +78,11 @@ A Python caller with the pinned kit available can construct `Reax13Engine(config
|
|
| 78 |
The HTTP wrapper does not invoke model-specific code itself: it calls `Reax13Engine(state, questions)`, checks that every requested option is present and normalized, and serializes that result. CPU contract tests use a mock native backend:
|
| 79 |
|
| 80 |
```bash
|
| 81 |
-
python3 -m unittest -v test_probabilities test_serve
|
| 82 |
```
|
| 83 |
|
|
|
|
|
|
|
| 84 |
## Public stand-in checkpoints
|
| 85 |
|
| 86 |
Validation uses these pinned public repositories, downloaded and hash-checked on the disposable pod:
|
|
@@ -90,4 +92,4 @@ Validation uses these pinned public repositories, downloaded and hash-checked on
|
|
| 90 |
|
| 91 |
They match the target layouts. Stand-in smoke and latency do not measure released REAX1.3 blended/fine-tuned weights or establish its quality. Release weights require their own end-to-end smoke and quality gates. No publication or production deployment is performed by this validation.
|
| 92 |
|
| 93 |
-
Code: Apache-2.0. Third-party files and external weights retain their licences; see THIRD_PARTY_NOTICES.md and third_party/.
|
|
|
|
| 2 |
|
| 3 |
`Reax13Engine` is a custom engine for the public [Decision Index kit](https://github.com/apolinario/decision-index/blob/25c6ec7b6b5f44a216c63c3a700de3a9c6d6699d/docs/engines.md). It loads both parts of one model repository on **one NVIDIA RTX PRO 6000 Blackwell 96 GB**. The language-model linear weights use dynamic fine-grained FP8, block128. Heads, non-language towers and small gated-delta projections remain BF16. Both parts must fit together; this is not two-GPU inference.
|
| 4 |
|
| 5 |
+
For every request, two host workers launch the native Gemma/KnowLine and Qwen/d3 paths on separate CUDA streams. Every question goes to both parts. Their per-option probabilities are averaged with weights 0.5/0.5, checked for completeness and normalized for rounding error. The result contains every supplied option. External requests are serialized. A host lock protects Triton’s shared mutable autotuner during tuning and dispatch; GPU kernels remain asynchronous after launch. Prompts, native temperature policies and option readouts are retained; no truncation or benchmark-specific rules. A request above the declared input cap (`max_input_proxy_tokens` in `reax13.json`, 32,000; size proxy = characters of the compact request JSON divided by 4) is refused as unsupported before any model work. A request has `request_timeout` = 1800 s to answer; rows of 24k-32k proxy tokens take minutes (measured 242-258 s on a stand-in box), so a client must allow at least that (the kit's `http` engine defaults to 600 s and needs a larger `timeout`). If the native backend dies (request timeout `request_timeout`, GPU memory failure, crash), later requests fail with an error and `/healthz` answers 503 until the container is restarted; nothing in this bundle restarts it (`docker run --rm`, no restart policy, no HEALTHCHECK), so the operator must. A bad or partial reply line also closes the backend.
|
| 6 |
|
| 7 |
The default `reax13` mode enables the official causal-conv1d kernel and within-request Qwen prefix/state reuse for multi-question prefixes up to24576 tokens. Prefixes above16384 wait for the Gemma forward before the cached Qwen pass to bound peak memory. An allocation failure retries the native uncached Qwen path. The cache is cleared between requests; no answers are cached. The conservative mode does not require miner projection kernels, CUDA graphs or RMSNorm compilation.
|
| 8 |
|
|
|
|
| 67 |
--rows /inputs/sample.jsonl --out /out/run --fresh
|
| 68 |
```
|
| 69 |
|
| 70 |
+
For index-style latency, pass `--option warmup_rows=/inputs/sample.jsonl`. This adds three longest rows and the first ten sample rows to the kit's two warmups (rows above the input cap are skipped before the longest rows are chosen), then measures each request once, sequentially. The official kit supports choice/noul. For a mixed sample containing native score questions, use the included compatibility driver with the same Engine API and timing boundaries: replace the final command above with `python /engine/benchmark.py --config /engine/reax13.json --rows /inputs/sample.jsonl --out /out/latency`. It adds score-distribution and expected-value validation without changing inputs. Startup/loading/warmup are excluded. The measured wall time includes IPC, both prompt renderers/tokenizers, all native model work, synchronization and probability combination. Engine raw output includes internal times and peak allocated/reserved GPU bytes. Peak GPU process memory is also sampled externally in the validation run.
|
| 71 |
|
| 72 |
A Python caller with the pinned kit available can construct `Reax13Engine(config=..., model=...)`, then call `engine(state, questions)` to get `(response, diagnostics)`. Use `engine.close()` when finished. The supplied adapter requires a container isolation marker. The experimental `optimization=plain` configuration is a sequential FP8 reference for parity diagnostics, not the fast mode. The public kit at this pin validates choice and noul; the backend also handles native score questions.
|
| 73 |
|
|
|
|
| 78 |
The HTTP wrapper does not invoke model-specific code itself: it calls `Reax13Engine(state, questions)`, checks that every requested option is present and normalized, and serializes that result. CPU contract tests use a mock native backend:
|
| 79 |
|
| 80 |
```bash
|
| 81 |
+
python3 -m unittest -v test_probabilities test_serve test_bundle
|
| 82 |
```
|
| 83 |
|
| 84 |
+
`test_bundle` also runs on CPU: a stub backend replaces the GPU process to test the input cap, dead-backend handling and the kernel dispatch guards. `test_fast_fp8_gemm.py` needs the GPU and the runtime image.
|
| 85 |
+
|
| 86 |
## Public stand-in checkpoints
|
| 87 |
|
| 88 |
Validation uses these pinned public repositories, downloaded and hash-checked on the disposable pod:
|
|
|
|
| 92 |
|
| 93 |
They match the target layouts. Stand-in smoke and latency do not measure released REAX1.3 blended/fine-tuned weights or establish its quality. Release weights require their own end-to-end smoke and quality gates. No publication or production deployment is performed by this validation.
|
| 94 |
|
| 95 |
+
Code: MIT (LICENSE), except fast_fp8_gemm.py, which stays Apache-2.0 as a derivative of finegrained-fp8 (full text in third_party/finegrained-fp8/LICENSE). Third-party files and external weights retain their licences; see THIRD_PARTY_NOTICES.md and third_party/.
|
engine/SEALED-CONTRACT.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
# REAX 1.3 HTTP serving contract
|
| 2 |
|
| 3 |
-
Status: CPU contract checks
|
| 4 |
|
| 5 |
## Image and build
|
| 6 |
|
|
@@ -45,10 +45,11 @@ Offline guarantees: `env -i` gives the Python process only the listed non-secret
|
|
| 45 |
|
| 46 |
## HTTP API
|
| 47 |
|
| 48 |
-
- `GET /healthz` returns ready only after both parts loaded and warmup completed.
|
|
|
|
| 49 |
- `POST /v1/systemone` accepts `application/json`, at most 16 MiB, with 1–64 keyed questions. A one-question request uses the same object shape as a joint request. Choice criteria allow 2–255 options; score criteria are an ordered list of 2–10 levels; noul criteria are optional. Unknown top-level fields, non-finite JSON numbers, unsupported types, and invalid shapes are rejected. HTTP access logs omit request bodies; the inherited backend traceback behavior on inference errors is unchanged.
|
| 50 |
- The HTTP adapter calls the same `Reax13Engine(state, questions)` instance and frozen `reax13.json` settings used by the Engine API. It does not alter model prompts, temperatures, ordering, or probability mixing. It validates the Engine response before returning it. For noul, it retains the native scalar `noul` field and adds a normalized `{ "false": 1-p, "true": p }` `probabilities` map for clients that consume all options uniformly.
|
| 51 |
-
- Capacity/context and malformed requests return HTTP 422. Unexpected inference errors return HTTP 500 without exposing traceback details. Calls are serialized against the single resident engine.
|
| 52 |
|
| 53 |
One-question request:
|
| 54 |
|
|
@@ -95,7 +96,7 @@ For choice and score questions, probabilities cover every supplied option/level
|
|
| 95 |
|
| 96 |
## Tokenizers, context, and native prompts
|
| 97 |
|
| 98 |
-
Both part maxima below are token counts for each rendered question, including chat-template tokens. The limits are upstream/model configuration values; acceptance at maximum context on the assembled REAX weights and the HTTP serving image is **UNVERIFIED**. The engine does not truncate input; an over-limit request must fail as unsupported.
|
| 99 |
|
| 100 |
| Part | Tokenizer and upstream context | Native prompt path |
|
| 101 |
|---|---|---|
|
|
@@ -106,6 +107,6 @@ Part G / KnowLine source pins: [`PelaAI/KnowLine-31B-Gen1` model card](https://h
|
|
| 106 |
|
| 107 |
## GPU measurements and verification status
|
| 108 |
|
| 109 |
-
Inherited Engine API baseline on one RTX PRO 6000 Blackwell Server Edition with 96 GB VRAM: 300 sequential requests over pinned public stand-ins; median 546.21 ms, mean 815.05 ms, p80 706.36 ms, p95 1,157.69 ms, max 23,778.34 ms. Model load and warmup excluded. Peak allocated memory was 83.95 GiB and peak reserved memory was 88.49 GiB.
|
| 110 |
|
| 111 |
-
The preceding slice's unmodified public harness smoke passed 8/8 and its Engine API benchmark passed 300/300; see `../r3-engine/RESULT.md` and the retained evidence under `/home/flori/jobs-data/hfdi-push-continue-20261101/r3-engine/`. The server wrapper's GPU startup, 5-request parity, HTTP latency overhead, and exact current image digest are **UNVERIFIED**.
|
|
|
|
| 1 |
# REAX 1.3 HTTP serving contract
|
| 2 |
|
| 3 |
+
Status: CPU contract checks pass for this tree (`test_bundle`, `test_probabilities`, `test_serve`). A GPU smoke of an earlier revision of this tree (before the IPC reader change) on pinned public stand-in weights was run once (52 rows, forced-timeout and over-cap checks); there is no T0 run of these exact bytes on release weights. The inherited baseline below is from the earlier engine. Every HTTP-server-on-GPU property below is **UNVERIFIED** until the wrapper is run against a GPU.
|
| 4 |
|
| 5 |
## Image and build
|
| 6 |
|
|
|
|
| 45 |
|
| 46 |
## HTTP API
|
| 47 |
|
| 48 |
+
- `GET /healthz` returns ready only after both parts loaded and warmup completed, and answers HTTP 503 `unavailable` once the native backend is gone (request timeout, GPU memory failure or crash). A request to a dead backend returns HTTP 503 `engine_unavailable`; the engine does not restart itself, restart the container.
|
| 49 |
+
- Reply protocol: a backend line is accepted as a reply only if it has the exact shape of backend.py's replies (ok: `Duo`, `timings_ms`, `peak_bytes`, `peak_reserved_bytes`, `optimization`; error: `error`, `exception`); any other line closes the backend and the request fails. **Known remaining limit:** a stray line with exactly the shape of a real reply cannot be detected without request ids (which would change `backend.py`); the stub tests cannot cover it.
|
| 50 |
- `POST /v1/systemone` accepts `application/json`, at most 16 MiB, with 1–64 keyed questions. A one-question request uses the same object shape as a joint request. Choice criteria allow 2–255 options; score criteria are an ordered list of 2–10 levels; noul criteria are optional. Unknown top-level fields, non-finite JSON numbers, unsupported types, and invalid shapes are rejected. HTTP access logs omit request bodies; the inherited backend traceback behavior on inference errors is unchanged.
|
| 51 |
- The HTTP adapter calls the same `Reax13Engine(state, questions)` instance and frozen `reax13.json` settings used by the Engine API. It does not alter model prompts, temperatures, ordering, or probability mixing. It validates the Engine response before returning it. For noul, it retains the native scalar `noul` field and adds a normalized `{ "false": 1-p, "true": p }` `probabilities` map for clients that consume all options uniformly.
|
| 52 |
+
- Capacity/context and malformed requests return HTTP 422, including a request whose size proxy (characters of the compact JSON of `{model:"jev-1.13.0", state, questions}` divided by 4, rounded down) exceeds `max_input_proxy_tokens` (32,000 in `reax13.json`); it is refused before any model work. Unexpected inference errors return HTTP 500 without exposing traceback details. Calls are serialized against the single resident engine.
|
| 53 |
|
| 54 |
One-question request:
|
| 55 |
|
|
|
|
| 96 |
|
| 97 |
## Tokenizers, context, and native prompts
|
| 98 |
|
| 99 |
+
Both part maxima below are token counts for each rendered question, including chat-template tokens. The limits are upstream/model configuration values; acceptance at maximum context on the assembled REAX weights and the HTTP serving image is **UNVERIFIED**. The engine does not truncate input; a request above the declared input cap (`max_input_proxy_tokens`, 32,000 proxy tokens) fails as unsupported before any model work, and an over-limit request must fail as unsupported.
|
| 100 |
|
| 101 |
| Part | Tokenizer and upstream context | Native prompt path |
|
| 102 |
|---|---|---|
|
|
|
|
| 107 |
|
| 108 |
## GPU measurements and verification status
|
| 109 |
|
| 110 |
+
Inherited Engine API baseline on one RTX PRO 6000 Blackwell Server Edition with 96 GB VRAM: 300 sequential requests over pinned public stand-ins; median 546.21 ms, mean 815.05 ms, p80 706.36 ms, p95 1,157.69 ms, max 23,778.34 ms. Model load and warmup excluded. Peak allocated memory was 83.95 GiB and peak reserved memory was 88.49 GiB. These numbers come from the previous engine, before the exact FP8 kernel, the kernel pin, the input cap and the request timeout; they are not measurements of this bundle. They included choice, noul, and score questions. These are Engine API figures, not HTTP round-trip latency.
|
| 111 |
|
| 112 |
+
The preceding slice's unmodified public harness smoke passed 8/8 and its Engine API benchmark passed 300/300; see `../r3-engine/RESULT.md` and the retained evidence under `/home/flori/jobs-data/hfdi-push-continue-20261101/r3-engine/`. The server wrapper's GPU startup, 5-request parity, HTTP latency overhead, and exact current image digest are **UNVERIFIED**. The server wrapper was exercised on a GPU only through the stand-in smoke named above.
|
engine/SHA256SUMS.json
CHANGED
|
@@ -1,42 +1,24 @@
|
|
| 1 |
{
|
| 2 |
"algorithm": "sha256",
|
|
|
|
| 3 |
"files": {
|
| 4 |
"Dockerfile": "2c19aa1728da1b1358ccd525c9ee47b70a6a1d646ef9b0d28d3b56a197607761",
|
| 5 |
"Dockerfile.conv": "487491b4907eb6c382c1e276a7cb488754382214f64e4ba134ff86876587b98c",
|
| 6 |
-
"LICENSE": "
|
| 7 |
-
"README.md": "
|
| 8 |
-
"SEALED-CONTRACT.md": "
|
| 9 |
-
"THIRD_PARTY_NOTICES.md": "
|
| 10 |
-
"autotune_guard.py": "
|
| 11 |
-
"backend.py": "
|
| 12 |
-
"benchmark.py": "
|
| 13 |
-
"
|
| 14 |
-
"
|
| 15 |
-
"
|
| 16 |
-
"
|
| 17 |
-
"
|
| 18 |
-
"
|
| 19 |
-
"
|
| 20 |
-
"
|
| 21 |
-
"
|
| 22 |
-
"third_party/finegrained-fp8/LICENSE": "561adc7696b191df9d4030782eba8f9968e88df4975c944212fc2dee8f27421c",
|
| 23 |
-
"third_party/finegrained-fp8/__init__.py": "01f34c9fac58ce8646492c0b68ca1748c989bcf128fe9f0d806c4b407e426b39",
|
| 24 |
-
"third_party/finegrained-fp8/_ops.py": "e5895a4c4adaacb383c46d537dd05171664f1d85a2f291b2060e0d945d338212",
|
| 25 |
-
"third_party/finegrained-fp8/_private_for_testing.py": "5ccf7e8e713659beb9d8603e5399f42cce95952b5a49c58a1f454855085fdfee",
|
| 26 |
-
"third_party/finegrained-fp8/batched.py": "d59bbd36d2b9798e09a3a86f9f3002438f6e98eb005646170fdd39a75fc175ea",
|
| 27 |
-
"third_party/finegrained-fp8/bayesian_autotuner.py": "8c1518bf27e9107b13060165a6dd93b397a6cfe092332be37354f151f64a56b8",
|
| 28 |
-
"third_party/finegrained-fp8/fused_batched.py": "e712ca6aff1d589a9162429d546c75e2e2fd168331a9e1c21153320407672034",
|
| 29 |
-
"third_party/finegrained-fp8/fused_grouped.py": "55893766cd9c34a4fe9b56254f3c8cdb0de2cdb8128bb5766724c1cefa6f31d0",
|
| 30 |
-
"third_party/finegrained-fp8/grouped.py": "382b03b38f0d0531d4fc6d2ce5e7441576ff9b9a38f3e3200f616ac060c69206",
|
| 31 |
-
"third_party/finegrained-fp8/matmul.py": "0f668b5d9e0e7db0675ea4518052b9320830f64f2468f2ecbcae7d0876336807",
|
| 32 |
-
"third_party/finegrained-fp8/metadata.json": "364d09ce040d25b30daf5dab953ffd3f2da4ea0ca29f36532e72b24531413607",
|
| 33 |
-
"third_party/finegrained-fp8/metadata.json.sigstore": "805119689a26b806e42b841fc1ab1136bbc52b4e7c751cc42ef28cd535b44126",
|
| 34 |
-
"third_party/finegrained-fp8/symbols.json": "664b5cff64e5597b9faff54fffb6d11a141c019b60b216799484ba2f00a551c4",
|
| 35 |
-
"third_party/finegrained-fp8/utils.py": "2cb87f98ab385e41c95fdf1e1bec777a25c8e0bb003a357a72c07a53c2b7b5d0",
|
| 36 |
-
"third_party/knowline/LICENSE": "ca64c7fa25098797fb01d2ade37443dd0f7e0f8527d3e7efdbeb4f75a3a7bf67",
|
| 37 |
-
"third_party/knowline/WEIGHT-LICENSE-Apache-2.0": "50cbab8a892c5f2993b8c7351a99182507472def3b1374558308605d99b86b32",
|
| 38 |
-
"third_party/knowline/knowline_layout.py": "2e839fa24f1ad9c980898b2a9aa62e2154ba3f57618bd5f2fedb5756f61f062f",
|
| 39 |
-
"third_party/knowline/knowline_server.py": "13ca50c4001fac0aa3899a35ff01e60f311132c571b17a8c3dc7aa6001ad2f3f",
|
| 40 |
-
"third_party/knowline/temperatures.json": "463a46c5ea253649bd5f43809655637d2d41939e503c81ddd01b28355d736f62"
|
| 41 |
}
|
| 42 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"algorithm": "sha256",
|
| 3 |
+
"note": "E* = shipped engine + ar-a exact fp8 kernel + w-qdrift Q pin (C) + request_timeout 1800 + declared input cap 32000 + dead-backend handling + fast_fp8_gemm guards/notice (w-bundle, 2026-11-03); round 3: code licence MIT (LICENSE), fast_fp8_gemm.py Apache-2.0 (w-bundle, 2026-11-03); round 4: reply-protocol check (any non-reply line closes the backend), test fixes, contract wording (w-bundle, 2026-11-03); round 5: exact-shape reply check (w-bundle, 2026-11-03)",
|
| 4 |
"files": {
|
| 5 |
"Dockerfile": "2c19aa1728da1b1358ccd525c9ee47b70a6a1d646ef9b0d28d3b56a197607761",
|
| 6 |
"Dockerfile.conv": "487491b4907eb6c382c1e276a7cb488754382214f64e4ba134ff86876587b98c",
|
| 7 |
+
"LICENSE": "6c6717154ca53121437292b4aaa544c0b54bc2b37bf4dffd9c0c9f320c071499",
|
| 8 |
+
"README.md": "cd5f4d6454ddbd8b97486f6b96e7241dfe06353cc98f95fc242e19bfacb511b3",
|
| 9 |
+
"SEALED-CONTRACT.md": "53488fe635373a71d15a050e3658d08dcdb5c79f43681b525c8382b9750e72c7",
|
| 10 |
+
"THIRD_PARTY_NOTICES.md": "93a6b264b7ccd1cd94d578b3d503e48d3e3d1cd707056881d87e0d85fb2860fe",
|
| 11 |
+
"autotune_guard.py": "88aeb73ab95b7571657337f24d8ff9474ef2376cd9e5cc069a2aede2cd66c5fe",
|
| 12 |
+
"backend.py": "5b27379cab7af5e12f7f5c3b6cd74096fd07211935aa8ad1a070c92689d6d0da",
|
| 13 |
+
"benchmark.py": "06b05053faa5c6a14d14cc1b3b06dccb5fc218fe735502e4a8ce901a16d0c8fc",
|
| 14 |
+
"fast_fp8_gemm.py": "e7a36d9d9a3c12546a9758eda40e90d2fcb1d0603b610080c7e361f85ec0d8c3",
|
| 15 |
+
"probabilities.py": "ba5106fbae0662011cc63ffa989da74eb36f5776f585bce20cba771c0aa29e1d",
|
| 16 |
+
"reax13.json": "09eb15a244298378050ef5f09aa94e3b76c1cf54a5245b40102ae0da4f581706",
|
| 17 |
+
"reax13_engine.py": "9e1c3bc6386f23f8a63759b84b62c6ebbfcbc56c9c216c63a799980385e1c6ed",
|
| 18 |
+
"serve.py": "b51bcd5af1ebcc75982b459e30e9ca1142e147306250c98ec862e0331088a628",
|
| 19 |
+
"test_bundle.py": "e3544f9fe7233e336ec0aed896bbea65a9b929a3041b907d07d02d0d2116c469",
|
| 20 |
+
"test_fast_fp8_gemm.py": "7faf334f074dbe5ad3c6d481c10daba2c83a4f2594561d83723d03996385be99",
|
| 21 |
+
"test_probabilities.py": "b338c4855272e25f3695c87dc30f4f8ee10ef32e40a3d1e4b23706506f19c345",
|
| 22 |
+
"test_serve.py": "d8164eacacaff6655a24e98f69c2f6fcafbc143bde451ac1f12547afea65b6e5"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
}
|
| 24 |
}
|
engine/SHA256SUMS.shipped.json
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"algorithm": "sha256",
|
| 3 |
+
"files": {
|
| 4 |
+
"Dockerfile": "2c19aa1728da1b1358ccd525c9ee47b70a6a1d646ef9b0d28d3b56a197607761",
|
| 5 |
+
"Dockerfile.conv": "487491b4907eb6c382c1e276a7cb488754382214f64e4ba134ff86876587b98c",
|
| 6 |
+
"LICENSE": "561adc7696b191df9d4030782eba8f9968e88df4975c944212fc2dee8f27421c",
|
| 7 |
+
"README.md": "039b20a35ca3a9586c53c047a2765f219d58d1b6dee7d267cd4c765240701a68",
|
| 8 |
+
"SEALED-CONTRACT.md": "dd5c1a7386bee52f255a188beaafa52b32d748094823c5dd7e44a4bd98e9e873",
|
| 9 |
+
"THIRD_PARTY_NOTICES.md": "a423913ef99a3027642d4de94487a81a583508abf394f918ced7c7cf5fad8940",
|
| 10 |
+
"autotune_guard.py": "724553b9e97fae1a020b097b8fb1c36f6c0d0d583405545a0cc53da281b3349b",
|
| 11 |
+
"backend.py": "3d62dcad6447c725312793af407f99753fac514dcabd647075bef48784fe5a19",
|
| 12 |
+
"benchmark.py": "52123b5dfdd975f9bb98f1736f2ed8d6650a60f21f0bd8c1175765ef3ae8fa3a",
|
| 13 |
+
"probabilities.py": "36737623314edaf6e78b36b5acea73d33cdfc97aff6c1ff56736eb8a375a85f0",
|
| 14 |
+
"reax13.json": "668647314226173db1fe816ac67c442c20fb633a68a85a091f69ebf3a45882a9",
|
| 15 |
+
"reax13_engine.py": "b96e01996610b6f0648d4983eeb5dcbe529b6ae016c4b89e284f3829ff20c57f",
|
| 16 |
+
"serve.py": "dac51111d4de210d5193c8e539ddcde309be1c29120f10518571e030ac5785b9",
|
| 17 |
+
"test_probabilities.py": "66b52436f07e3e20e1e2c4eb3c7059beac23e7921313614916827c0c7b3b80f5",
|
| 18 |
+
"test_serve.py": "063e19044b4b5b704cadd8a4f47e2ad46284e2a55ec85bc93d69df796954b4cd",
|
| 19 |
+
"third_party/SHA256SUMS.json": "8912d47515d8753e42bb59037c434cd1024e5720653b4ff16770fc481603c96b",
|
| 20 |
+
"third_party/d3/LICENSE": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4",
|
| 21 |
+
"third_party/d3/NOTICE": "d7522881de6b8417b997953c9b82d54e1be61952571a163c2fcf77f1e199406d",
|
| 22 |
+
"third_party/finegrained-fp8/LICENSE": "561adc7696b191df9d4030782eba8f9968e88df4975c944212fc2dee8f27421c",
|
| 23 |
+
"third_party/finegrained-fp8/__init__.py": "01f34c9fac58ce8646492c0b68ca1748c989bcf128fe9f0d806c4b407e426b39",
|
| 24 |
+
"third_party/finegrained-fp8/_ops.py": "e5895a4c4adaacb383c46d537dd05171664f1d85a2f291b2060e0d945d338212",
|
| 25 |
+
"third_party/finegrained-fp8/_private_for_testing.py": "5ccf7e8e713659beb9d8603e5399f42cce95952b5a49c58a1f454855085fdfee",
|
| 26 |
+
"third_party/finegrained-fp8/batched.py": "d59bbd36d2b9798e09a3a86f9f3002438f6e98eb005646170fdd39a75fc175ea",
|
| 27 |
+
"third_party/finegrained-fp8/bayesian_autotuner.py": "8c1518bf27e9107b13060165a6dd93b397a6cfe092332be37354f151f64a56b8",
|
| 28 |
+
"third_party/finegrained-fp8/fused_batched.py": "e712ca6aff1d589a9162429d546c75e2e2fd168331a9e1c21153320407672034",
|
| 29 |
+
"third_party/finegrained-fp8/fused_grouped.py": "55893766cd9c34a4fe9b56254f3c8cdb0de2cdb8128bb5766724c1cefa6f31d0",
|
| 30 |
+
"third_party/finegrained-fp8/grouped.py": "382b03b38f0d0531d4fc6d2ce5e7441576ff9b9a38f3e3200f616ac060c69206",
|
| 31 |
+
"third_party/finegrained-fp8/matmul.py": "0f668b5d9e0e7db0675ea4518052b9320830f64f2468f2ecbcae7d0876336807",
|
| 32 |
+
"third_party/finegrained-fp8/metadata.json": "364d09ce040d25b30daf5dab953ffd3f2da4ea0ca29f36532e72b24531413607",
|
| 33 |
+
"third_party/finegrained-fp8/metadata.json.sigstore": "805119689a26b806e42b841fc1ab1136bbc52b4e7c751cc42ef28cd535b44126",
|
| 34 |
+
"third_party/finegrained-fp8/symbols.json": "664b5cff64e5597b9faff54fffb6d11a141c019b60b216799484ba2f00a551c4",
|
| 35 |
+
"third_party/finegrained-fp8/utils.py": "2cb87f98ab385e41c95fdf1e1bec777a25c8e0bb003a357a72c07a53c2b7b5d0",
|
| 36 |
+
"third_party/knowline/LICENSE": "ca64c7fa25098797fb01d2ade37443dd0f7e0f8527d3e7efdbeb4f75a3a7bf67",
|
| 37 |
+
"third_party/knowline/WEIGHT-LICENSE-Apache-2.0": "50cbab8a892c5f2993b8c7351a99182507472def3b1374558308605d99b86b32",
|
| 38 |
+
"third_party/knowline/knowline_layout.py": "2e839fa24f1ad9c980898b2a9aa62e2154ba3f57618bd5f2fedb5756f61f062f",
|
| 39 |
+
"third_party/knowline/knowline_server.py": "13ca50c4001fac0aa3899a35ff01e60f311132c571b17a8c3dc7aa6001ad2f3f",
|
| 40 |
+
"third_party/knowline/temperatures.json": "463a46c5ea253649bd5f43809655637d2d41939e503c81ddd01b28355d736f62"
|
| 41 |
+
}
|
| 42 |
+
}
|
engine/THIRD_PARTY_NOTICES.md
CHANGED
|
@@ -1,11 +1,12 @@
|
|
| 1 |
# Third-party notices
|
| 2 |
|
| 3 |
-
REAX
|
| 4 |
|
| 5 |
| Component | Source and pin | Licence | Treatment |
|
| 6 |
|---|---|---|---|
|
| 7 |
| KnowLine frontend/layout | PelaAI/KnowLine-31B-Gen1, 8f052a8130e599b53e5324c34f0f5ff90696e1c6 | MIT, file headers; adapted from llm2jev0.6.1/OpenJev, AnyJev contributors | Unchanged files in third_party/knowline; native temperatures |
|
| 8 |
| Finegrained FP8 | kernels-community/finegrained-fp8, 81ed101eb51b0801a4d9469541eda6ef73b0b6f6, kernel5fe9dbb3d45f6f2652958c1f70e5381891fa7aa0, version4 | Apache-2.0, upstream metadata.json | Unchanged Python/Triton in third_party/finegrained-fp8, all11 digests verified; includes i64 addressing |
|
|
|
|
| 9 |
| d3 native runtime | vllm-sr/d3-ultra, 52ba9e7a00402f9560c7fba8c039608e7ddee7fb | Apache-2.0, repo LICENSE | Loaded from part-q; not vendored or relicensed here; keep full upstream licence and model metadata |
|
| 10 |
| causal-conv1d | Dao-AILab/causal-conv1d v1.7.0 / PyPI1.7.0 | BSD-3-Clause | Pinned dependency; installed with upstream licence metadata |
|
| 11 |
| Decision Index kit | apolinario/decision-index, 25c6ec7b6b5f44a216c63c3a700de3a9c6d6699d | MIT, Copyright2026 Apolinário Passos | External kit; installed/supplied separately; no kit source distributed inside engine/ |
|
|
|
|
| 1 |
# Third-party notices
|
| 2 |
|
| 3 |
+
REAX code is MIT (LICENSE); fast_fp8_gemm.py stays Apache-2.0 as a derivative of finegrained-fp8. Third-party code below keeps its own licence; the root licence does not relicense it.
|
| 4 |
|
| 5 |
| Component | Source and pin | Licence | Treatment |
|
| 6 |
|---|---|---|---|
|
| 7 |
| KnowLine frontend/layout | PelaAI/KnowLine-31B-Gen1, 8f052a8130e599b53e5324c34f0f5ff90696e1c6 | MIT, file headers; adapted from llm2jev0.6.1/OpenJev, AnyJev contributors | Unchanged files in third_party/knowline; native temperatures |
|
| 8 |
| Finegrained FP8 | kernels-community/finegrained-fp8, 81ed101eb51b0801a4d9469541eda6ef73b0b6f6, kernel5fe9dbb3d45f6f2652958c1f70e5381891fa7aa0, version4 | Apache-2.0, upstream metadata.json | Unchanged Python/Triton in third_party/finegrained-fp8, all11 digests verified; includes i64 addressing |
|
| 9 |
+
| Finegrained FP8 GEMM, derived | fast_fp8_gemm.py, derived from w8a8_block_dynamic_fp8_matmul in the finegrained-fp8 row above (copyright 2025 The HuggingFace Inc. team) | Apache-2.0 | Modified in this package: activation quantization split into its own kernel, GEMM rewritten around it, own tile parameters, routing wrapper; arithmetic unchanged, bits compared in test_fast_fp8_gemm.py. Copyright line, licence notice and statement of changes are in the file header |
|
| 10 |
| d3 native runtime | vllm-sr/d3-ultra, 52ba9e7a00402f9560c7fba8c039608e7ddee7fb | Apache-2.0, repo LICENSE | Loaded from part-q; not vendored or relicensed here; keep full upstream licence and model metadata |
|
| 11 |
| causal-conv1d | Dao-AILab/causal-conv1d v1.7.0 / PyPI1.7.0 | BSD-3-Clause | Pinned dependency; installed with upstream licence metadata |
|
| 12 |
| Decision Index kit | apolinario/decision-index, 25c6ec7b6b5f44a216c63c3a700de3a9c6d6699d | MIT, Copyright2026 Apolinário Passos | External kit; installed/supplied separately; no kit source distributed inside engine/ |
|
engine/autotune_guard.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
# SPDX-License-Identifier:
|
| 2 |
"""Protect Triton's shared host autotuner bookkeeping; GPU streams remain independent."""
|
| 3 |
import functools
|
| 4 |
import threading
|
|
@@ -21,3 +21,26 @@ def install():
|
|
| 21 |
|
| 22 |
guarded._reax_thread_safe = True
|
| 23 |
Autotuner.run = guarded
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: MIT
|
| 2 |
"""Protect Triton's shared host autotuner bookkeeping; GPU streams remain independent."""
|
| 3 |
import functools
|
| 4 |
import threading
|
|
|
|
| 21 |
|
| 22 |
guarded._reax_thread_safe = True
|
| 23 |
Autotuner.run = guarded
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
# Part Q's answers depend on these configs (measured), and stock autotune
|
| 27 |
+
# chooses them per process by timing. One config per kernel makes Part Q's answers independent of the process.
|
| 28 |
+
_ANSWER_CONFIGS = (
|
| 29 |
+
('fla.modules.l2norm', 'l2norm_fwd_kernel', {'BT': 16, 'num_warps': 8}),
|
| 30 |
+
('fla.ops.common.chunk_o', 'chunk_fwd_kernel_o', {'BK': 32, 'BV': 32}),
|
| 31 |
+
('fla.ops.common.chunk_delta_h', 'chunk_gated_delta_rule_fwd_kernel_h_blockdim64', {'BV': 32, 'num_stages': 2}),
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def pin_answer_configs():
|
| 36 |
+
import importlib
|
| 37 |
+
from triton.runtime.autotuner import Autotuner
|
| 38 |
+
for module, name, wanted in _ANSWER_CONFIGS:
|
| 39 |
+
tuner = getattr(importlib.import_module(module), name)
|
| 40 |
+
while not isinstance(tuner, Autotuner):
|
| 41 |
+
tuner = tuner.fn
|
| 42 |
+
hits = [c for c in tuner.configs if all(c.all_kwargs().get(k) == v for k, v in wanted.items())]
|
| 43 |
+
if not hits:
|
| 44 |
+
raise RuntimeError(f'answer config {name} {wanted} not in the kernel grid')
|
| 45 |
+
tuner.configs = hits[:1]
|
| 46 |
+
tuner.cache.clear()
|
engine/backend.py
CHANGED
|
@@ -1,11 +1,13 @@
|
|
| 1 |
-
# SPDX-License-Identifier:
|
| 2 |
"""Native K and U frontends in one isolated process/GPU, changed weight storage only."""
|
| 3 |
import os,sys,json,time,threading,traceback,contextlib
|
| 4 |
from pathlib import Path
|
| 5 |
assert Path('/.dockerenv').exists() and os.environ.get('DUO_ISOLATED')=='1'
|
| 6 |
import torch,numpy as np
|
| 7 |
-
from autotune_guard import install as install_autotune_guard
|
| 8 |
install_autotune_guard()
|
|
|
|
|
|
|
| 9 |
import transformers
|
| 10 |
from concurrent.futures import ThreadPoolExecutor
|
| 11 |
K_STREAM=None
|
|
@@ -72,6 +74,7 @@ try:
|
|
| 72 |
try:return original_run(prepared)
|
| 73 |
finally:u.runtime.batch_size=wanted
|
| 74 |
u.runtime._run=bounded_run
|
|
|
|
| 75 |
ba_patches=[]
|
| 76 |
state_once_backend=d3_runtime.StateOnce(u.backbone.language_model)
|
| 77 |
opt_active='plain'
|
|
@@ -91,6 +94,7 @@ try:
|
|
| 91 |
for model in [km]+([u.backbone] if u else []):
|
| 92 |
assert any('float8' in str(p.dtype) or p.dtype==torch.int8 for p in model.parameters()), 'quantization silently dequantized'
|
| 93 |
torch.cuda.synchronize()
|
|
|
|
| 94 |
print(json.dumps({'ready':True,'autotuner_guard':True,'mode':mode,'scheme':scheme,'cache_parity_max_abs':cache_parity,'gpu':torch.cuda.get_device_name(0),'k':info(km),'u':info(u.backbone) if u else None,'allocated':torch.cuda.memory_allocated(),'reserved':torch.cuda.memory_reserved(),'peak_load_bytes':torch.cuda.max_memory_allocated()}),flush=True)
|
| 95 |
except BaseException:
|
| 96 |
traceback.print_exc();raise
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: MIT
|
| 2 |
"""Native K and U frontends in one isolated process/GPU, changed weight storage only."""
|
| 3 |
import os,sys,json,time,threading,traceback,contextlib
|
| 4 |
from pathlib import Path
|
| 5 |
assert Path('/.dockerenv').exists() and os.environ.get('DUO_ISOLATED')=='1'
|
| 6 |
import torch,numpy as np
|
| 7 |
+
from autotune_guard import install as install_autotune_guard,pin_answer_configs
|
| 8 |
install_autotune_guard()
|
| 9 |
+
import fast_fp8_gemm
|
| 10 |
+
fast_fp8_gemm.install()
|
| 11 |
import transformers
|
| 12 |
from concurrent.futures import ThreadPoolExecutor
|
| 13 |
K_STREAM=None
|
|
|
|
| 74 |
try:return original_run(prepared)
|
| 75 |
finally:u.runtime.batch_size=wanted
|
| 76 |
u.runtime._run=bounded_run
|
| 77 |
+
pin_answer_configs()
|
| 78 |
ba_patches=[]
|
| 79 |
state_once_backend=d3_runtime.StateOnce(u.backbone.language_model)
|
| 80 |
opt_active='plain'
|
|
|
|
| 94 |
for model in [km]+([u.backbone] if u else []):
|
| 95 |
assert any('float8' in str(p.dtype) or p.dtype==torch.int8 for p in model.parameters()), 'quantization silently dequantized'
|
| 96 |
torch.cuda.synchronize()
|
| 97 |
+
fast_fp8_gemm.precompile()
|
| 98 |
print(json.dumps({'ready':True,'autotuner_guard':True,'mode':mode,'scheme':scheme,'cache_parity_max_abs':cache_parity,'gpu':torch.cuda.get_device_name(0),'k':info(km),'u':info(u.backbone) if u else None,'allocated':torch.cuda.memory_allocated(),'reserved':torch.cuda.memory_reserved(),'peak_load_bytes':torch.cuda.max_memory_allocated()}),flush=True)
|
| 99 |
except BaseException:
|
| 100 |
traceback.print_exc();raise
|
engine/benchmark.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
# SPDX-License-Identifier:
|
| 2 |
"""Index-style sequential Engine benchmark, including native score question validation.
|
| 3 |
|
| 4 |
The pinned official kit only validates choice/noul. This driver retains its Engine
|
|
@@ -12,6 +12,7 @@ from pathlib import Path
|
|
| 12 |
import time
|
| 13 |
|
| 14 |
from probabilities import distribution
|
|
|
|
| 15 |
from reax13_engine import Reax13Engine
|
| 16 |
|
| 17 |
|
|
@@ -60,13 +61,22 @@ def main():
|
|
| 60 |
engine.synchronize()
|
| 61 |
status(event='ready')
|
| 62 |
start = time.perf_counter()
|
|
|
|
| 63 |
with (out/'results.jsonl').open('w') as f:
|
| 64 |
for i, row in enumerate(rows):
|
| 65 |
# Same clock boundaries as official runner: synchronize, request,
|
| 66 |
# synchronize, validate. Includes JSON IPC and all native work.
|
| 67 |
tick = time.perf_counter()
|
| 68 |
engine.synchronize()
|
| 69 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
engine.synchronize()
|
| 71 |
validate(row['questions'], response)
|
| 72 |
elapsed = (time.perf_counter()-tick)*1000
|
|
@@ -75,9 +85,10 @@ def main():
|
|
| 75 |
'payload':{'state':row['state'],'questions':row['questions']}}
|
| 76 |
f.write(json.dumps(result)+'\n')
|
| 77 |
f.flush()
|
|
|
|
| 78 |
if (i+1)%10 == 0:
|
| 79 |
-
status(event='progress',completed=i+1,counts=
|
| 80 |
-
status(event='complete',completed=len(rows),counts=
|
| 81 |
finally:
|
| 82 |
if engine is not None:
|
| 83 |
engine.close()
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: MIT
|
| 2 |
"""Index-style sequential Engine benchmark, including native score question validation.
|
| 3 |
|
| 4 |
The pinned official kit only validates choice/noul. This driver retains its Engine
|
|
|
|
| 12 |
import time
|
| 13 |
|
| 14 |
from probabilities import distribution
|
| 15 |
+
from decision_index.engines import Unsupported
|
| 16 |
from reax13_engine import Reax13Engine
|
| 17 |
|
| 18 |
|
|
|
|
| 61 |
engine.synchronize()
|
| 62 |
status(event='ready')
|
| 63 |
start = time.perf_counter()
|
| 64 |
+
counts = {'ok':0, 'unsupported':0}
|
| 65 |
with (out/'results.jsonl').open('w') as f:
|
| 66 |
for i, row in enumerate(rows):
|
| 67 |
# Same clock boundaries as official runner: synchronize, request,
|
| 68 |
# synchronize, validate. Includes JSON IPC and all native work.
|
| 69 |
tick = time.perf_counter()
|
| 70 |
engine.synchronize()
|
| 71 |
+
try:
|
| 72 |
+
response, raw = engine(row['state'], row['questions'])
|
| 73 |
+
except Unsupported as exc:
|
| 74 |
+
elapsed = (time.perf_counter()-tick)*1000
|
| 75 |
+
f.write(json.dumps({**row['_evaluation'], 'status':'unsupported', 'error':str(exc), 'total_wall_ms':elapsed,
|
| 76 |
+
'payload':{'state':row['state'],'questions':row['questions']}})+'\n')
|
| 77 |
+
f.flush()
|
| 78 |
+
counts['unsupported'] += 1
|
| 79 |
+
continue
|
| 80 |
engine.synchronize()
|
| 81 |
validate(row['questions'], response)
|
| 82 |
elapsed = (time.perf_counter()-tick)*1000
|
|
|
|
| 85 |
'payload':{'state':row['state'],'questions':row['questions']}}
|
| 86 |
f.write(json.dumps(result)+'\n')
|
| 87 |
f.flush()
|
| 88 |
+
counts['ok'] += 1
|
| 89 |
if (i+1)%10 == 0:
|
| 90 |
+
status(event='progress',completed=i+1,counts=dict(counts),elapsed_seconds=time.perf_counter()-start)
|
| 91 |
+
status(event='complete',completed=len(rows),counts=dict(counts),elapsed_seconds=time.perf_counter()-start)
|
| 92 |
finally:
|
| 93 |
if engine is not None:
|
| 94 |
engine.close()
|
engine/fast_fp8_gemm.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
# Copyright 2025 The HuggingFace Inc. team. All rights reserved (upstream finegrained-fp8, matmul.py).
|
| 3 |
+
# Modifications Copyright 2026 The REAX Engine authors.
|
| 4 |
+
#
|
| 5 |
+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.
|
| 6 |
+
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 (full text: LICENSE and third_party/finegrained-fp8/LICENSE).
|
| 7 |
+
# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
|
| 8 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 9 |
+
#
|
| 10 |
+
# STATEMENT OF CHANGES (Apache-2.0 section 4(b)). This file is derived from w8a8_block_dynamic_fp8_matmul in
|
| 11 |
+
# third_party/finegrained-fp8/matmul.py (kernels-community/finegrained-fp8, pin in THIRD_PARTY_NOTICES.md). Changes: the activation
|
| 12 |
+
# quantization is a separate Triton kernel run once per call instead of inline for every N tile; the GEMM kernel is rewritten to read the
|
| 13 |
+
# quantized activations and their scales; tile sizes and launch parameters are fixed here; install() routes transformers' fp8_linear
|
| 14 |
+
# to this kernel only for the shapes and dtypes it supports and sends everything else to the original dispatcher with DeepGEMM disabled;
|
| 15 |
+
# the upstream K and Bs shape asserts are kept. Arithmetic is unchanged (test_fast_fp8_gemm.py compares bits).
|
| 16 |
+
"""Same block-scaled FP8 GEMM as finegrained-fp8 w8a8_block_dynamic_fp8_matmul (Apache-2.0, The HuggingFace Inc. team, third_party/finegrained-fp8/LICENSE),
|
| 17 |
+
with the activation quantization run once per call in its own kernel. Upstream repeats it inline for every N tile. Same fp32 scale and cast arithmetic,
|
| 18 |
+
same K tiles, same multiply order (dot*a_s)*b_s: results are bit-identical (test_fast_fp8_gemm.py)."""
|
| 19 |
+
import torch,triton,triton.language as tl
|
| 20 |
+
K_T=128
|
| 21 |
+
@triton.jit
|
| 22 |
+
def _quant(A,Q,S,M,K,sa_m,sa_k,sq_m,sq_k,ss_m,ss_k,BLOCK_M:tl.constexpr,BLOCK_K:tl.constexpr):
|
| 23 |
+
pid_m=tl.program_id(0);pid_k=tl.program_id(1)
|
| 24 |
+
offs_m=pid_m*BLOCK_M+tl.arange(0,BLOCK_M);offs_k=pid_k*BLOCK_K+tl.arange(0,BLOCK_K)
|
| 25 |
+
mask_m=offs_m<M
|
| 26 |
+
a_raw=tl.load(A+offs_m[:,None]*sa_m+offs_k[None,:]*sa_k,mask=mask_m[:,None]&(offs_k[None,:]<K),other=0.0).to(tl.float32)
|
| 27 |
+
a_s=tl.max(tl.abs(a_raw),axis=1)/448.0
|
| 28 |
+
q=(a_raw/tl.maximum(a_s[:,None],1e-12)).to(tl.float8e4nv)
|
| 29 |
+
tl.store(Q+offs_m[:,None]*sq_m+offs_k[None,:]*sq_k,q,mask=mask_m[:,None]&(offs_k[None,:]<K))
|
| 30 |
+
tl.store(S+offs_m*ss_m+pid_k*ss_k,a_s,mask=mask_m)
|
| 31 |
+
@triton.jit
|
| 32 |
+
def _gemm(Q,B,C,S,Bs,M,N,K,sq_m,sq_k,sb_k,sb_n,sc_m,sc_n,ss_m,ss_k,sbs_k,sbs_n,BLOCK_M:tl.constexpr,BLOCK_N:tl.constexpr,BLOCK_K:tl.constexpr,GROUP:tl.constexpr):
|
| 33 |
+
pid_m=tl.program_id(0);pid_n=tl.program_id(1)
|
| 34 |
+
pid_m,pid_n=tl.swizzle2d(pid_m,pid_n,tl.cdiv(M,BLOCK_M),tl.cdiv(N,BLOCK_N),GROUP)
|
| 35 |
+
offs_m=(pid_m*BLOCK_M+tl.arange(0,BLOCK_M))%M;offs_n=(pid_n*BLOCK_N+tl.arange(0,BLOCK_N))%N;offs_k=tl.arange(0,BLOCK_K)
|
| 36 |
+
a_ptrs=Q+offs_m[:,None]*sq_m+offs_k[None,:]*sq_k;b_ptrs=B+offs_k[:,None]*sb_k+offs_n[None,:]*sb_n
|
| 37 |
+
s_ptrs=S+offs_m*ss_m;bs_ptr=Bs+pid_n*sbs_n
|
| 38 |
+
acc=tl.zeros((BLOCK_M,BLOCK_N),dtype=tl.float32)
|
| 39 |
+
for k in range(0,tl.cdiv(K,BLOCK_K)):
|
| 40 |
+
a=tl.load(a_ptrs);b=tl.load(b_ptrs);a_s=tl.load(s_ptrs)
|
| 41 |
+
acc+=tl.dot(a,b)*a_s[:,None]*tl.load(bs_ptr)
|
| 42 |
+
a_ptrs+=BLOCK_K*sq_k;b_ptrs+=BLOCK_K*sb_k;s_ptrs+=ss_k;bs_ptr+=sbs_k
|
| 43 |
+
c=acc.to(C.dtype.element_ty);om=pid_m*BLOCK_M+tl.arange(0,BLOCK_M);on=pid_n*BLOCK_N+tl.arange(0,BLOCK_N)
|
| 44 |
+
tl.store(C+sc_m*om[:,None]+sc_n*on[None,:],c,mask=(om[:,None]<M)&(on[None,:]<N))
|
| 45 |
+
def _bm(M):return min(max(triton.next_power_of_2(M),16),128)
|
| 46 |
+
def matmul(A,B,Bs,block,out_dtype):
|
| 47 |
+
assert len(block)==2,f"block_size must be [block_n, block_k], got {block}"
|
| 48 |
+
bn,bk=block
|
| 49 |
+
assert A.shape[-1]==B.shape[-1],f"K mismatch: A has K={A.shape[-1]}, B has K={B.shape[-1]}"
|
| 50 |
+
assert B.ndim==2,f"B must be 2D (N, K), got ndim={B.ndim}"
|
| 51 |
+
M=A.numel()//A.shape[-1];K=A.shape[-1];N=B.shape[0];A2=A.reshape(M,K)
|
| 52 |
+
assert Bs.ndim==2 and tuple(Bs.shape)==(triton.cdiv(N,bn),triton.cdiv(K,bk)),f"Bs shape {tuple(Bs.shape)} != expected ({triton.cdiv(N,bn)}, {triton.cdiv(K,bk)})"
|
| 53 |
+
Q=torch.empty(M,K,device=A.device,dtype=torch.float8_e4m3fn);S=torch.empty(M,K//K_T,device=A.device,dtype=torch.float32)
|
| 54 |
+
_quant[(triton.cdiv(M,64),K//K_T)](A2,Q,S,M,K,A2.stride(0),A2.stride(1),Q.stride(0),Q.stride(1),S.stride(0),S.stride(1),BLOCK_M=64,BLOCK_K=K_T,num_warps=8,num_stages=3)
|
| 55 |
+
BM=_bm(M);C=torch.empty(*A.shape[:-1],N,device=A.device,dtype=out_dtype)
|
| 56 |
+
_gemm[(triton.cdiv(M,BM),triton.cdiv(N,bn))](Q,B,C,S,Bs,M,N,K,Q.stride(0),Q.stride(1),B.stride(1),B.stride(0),C.stride(-2),C.stride(-1),S.stride(0),S.stride(1),Bs.stride(1),Bs.stride(0),BLOCK_M=BM,BLOCK_N=bn,BLOCK_K=bk,GROUP=8,num_warps=16,num_stages=3)
|
| 57 |
+
return C
|
| 58 |
+
def install():
|
| 59 |
+
"""Route FP8Linear through this kernel for the shape it supports; everything else keeps the original path."""
|
| 60 |
+
import transformers.integrations.finegrained_fp8 as tf
|
| 61 |
+
orig=tf.fp8_linear
|
| 62 |
+
# Whole-weight blocks (N,K equal to the block) are tensor-wide upstream; not a shape this kernel was written for. DeepGEMM is never used: every call is this kernel or upstream Triton.
|
| 63 |
+
def fp8_linear(input,weight,weight_scale_inv,block_size=None,bias=None,activation_scale=None,allow_deepgemm=True):
|
| 64 |
+
if activation_scale is None and bias is None and block_size is not None and block_size[1]==K_T and input.ndim>=2 and input.numel()>0 and input.shape[-1]%K_T==0 and input.is_contiguous() and weight.dtype==torch.float8_e4m3fn and weight.is_contiguous() and weight.ndim==2 and not (weight.shape[0]==block_size[0] and weight.shape[1]==block_size[1]) and weight_scale_inv.dtype==torch.float32 and weight_scale_inv.ndim==2 and input.dtype==torch.bfloat16:
|
| 65 |
+
return matmul(input,weight,weight_scale_inv,block_size,input.dtype)
|
| 66 |
+
return orig(input,weight,weight_scale_inv,block_size,bias,activation_scale,False)
|
| 67 |
+
tf.fp8_linear=fp8_linear
|
| 68 |
+
def precompile():
|
| 69 |
+
"""Compile every tile before serving: a first compile inside a request costs hundreds of ms."""
|
| 70 |
+
W=torch.zeros(128,128,device='cuda',dtype=torch.float8_e4m3fn);S=torch.ones(1,1,device='cuda',dtype=torch.float32)
|
| 71 |
+
for M in (16,32,64,128):matmul(torch.ones(M,128,device='cuda',dtype=torch.bfloat16),W,S,[128,128],torch.bfloat16)
|
| 72 |
+
torch.cuda.synchronize()
|
engine/probabilities.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
# SPDX-License-Identifier:
|
| 2 |
"""Validate each native distribution, then average probabilities at equal weight."""
|
| 3 |
import math
|
| 4 |
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: MIT
|
| 2 |
"""Validate each native distribution, then average probabilities at equal weight."""
|
| 3 |
import math
|
| 4 |
|
engine/reax13.json
CHANGED
|
@@ -7,5 +7,6 @@
|
|
| 7 |
"optimization": "fastconv_longcache",
|
| 8 |
"u_batch_size": 8,
|
| 9 |
"startup_timeout": 1800,
|
| 10 |
-
"request_timeout":
|
|
|
|
| 11 |
}
|
|
|
|
| 7 |
"optimization": "fastconv_longcache",
|
| 8 |
"u_batch_size": 8,
|
| 9 |
"startup_timeout": 1800,
|
| 10 |
+
"request_timeout": 1800,
|
| 11 |
+
"max_input_proxy_tokens": 32000
|
| 12 |
}
|
engine/reax13_engine.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
# SPDX-License-Identifier:
|
| 2 |
"""Decision Index adapter. Start one persistent isolated GPU backend per engine."""
|
| 3 |
import atexit
|
| 4 |
import json
|
|
@@ -8,10 +8,22 @@ import select
|
|
| 8 |
import subprocess
|
| 9 |
import sys
|
| 10 |
import threading
|
|
|
|
| 11 |
|
| 12 |
from decision_index.engines import Engine, Unsupported
|
| 13 |
|
| 14 |
ROOT = Path(__file__).resolve().parent
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
|
| 17 |
class Reax13Engine(Engine):
|
|
@@ -24,6 +36,9 @@ class Reax13Engine(Engine):
|
|
| 24 |
super().__init__(**options)
|
| 25 |
self.config = json.loads(Path(config or ROOT / 'reax13.json').read_text())
|
| 26 |
self.config.update(options)
|
|
|
|
|
|
|
|
|
|
| 27 |
if self.config['mode'] != 'reax13' or self.config['quantization'] != 'fp8':
|
| 28 |
raise ValueError('this engine supports reax13 FP8 only')
|
| 29 |
if self.config['optimization'] not in ('fastconv_longcache', 'plain'):
|
|
@@ -36,6 +51,7 @@ class Reax13Engine(Engine):
|
|
| 36 |
raise FileNotFoundError('model root must contain both part-g/ and part-q/')
|
| 37 |
env = dict(os.environ, REAX_PART_G=str(paths['part_g']), REAX_PART_Q=str(paths['part_q']))
|
| 38 |
self.lock = threading.Lock()
|
|
|
|
| 39 |
self.process = subprocess.Popen([sys.executable, '-u', str(ROOT / 'backend.py')],
|
| 40 |
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
| 41 |
text=True, bufsize=1, env=env)
|
|
@@ -55,13 +71,49 @@ class Reax13Engine(Engine):
|
|
| 55 |
'policy': 'native prompts and temperatures; no item-specific rules or truncation'}
|
| 56 |
|
| 57 |
def _read(self, timeout):
|
| 58 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
self.close()
|
| 60 |
-
raise
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
|
| 66 |
def __call__(self, state, questions):
|
| 67 |
if not questions or len(questions) > 64:
|
|
@@ -71,13 +123,24 @@ class Reax13Engine(Engine):
|
|
| 71 |
raise Unsupported('unsupported question type ' + question['type'])
|
| 72 |
if question['type'] == 'choice' and not 2 <= len(question['criteria']) <= 255:
|
| 73 |
raise Unsupported('options per choice must be 2..255')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
with self.lock:
|
|
|
|
|
|
|
| 75 |
request = {'state': state, 'questions': questions,
|
| 76 |
'_kind': 'concurrent' if self.config['optimization'] != 'plain' else 'sequential',
|
| 77 |
'_ubatch': int(self.config['u_batch_size']), '_opt': self.config['optimization']}
|
| 78 |
self.process.stdin.write(json.dumps(request) + '\n')
|
| 79 |
self.process.stdin.flush()
|
| 80 |
reply = self._read(float(self.config['request_timeout']))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
if reply.get('status') != 'ok':
|
| 82 |
message = reply.get('error', 'native backend error')
|
| 83 |
if reply.get('exception') in ('OutOfMemoryError', 'AcceleratorError') or 'device-side assert' in message:
|
|
@@ -93,9 +156,13 @@ class Reax13Engine(Engine):
|
|
| 93 |
path = self.config.get('warmup_rows')
|
| 94 |
if path:
|
| 95 |
rows = [json.loads(line) for line in Path(path).read_text().splitlines()]
|
| 96 |
-
|
|
|
|
| 97 |
for row in warm:
|
| 98 |
-
|
|
|
|
|
|
|
|
|
|
| 99 |
|
| 100 |
def runtime(self):
|
| 101 |
return self.ready
|
|
@@ -104,6 +171,7 @@ class Reax13Engine(Engine):
|
|
| 104 |
process = getattr(self, 'process', None)
|
| 105 |
if process is not None:
|
| 106 |
self.process = None
|
|
|
|
| 107 |
if process.poll() is None:
|
| 108 |
process.stdin.close()
|
| 109 |
try:
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: MIT
|
| 2 |
"""Decision Index adapter. Start one persistent isolated GPU backend per engine."""
|
| 3 |
import atexit
|
| 4 |
import json
|
|
|
|
| 8 |
import subprocess
|
| 9 |
import sys
|
| 10 |
import threading
|
| 11 |
+
import time
|
| 12 |
|
| 13 |
from decision_index.engines import Engine, Unsupported
|
| 14 |
|
| 15 |
ROOT = Path(__file__).resolve().parent
|
| 16 |
+
PROXY_MODEL = 'jev-1.13.0'
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class EngineUnavailable(RuntimeError):
|
| 20 |
+
"""The native backend is gone (timeout, GPU failure or crash); this engine cannot serve until restarted."""
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def proxy_tokens(state, questions):
|
| 24 |
+
"""A character-count size proxy: length of the compact request JSON // 4 (the literal model name is fixed, not looked up). No tokenizer, no clock, no answer-dependent step."""
|
| 25 |
+
return max(1, len(json.dumps({'model': PROXY_MODEL, 'state': state, 'questions': questions},
|
| 26 |
+
ensure_ascii=False, separators=(',', ':'))) // 4)
|
| 27 |
|
| 28 |
|
| 29 |
class Reax13Engine(Engine):
|
|
|
|
| 36 |
super().__init__(**options)
|
| 37 |
self.config = json.loads(Path(config or ROOT / 'reax13.json').read_text())
|
| 38 |
self.config.update(options)
|
| 39 |
+
self.max_input_proxy_tokens = int(self.config['max_input_proxy_tokens'])
|
| 40 |
+
if self.max_input_proxy_tokens <= 0:
|
| 41 |
+
raise ValueError('max_input_proxy_tokens must be positive')
|
| 42 |
if self.config['mode'] != 'reax13' or self.config['quantization'] != 'fp8':
|
| 43 |
raise ValueError('this engine supports reax13 FP8 only')
|
| 44 |
if self.config['optimization'] not in ('fastconv_longcache', 'plain'):
|
|
|
|
| 51 |
raise FileNotFoundError('model root must contain both part-g/ and part-q/')
|
| 52 |
env = dict(os.environ, REAX_PART_G=str(paths['part_g']), REAX_PART_Q=str(paths['part_q']))
|
| 53 |
self.lock = threading.Lock()
|
| 54 |
+
self._buf = b''
|
| 55 |
self.process = subprocess.Popen([sys.executable, '-u', str(ROOT / 'backend.py')],
|
| 56 |
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
| 57 |
text=True, bufsize=1, env=env)
|
|
|
|
| 71 |
'policy': 'native prompts and temperatures; no item-specific rules or truncation'}
|
| 72 |
|
| 73 |
def _read(self, timeout):
|
| 74 |
+
"""One reply line within `timeout` seconds in total (also across a partial line). A bad line closes the backend: a stale reply must never be matched to a later request."""
|
| 75 |
+
deadline = time.monotonic() + timeout
|
| 76 |
+
fd = self.process.stdout.fileno()
|
| 77 |
+
while b'\n' not in self._buf:
|
| 78 |
+
remaining = deadline - time.monotonic()
|
| 79 |
+
if remaining <= 0 or not select.select([fd], [], [], remaining)[0]:
|
| 80 |
+
self.close()
|
| 81 |
+
raise TimeoutError('REAX backend response deadline exceeded; backend closed')
|
| 82 |
+
chunk = os.read(fd, 1 << 20)
|
| 83 |
+
if not chunk:
|
| 84 |
+
status = self.process.poll()
|
| 85 |
+
self.close()
|
| 86 |
+
raise RuntimeError('REAX backend exited: ' + str(status))
|
| 87 |
+
self._buf += chunk
|
| 88 |
+
line, _, self._buf = self._buf.partition(b'\n')
|
| 89 |
+
try:
|
| 90 |
+
reply = json.loads(line.decode('utf-8'))
|
| 91 |
+
except ValueError:
|
| 92 |
+
self.close()
|
| 93 |
+
raise RuntimeError('REAX backend sent an unreadable reply line; backend closed')
|
| 94 |
+
if not isinstance(reply, dict):
|
| 95 |
self.close()
|
| 96 |
+
raise RuntimeError('REAX backend sent a non-object reply line; backend closed')
|
| 97 |
+
return reply
|
| 98 |
+
|
| 99 |
+
# Exact reply shapes of backend.py (the 'ok' and 'error' prints at the end of its request loop).
|
| 100 |
+
OK_KEYS = ('Duo', 'timings_ms', 'peak_bytes', 'peak_reserved_bytes', 'optimization')
|
| 101 |
+
ERROR_KEYS = ('error', 'exception')
|
| 102 |
+
|
| 103 |
+
@classmethod
|
| 104 |
+
def _is_reply(cls, reply):
|
| 105 |
+
"""True only for a line shaped like one of backend.py's two replies. Known limit: a stray line with exactly the shape of a real
|
| 106 |
+
reply cannot be detected without request ids; only a line that is not reply-shaped closes the backend."""
|
| 107 |
+
status = reply.get('status')
|
| 108 |
+
if status == 'ok':
|
| 109 |
+
return all(key in reply for key in cls.OK_KEYS)
|
| 110 |
+
if status == 'error':
|
| 111 |
+
return all(key in reply for key in cls.ERROR_KEYS)
|
| 112 |
+
return False
|
| 113 |
+
|
| 114 |
+
def alive(self):
|
| 115 |
+
process = getattr(self, 'process', None)
|
| 116 |
+
return process is not None and process.poll() is None
|
| 117 |
|
| 118 |
def __call__(self, state, questions):
|
| 119 |
if not questions or len(questions) > 64:
|
|
|
|
| 123 |
raise Unsupported('unsupported question type ' + question['type'])
|
| 124 |
if question['type'] == 'choice' and not 2 <= len(question['criteria']) <= 255:
|
| 125 |
raise Unsupported('options per choice must be 2..255')
|
| 126 |
+
size = proxy_tokens(state, questions)
|
| 127 |
+
if size > self.max_input_proxy_tokens:
|
| 128 |
+
# Declared input cap: refused before any model work, never truncated.
|
| 129 |
+
raise Unsupported('request of %d proxy tokens exceeds this engine\'s declared maximum context length of %d proxy tokens; input is never truncated'
|
| 130 |
+
% (size, self.max_input_proxy_tokens))
|
| 131 |
with self.lock:
|
| 132 |
+
if not self.alive():
|
| 133 |
+
raise EngineUnavailable('REAX backend is not running (closed after a timeout, GPU failure or crash); restart the engine')
|
| 134 |
request = {'state': state, 'questions': questions,
|
| 135 |
'_kind': 'concurrent' if self.config['optimization'] != 'plain' else 'sequential',
|
| 136 |
'_ubatch': int(self.config['u_batch_size']), '_opt': self.config['optimization']}
|
| 137 |
self.process.stdin.write(json.dumps(request) + '\n')
|
| 138 |
self.process.stdin.flush()
|
| 139 |
reply = self._read(float(self.config['request_timeout']))
|
| 140 |
+
if not self._is_reply(reply):
|
| 141 |
+
# Not a reply (e.g. a stray JSON object printed by a native library): the real reply is still queued.
|
| 142 |
+
self.close()
|
| 143 |
+
raise RuntimeError('REAX backend sent a line that is not a reply; backend closed')
|
| 144 |
if reply.get('status') != 'ok':
|
| 145 |
message = reply.get('error', 'native backend error')
|
| 146 |
if reply.get('exception') in ('OutOfMemoryError', 'AcceleratorError') or 'device-side assert' in message:
|
|
|
|
| 156 |
path = self.config.get('warmup_rows')
|
| 157 |
if path:
|
| 158 |
rows = [json.loads(line) for line in Path(path).read_text().splitlines()]
|
| 159 |
+
fits = [row for row in rows if proxy_tokens(row['state'], row['questions']) <= self.max_input_proxy_tokens]
|
| 160 |
+
warm = sorted(fits, key=lambda row: len(json.dumps(row)), reverse=True)[:3] + fits[:10]
|
| 161 |
for row in warm:
|
| 162 |
+
try:
|
| 163 |
+
self(row['state'], row['questions'])
|
| 164 |
+
except Unsupported:
|
| 165 |
+
pass
|
| 166 |
|
| 167 |
def runtime(self):
|
| 168 |
return self.ready
|
|
|
|
| 171 |
process = getattr(self, 'process', None)
|
| 172 |
if process is not None:
|
| 173 |
self.process = None
|
| 174 |
+
self._buf = b''
|
| 175 |
if process.poll() is None:
|
| 176 |
process.stdin.close()
|
| 177 |
try:
|
engine/serve.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
# SPDX-License-Identifier:
|
| 2 |
"""HTTP adapter for the existing Reax13Engine path.
|
| 3 |
|
| 4 |
The model is loaded once at process start. Each POST body is passed directly to
|
|
@@ -181,6 +181,10 @@ class ReaxHandler(BaseHTTPRequestHandler):
|
|
| 181 |
|
| 182 |
def do_GET(self) -> None:
|
| 183 |
if urlsplit(self.path).path == "/healthz":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 184 |
self._send_json(HTTPStatus.OK, {"status": "ready", "model": "reax13", "mode": "reax13"})
|
| 185 |
return
|
| 186 |
self._send_json(HTTPStatus.NOT_FOUND, {"error": {"code": "not_found", "message": "unknown endpoint"}})
|
|
@@ -218,6 +222,13 @@ class ReaxHandler(BaseHTTPRequestHandler):
|
|
| 218 |
except RequestError as exc:
|
| 219 |
self._send_json(HTTPStatus.UNPROCESSABLE_ENTITY, {"error": {"code": "invalid_request", "message": str(exc)}})
|
| 220 |
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 221 |
except ValueError as exc:
|
| 222 |
# Reax13Engine uses Decision Index's Unsupported (a ValueError)
|
| 223 |
# for native capacity/context refusals.
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: MIT
|
| 2 |
"""HTTP adapter for the existing Reax13Engine path.
|
| 3 |
|
| 4 |
The model is loaded once at process start. Each POST body is passed directly to
|
|
|
|
| 181 |
|
| 182 |
def do_GET(self) -> None:
|
| 183 |
if urlsplit(self.path).path == "/healthz":
|
| 184 |
+
alive = getattr(self.server.service.engine, "alive", None)
|
| 185 |
+
if callable(alive) and not alive():
|
| 186 |
+
self._send_json(HTTPStatus.SERVICE_UNAVAILABLE, {"status": "unavailable", "model": "reax13", "mode": "reax13"})
|
| 187 |
+
return
|
| 188 |
self._send_json(HTTPStatus.OK, {"status": "ready", "model": "reax13", "mode": "reax13"})
|
| 189 |
return
|
| 190 |
self._send_json(HTTPStatus.NOT_FOUND, {"error": {"code": "not_found", "message": "unknown endpoint"}})
|
|
|
|
| 222 |
except RequestError as exc:
|
| 223 |
self._send_json(HTTPStatus.UNPROCESSABLE_ENTITY, {"error": {"code": "invalid_request", "message": str(exc)}})
|
| 224 |
return
|
| 225 |
+
except RuntimeError as exc:
|
| 226 |
+
if type(exc).__name__ == "EngineUnavailable":
|
| 227 |
+
self._send_json(HTTPStatus.SERVICE_UNAVAILABLE, {"error": {"code": "engine_unavailable", "message": "native backend is not running"}})
|
| 228 |
+
return
|
| 229 |
+
self.log_error("inference failed (%s)", type(exc).__name__)
|
| 230 |
+
self._send_json(HTTPStatus.INTERNAL_SERVER_ERROR, {"error": {"code": "inference_error", "message": "inference failed"}})
|
| 231 |
+
return
|
| 232 |
except ValueError as exc:
|
| 233 |
# Reax13Engine uses Decision Index's Unsupported (a ValueError)
|
| 234 |
# for native capacity/context refusals.
|
engine/test_bundle.py
ADDED
|
@@ -0,0 +1,475 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: MIT
|
| 2 |
+
"""CPU tests for the declared input cap, the size-routing predicate and dead-backend handling.
|
| 3 |
+
|
| 4 |
+
A stub backend speaking backend.py's JSON line protocol stands in for the GPU process. Needs the Decision Index kit on
|
| 5 |
+
PYTHONPATH (as in the runtime image); falls back to a minimal copy of its Engine/Unsupported when it is absent.
|
| 6 |
+
"""
|
| 7 |
+
import ast
|
| 8 |
+
import json
|
| 9 |
+
import os
|
| 10 |
+
import sys
|
| 11 |
+
import tempfile
|
| 12 |
+
import threading
|
| 13 |
+
import types
|
| 14 |
+
import unittest
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
from unittest import mock
|
| 17 |
+
from urllib.error import HTTPError
|
| 18 |
+
from urllib.request import Request, urlopen
|
| 19 |
+
|
| 20 |
+
HERE = Path(__file__).resolve().parent
|
| 21 |
+
try:
|
| 22 |
+
import decision_index.engines # noqa: F401
|
| 23 |
+
except ImportError:
|
| 24 |
+
pkg = types.ModuleType('decision_index')
|
| 25 |
+
eng = types.ModuleType('decision_index.engines')
|
| 26 |
+
|
| 27 |
+
class Unsupported(ValueError):
|
| 28 |
+
pass
|
| 29 |
+
|
| 30 |
+
class Engine:
|
| 31 |
+
def __init__(self, **options):
|
| 32 |
+
self.options = options
|
| 33 |
+
|
| 34 |
+
def warmup(self):
|
| 35 |
+
pass
|
| 36 |
+
|
| 37 |
+
eng.Engine, eng.Unsupported = Engine, Unsupported
|
| 38 |
+
pkg.engines = eng
|
| 39 |
+
sys.modules.update({'decision_index': pkg, 'decision_index.engines': eng})
|
| 40 |
+
|
| 41 |
+
import reax13_engine as module # noqa: E402
|
| 42 |
+
from serve import build_server # noqa: E402
|
| 43 |
+
|
| 44 |
+
CONFIG = json.loads((HERE / 'reax13.json').read_text())
|
| 45 |
+
HAS_ROUTE = 'route_u_px' in CONFIG
|
| 46 |
+
|
| 47 |
+
STUB = r'''
|
| 48 |
+
import json, sys, time
|
| 49 |
+
print(json.dumps({"ready": True}), flush=True)
|
| 50 |
+
log = open(sys.argv[1], "a")
|
| 51 |
+
for line in sys.stdin:
|
| 52 |
+
r = json.loads(line)
|
| 53 |
+
state = r["state"]
|
| 54 |
+
log.write(json.dumps({**{k: v for k, v in r.items() if k.startswith("_")}, "_n": len(state) if isinstance(state, str) else None, "_head": state[:8] if isinstance(state, str) else None, "_tail": state[-8:] if isinstance(state, str) else None}) + "\n"); log.flush()
|
| 55 |
+
if state == "SLEEP":
|
| 56 |
+
time.sleep(30)
|
| 57 |
+
if state == "STRAY":
|
| 58 |
+
print("stray native library output", flush=True)
|
| 59 |
+
if state == "STRAYDICT":
|
| 60 |
+
print(json.dumps({"event": "autotune"}), flush=True)
|
| 61 |
+
if state == "STRAYOK":
|
| 62 |
+
print(json.dumps({"status": "ok"}), flush=True)
|
| 63 |
+
if state == "STRAYINFO":
|
| 64 |
+
print(json.dumps({"status": "info", "note": "tuning"}), flush=True)
|
| 65 |
+
if state == "STRAYERR":
|
| 66 |
+
print(json.dumps({"status": "error", "error": "x"}), flush=True)
|
| 67 |
+
if state == "AFTERDICT":
|
| 68 |
+
print(json.dumps({"status": "ok", "Duo": {"model": "reax13", "answers": {}}, "timings_ms": {}, "peak_bytes": 0, "peak_reserved_bytes": 0, "optimization": {"name": "stub"}}), flush=True)
|
| 69 |
+
print(json.dumps({"event": "autotune"}), flush=True)
|
| 70 |
+
continue
|
| 71 |
+
if state == "ARRAY":
|
| 72 |
+
print("[1, 2]", flush=True)
|
| 73 |
+
continue
|
| 74 |
+
if state == "PARTIAL":
|
| 75 |
+
sys.stdout.write('{"status": "ok", "Duo'); sys.stdout.flush()
|
| 76 |
+
time.sleep(3)
|
| 77 |
+
if state == "CRASH":
|
| 78 |
+
sys.exit(3)
|
| 79 |
+
if state == "OOM":
|
| 80 |
+
print(json.dumps({"status": "error", "error": "CUDA out of memory", "exception": "OutOfMemoryError"}), flush=True)
|
| 81 |
+
continue
|
| 82 |
+
answers = {}
|
| 83 |
+
for qid, q in r["questions"].items():
|
| 84 |
+
keys = list(q["criteria"])
|
| 85 |
+
answers[qid] = {"type": "choice", "choice": sorted(keys)[0], "probabilities": {k: (0.6 if k == sorted(keys)[0] else 0.4 / (len(keys) - 1)) for k in keys}}
|
| 86 |
+
print(json.dumps({"status": "ok", "Duo": {"model": "reax13", "answers": answers}, "timings_ms": {}, "peak_bytes": 0,
|
| 87 |
+
"peak_reserved_bytes": 0, "optimization": {"name": "stub"}}), flush=True)
|
| 88 |
+
'''
|
| 89 |
+
|
| 90 |
+
Q = {'q': {'type': 'choice', 'instructions': 'Pick', 'criteria': {'a': 'A', 'b': 'B'}}}
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def state_for(target):
|
| 94 |
+
"""A state string whose request has exactly `target` proxy tokens (floor(len/4))."""
|
| 95 |
+
base = len(json.dumps({'model': 'jev-1.13.0', 'state': '', 'questions': Q}, ensure_ascii=False, separators=(',', ':')))
|
| 96 |
+
state = 'x' * (4 * target - base)
|
| 97 |
+
assert module.proxy_tokens(state, Q) == target
|
| 98 |
+
return state
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
class StubEngineCase(unittest.TestCase):
|
| 102 |
+
def make(self, **options):
|
| 103 |
+
tmp = tempfile.TemporaryDirectory()
|
| 104 |
+
self.addCleanup(tmp.cleanup)
|
| 105 |
+
root = Path(tmp.name)
|
| 106 |
+
for part in ('part-g', 'part-q'):
|
| 107 |
+
(root / part).mkdir()
|
| 108 |
+
(root / 'stub.py').write_text(STUB)
|
| 109 |
+
self.log = root / 'requests.log'
|
| 110 |
+
real_popen = module.subprocess.Popen
|
| 111 |
+
real_exists = Path.exists
|
| 112 |
+
|
| 113 |
+
def popen(cmd, **kw):
|
| 114 |
+
return real_popen([sys.executable, '-u', str(root / 'stub.py'), str(self.log)], **kw)
|
| 115 |
+
|
| 116 |
+
env = {'DUO_ISOLATED': '1'}
|
| 117 |
+
env.update(options.pop('env', {}))
|
| 118 |
+
with mock.patch.dict(os.environ, env), mock.patch.object(module.subprocess, 'Popen', popen), \
|
| 119 |
+
mock.patch.object(Path, 'exists', lambda p: True if str(p) == '/.dockerenv' else real_exists(p)):
|
| 120 |
+
engine = module.Reax13Engine(config=str(HERE / 'reax13.json'), model=str(root), **options)
|
| 121 |
+
self.addCleanup(engine.close)
|
| 122 |
+
return engine
|
| 123 |
+
|
| 124 |
+
def seen(self):
|
| 125 |
+
return [json.loads(x) for x in self.log.read_text().splitlines()] if self.log.exists() else []
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
class CapTests(StubEngineCase):
|
| 129 |
+
def test_config_value(self):
|
| 130 |
+
self.assertEqual(CONFIG['max_input_proxy_tokens'], 32000)
|
| 131 |
+
self.assertEqual(self.make().max_input_proxy_tokens, 32000)
|
| 132 |
+
|
| 133 |
+
def test_boundary_32000_ok_32001_unsupported_before_model_work(self):
|
| 134 |
+
engine = self.make()
|
| 135 |
+
engine(state_for(32000), Q)
|
| 136 |
+
self.assertEqual(len(self.seen()), 1)
|
| 137 |
+
with self.assertRaises(module.Unsupported):
|
| 138 |
+
engine(state_for(32001), Q)
|
| 139 |
+
self.assertEqual(len(self.seen()), 1, 'over-cap request must not reach the backend')
|
| 140 |
+
self.assertTrue(engine.alive())
|
| 141 |
+
engine('short', Q)
|
| 142 |
+
self.assertEqual(len(self.seen()), 2)
|
| 143 |
+
|
| 144 |
+
def test_never_truncates(self):
|
| 145 |
+
engine = self.make()
|
| 146 |
+
state = 'HEAD1234' + state_for(32000)[16:] + 'TAIL5678'
|
| 147 |
+
self.assertEqual(module.proxy_tokens(state, Q), 32000)
|
| 148 |
+
engine(state, Q)
|
| 149 |
+
seen = self.seen()[-1]
|
| 150 |
+
self.assertEqual((seen['_n'], seen['_head'], seen['_tail']), (len(state), 'HEAD1234', 'TAIL5678'))
|
| 151 |
+
|
| 152 |
+
def test_cap_is_configurable(self):
|
| 153 |
+
engine = self.make(max_input_proxy_tokens=50)
|
| 154 |
+
with self.assertRaises(module.Unsupported):
|
| 155 |
+
engine(state_for(51), Q)
|
| 156 |
+
engine(state_for(50), Q)
|
| 157 |
+
|
| 158 |
+
def test_proxy_matches_suite_formula(self):
|
| 159 |
+
for state in ('', 'abc', {'k': 'äöü€'}, ['x'] * 50):
|
| 160 |
+
expected = max(1, len(json.dumps({'model': 'jev-1.13.0', 'state': state, 'questions': Q}, ensure_ascii=False, separators=(',', ':'))) // 4)
|
| 161 |
+
self.assertEqual(module.proxy_tokens(state, Q), expected)
|
| 162 |
+
|
| 163 |
+
def test_http_422_unsupported(self):
|
| 164 |
+
engine = self.make()
|
| 165 |
+
server = build_server(engine, '127.0.0.1', 0)
|
| 166 |
+
threading.Thread(target=server.serve_forever, daemon=True).start()
|
| 167 |
+
self.addCleanup(server.server_close)
|
| 168 |
+
self.addCleanup(server.shutdown)
|
| 169 |
+
port = server.server_address[1]
|
| 170 |
+
|
| 171 |
+
def post(state):
|
| 172 |
+
body = json.dumps({'state': state, 'questions': Q}).encode()
|
| 173 |
+
try:
|
| 174 |
+
with urlopen(Request(f'http://127.0.0.1:{port}/v1/systemone', body, {'Content-Type': 'application/json'})) as r:
|
| 175 |
+
return r.status, json.loads(r.read())
|
| 176 |
+
except HTTPError as e:
|
| 177 |
+
return e.code, json.loads(e.read())
|
| 178 |
+
|
| 179 |
+
self.assertEqual(post(state_for(32000))[0], 200)
|
| 180 |
+
status, body = post(state_for(32001))
|
| 181 |
+
self.assertEqual((status, body['error']['code']), (422, 'unsupported'))
|
| 182 |
+
|
| 183 |
+
def test_http_422_text_carries_kit_capacity_marker(self):
|
| 184 |
+
markers = ("options per choice", "a choice needs at least two options", "a score takes 2 to 10 levels", "the canvas holds",
|
| 185 |
+
"maximum context length", "maximum model length", "longer than the maximum model length", "context window", "too many tokens")
|
| 186 |
+
engine = self.make()
|
| 187 |
+
with self.assertRaises(module.Unsupported) as ctx:
|
| 188 |
+
engine(state_for(32001), Q)
|
| 189 |
+
self.assertTrue(any(m in str(ctx.exception) for m in markers), str(ctx.exception))
|
| 190 |
+
server = build_server(engine, '127.0.0.1', 0)
|
| 191 |
+
threading.Thread(target=server.serve_forever, daemon=True).start()
|
| 192 |
+
self.addCleanup(server.server_close)
|
| 193 |
+
self.addCleanup(server.shutdown)
|
| 194 |
+
body = json.dumps({'state': state_for(32001), 'questions': Q}).encode()
|
| 195 |
+
with self.assertRaises(HTTPError) as http:
|
| 196 |
+
urlopen(Request(f'http://127.0.0.1:{server.server_address[1]}/v1/systemone', body, {'Content-Type': 'application/json'}))
|
| 197 |
+
self.assertEqual(http.exception.code, 422)
|
| 198 |
+
body = http.exception.read().decode()
|
| 199 |
+
self.assertTrue(any(m in body for m in markers), body)
|
| 200 |
+
|
| 201 |
+
def test_warmup_filters_over_cap_before_picking_longest(self):
|
| 202 |
+
rows = Path(tempfile.mkdtemp()) / 'rows.jsonl'
|
| 203 |
+
lines = [json.dumps({'state': state_for(32001), 'questions': Q})] + [json.dumps({'state': 's' * (i + 1), 'questions': Q}) for i in range(14)]
|
| 204 |
+
rows.write_text('\n'.join(lines) + '\n')
|
| 205 |
+
engine = self.make(warmup_rows=str(rows))
|
| 206 |
+
# The kit's own Engine.warmup (present in the runtime image) sends rows of its own; count only this engine's.
|
| 207 |
+
with mock.patch.object(module.Engine, 'warmup', lambda self: None):
|
| 208 |
+
engine.warmup()
|
| 209 |
+
sent = self.seen()
|
| 210 |
+
self.assertEqual(len(sent), 13, 'over-cap row skipped, then 3 longest + first 10 fitting rows')
|
| 211 |
+
self.assertEqual(sorted(s['_n'] for s in sent[:3]), [12, 13, 14])
|
| 212 |
+
self.assertNotIn(len('x' * 130000), [s['_n'] for s in sent])
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
class DeadEngineTests(StubEngineCase):
|
| 216 |
+
def http(self, engine):
|
| 217 |
+
server = build_server(engine, '127.0.0.1', 0)
|
| 218 |
+
threading.Thread(target=server.serve_forever, daemon=True).start()
|
| 219 |
+
self.addCleanup(server.server_close)
|
| 220 |
+
self.addCleanup(server.shutdown)
|
| 221 |
+
port = server.server_address[1]
|
| 222 |
+
|
| 223 |
+
def get():
|
| 224 |
+
try:
|
| 225 |
+
with urlopen(f'http://127.0.0.1:{port}/healthz') as r:
|
| 226 |
+
return r.status, json.loads(r.read())
|
| 227 |
+
except HTTPError as e:
|
| 228 |
+
return e.code, json.loads(e.read())
|
| 229 |
+
|
| 230 |
+
def post(state):
|
| 231 |
+
body = json.dumps({'state': state, 'questions': Q}).encode()
|
| 232 |
+
try:
|
| 233 |
+
with urlopen(Request(f'http://127.0.0.1:{port}/v1/systemone', body, {'Content-Type': 'application/json'})) as r:
|
| 234 |
+
return r.status, json.loads(r.read())
|
| 235 |
+
except HTTPError as e:
|
| 236 |
+
return e.code, json.loads(e.read())
|
| 237 |
+
return get, post
|
| 238 |
+
|
| 239 |
+
def test_normal_rows_unchanged_and_healthy(self):
|
| 240 |
+
engine = self.make()
|
| 241 |
+
first = engine('s', Q)[0]
|
| 242 |
+
get, post = self.http(engine)
|
| 243 |
+
self.assertEqual(get()[0], 200)
|
| 244 |
+
self.assertEqual(post('s')[1]['answers'], first['answers'])
|
| 245 |
+
|
| 246 |
+
def test_timeout_then_loud_failure_and_unhealthy(self):
|
| 247 |
+
engine = self.make(request_timeout=0.5)
|
| 248 |
+
engine('before', Q)
|
| 249 |
+
with self.assertRaises(TimeoutError):
|
| 250 |
+
engine('SLEEP', Q)
|
| 251 |
+
self.assertFalse(engine.alive())
|
| 252 |
+
with self.assertRaises(module.EngineUnavailable) as ctx:
|
| 253 |
+
engine('after', Q)
|
| 254 |
+
self.assertNotIsInstance(ctx.exception, AttributeError)
|
| 255 |
+
get, post = self.http(engine)
|
| 256 |
+
self.assertEqual(get(), (503, {'status': 'unavailable', 'model': 'reax13', 'mode': 'reax13'}))
|
| 257 |
+
status, body = post('after')
|
| 258 |
+
self.assertEqual((status, body['error']['code']), (503, 'engine_unavailable'))
|
| 259 |
+
|
| 260 |
+
def test_oom_closes_engine_then_loud_failure(self):
|
| 261 |
+
engine = self.make()
|
| 262 |
+
with self.assertRaises(RuntimeError):
|
| 263 |
+
engine('OOM', Q)
|
| 264 |
+
self.assertFalse(engine.alive())
|
| 265 |
+
with self.assertRaises(module.EngineUnavailable):
|
| 266 |
+
engine('after', Q)
|
| 267 |
+
|
| 268 |
+
def test_backend_crash_is_unhealthy(self):
|
| 269 |
+
engine = self.make()
|
| 270 |
+
with self.assertRaises(RuntimeError):
|
| 271 |
+
engine('CRASH', Q)
|
| 272 |
+
self.assertFalse(engine.alive())
|
| 273 |
+
with self.assertRaises(module.EngineUnavailable):
|
| 274 |
+
engine('after', Q)
|
| 275 |
+
self.assertEqual(self.http(engine)[0]()[0], 503)
|
| 276 |
+
|
| 277 |
+
def test_stray_or_non_object_reply_line_closes_backend_no_desync(self):
|
| 278 |
+
for bad in ('STRAY', 'ARRAY'):
|
| 279 |
+
engine = self.make()
|
| 280 |
+
engine('before', Q)
|
| 281 |
+
with self.assertRaises(RuntimeError):
|
| 282 |
+
engine(bad, Q)
|
| 283 |
+
self.assertFalse(engine.alive(), bad)
|
| 284 |
+
with self.assertRaises(module.EngineUnavailable):
|
| 285 |
+
engine('row-3', Q)
|
| 286 |
+
|
| 287 |
+
def test_stray_json_object_line_never_shifts_later_replies(self):
|
| 288 |
+
# A line that parses as a JSON object but is not a reply: the real reply stays queued, so the backend must close.
|
| 289 |
+
engine = self.make()
|
| 290 |
+
with self.assertRaises(RuntimeError):
|
| 291 |
+
engine('STRAYDICT', Q)
|
| 292 |
+
self.assertFalse(engine.alive())
|
| 293 |
+
for row in ('row-3', 'row-4'):
|
| 294 |
+
with self.assertRaises(module.EngineUnavailable):
|
| 295 |
+
engine(row, Q)
|
| 296 |
+
|
| 297 |
+
def test_stray_status_lines_not_reply_shaped_close_backend(self):
|
| 298 |
+
# Lines with a `status` key but not the exact shape of backend.py's replies must not shift later answers.
|
| 299 |
+
for bad in ('STRAYOK', 'STRAYINFO', 'STRAYERR'):
|
| 300 |
+
engine = self.make()
|
| 301 |
+
engine('before', Q)
|
| 302 |
+
with self.assertRaises(RuntimeError):
|
| 303 |
+
engine(bad, Q)
|
| 304 |
+
self.assertFalse(engine.alive(), bad)
|
| 305 |
+
for row in ('row-3', 'row-4'):
|
| 306 |
+
with self.assertRaises(module.EngineUnavailable):
|
| 307 |
+
engine(row, Q)
|
| 308 |
+
|
| 309 |
+
def test_stray_json_object_line_right_after_a_good_reply(self):
|
| 310 |
+
engine = self.make()
|
| 311 |
+
self.assertEqual(engine('AFTERDICT', Q)[0]['model'], 'reax13')
|
| 312 |
+
with self.assertRaises(RuntimeError):
|
| 313 |
+
engine('row-2', Q)
|
| 314 |
+
self.assertFalse(engine.alive())
|
| 315 |
+
with self.assertRaises(module.EngineUnavailable):
|
| 316 |
+
engine('row-3', Q)
|
| 317 |
+
self.assertEqual(self.http(engine)[0]()[0], 503)
|
| 318 |
+
|
| 319 |
+
def test_deadline_enforced_across_partial_line(self):
|
| 320 |
+
engine = self.make(request_timeout=0.7)
|
| 321 |
+
with self.assertRaises(TimeoutError):
|
| 322 |
+
engine('PARTIAL', Q)
|
| 323 |
+
self.assertFalse(engine.alive())
|
| 324 |
+
|
| 325 |
+
def test_close_is_idempotent_and_requests_after_close_fail_loudly(self):
|
| 326 |
+
engine = self.make()
|
| 327 |
+
engine.close()
|
| 328 |
+
engine.close()
|
| 329 |
+
with self.assertRaises(module.EngineUnavailable):
|
| 330 |
+
engine('x', Q)
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
class BenchmarkDriverTests(unittest.TestCase):
|
| 334 |
+
def test_over_cap_row_is_recorded_unsupported_and_run_continues(self):
|
| 335 |
+
import benchmark
|
| 336 |
+
|
| 337 |
+
class Fake:
|
| 338 |
+
provenance, latency = {}, 'x'
|
| 339 |
+
|
| 340 |
+
def __init__(self, **kw):
|
| 341 |
+
pass
|
| 342 |
+
|
| 343 |
+
def runtime(self):
|
| 344 |
+
return {}
|
| 345 |
+
|
| 346 |
+
def warmup(self):
|
| 347 |
+
pass
|
| 348 |
+
|
| 349 |
+
def synchronize(self):
|
| 350 |
+
pass
|
| 351 |
+
|
| 352 |
+
def close(self):
|
| 353 |
+
pass
|
| 354 |
+
|
| 355 |
+
def __call__(self, state, questions):
|
| 356 |
+
if state == 'BIG':
|
| 357 |
+
raise module.Unsupported('exceeds this engine\'s declared maximum context length')
|
| 358 |
+
return {'model': 'reax13', 'answers': {'q': {'type': 'choice', 'choice': 'a', 'probabilities': {'a': 0.6, 'b': 0.4}}}}, {}
|
| 359 |
+
|
| 360 |
+
tmp = Path(tempfile.mkdtemp())
|
| 361 |
+
rows = tmp / 'rows.jsonl'
|
| 362 |
+
rows.write_text('\n'.join(json.dumps({'state': s, 'questions': Q, '_evaluation': {'id': s}}) for s in ('a', 'BIG', 'c')) + '\n')
|
| 363 |
+
with mock.patch.object(benchmark, 'Reax13Engine', Fake), mock.patch.object(sys, 'argv', ['benchmark', '--rows', str(rows), '--out', str(tmp / 'out')]):
|
| 364 |
+
benchmark.main()
|
| 365 |
+
out = [json.loads(x) for x in (tmp / 'out' / 'results.jsonl').read_text().splitlines()]
|
| 366 |
+
self.assertEqual([(o['id'], o['status']) for o in out], [('a', 'ok'), ('BIG', 'unsupported'), ('c', 'ok')])
|
| 367 |
+
self.assertEqual(json.loads((tmp / 'out' / 'status.json').read_text())['counts'], {'ok': 2, 'unsupported': 1})
|
| 368 |
+
|
| 369 |
+
|
| 370 |
+
def backend_proxy_tokens():
|
| 371 |
+
"""Compile the backend's own proxy_tokens source (backend.py needs a GPU to import)."""
|
| 372 |
+
tree = ast.parse((HERE / 'backend.py').read_text())
|
| 373 |
+
for node in tree.body:
|
| 374 |
+
if isinstance(node, ast.FunctionDef) and node.name == 'proxy_tokens':
|
| 375 |
+
scope = {'json': json}
|
| 376 |
+
exec(compile(ast.Module([node], []), 'backend.py', 'exec'), scope)
|
| 377 |
+
return scope['proxy_tokens']
|
| 378 |
+
return None
|
| 379 |
+
|
| 380 |
+
|
| 381 |
+
@unittest.skipUnless(HAS_ROUTE, 'routing rule only in engine-EC-R')
|
| 382 |
+
class RouteTests(StubEngineCase):
|
| 383 |
+
def test_default_1000_in_json(self):
|
| 384 |
+
self.assertEqual(CONFIG['route_u_px'], 1000)
|
| 385 |
+
self.assertEqual(self.make().route_u_px, 1000)
|
| 386 |
+
|
| 387 |
+
def test_env_overrides_and_zero_is_off(self):
|
| 388 |
+
self.assertEqual(self.make(env={'REAX_ROUTE_U_PX': '5000'}).route_u_px, 5000)
|
| 389 |
+
self.assertEqual(self.make(env={'REAX_ROUTE_U_PX': '0'}).route_u_px, 0)
|
| 390 |
+
|
| 391 |
+
def test_engine_sends_threshold_to_backend(self):
|
| 392 |
+
engine = self.make()
|
| 393 |
+
engine('s', Q)
|
| 394 |
+
self.assertEqual(self.seen()[-1]['_route_u_px'], 1000)
|
| 395 |
+
|
| 396 |
+
def test_predicate_boundaries(self):
|
| 397 |
+
fn = backend_proxy_tokens()
|
| 398 |
+
self.assertIsNotNone(fn)
|
| 399 |
+
for target, routed_at_1000, over_cap in ((1000, False, False), (1001, True, False), (32000, True, False), (32001, True, True)):
|
| 400 |
+
state = state_for(target)
|
| 401 |
+
r = {'state': state, 'questions': Q}
|
| 402 |
+
self.assertEqual(fn(r), target)
|
| 403 |
+
self.assertEqual(fn(r) > 1000, routed_at_1000, target)
|
| 404 |
+
self.assertEqual(fn(r) > CONFIG['max_input_proxy_tokens'], over_cap, target)
|
| 405 |
+
|
| 406 |
+
def test_backend_and_engine_predicates_agree(self):
|
| 407 |
+
fn = backend_proxy_tokens()
|
| 408 |
+
for state in ('', 'abc', {'k': 'äöü€' * 100}, ['x'] * 50):
|
| 409 |
+
self.assertEqual(fn({'state': state, 'questions': Q}), module.proxy_tokens(state, Q))
|
| 410 |
+
|
| 411 |
+
def test_backend_routing_source(self):
|
| 412 |
+
text = (HERE / 'backend.py').read_text()
|
| 413 |
+
self.assertIn("routed=kind=='concurrent' and route_px>0 and proxy_tokens(r)>route_px", text)
|
| 414 |
+
|
| 415 |
+
|
| 416 |
+
@unittest.skipIf(HAS_ROUTE, 'engine-EC has no routing rule')
|
| 417 |
+
class NoRouteTests(StubEngineCase):
|
| 418 |
+
def test_ec_sends_no_route_and_backend_has_none(self):
|
| 419 |
+
engine = self.make()
|
| 420 |
+
engine('s', Q)
|
| 421 |
+
self.assertNotIn('_route_u_px', self.seen()[-1])
|
| 422 |
+
self.assertNotIn('_route_u_px', (HERE / 'backend.py').read_text())
|
| 423 |
+
self.assertNotIn('REAX_ROUTE_U_PX', (HERE / 'reax13_engine.py').read_text())
|
| 424 |
+
|
| 425 |
+
|
| 426 |
+
class KernelGuardTests(unittest.TestCase):
|
| 427 |
+
"""fast_fp8_gemm.py wrapper and asserts, CPU only (the asserts fire before any kernel launch)."""
|
| 428 |
+
|
| 429 |
+
def setUp(self):
|
| 430 |
+
try:
|
| 431 |
+
import torch
|
| 432 |
+
import fast_fp8_gemm
|
| 433 |
+
except ImportError as exc:
|
| 434 |
+
self.skipTest('torch/triton not importable: %s' % exc)
|
| 435 |
+
self.torch, self.f = torch, fast_fp8_gemm
|
| 436 |
+
|
| 437 |
+
def weights(self, n, k):
|
| 438 |
+
t = self.torch
|
| 439 |
+
return t.zeros(n, k, dtype=t.float8_e4m3fn), t.ones((n + 127) // 128, (k + 127) // 128, dtype=t.float32)
|
| 440 |
+
|
| 441 |
+
def test_upstream_k_and_bs_asserts_kept(self):
|
| 442 |
+
t = self.torch
|
| 443 |
+
W, S = self.weights(256, 256)
|
| 444 |
+
with self.assertRaisesRegex(AssertionError, 'K mismatch'):
|
| 445 |
+
self.f.matmul(t.zeros(4, 128, dtype=t.bfloat16), W, S, [128, 128], t.bfloat16)
|
| 446 |
+
with self.assertRaisesRegex(AssertionError, 'Bs shape'):
|
| 447 |
+
self.f.matmul(t.zeros(4, 256, dtype=t.bfloat16), W, S[:1], [128, 128], t.bfloat16)
|
| 448 |
+
with self.assertRaisesRegex(AssertionError, 'block_size'):
|
| 449 |
+
self.f.matmul(t.zeros(4, 256, dtype=t.bfloat16), W, S, [128], t.bfloat16)
|
| 450 |
+
|
| 451 |
+
def test_dispatch_guards(self):
|
| 452 |
+
t = self.torch
|
| 453 |
+
calls = []
|
| 454 |
+
fake = types.ModuleType('transformers.integrations.finegrained_fp8')
|
| 455 |
+
fake.fp8_linear = lambda *a: calls.append(('orig', a[-1])) or 'orig'
|
| 456 |
+
mods = {'transformers': types.ModuleType('transformers'), 'transformers.integrations': types.ModuleType('transformers.integrations'),
|
| 457 |
+
'transformers.integrations.finegrained_fp8': fake}
|
| 458 |
+
mods['transformers'].integrations = mods['transformers.integrations']
|
| 459 |
+
mods['transformers.integrations'].finegrained_fp8 = fake
|
| 460 |
+
with mock.patch.dict(sys.modules, mods), mock.patch.object(self.f, 'matmul', lambda *a: calls.append('fast') or 'fast'):
|
| 461 |
+
self.f.install()
|
| 462 |
+
x = t.zeros(3, 256, dtype=t.bfloat16)
|
| 463 |
+
W, S = self.weights(256, 256)
|
| 464 |
+
self.assertEqual(fake.fp8_linear(x, W, S, [128, 128]), 'fast')
|
| 465 |
+
# whole-weight block (N=K=128): not this kernel's shape -> upstream, DeepGEMM off
|
| 466 |
+
Wt, St = self.weights(128, 128)
|
| 467 |
+
self.assertEqual(fake.fp8_linear(t.zeros(3, 128, dtype=t.bfloat16), Wt, St, [128, 128]), 'orig')
|
| 468 |
+
# a call that falls through must not allow DeepGEMM even when the caller asks for it
|
| 469 |
+
self.assertEqual(fake.fp8_linear(x, W, S, [128, 128], None, None, True), 'fast')
|
| 470 |
+
self.assertEqual(fake.fp8_linear(x, W, S, [128, 128], t.ones(256, dtype=t.bfloat16), None, True), 'orig')
|
| 471 |
+
self.assertEqual(calls, ['fast', ('orig', False), 'fast', ('orig', False)])
|
| 472 |
+
|
| 473 |
+
|
| 474 |
+
if __name__ == '__main__':
|
| 475 |
+
unittest.main()
|
engine/test_fast_fp8_gemm.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: MIT
|
| 2 |
+
"""GPU check: fast_fp8_gemm returns the same bits as the finegrained-fp8 kernel. Run inside the runtime container."""
|
| 3 |
+
import torch
|
| 4 |
+
import fast_fp8_gemm as f
|
| 5 |
+
from autotune_guard import install
|
| 6 |
+
install()
|
| 7 |
+
from transformers.integrations.finegrained_fp8 import load_finegrained_fp8_kernel
|
| 8 |
+
def test_bits():
|
| 9 |
+
mm=load_finegrained_fp8_kernel().matmul;torch.manual_seed(0)
|
| 10 |
+
for N,K in [(21504,5376),(5376,21504),(8192,5376),(4096,5376),(5376,8192),(16384,5376),(2048,5376),(200,256)]:
|
| 11 |
+
w=torch.randn(N,K,device='cuda')*.02;gn,gk=(N+127)//128,K//128
|
| 12 |
+
s=torch.stack([torch.stack([w[i*128:(i+1)*128,j*128:(j+1)*128].abs().amax() for j in range(gk)]) for i in range(gn)])/448
|
| 13 |
+
W=(w/s.repeat_interleave(128,0)[:N].repeat_interleave(128,1)).to(torch.float8_e4m3fn)
|
| 14 |
+
for shape in [(1,1),(5,1),(1,17),(129,1),(1,1000),(4,300)]:
|
| 15 |
+
A=(torch.randn(*shape,K,device='cuda')*torch.exp(torch.randn(*shape,K,device='cuda'))).to(torch.bfloat16)
|
| 16 |
+
a=mm(A,W,s,[128,128],A.dtype);b=f.matmul(A,W,s,[128,128],A.dtype);torch.cuda.synchronize()
|
| 17 |
+
assert torch.equal(a,b),(N,K,shape,int((a!=b).sum()))
|
| 18 |
+
# zero rows, the 1e-12 floor
|
| 19 |
+
A=torch.zeros(3,K,device='cuda',dtype=torch.bfloat16);assert torch.equal(mm(A,W,s,[128,128],A.dtype),f.matmul(A,W,s,[128,128],A.dtype))
|
| 20 |
+
if __name__=='__main__':
|
| 21 |
+
test_bits();print('PASS')
|
engine/test_probabilities.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
# SPDX-License-Identifier:
|
| 2 |
import math
|
| 3 |
import unittest
|
| 4 |
from probabilities import mix
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: MIT
|
| 2 |
import math
|
| 3 |
import unittest
|
| 4 |
from probabilities import mix
|
engine/test_serve.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
# SPDX-License-Identifier:
|
| 2 |
import json
|
| 3 |
import threading
|
| 4 |
import unittest
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: MIT
|
| 2 |
import json
|
| 3 |
import threading
|
| 4 |
import unittest
|