Add ExecuTorch WebGPU program for qwen2.5-0.5b-instruct

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+qwen2_5_0_5b_instruct_q4gsw.pte filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

NOTICE

@@ -0,0 +1,6 @@
+This repository contains a modified version of Qwen/Qwen2.5-0.5B-Instruct.
+
+Modifications by Meta Platforms, Inc.: conversion to the ExecuTorch program format (.pte) for the
+ExecuTorch WebGPU backend, and quantization as described in README.md.
+
+The original work and these modifications are distributed under the license in LICENSE.

README.md

@@ -0,0 +1,58 @@
+---
+license: apache-2.0
+base_model:
+- Qwen/Qwen2.5-0.5B-Instruct
+base_model_relation: quantized
+library_name: executorch
+pipeline_tag: text-generation
+tags:
+- executorch
+- webgpu
+- on-device
+- browser
+---
+# Qwen2.5 0.5B Instruct for ExecuTorch WebGPU
+
+This repository contains an [ExecuTorch](https://github.com/pytorch/executorch) conversion of
+[Qwen/Qwen2.5-0.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct) for the ExecuTorch **WebGPU** backend, so the model runs entirely in a web browser on
+the visitor's GPU. It powers the corresponding demo in the
+[ExecuTorch Playground](https://executorch.ai/playground/webgpu/).
+
+- **Base model:** [Qwen/Qwen2.5-0.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct)
+- **Format:** ExecuTorch `.pte` programs delegated to the WebGPU backend
+- **Quantization:** 4-bit groupwise weights (q4gsw), fp16 KV cache
+- **Total size:** 296.5 MB
+
+## Files
+
+| File | Size | SHA-256 |
+|---|---|---|
+| `qwen2_5_0_5b_instruct_q4gsw.pte` | 285.6 MB | `474274b934978f6c…` |
+| `tokenizer/config.json` | 659 B | `18e18afcaccafade…` |
+| `tokenizer/generation_config.json` | 242 B | `e558847a8b440261…` |
+| `tokenizer/merges.txt` | 1.6 MB | `599bab5407508877…` |
+| `tokenizer/tokenizer.json` | 6.7 MB | `c0382117ea329cdf…` |
+| `tokenizer/tokenizer_config.json` | 7.1 KB | `5b5d4f65d0acd3b2…` |
+| `tokenizer/vocab.json` | 2.6 MB | `ca10d7e9fb3ed185…` |
+
+Full checksums are in `manifest.json`. The playground verifies them after download.
+
+## Running it
+
+These programs need the ExecuTorch WebGPU runtime compiled to WebAssembly
+([backends/webgpu](https://github.com/pytorch/executorch/tree/main/backends/webgpu)). The easiest way to
+try the model is the [ExecuTorch Playground](https://executorch.ai/playground/webgpu/), which downloads these
+files once, caches them in the browser, and runs everything locally.
+
+## Provenance
+
+- **Upstream:** `Qwen/Qwen2.5-0.5B-Instruct` at revision `7ae557604adf67be50417f59c2c2f167def9a775` (official)
+- **Exported with:** executorch 1.5.1+cpu, torch 2.14.1+cpu, torchao 0.18.0+cpu, transformers 5.19.0 (executorch git `3b6068392324`)
+- **Reproduce:** `python recipes/qwen2.5-0.5b-instruct.py --out out/qwen2.5-0.5b-instruct` (recipe in `export/`)
+- Re-running the recipe produces byte-identical files.
+
+## License
+
+This is a derivative of the upstream model and is distributed under the same terms
+(`apache-2.0`); see `LICENSE`.
+Modifications (conversion to the ExecuTorch format and quantization) by Meta Platforms, Inc.

export/_common.py

@@ -0,0 +1,330 @@
+"""Shared helpers for the ExecuTorch WebGPU export recipes in this directory.
+
+Each recipe builds an eager model from a pinned upstream revision and calls
+`export_vision` (or its own LLM path), which writes:
+
+  <out>/<name>_webgpu.pte    program lowered with WebGPUPartitioner (delegate id "VulkanBackend")
+  <out>/<name>_portable.pte  same exported graph without a delegate, for CPU validation only
+  <out>/recipe.json          upstream id + revision, versions, shapes, dtype, command line
+  <out>/validation.json      eager vs exported-program vs portable .pte (and goldens, if given)
+"""
+
+import hashlib
+import io
+import json
+import platform
+import shlex
+import sys
+import urllib.request
+import zipfile
+from pathlib import Path
+
+import numpy as np
+import torch
+
+PROXY_HINT = (
+    "set https_proxy if the machine needs a proxy for huggingface.co / pypi.org"
+)
+
+
+def versions():
+    import executorch
+    import torchao
+    from executorch.version import git_version
+
+    out = {
+        "python": platform.python_version(),
+        "torch": torch.__version__,
+        "torchao": torchao.__version__,
+        "executorch": getattr(executorch, "__version__", None)
+        or __import__("executorch.version").version.__version__,
+        "executorch_git": git_version,
+    }
+    try:
+        import transformers
+
+        out["transformers"] = transformers.__version__
+    except ImportError:
+        pass
+    try:
+        import huggingface_hub
+
+        out["huggingface_hub"] = huggingface_hub.__version__
+    except ImportError:
+        pass
+    return out
+
+
+def sha256_file(path):
+    h = hashlib.sha256()
+    with open(path, "rb") as fh:
+        for chunk in iter(lambda: fh.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def hf_file(repo_id, filename, revision, expected_sha256=None, cache_dir=None):
+    from huggingface_hub import hf_hub_download
+
+    path = hf_hub_download(
+        repo_id=repo_id, filename=filename, revision=revision, cache_dir=cache_dir
+    )
+    digest = sha256_file(path)
+    if expected_sha256 and digest != expected_sha256:
+        sys.exit(
+            f"{repo_id}@{revision}:{filename} sha256 {digest} != {expected_sha256}"
+        )
+    return Path(path), digest
+
+
+def pypi_wheel_member(url, expected_sha256, member):
+    """Read one file out of a PyPI wheel, verifying the wheel's sha256."""
+    with urllib.request.urlopen(url, timeout=120) as resp:
+        blob = resp.read()
+    digest = hashlib.sha256(blob).hexdigest()
+    if digest != expected_sha256:
+        sys.exit(f"{url}: sha256 {digest} != {expected_sha256} ({PROXY_HINT})")
+    with zipfile.ZipFile(io.BytesIO(blob)) as zf:
+        return zf.read(member).decode()
+
+
+def scrub(text, out_dir):
+    """Replace machine-specific paths so recipe.json can ship with the model."""
+    text = text.replace(str(Path(out_dir).resolve()), "<out>")
+    text = text.replace(str(Path(__file__).resolve().parent), "recipes")
+    return text.replace(str(Path.home()), "~")
+
+
+def command_line(out_dir):
+    argv = [f"recipes/{Path(sys.argv[0]).name}", *sys.argv[1:]]
+    args = []
+    for a in argv:
+        if a.startswith("/") or a.startswith("."):
+            a = scrub(str(Path(a).resolve()), out_dir)
+        args.append(a)
+    return scrub(" ".join(["python", *(shlex.quote(a) for a in args)]), out_dir)
+
+
+def lower_webgpu(ep):
+    from executorch.backends.webgpu.partitioner import WebGPUPartitioner
+    from executorch.exir import to_edge_transform_and_lower
+
+    return to_edge_transform_and_lower(
+        ep, partitioner=[WebGPUPartitioner()]
+    ).to_executorch()
+
+
+def lower_portable(ep):
+    from executorch.exir import to_edge_transform_and_lower
+
+    return to_edge_transform_and_lower(ep).to_executorch()
+
+
+def delegate_summary(pte_bytes):
+    """Non-delegated kernels and per-delegate op histograms of a serialized program."""
+    import collections
+
+    from executorch.backends.vulkan.serialization.vulkan_graph_serialize import (
+        extract_vk_flatbuffer,
+        flatbuffer_to_vk_graph,
+    )
+    from executorch.exir._serialize._program import deserialize_pte_binary
+
+    pte = deserialize_pte_binary(pte_bytes)
+    prog = pte.program if hasattr(pte, "program") else pte
+    seg_base = int.from_bytes(pte_bytes[24:32], "little")
+    plan = prog.execution_plan[0]
+    ops = [f"{o.name}.{o.overload}" for o in plan.operators]
+    kernels = collections.Counter()
+    for chain in plan.chains:
+        for ins in chain.instructions:
+            if type(ins.instr_args).__name__ == "KernelCall":
+                kernels[ops[ins.instr_args.op_index]] += 1
+    delegates = []
+    for d in plan.delegates:
+        proc = d.processed
+        if proc.location.name == "INLINE":
+            blob = prog.backend_delegate_data[proc.index].data
+        else:
+            seg = prog.segments[proc.index]
+            blob = pte_bytes[seg_base + seg.offset : seg_base + seg.offset + seg.size]
+        g = flatbuffer_to_vk_graph(extract_vk_flatbuffer(blob))
+        delegates.append(
+            {
+                "id": d.id,
+                "ops": dict(collections.Counter(c.name for c in g.chain)),
+                "inputs": len(g.input_ids),
+                "outputs": len(g.output_ids),
+            }
+        )
+    return {"non_delegated_kernels": dict(kernels), "delegates": delegates}
+
+
+def compare(a, b):
+    a = np.asarray(a, dtype=np.float64).ravel()
+    b = np.asarray(b, dtype=np.float64).ravel()
+    n = min(a.size, b.size)
+    a, b = a[:n], b[:n]
+    cos = float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b) + 1e-30))
+    return {
+        "max_abs": float(np.max(np.abs(a - b))),
+        "mean_abs": float(np.mean(np.abs(a - b))),
+        "cosine": cos,
+    }
+
+
+def run_pte(pte_bytes, inputs):
+    from executorch.extension.pybindings.portable_lib import (
+        _load_for_executorch_from_buffer,
+    )
+
+    m = _load_for_executorch_from_buffer(pte_bytes)
+    return [
+        o.detach().numpy() if isinstance(o, torch.Tensor) else o
+        for o in m.forward(inputs)
+    ]
+
+
+def try_run_webgpu_pte_on_cpu(pte_bytes, inputs):
+    try:
+        return run_pte(pte_bytes, inputs), None
+    except Exception as e:  # noqa: BLE001 - the error text is the result
+        return None, f"{type(e).__name__}: {str(e).splitlines()[0][:300]}"
+
+
+def export_vision(
+    *,
+    name,
+    model,
+    example_input,
+    out_dir,
+    upstream,
+    extra_recipe,
+    golden=None,
+    output_index=None,
+):
+    """Export, lower, write both .pte files, recipe.json, and validation.json.
+
+    `golden` is an optional (input ndarray, output ndarray) pair; when its input
+    shape matches `example_input`, all paths are also compared against it.
+    `output_index` picks one tensor when the eager model returns a tuple.
+    """
+    out_dir = Path(out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    torch.manual_seed(0)
+    model = model.eval()
+
+    class Wrapped(torch.nn.Module):
+        def __init__(self, m):
+            super().__init__()
+            self.m = m
+
+        def forward(self, x):
+            y = self.m(x)
+            return y[output_index] if output_index is not None else y
+
+    wrapped = Wrapped(model).eval()
+    with torch.no_grad():
+        ep = torch.export.export(wrapped, (example_input,), strict=False)
+        webgpu = lower_webgpu(ep)
+        portable = lower_portable(ep)
+
+    webgpu_path = out_dir / f"{name}_webgpu.pte"
+    portable_path = out_dir / f"{name}_portable.pte"
+    webgpu_path.write_bytes(webgpu.buffer)
+    portable_path.write_bytes(portable.buffer)
+
+    summary = delegate_summary(webgpu.buffer)
+    out_shape = list(ep.module()(example_input).shape)
+    recipe = {
+        "model": name,
+        "upstream": upstream,
+        "versions": versions(),
+        "export": {
+            "api": "torch.export.export(strict=False) -> to_edge_transform_and_lower(partitioner=[WebGPUPartitioner()]) -> to_executorch()",
+            "partitioner": "executorch.backends.webgpu.partitioner.WebGPUPartitioner() (default compile options; wraps VulkanPartitioner)",
+            "inputs": [
+                {
+                    "shape": list(example_input.shape),
+                    "dtype": "float32",
+                    "layout": "NCHW",
+                }
+            ],
+            "outputs": [{"shape": out_shape, "dtype": "float32"}],
+            "dtype": "float32",
+            "quantization": "none (fp32)",
+            **extra_recipe,
+        },
+        "files": {
+            webgpu_path.name: {
+                "bytes": webgpu_path.stat().st_size,
+                "sha256": sha256_file(webgpu_path),
+            },
+            portable_path.name: {
+                "bytes": portable_path.stat().st_size,
+                "sha256": sha256_file(portable_path),
+                "note": "CPU-only validation artifact (portable kernels, no delegate); not for release",
+            },
+        },
+        "graph": summary,
+        "command": command_line(out_dir),
+    }
+    (out_dir / "recipe.json").write_text(json.dumps(recipe, indent=2) + "\n")
+
+    # Validation on CPU.
+    gen = torch.Generator().manual_seed(1234)
+    rand_input = torch.rand(example_input.shape, generator=gen)
+    cases = {"random": rand_input}
+    if golden is not None and tuple(golden[0].shape) == tuple(example_input.shape):
+        cases["golden"] = torch.from_numpy(golden[0].copy())
+    report = {
+        "cpu_runtime": "executorch pybindings (portable_lib) from the pip wheel",
+        "cases": {},
+    }
+    webgpu_cpu_error = None
+    for label, x in cases.items():
+        with torch.no_grad():
+            eager = wrapped(x).numpy()
+            exported = ep.module()(x).numpy()
+        port = run_pte(portable.buffer, (x,))[0]
+        res = {
+            "exported_vs_eager": compare(exported, eager),
+            "portable_pte_vs_eager": compare(port, eager),
+        }
+        wg, err = try_run_webgpu_pte_on_cpu(webgpu.buffer, (x,))
+        if wg is not None:
+            res["webgpu_pte_vs_eager"] = compare(wg[0], eager)
+        else:
+            webgpu_cpu_error = err
+        if label == "golden":
+            ref = golden[1]
+            res["internal_golden_vs_eager"] = compare(eager, ref)
+            res["internal_golden_vs_portable_pte"] = compare(port, ref)
+            res["golden_note"] = (
+                "reference output from the internal goldens.manifest.json"
+            )
+        report["cases"][label] = res
+    report["webgpu_pte_on_cpu"] = (
+        "ran"
+        if webgpu_cpu_error is None
+        else f"not runnable on CPU pybindings: {webgpu_cpu_error}"
+    )
+    (out_dir / "validation.json").write_text(json.dumps(report, indent=2) + "\n")
+    print(
+        json.dumps(
+            {"recipe": recipe["files"], "graph": summary, "validation": report},
+            indent=2,
+        )
+    )
+    return ep, webgpu, portable
+
+
+def load_golden(manifest_path, key):
+    man = json.loads(Path(manifest_path).read_text())[key]["tensors"]
+    base = Path(manifest_path).parent
+
+    def ld(t):
+        return np.fromfile(base / t["file"], dtype=np.float32).reshape(t["shape"])
+
+    inp = man.get("input")
+    return ld(inp), ld(man["output"])

export/_llm.py

@@ -0,0 +1,157 @@
+"""Shared export path for the Qwen LLM recipes: pinned HF snapshot -> ExecuTorch checkpoint
+-> export_llm (Vulkan partitioner, which WebGPUPartitioner wraps unchanged) -> .pte.
+
+Graph contract (matches the internal WebGPU LLM artifacts): forward(tokens[1, <=prefill_max]
+int64, input_pos[1] int64) -> logits[1, vocab] float32 for the last position, KV cache inside the
+program (fp32 buffers, max_context_length positions), 4-bit groupwise weight-only linears
+(et_vk.linear_q4gsw) and 4-bit groupwise embedding (et_vk.embedding_q4gsw).
+"""
+
+import importlib
+import json
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+import _common
+
+SHIMS = Path(__file__).resolve().parent / "_shims"
+
+TOKENIZER_FILES = [
+    "tokenizer.json",
+    "tokenizer_config.json",
+    "vocab.json",
+    "merges.txt",
+    "generation_config.json",
+    "config.json",
+    "LICENSE",
+]
+
+
+def export(
+    *, name, upstream_repo, revision, model_class, params, converter, max_context, args
+):
+    from huggingface_hub import snapshot_download
+
+    out = Path(args.out).resolve()
+    work = out / "_work"
+    work.mkdir(parents=True, exist_ok=True)
+    snap = Path(
+        snapshot_download(
+            upstream_repo,
+            revision=revision,
+            allow_patterns=["*.safetensors", "*.json", "merges.txt", "LICENSE"],
+        )
+    )
+
+    sys.path.insert(0, str(SHIMS))
+    env = {**os.environ, "PYTHONPATH": os.pathsep.join(filter(None, [str(SHIMS), os.environ.get("PYTHONPATH")]))}
+    ckpt = work / "checkpoint.pth"
+    if not ckpt.exists():
+        importlib.import_module(converter).convert_weights(str(snap), str(ckpt))
+    params_path = work / "params.json"
+    params_path.write_text(json.dumps(params, indent=2) + "\n")
+
+    output_name = f"{name}.pte"
+    overrides = [
+        f"base.model_class={model_class}",
+        f"base.checkpoint={ckpt}",
+        f"base.params={params_path}",
+        "model.use_kv_cache=true",
+        "model.use_sdpa_with_kv_cache=true",
+        "model.enable_dynamic_shape=true",
+        "model.dtype_override=fp32",
+        f"export.max_seq_length={args.prefill_max + 1}",
+        f"export.max_context_length={max_context}",
+        f"export.output_dir={out}",
+        f"export.output_name={out / output_name}",
+        "quantization.qmode=4w",
+        f"quantization.group_size={args.group_size}",
+        f"quantization.embedding_quantize='4,{args.group_size}'",
+        "backend.vulkan.enabled=true",
+    ]
+    cmd = [
+        sys.executable,
+        "-m",
+        "executorch.extension.llm.export.export_llm",
+        *overrides,
+    ]
+    print("+", " ".join(shlex.quote(c) for c in cmd), flush=True)
+    subprocess.run(cmd, check=True, env=env)
+
+    pte = out / output_name
+    tok_dir = out / "tokenizer"
+    tok_dir.mkdir(exist_ok=True)
+    tokenizer_files = {}
+    for f in TOKENIZER_FILES:
+        if (snap / f).exists():
+            shutil.copyfile(snap / f, tok_dir / f)
+            tokenizer_files[f] = _common.sha256_file(tok_dir / f)
+    shutil.rmtree(work)
+
+    data = pte.read_bytes()
+    recipe = {
+        "model": name,
+        "upstream": {
+            "model": upstream_repo,
+            "revision": revision,
+            "official": True,
+            "safetensors_sha256": {
+                p.name: _common.sha256_file(p)
+                for p in sorted(snap.glob("*.safetensors"))
+            },
+        },
+        "versions": _common.versions(),
+        "export": {
+            "api": "python -m executorch.extension.llm.export.export_llm (hydra overrides below)",
+            "overrides": [_common.scrub(o, out).replace(str(work.resolve()), "<work>") for o in overrides],
+            "checkpoint_conversion": f"{converter}.convert_weights(<snapshot>, checkpoint.pth)",
+            "import_shim": "recipes/_shims/torchtune (re-exports executorch.examples.models.checkpoint.get_mapped_key; torchtune itself is not installed)",
+            "params": params,
+            "partitioner": "VulkanPartitioner via backend.vulkan.enabled (WebGPUPartitioner is a pass-through wrapper of it; delegate id VulkanBackend)",
+            "inputs": [
+                {
+                    "name": "tokens",
+                    "shape": [1, f"1..{args.prefill_max}"],
+                    "dtype": "int64",
+                },
+                {"name": "input_pos", "shape": [1], "dtype": "int64"},
+            ],
+            "outputs": [
+                {
+                    "name": "logits",
+                    "shape": [1, params["vocab_size"]],
+                    "dtype": "float32",
+                    "note": "last position only",
+                }
+            ],
+            "prefill_max": args.prefill_max,
+            "max_seq_len_metadata": args.prefill_max + 1,
+            "max_context_length": max_context,
+            "dtype": "float32 activations",
+            "quantization": f"4-bit groupwise weight-only linears (torchao IntxWeightOnlyConfig int4, group {args.group_size}) and 4-bit groupwise embedding (group {args.group_size}); output head quantized too",
+            "kv_cache": "fp32 buffers in the program; fp16 KV is a runtime option of the WebGPU runtime (et_set_kv_f16), not an export flag",
+        },
+        "files": {
+            pte.name: {"bytes": len(data), "sha256": _common.sha256_file(pte)},
+            **{f"tokenizer/{k}": {"sha256": v} for k, v in tokenizer_files.items()},
+        },
+        "graph": _common.delegate_summary(data),
+        "command": _common.command_line(out),
+    }
+    (out / "recipe.json").write_text(json.dumps(recipe, indent=2) + "\n")
+    print(json.dumps({"files": recipe["files"], "graph": recipe["graph"]}, indent=2))
+
+
+def add_args(ap, max_context):
+    ap.add_argument("--out", required=True, help="output directory")
+    ap.add_argument(
+        "--prefill-max", type=int, default=512, help="max tokens per forward call"
+    )
+    ap.add_argument(
+        "--max-context", type=int, default=max_context, help="KV cache length"
+    )
+    ap.add_argument("--group-size", type=int, default=64)

export/_shims/torchtune/__init__.py

@@ -0,0 +1,8 @@
+"""Import shim for `torchtune.models.convert_weights.get_mapped_key`.
+
+executorch 1.5.1's examples/models/qwen2_5 imports that single helper from torchtune, which
+ExecuTorch pins to a GitHub commit (unreachable from the export host); the PyPI and nightly
+torchtune builds do not import against torchao 0.18. ExecuTorch ships an equivalent helper in
+executorch.examples.models.checkpoint (already used by its qwen3 converter); this shim
+re-exports it. Only on PYTHONPATH while the Qwen2.5 recipe runs.
+"""

export/_shims/torchtune/models/convert_weights.py

@@ -0,0 +1 @@
+from executorch.examples.models.checkpoint import get_mapped_key  # noqa: F401

export/qwen2.5-0.5b-instruct.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+"""Qwen2.5 0.5B Instruct -> ExecuTorch WebGPU (4-bit groupwise weights, KV cache in program)."""
+
+import argparse
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import _llm  # noqa: E402
+
+# Same values as executorch examples/models/qwen2_5/config/0_5b_config.json.
+PARAMS = {
+    "dim": 896,
+    "ffn_dim_multiplier": 1,
+    "hidden_dim": 4864,
+    "n_heads": 14,
+    "n_kv_heads": 2,
+    "n_layers": 24,
+    "norm_eps": 1e-06,
+    "rope_theta": 1000000.0,
+    "use_scaled_rope": False,
+    "vocab_size": 151936,
+    "use_hf_rope": True,
+    "attention_qkv_bias": True,
+}
+
+SPEC = {
+    "name": "qwen2_5_0_5b_instruct_q4gsw",
+    "upstream_repo": "Qwen/Qwen2.5-0.5B-Instruct",
+    "revision": "7ae557604adf67be50417f59c2c2f167def9a775",
+    "model_class": "qwen2_5_0_5b",
+    "params": PARAMS,
+    "converter": "executorch.examples.models.qwen2_5.convert_weights",
+}
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__)
+    _llm.add_args(ap, max_context=8192)
+    args = ap.parse_args()
+    _llm.export(**SPEC, max_context=args.max_context, args=args)
+
+
+if __name__ == "__main__":
+    main()

export/recipe.json

@@ -0,0 +1,144 @@
+{
+  "model": "qwen2_5_0_5b_instruct_q4gsw",
+  "upstream": {
+    "model": "Qwen/Qwen2.5-0.5B-Instruct",
+    "revision": "7ae557604adf67be50417f59c2c2f167def9a775",
+    "official": true,
+    "safetensors_sha256": {
+      "model.safetensors": "fdf756fa7fcbe7404d5c60e26bff1a0c8b8aa1f72ced49e7dd0210fe288fb7fe"
+    }
+  },
+  "versions": {
+    "python": "3.12.15+meta",
+    "torch": "2.14.1+cpu",
+    "torchao": "0.18.0+cpu",
+    "executorch": "1.5.1+cpu",
+    "executorch_git": "3b60683923245cf472b7323426920e15623ba361",
+    "transformers": "5.19.0",
+    "huggingface_hub": "2.2.0"
+  },
+  "export": {
+    "api": "python -m executorch.extension.llm.export.export_llm (hydra overrides below)",
+    "overrides": [
+      "base.model_class=qwen2_5_0_5b",
+      "base.checkpoint=<out>/_work/checkpoint.pth",
+      "base.params=<out>/_work/params.json",
+      "model.use_kv_cache=true",
+      "model.use_sdpa_with_kv_cache=true",
+      "model.enable_dynamic_shape=true",
+      "model.dtype_override=fp32",
+      "export.max_seq_length=513",
+      "export.max_context_length=8192",
+      "export.output_dir=<out>",
+      "export.output_name=<out>/qwen2_5_0_5b_instruct_q4gsw.pte",
+      "quantization.qmode=4w",
+      "quantization.group_size=64",
+      "quantization.embedding_quantize='4,64'",
+      "backend.vulkan.enabled=true"
+    ],
+    "checkpoint_conversion": "executorch.examples.models.qwen2_5.convert_weights.convert_weights(<snapshot>, checkpoint.pth)",
+    "import_shim": "recipes/_shims/torchtune (re-exports executorch.examples.models.checkpoint.get_mapped_key; torchtune itself is not installed)",
+    "params": {
+      "dim": 896,
+      "ffn_dim_multiplier": 1,
+      "hidden_dim": 4864,
+      "n_heads": 14,
+      "n_kv_heads": 2,
+      "n_layers": 24,
+      "norm_eps": 1e-06,
+      "rope_theta": 1000000.0,
+      "use_scaled_rope": false,
+      "vocab_size": 151936,
+      "use_hf_rope": true,
+      "attention_qkv_bias": true
+    },
+    "partitioner": "VulkanPartitioner via backend.vulkan.enabled (WebGPUPartitioner is a pass-through wrapper of it; delegate id VulkanBackend)",
+    "inputs": [
+      {
+        "name": "tokens",
+        "shape": [
+          1,
+          "1..512"
+        ],
+        "dtype": "int64"
+      },
+      {
+        "name": "input_pos",
+        "shape": [
+          1
+        ],
+        "dtype": "int64"
+      }
+    ],
+    "outputs": [
+      {
+        "name": "logits",
+        "shape": [
+          1,
+          151936
+        ],
+        "dtype": "float32",
+        "note": "last position only"
+      }
+    ],
+    "prefill_max": 512,
+    "max_seq_len_metadata": 513,
+    "max_context_length": 8192,
+    "dtype": "float32 activations",
+    "quantization": "4-bit groupwise weight-only linears (torchao IntxWeightOnlyConfig int4, group 64) and 4-bit groupwise embedding (group 64); output head quantized too",
+    "kv_cache": "fp32 buffers in the program; fp16 KV is a runtime option of the WebGPU runtime (et_set_kv_f16), not an export flag"
+  },
+  "files": {
+    "qwen2_5_0_5b_instruct_q4gsw.pte": {
+      "bytes": 299447808,
+      "sha256": "474274b934978f6c11a842d575a07417d6687eb4d93f6b12d30d144dce729faa"
+    },
+    "tokenizer/tokenizer.json": {
+      "sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539"
+    },
+    "tokenizer/tokenizer_config.json": {
+      "sha256": "5b5d4f65d0acd3b2d56a35b56d374a36cbc1c8fa5cf3b3febbbfabf22f359583"
+    },
+    "tokenizer/vocab.json": {
+      "sha256": "ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910"
+    },
+    "tokenizer/merges.txt": {
+      "sha256": "599bab54075088774b1733fde865d5bd747cbcc7a547c5bc12610e874e26f5e3"
+    },
+    "tokenizer/generation_config.json": {
+      "sha256": "e558847a8b4402616f1273797b015104dc266fe4b520056fca88823ba8f8ebe6"
+    },
+    "tokenizer/config.json": {
+      "sha256": "18e18afcaccafade98daf13a54092927904649e1dd4eba8299ab717d5d94ff45"
+    },
+    "tokenizer/LICENSE": {
+      "sha256": "832dd9e00a68dd83b3c3fb9f5588dad7dcf337a0db50f7d9483f310cd292e92e"
+    }
+  },
+  "graph": {
+    "non_delegated_kernels": {},
+    "delegates": [
+      {
+        "id": "VulkanBackend",
+        "ops": {
+          "et_vk.prepack.default": 2,
+          "sym_size.int": 1,
+          "et_vk.embedding_q4gsw.default": 1,
+          "et_vk.select_as_symint.default": 2,
+          "et_vk.rms_norm.default": 49,
+          "et_vk.linear_q4gsw.default": 169,
+          "aten.view_copy.default": 192,
+          "et_vk.apply_rotary_emb_hf.default": 24,
+          "sdpa_with_kv_cache.default": 24,
+          "aten.add.Tensor": 48,
+          "aten.sigmoid.default": 24,
+          "aten.mul.Tensor": 48,
+          "aten.select_copy.int": 1
+        },
+        "inputs": 2,
+        "outputs": 1
+      }
+    ]
+  },
+  "command": "python recipes/qwen2.5-0.5b-instruct.py --out out/qwen2.5-0.5b-instruct"
+}

export/requirements.txt

@@ -0,0 +1,15 @@
+# Environment the re-exports in REEXPORT.md were produced with (Python 3.12, Linux x86_64, CPU only).
+#   python -m venv venv
+#   venv/bin/pip install -r recipes/requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu
+# executorch 1.5.1 = pytorch/executorch git 3b60683923245cf472b7323426920e15623ba361; its wheel ships
+# backends/webgpu (WebGPUPartitioner) and examples/models/llama (export_llm).
+executorch==1.5.1
+torch==2.14.1+cpu
+torchvision==0.29.1+cpu
+torchao==0.18.0+cpu
+transformers==5.19.0
+huggingface_hub==2.2.0
+safetensors==0.8.0
+tokenizers==0.23.3
+numpy==2.5.4
+pillow==12.3.0

manifest.json

@@ -0,0 +1,41 @@
+{
+  "model": "qwen2.5-0.5b-instruct",
+  "repo": "executorch-community/Qwen2.5-0.5B-Instruct-ExecuTorch-WebGPU",
+  "files": [
+    {
+      "path": "qwen2_5_0_5b_instruct_q4gsw.pte",
+      "bytes": 299447808,
+      "sha256": "474274b934978f6c11a842d575a07417d6687eb4d93f6b12d30d144dce729faa"
+    },
+    {
+      "path": "tokenizer/config.json",
+      "bytes": 659,
+      "sha256": "18e18afcaccafade98daf13a54092927904649e1dd4eba8299ab717d5d94ff45"
+    },
+    {
+      "path": "tokenizer/generation_config.json",
+      "bytes": 242,
+      "sha256": "e558847a8b4402616f1273797b015104dc266fe4b520056fca88823ba8f8ebe6"
+    },
+    {
+      "path": "tokenizer/merges.txt",
+      "bytes": 1671839,
+      "sha256": "599bab54075088774b1733fde865d5bd747cbcc7a547c5bc12610e874e26f5e3"
+    },
+    {
+      "path": "tokenizer/tokenizer.json",
+      "bytes": 7031645,
+      "sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539"
+    },
+    {
+      "path": "tokenizer/tokenizer_config.json",
+      "bytes": 7305,
+      "sha256": "5b5d4f65d0acd3b2d56a35b56d374a36cbc1c8fa5cf3b3febbbfabf22f359583"
+    },
+    {
+      "path": "tokenizer/vocab.json",
+      "bytes": 2776833,
+      "sha256": "ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910"
+    }
+  ]
+}

qwen2_5_0_5b_instruct_q4gsw.pte

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:474274b934978f6c11a842d575a07417d6687eb4d93f6b12d30d144dce729faa
+size 299447808

tokenizer/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

tokenizer/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "pad_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_p": 0.8,
+  "top_k": 20,
+  "transformers_version": "4.37.0"
+}

tokenizer/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}