Text Generation
Transformers
Safetensors
English
wildai
nanochat
scaling-laws
ai-generated-text
pretraining-data
Instructions to use pangram/WildAI-models with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use pangram/WildAI-models with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="pangram/WildAI-models")# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("pangram/WildAI-models", device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use pangram/WildAI-models with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "pangram/WildAI-models" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "pangram/WildAI-models", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker
docker model run hf.co/pangram/WildAI-models
- SGLang
How to use pangram/WildAI-models with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "pangram/WildAI-models" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "pangram/WildAI-models", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "pangram/WildAI-models" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "pangram/WildAI-models", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }' - Docker Model Runner
How to use pangram/WildAI-models with Docker Model Runner:
docker model run hf.co/pangram/WildAI-models
Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- 268m-g072-ai-r0.01/LICENSE +438 -0
- 268m-g072-ai-r0.01/README.md +105 -0
- 268m-g072-ai-r0.01/config.json +46 -0
- 268m-g072-ai-r0.01/configuration_wildai.py +96 -0
- 268m-g072-ai-r0.01/generation_config.json +10 -0
- 268m-g072-ai-r0.01/modeling_wildai.py +340 -0
- 268m-g072-ai-r0.01/tokenizer.json +0 -0
- 268m-g072-ai-r0.01/tokenizer_config.json +11 -0
- 268m-g072-ai-r0.5/LICENSE +438 -0
- 268m-g072-ai-r0.5/README.md +105 -0
- 268m-g072-ai-r0.5/config.json +46 -0
- 268m-g072-ai-r0.5/configuration_wildai.py +96 -0
- 268m-g072-ai-r0.5/generation_config.json +10 -0
- 268m-g072-ai-r0.5/modeling_wildai.py +340 -0
- 268m-g072-ai-r0.5/tokenizer.json +0 -0
- 268m-g072-ai-r0.5/tokenizer_config.json +11 -0
- 268m-g072-human-r0.005/LICENSE +438 -0
- 268m-g072-human-r0.005/README.md +105 -0
- 268m-g072-human-r0.005/config.json +46 -0
- 268m-g072-human-r0.005/configuration_wildai.py +96 -0
- 268m-g072-human-r0.005/generation_config.json +10 -0
- 268m-g072-human-r0.005/modeling_wildai.py +340 -0
- 268m-g072-human-r0.005/tokenizer.json +0 -0
- 268m-g072-human-r0.005/tokenizer_config.json +11 -0
- 268m-g073-ai-r0.05/LICENSE +438 -0
- 268m-g073-ai-r0.05/README.md +105 -0
- 268m-g073-ai-r0.05/configuration_wildai.py +96 -0
- 268m-g073-ai-r0.05/generation_config.json +10 -0
- 268m-g073-ai-r0.05/modeling_wildai.py +340 -0
- 268m-g073-ai-r0.05/tokenizer.json +0 -0
- 268m-g073-human-r0.025/LICENSE +438 -0
- 268m-g073-human-r0.025/README.md +105 -0
- 268m-g073-human-r0.025/config.json +46 -0
- 268m-g073-human-r0.025/configuration_wildai.py +96 -0
- 268m-g073-human-r0.025/generation_config.json +10 -0
- 268m-g073-human-r0.025/modeling_wildai.py +340 -0
- 268m-g073-human-r0.025/tokenizer.json +0 -0
- 268m-g073-human-r0.025/tokenizer_config.json +11 -0
- 268m-g073-human-r0.05/LICENSE +438 -0
- 268m-g073-human-r0.05/README.md +105 -0
- 268m-g073-human-r0.05/config.json +46 -0
- 268m-g073-human-r0.05/configuration_wildai.py +96 -0
- 268m-g073-human-r0.05/generation_config.json +10 -0
- 268m-g073-human-r0.05/modeling_wildai.py +340 -0
- 268m-g073-human-r0.05/tokenizer.json +0 -0
- 268m-g073-human-r0.05/tokenizer_config.json +11 -0
- 268m-g074-human-r0.025/LICENSE +438 -0
- 268m-g074-human-r0.025/README.md +105 -0
- 268m-g074-human-r0.025/config.json +46 -0
- 268m-g074-human-r0.025/configuration_wildai.py +96 -0
268m-g072-ai-r0.01/LICENSE
ADDED
|
@@ -0,0 +1,438 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Attribution-NonCommercial-ShareAlike 4.0 International
|
| 2 |
+
|
| 3 |
+
=======================================================================
|
| 4 |
+
|
| 5 |
+
Creative Commons Corporation ("Creative Commons") is not a law firm and
|
| 6 |
+
does not provide legal services or legal advice. Distribution of
|
| 7 |
+
Creative Commons public licenses does not create a lawyer-client or
|
| 8 |
+
other relationship. Creative Commons makes its licenses and related
|
| 9 |
+
information available on an "as-is" basis. Creative Commons gives no
|
| 10 |
+
warranties regarding its licenses, any material licensed under their
|
| 11 |
+
terms and conditions, or any related information. Creative Commons
|
| 12 |
+
disclaims all liability for damages resulting from their use to the
|
| 13 |
+
fullest extent possible.
|
| 14 |
+
|
| 15 |
+
Using Creative Commons Public Licenses
|
| 16 |
+
|
| 17 |
+
Creative Commons public licenses provide a standard set of terms and
|
| 18 |
+
conditions that creators and other rights holders may use to share
|
| 19 |
+
original works of authorship and other material subject to copyright
|
| 20 |
+
and certain other rights specified in the public license below. The
|
| 21 |
+
following considerations are for informational purposes only, are not
|
| 22 |
+
exhaustive, and do not form part of our licenses.
|
| 23 |
+
|
| 24 |
+
Considerations for licensors: Our public licenses are
|
| 25 |
+
intended for use by those authorized to give the public
|
| 26 |
+
permission to use material in ways otherwise restricted by
|
| 27 |
+
copyright and certain other rights. Our licenses are
|
| 28 |
+
irrevocable. Licensors should read and understand the terms
|
| 29 |
+
and conditions of the license they choose before applying it.
|
| 30 |
+
Licensors should also secure all rights necessary before
|
| 31 |
+
applying our licenses so that the public can reuse the
|
| 32 |
+
material as expected. Licensors should clearly mark any
|
| 33 |
+
material not subject to the license. This includes other CC-
|
| 34 |
+
licensed material, or material used under an exception or
|
| 35 |
+
limitation to copyright. More considerations for licensors:
|
| 36 |
+
wiki.creativecommons.org/Considerations_for_licensors
|
| 37 |
+
|
| 38 |
+
Considerations for the public: By using one of our public
|
| 39 |
+
licenses, a licensor grants the public permission to use the
|
| 40 |
+
licensed material under specified terms and conditions. If
|
| 41 |
+
the licensor's permission is not necessary for any reason--for
|
| 42 |
+
example, because of any applicable exception or limitation to
|
| 43 |
+
copyright--then that use is not regulated by the license. Our
|
| 44 |
+
licenses grant only permissions under copyright and certain
|
| 45 |
+
other rights that a licensor has authority to grant. Use of
|
| 46 |
+
the licensed material may still be restricted for other
|
| 47 |
+
reasons, including because others have copyright or other
|
| 48 |
+
rights in the material. A licensor may make special requests,
|
| 49 |
+
such as asking that all changes be marked or described.
|
| 50 |
+
Although not required by our licenses, you are encouraged to
|
| 51 |
+
respect those requests where reasonable. More considerations
|
| 52 |
+
for the public:
|
| 53 |
+
wiki.creativecommons.org/Considerations_for_licensees
|
| 54 |
+
|
| 55 |
+
=======================================================================
|
| 56 |
+
|
| 57 |
+
Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International
|
| 58 |
+
Public License
|
| 59 |
+
|
| 60 |
+
By exercising the Licensed Rights (defined below), You accept and agree
|
| 61 |
+
to be bound by the terms and conditions of this Creative Commons
|
| 62 |
+
Attribution-NonCommercial-ShareAlike 4.0 International Public License
|
| 63 |
+
("Public License"). To the extent this Public License may be
|
| 64 |
+
interpreted as a contract, You are granted the Licensed Rights in
|
| 65 |
+
consideration of Your acceptance of these terms and conditions, and the
|
| 66 |
+
Licensor grants You such rights in consideration of benefits the
|
| 67 |
+
Licensor receives from making the Licensed Material available under
|
| 68 |
+
these terms and conditions.
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
Section 1 -- Definitions.
|
| 72 |
+
|
| 73 |
+
a. Adapted Material means material subject to Copyright and Similar
|
| 74 |
+
Rights that is derived from or based upon the Licensed Material
|
| 75 |
+
and in which the Licensed Material is translated, altered,
|
| 76 |
+
arranged, transformed, or otherwise modified in a manner requiring
|
| 77 |
+
permission under the Copyright and Similar Rights held by the
|
| 78 |
+
Licensor. For purposes of this Public License, where the Licensed
|
| 79 |
+
Material is a musical work, performance, or sound recording,
|
| 80 |
+
Adapted Material is always produced where the Licensed Material is
|
| 81 |
+
synched in timed relation with a moving image.
|
| 82 |
+
|
| 83 |
+
b. Adapter's License means the license You apply to Your Copyright
|
| 84 |
+
and Similar Rights in Your contributions to Adapted Material in
|
| 85 |
+
accordance with the terms and conditions of this Public License.
|
| 86 |
+
|
| 87 |
+
c. BY-NC-SA Compatible License means a license listed at
|
| 88 |
+
creativecommons.org/compatiblelicenses, approved by Creative
|
| 89 |
+
Commons as essentially the equivalent of this Public License.
|
| 90 |
+
|
| 91 |
+
d. Copyright and Similar Rights means copyright and/or similar rights
|
| 92 |
+
closely related to copyright including, without limitation,
|
| 93 |
+
performance, broadcast, sound recording, and Sui Generis Database
|
| 94 |
+
Rights, without regard to how the rights are labeled or
|
| 95 |
+
categorized. For purposes of this Public License, the rights
|
| 96 |
+
specified in Section 2(b)(1)-(2) are not Copyright and Similar
|
| 97 |
+
Rights.
|
| 98 |
+
|
| 99 |
+
e. Effective Technological Measures means those measures that, in the
|
| 100 |
+
absence of proper authority, may not be circumvented under laws
|
| 101 |
+
fulfilling obligations under Article 11 of the WIPO Copyright
|
| 102 |
+
Treaty adopted on December 20, 1996, and/or similar international
|
| 103 |
+
agreements.
|
| 104 |
+
|
| 105 |
+
f. Exceptions and Limitations means fair use, fair dealing, and/or
|
| 106 |
+
any other exception or limitation to Copyright and Similar Rights
|
| 107 |
+
that applies to Your use of the Licensed Material.
|
| 108 |
+
|
| 109 |
+
g. License Elements means the license attributes listed in the name
|
| 110 |
+
of a Creative Commons Public License. The License Elements of this
|
| 111 |
+
Public License are Attribution, NonCommercial, and ShareAlike.
|
| 112 |
+
|
| 113 |
+
h. Licensed Material means the artistic or literary work, database,
|
| 114 |
+
or other material to which the Licensor applied this Public
|
| 115 |
+
License.
|
| 116 |
+
|
| 117 |
+
i. Licensed Rights means the rights granted to You subject to the
|
| 118 |
+
terms and conditions of this Public License, which are limited to
|
| 119 |
+
all Copyright and Similar Rights that apply to Your use of the
|
| 120 |
+
Licensed Material and that the Licensor has authority to license.
|
| 121 |
+
|
| 122 |
+
j. Licensor means the individual(s) or entity(ies) granting rights
|
| 123 |
+
under this Public License.
|
| 124 |
+
|
| 125 |
+
k. NonCommercial means not primarily intended for or directed towards
|
| 126 |
+
commercial advantage or monetary compensation. For purposes of
|
| 127 |
+
this Public License, the exchange of the Licensed Material for
|
| 128 |
+
other material subject to Copyright and Similar Rights by digital
|
| 129 |
+
file-sharing or similar means is NonCommercial provided there is
|
| 130 |
+
no payment of monetary compensation in connection with the
|
| 131 |
+
exchange.
|
| 132 |
+
|
| 133 |
+
l. Share means to provide material to the public by any means or
|
| 134 |
+
process that requires permission under the Licensed Rights, such
|
| 135 |
+
as reproduction, public display, public performance, distribution,
|
| 136 |
+
dissemination, communication, or importation, and to make material
|
| 137 |
+
available to the public including in ways that members of the
|
| 138 |
+
public may access the material from a place and at a time
|
| 139 |
+
individually chosen by them.
|
| 140 |
+
|
| 141 |
+
m. Sui Generis Database Rights means rights other than copyright
|
| 142 |
+
resulting from Directive 96/9/EC of the European Parliament and of
|
| 143 |
+
the Council of 11 March 1996 on the legal protection of databases,
|
| 144 |
+
as amended and/or succeeded, as well as other essentially
|
| 145 |
+
equivalent rights anywhere in the world.
|
| 146 |
+
|
| 147 |
+
n. You means the individual or entity exercising the Licensed Rights
|
| 148 |
+
under this Public License. Your has a corresponding meaning.
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
Section 2 -- Scope.
|
| 152 |
+
|
| 153 |
+
a. License grant.
|
| 154 |
+
|
| 155 |
+
1. Subject to the terms and conditions of this Public License,
|
| 156 |
+
the Licensor hereby grants You a worldwide, royalty-free,
|
| 157 |
+
non-sublicensable, non-exclusive, irrevocable license to
|
| 158 |
+
exercise the Licensed Rights in the Licensed Material to:
|
| 159 |
+
|
| 160 |
+
a. reproduce and Share the Licensed Material, in whole or
|
| 161 |
+
in part, for NonCommercial purposes only; and
|
| 162 |
+
|
| 163 |
+
b. produce, reproduce, and Share Adapted Material for
|
| 164 |
+
NonCommercial purposes only.
|
| 165 |
+
|
| 166 |
+
2. Exceptions and Limitations. For the avoidance of doubt, where
|
| 167 |
+
Exceptions and Limitations apply to Your use, this Public
|
| 168 |
+
License does not apply, and You do not need to comply with
|
| 169 |
+
its terms and conditions.
|
| 170 |
+
|
| 171 |
+
3. Term. The term of this Public License is specified in Section
|
| 172 |
+
6(a).
|
| 173 |
+
|
| 174 |
+
4. Media and formats; technical modifications allowed. The
|
| 175 |
+
Licensor authorizes You to exercise the Licensed Rights in
|
| 176 |
+
all media and formats whether now known or hereafter created,
|
| 177 |
+
and to make technical modifications necessary to do so. The
|
| 178 |
+
Licensor waives and/or agrees not to assert any right or
|
| 179 |
+
authority to forbid You from making technical modifications
|
| 180 |
+
necessary to exercise the Licensed Rights, including
|
| 181 |
+
technical modifications necessary to circumvent Effective
|
| 182 |
+
Technological Measures. For purposes of this Public License,
|
| 183 |
+
simply making modifications authorized by this Section 2(a)
|
| 184 |
+
(4) never produces Adapted Material.
|
| 185 |
+
|
| 186 |
+
5. Downstream recipients.
|
| 187 |
+
|
| 188 |
+
a. Offer from the Licensor -- Licensed Material. Every
|
| 189 |
+
recipient of the Licensed Material automatically
|
| 190 |
+
receives an offer from the Licensor to exercise the
|
| 191 |
+
Licensed Rights under the terms and conditions of this
|
| 192 |
+
Public License.
|
| 193 |
+
|
| 194 |
+
b. Additional offer from the Licensor -- Adapted Material.
|
| 195 |
+
Every recipient of Adapted Material from You
|
| 196 |
+
automatically receives an offer from the Licensor to
|
| 197 |
+
exercise the Licensed Rights in the Adapted Material
|
| 198 |
+
under the conditions of the Adapter's License You apply.
|
| 199 |
+
|
| 200 |
+
c. No downstream restrictions. You may not offer or impose
|
| 201 |
+
any additional or different terms or conditions on, or
|
| 202 |
+
apply any Effective Technological Measures to, the
|
| 203 |
+
Licensed Material if doing so restricts exercise of the
|
| 204 |
+
Licensed Rights by any recipient of the Licensed
|
| 205 |
+
Material.
|
| 206 |
+
|
| 207 |
+
6. No endorsement. Nothing in this Public License constitutes or
|
| 208 |
+
may be construed as permission to assert or imply that You
|
| 209 |
+
are, or that Your use of the Licensed Material is, connected
|
| 210 |
+
with, or sponsored, endorsed, or granted official status by,
|
| 211 |
+
the Licensor or others designated to receive attribution as
|
| 212 |
+
provided in Section 3(a)(1)(A)(i).
|
| 213 |
+
|
| 214 |
+
b. Other rights.
|
| 215 |
+
|
| 216 |
+
1. Moral rights, such as the right of integrity, are not
|
| 217 |
+
licensed under this Public License, nor are publicity,
|
| 218 |
+
privacy, and/or other similar personality rights; however, to
|
| 219 |
+
the extent possible, the Licensor waives and/or agrees not to
|
| 220 |
+
assert any such rights held by the Licensor to the limited
|
| 221 |
+
extent necessary to allow You to exercise the Licensed
|
| 222 |
+
Rights, but not otherwise.
|
| 223 |
+
|
| 224 |
+
2. Patent and trademark rights are not licensed under this
|
| 225 |
+
Public License.
|
| 226 |
+
|
| 227 |
+
3. To the extent possible, the Licensor waives any right to
|
| 228 |
+
collect royalties from You for the exercise of the Licensed
|
| 229 |
+
Rights, whether directly or through a collecting society
|
| 230 |
+
under any voluntary or waivable statutory or compulsory
|
| 231 |
+
licensing scheme. In all other cases the Licensor expressly
|
| 232 |
+
reserves any right to collect such royalties, including when
|
| 233 |
+
the Licensed Material is used other than for NonCommercial
|
| 234 |
+
purposes.
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
Section 3 -- License Conditions.
|
| 238 |
+
|
| 239 |
+
Your exercise of the Licensed Rights is expressly made subject to the
|
| 240 |
+
following conditions.
|
| 241 |
+
|
| 242 |
+
a. Attribution.
|
| 243 |
+
|
| 244 |
+
1. If You Share the Licensed Material (including in modified
|
| 245 |
+
form), You must:
|
| 246 |
+
|
| 247 |
+
a. retain the following if it is supplied by the Licensor
|
| 248 |
+
with the Licensed Material:
|
| 249 |
+
|
| 250 |
+
i. identification of the creator(s) of the Licensed
|
| 251 |
+
Material and any others designated to receive
|
| 252 |
+
attribution, in any reasonable manner requested by
|
| 253 |
+
the Licensor (including by pseudonym if
|
| 254 |
+
designated);
|
| 255 |
+
|
| 256 |
+
ii. a copyright notice;
|
| 257 |
+
|
| 258 |
+
iii. a notice that refers to this Public License;
|
| 259 |
+
|
| 260 |
+
iv. a notice that refers to the disclaimer of
|
| 261 |
+
warranties;
|
| 262 |
+
|
| 263 |
+
v. a URI or hyperlink to the Licensed Material to the
|
| 264 |
+
extent reasonably practicable;
|
| 265 |
+
|
| 266 |
+
b. indicate if You modified the Licensed Material and
|
| 267 |
+
retain an indication of any previous modifications; and
|
| 268 |
+
|
| 269 |
+
c. indicate the Licensed Material is licensed under this
|
| 270 |
+
Public License, and include the text of, or the URI or
|
| 271 |
+
hyperlink to, this Public License.
|
| 272 |
+
|
| 273 |
+
2. You may satisfy the conditions in Section 3(a)(1) in any
|
| 274 |
+
reasonable manner based on the medium, means, and context in
|
| 275 |
+
which You Share the Licensed Material. For example, it may be
|
| 276 |
+
reasonable to satisfy the conditions by providing a URI or
|
| 277 |
+
hyperlink to a resource that includes the required
|
| 278 |
+
information.
|
| 279 |
+
3. If requested by the Licensor, You must remove any of the
|
| 280 |
+
information required by Section 3(a)(1)(A) to the extent
|
| 281 |
+
reasonably practicable.
|
| 282 |
+
|
| 283 |
+
b. ShareAlike.
|
| 284 |
+
|
| 285 |
+
In addition to the conditions in Section 3(a), if You Share
|
| 286 |
+
Adapted Material You produce, the following conditions also apply.
|
| 287 |
+
|
| 288 |
+
1. The Adapter's License You apply must be a Creative Commons
|
| 289 |
+
license with the same License Elements, this version or
|
| 290 |
+
later, or a BY-NC-SA Compatible License.
|
| 291 |
+
|
| 292 |
+
2. You must include the text of, or the URI or hyperlink to, the
|
| 293 |
+
Adapter's License You apply. You may satisfy this condition
|
| 294 |
+
in any reasonable manner based on the medium, means, and
|
| 295 |
+
context in which You Share Adapted Material.
|
| 296 |
+
|
| 297 |
+
3. You may not offer or impose any additional or different terms
|
| 298 |
+
or conditions on, or apply any Effective Technological
|
| 299 |
+
Measures to, Adapted Material that restrict exercise of the
|
| 300 |
+
rights granted under the Adapter's License You apply.
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
Section 4 -- Sui Generis Database Rights.
|
| 304 |
+
|
| 305 |
+
Where the Licensed Rights include Sui Generis Database Rights that
|
| 306 |
+
apply to Your use of the Licensed Material:
|
| 307 |
+
|
| 308 |
+
a. for the avoidance of doubt, Section 2(a)(1) grants You the right
|
| 309 |
+
to extract, reuse, reproduce, and Share all or a substantial
|
| 310 |
+
portion of the contents of the database for NonCommercial purposes
|
| 311 |
+
only;
|
| 312 |
+
|
| 313 |
+
b. if You include all or a substantial portion of the database
|
| 314 |
+
contents in a database in which You have Sui Generis Database
|
| 315 |
+
Rights, then the database in which You have Sui Generis Database
|
| 316 |
+
Rights (but not its individual contents) is Adapted Material,
|
| 317 |
+
including for purposes of Section 3(b); and
|
| 318 |
+
|
| 319 |
+
c. You must comply with the conditions in Section 3(a) if You Share
|
| 320 |
+
all or a substantial portion of the contents of the database.
|
| 321 |
+
|
| 322 |
+
For the avoidance of doubt, this Section 4 supplements and does not
|
| 323 |
+
replace Your obligations under this Public License where the Licensed
|
| 324 |
+
Rights include other Copyright and Similar Rights.
|
| 325 |
+
|
| 326 |
+
|
| 327 |
+
Section 5 -- Disclaimer of Warranties and Limitation of Liability.
|
| 328 |
+
|
| 329 |
+
a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
|
| 330 |
+
EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
|
| 331 |
+
AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
|
| 332 |
+
ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
|
| 333 |
+
IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
|
| 334 |
+
WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
|
| 335 |
+
PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
|
| 336 |
+
ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
|
| 337 |
+
KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
|
| 338 |
+
ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
|
| 339 |
+
|
| 340 |
+
b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
|
| 341 |
+
TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
|
| 342 |
+
NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
|
| 343 |
+
INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
|
| 344 |
+
COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
|
| 345 |
+
USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
|
| 346 |
+
ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
|
| 347 |
+
DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
|
| 348 |
+
IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
|
| 349 |
+
|
| 350 |
+
c. The disclaimer of warranties and limitation of liability provided
|
| 351 |
+
above shall be interpreted in a manner that, to the extent
|
| 352 |
+
possible, most closely approximates an absolute disclaimer and
|
| 353 |
+
waiver of all liability.
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
Section 6 -- Term and Termination.
|
| 357 |
+
|
| 358 |
+
a. This Public License applies for the term of the Copyright and
|
| 359 |
+
Similar Rights licensed here. However, if You fail to comply with
|
| 360 |
+
this Public License, then Your rights under this Public License
|
| 361 |
+
terminate automatically.
|
| 362 |
+
|
| 363 |
+
b. Where Your right to use the Licensed Material has terminated under
|
| 364 |
+
Section 6(a), it reinstates:
|
| 365 |
+
|
| 366 |
+
1. automatically as of the date the violation is cured, provided
|
| 367 |
+
it is cured within 30 days of Your discovery of the
|
| 368 |
+
violation; or
|
| 369 |
+
|
| 370 |
+
2. upon express reinstatement by the Licensor.
|
| 371 |
+
|
| 372 |
+
For the avoidance of doubt, this Section 6(b) does not affect any
|
| 373 |
+
right the Licensor may have to seek remedies for Your violations
|
| 374 |
+
of this Public License.
|
| 375 |
+
|
| 376 |
+
c. For the avoidance of doubt, the Licensor may also offer the
|
| 377 |
+
Licensed Material under separate terms or conditions or stop
|
| 378 |
+
distributing the Licensed Material at any time; however, doing so
|
| 379 |
+
will not terminate this Public License.
|
| 380 |
+
|
| 381 |
+
d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
|
| 382 |
+
License.
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
Section 7 -- Other Terms and Conditions.
|
| 386 |
+
|
| 387 |
+
a. The Licensor shall not be bound by any additional or different
|
| 388 |
+
terms or conditions communicated by You unless expressly agreed.
|
| 389 |
+
|
| 390 |
+
b. Any arrangements, understandings, or agreements regarding the
|
| 391 |
+
Licensed Material not stated herein are separate from and
|
| 392 |
+
independent of the terms and conditions of this Public License.
|
| 393 |
+
|
| 394 |
+
|
| 395 |
+
Section 8 -- Interpretation.
|
| 396 |
+
|
| 397 |
+
a. For the avoidance of doubt, this Public License does not, and
|
| 398 |
+
shall not be interpreted to, reduce, limit, restrict, or impose
|
| 399 |
+
conditions on any use of the Licensed Material that could lawfully
|
| 400 |
+
be made without permission under this Public License.
|
| 401 |
+
|
| 402 |
+
b. To the extent possible, if any provision of this Public License is
|
| 403 |
+
deemed unenforceable, it shall be automatically reformed to the
|
| 404 |
+
minimum extent necessary to make it enforceable. If the provision
|
| 405 |
+
cannot be reformed, it shall be severed from this Public License
|
| 406 |
+
without affecting the enforceability of the remaining terms and
|
| 407 |
+
conditions.
|
| 408 |
+
|
| 409 |
+
c. No term or condition of this Public License will be waived and no
|
| 410 |
+
failure to comply consented to unless expressly agreed to by the
|
| 411 |
+
Licensor.
|
| 412 |
+
|
| 413 |
+
d. Nothing in this Public License constitutes or may be interpreted
|
| 414 |
+
as a limitation upon, or waiver of, any privileges and immunities
|
| 415 |
+
that apply to the Licensor or You, including from the legal
|
| 416 |
+
processes of any jurisdiction or authority.
|
| 417 |
+
|
| 418 |
+
=======================================================================
|
| 419 |
+
|
| 420 |
+
Creative Commons is not a party to its public
|
| 421 |
+
licenses. Notwithstanding, Creative Commons may elect to apply one of
|
| 422 |
+
its public licenses to material it publishes and in those instances
|
| 423 |
+
will be considered the “Licensor.” The text of the Creative Commons
|
| 424 |
+
public licenses is dedicated to the public domain under the CC0 Public
|
| 425 |
+
Domain Dedication. Except for the limited purpose of indicating that
|
| 426 |
+
material is shared under a Creative Commons public license or as
|
| 427 |
+
otherwise permitted by the Creative Commons policies published at
|
| 428 |
+
creativecommons.org/policies, Creative Commons does not authorize the
|
| 429 |
+
use of the trademark "Creative Commons" or any other trademark or logo
|
| 430 |
+
of Creative Commons without its prior written consent including,
|
| 431 |
+
without limitation, in connection with any unauthorized modifications
|
| 432 |
+
to any of its public licenses or any other arrangements,
|
| 433 |
+
understandings, or agreements concerning use of licensed material. For
|
| 434 |
+
the avoidance of doubt, this paragraph does not form part of the
|
| 435 |
+
public licenses.
|
| 436 |
+
|
| 437 |
+
Creative Commons may be contacted at creativecommons.org.
|
| 438 |
+
|
268m-g072-ai-r0.01/README.md
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: cc-by-nc-sa-4.0
|
| 3 |
+
library_name: transformers
|
| 4 |
+
pipeline_tag: text-generation
|
| 5 |
+
language:
|
| 6 |
+
- en
|
| 7 |
+
datasets:
|
| 8 |
+
- pangram/WildAI
|
| 9 |
+
tags:
|
| 10 |
+
- wildai
|
| 11 |
+
- nanochat
|
| 12 |
+
- scaling-laws
|
| 13 |
+
- ai-generated-text
|
| 14 |
+
- pretraining-data
|
| 15 |
+
---
|
| 16 |
+
|
| 17 |
+
# 268m-g072-ai-r0.01
|
| 18 |
+
|
| 19 |
+
> **License: [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/).** The WildAI code, models and dataset are released under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license: you may use and adapt them for non-commercial purposes, with attribution, and must share adaptations under the same license.
|
| 20 |
+
|
| 21 |
+
A 268M decoder-only language model trained from scratch for [*How Much Is an AI Token Worth? Scaling Laws for Wild AI-Generated Web Text*](https://arxiv.org/abs/2609.40295), one of the paper's models measuring how wild AI-generated web text in pretraining data changes a model's loss. It is a research artifact for studying scaling laws, not an assistant: it has had no instruction tuning or safety training.
|
| 22 |
+
|
| 23 |
+
## Training data
|
| 24 |
+
|
| 25 |
+
The human documents of `268m-g072-control` plus wild AI-generated web documents: 1.17B human and 11.9M AI tokens trained on, an AI-to-human token ratio r = 0.0101 (design 0.01), so 1.0% of the training tokens are AI-generated.
|
| 26 |
+
|
| 27 |
+
| | |
|
| 28 |
+
|---|---|
|
| 29 |
+
| Parameters (N, input embedding included, value embeddings excluded) | 268.4M |
|
| 30 |
+
| Layers / width | 16 / 1024 |
|
| 31 |
+
| Training tokens | 1.19B (2,262 optimizer steps) |
|
| 32 |
+
| Human / AI tokens | 1.17B / 11.9M |
|
| 33 |
+
| Group / arm / seed | g072 / ai / 1337 |
|
| 34 |
+
| Used in the paper for | fitting the scaling laws (726 models, 19.9M to 268M) |
|
| 35 |
+
|
| 36 |
+
## Held-out loss
|
| 37 |
+
|
| 38 |
+
| Set | Bits per byte | Text |
|
| 39 |
+
|---|---|---|
|
| 40 |
+
| C4 | 0.9963 | C4 validation text (human-written, pre-2022) |
|
| 41 |
+
| FW22 | 0.9808 | held-out FineWeb documents crawled before 2022 |
|
| 42 |
+
| Paloma | 1.3601 | macro average over 16 Paloma sources |
|
| 43 |
+
| FW26 | 0.8662 | held-out 2026 FineWeb-style crawl (22.3% AI tokens) |
|
| 44 |
+
| FW26-H | 0.9136 | the human-labeled documents of FW26 |
|
| 45 |
+
| FW26-AI | 0.6916 | the AI-labeled documents of FW26 |
|
| 46 |
+
| Cosmo | 0.6895 | Cosmopedia (AI-generated textbooks and stories) |
|
| 47 |
+
|
| 48 |
+
<details><summary>Paloma sources</summary>
|
| 49 |
+
|
| 50 |
+
| Source | Bits per byte |
|
| 51 |
+
|---|---|
|
| 52 |
+
| 4chan | 1.3672 |
|
| 53 |
+
| c4 | 1.0145 |
|
| 54 |
+
| c4_100_domains | 0.9973 |
|
| 55 |
+
| dolma | 1.2541 |
|
| 56 |
+
| falcon_refinedweb | 1.1085 |
|
| 57 |
+
| gab | 1.8961 |
|
| 58 |
+
| m2d2_s2orc | 1.1691 |
|
| 59 |
+
| m2d2_wikipedia | 1.0735 |
|
| 60 |
+
| manosphere | 1.3775 |
|
| 61 |
+
| mc4_en | 1.1013 |
|
| 62 |
+
| programming | 1.9370 |
|
| 63 |
+
| ptb | 1.2300 |
|
| 64 |
+
| reddit_100 | 1.2488 |
|
| 65 |
+
| redpajama | 1.4182 |
|
| 66 |
+
| twitteraae | 2.4931 |
|
| 67 |
+
| wikitext_103 | 1.0749 |
|
| 68 |
+
|
| 69 |
+
</details>
|
| 70 |
+
|
| 71 |
+
## Usage
|
| 72 |
+
|
| 73 |
+
```python
|
| 74 |
+
import torch
|
| 75 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 76 |
+
|
| 77 |
+
tokenizer = AutoTokenizer.from_pretrained("pangram/WildAI-models", subfolder="268m-g072-ai-r0.01", trust_remote_code=True)
|
| 78 |
+
model = AutoModelForCausalLM.from_pretrained("pangram/WildAI-models", subfolder="268m-g072-ai-r0.01", trust_remote_code=True, dtype=torch.bfloat16)
|
| 79 |
+
|
| 80 |
+
inputs = tokenizer("The history of the printing press", return_tensors="pt")
|
| 81 |
+
output = model.generate(**inputs, max_new_tokens=40, do_sample=False)
|
| 82 |
+
print(tokenizer.decode(output[0], skip_special_tokens=True))
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
The tokenizer prepends `<|bos|>` to every text, as in training; pass `add_special_tokens=False` to skip it. The model predicts `<|bos|>` where a document ends, and generation stops there. Weights are stored in bfloat16, the precision the paper's losses were computed in; `dtype=torch.float32` computes in float32 instead.
|
| 86 |
+
|
| 87 |
+
## Architecture
|
| 88 |
+
|
| 89 |
+
The nanochat GPT recipe: rotary embeddings, RMSNorm without weights (also on queries and keys), squared-ReLU MLP, attention windows in a repeating short-short-short-long pattern (512 tokens, then the full 2,048-token context; the last layer always full), gated value embeddings on alternating layers, a previous-token smear, per-layer residual scalars, a mid-network backout, untied input and output embeddings and logits soft-capped at 15. Vocabulary 32,768 tokens, context 2,048. The code ships with the weights (`trust_remote_code=True`) and matches the training code's forward pass.
|
| 90 |
+
|
| 91 |
+
## License
|
| 92 |
+
|
| 93 |
+
**License: [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/).** The WildAI code, models and dataset are released under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license: you may use and adapt them for non-commercial purposes, with attribution, and must share adaptations under the same license. The full text is in `LICENSE`.
|
| 94 |
+
|
| 95 |
+
## Citation
|
| 96 |
+
|
| 97 |
+
```bibtex
|
| 98 |
+
@article{russell2026wildai,
|
| 99 |
+
title = {How Much Is an AI Token Worth? Scaling Laws for Wild AI-Generated Web Text},
|
| 100 |
+
author = {Russell, Jenna and Glickenhaus, Ben and Thai, Katherine and Wieting, John and Iyyer, Mohit and Spero, Max and Emi, Bradley},
|
| 101 |
+
journal = {arXiv preprint arXiv:2609.40295},
|
| 102 |
+
year = {2026},
|
| 103 |
+
url = {https://arxiv.org/abs/2609.40295}
|
| 104 |
+
}
|
| 105 |
+
```
|
268m-g072-ai-r0.01/config.json
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"WildAIForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"auto_map": {
|
| 6 |
+
"AutoConfig": "configuration_wildai.WildAIConfig",
|
| 7 |
+
"AutoModelForCausalLM": "modeling_wildai.WildAIForCausalLM"
|
| 8 |
+
},
|
| 9 |
+
"bos_token_id": 32759,
|
| 10 |
+
"dtype": "bfloat16",
|
| 11 |
+
"eos_token_id": 32759,
|
| 12 |
+
"hidden_size": 1024,
|
| 13 |
+
"logit_softcap": 15.0,
|
| 14 |
+
"max_position_embeddings": 2048,
|
| 15 |
+
"model_type": "wildai",
|
| 16 |
+
"num_attention_heads": 8,
|
| 17 |
+
"num_hidden_layers": 16,
|
| 18 |
+
"num_key_value_heads": 8,
|
| 19 |
+
"pad_token_id": 32759,
|
| 20 |
+
"qk_scale": 1.2,
|
| 21 |
+
"release": {
|
| 22 |
+
"added_ratio": 0.01,
|
| 23 |
+
"ai_ratio": 0.01009470427061251,
|
| 24 |
+
"ai_tokens": 11852065,
|
| 25 |
+
"arm": "ai",
|
| 26 |
+
"depth": 16,
|
| 27 |
+
"group": "g072",
|
| 28 |
+
"human_tokens": 1174087391,
|
| 29 |
+
"n_params": 268436282,
|
| 30 |
+
"name": "268m-g072-ai-r0.01",
|
| 31 |
+
"seed": 1337,
|
| 32 |
+
"size": "268m",
|
| 33 |
+
"split": "fit",
|
| 34 |
+
"steps": 2262,
|
| 35 |
+
"total_tokens": 1185939456
|
| 36 |
+
},
|
| 37 |
+
"rotary_base": 100000.0,
|
| 38 |
+
"smear_channels": 24,
|
| 39 |
+
"tie_word_embeddings": false,
|
| 40 |
+
"transformers_version": "5.17.0",
|
| 41 |
+
"use_cache": true,
|
| 42 |
+
"value_embed_gate_channels": 12,
|
| 43 |
+
"value_embed_gate_range": 3.0,
|
| 44 |
+
"vocab_size": 32768,
|
| 45 |
+
"window_pattern": "SSSL"
|
| 46 |
+
}
|
268m-g072-ai-r0.01/configuration_wildai.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Configuration of the WildAI language models (a nanochat GPT variant) for Hugging Face ``transformers``.
|
| 2 |
+
|
| 3 |
+
This file is copied verbatim into every released model repository and loaded with ``trust_remote_code=True``.
|
| 4 |
+
It depends only on ``transformers``.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from typing import ClassVar
|
| 10 |
+
|
| 11 |
+
from transformers import PretrainedConfig
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class WildAIConfig(PretrainedConfig):
|
| 15 |
+
"""Architecture of a WildAI model. Only width and depth vary across the released models.
|
| 16 |
+
|
| 17 |
+
Every layer uses causal attention limited to a left window: a query attends to itself and the ``window`` tokens before
|
| 18 |
+
it. ``window_pattern`` is tiled over the layers ("S" = a quarter of the context rounded up to a multiple of 128 tokens,
|
| 19 |
+
"L" = the full context) and the last layer always uses "L".
|
| 20 |
+
|
| 21 |
+
``release`` holds the model's entry in the release catalog (name, token counts, AI ratio, ...); it does not affect
|
| 22 |
+
the forward pass.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
model_type = "wildai"
|
| 26 |
+
keys_to_ignore_at_inference: ClassVar[list[str]] = ["past_key_values"]
|
| 27 |
+
|
| 28 |
+
def __init__(
|
| 29 |
+
self,
|
| 30 |
+
vocab_size: int = 32768,
|
| 31 |
+
hidden_size: int = 768,
|
| 32 |
+
num_hidden_layers: int = 12,
|
| 33 |
+
num_attention_heads: int = 6,
|
| 34 |
+
num_key_value_heads: int = 6,
|
| 35 |
+
max_position_embeddings: int = 2048,
|
| 36 |
+
window_pattern: str = "SSSL",
|
| 37 |
+
rotary_base: float = 100000.0,
|
| 38 |
+
qk_scale: float = 1.2,
|
| 39 |
+
logit_softcap: float = 15.0,
|
| 40 |
+
value_embed_gate_channels: int = 12,
|
| 41 |
+
value_embed_gate_range: float = 3.0,
|
| 42 |
+
smear_channels: int = 24,
|
| 43 |
+
bos_token_id: int = 32759,
|
| 44 |
+
eos_token_id: int = 32759,
|
| 45 |
+
use_cache: bool = True,
|
| 46 |
+
release: dict[str, str | int | float | None] | None = None,
|
| 47 |
+
**kwargs: object,
|
| 48 |
+
) -> None:
|
| 49 |
+
if hidden_size % num_attention_heads != 0 or num_attention_heads % num_key_value_heads != 0:
|
| 50 |
+
raise ValueError("hidden_size must divide into num_attention_heads, and those into num_key_value_heads")
|
| 51 |
+
if not window_pattern or set(window_pattern.upper()) - {"S", "L"}:
|
| 52 |
+
raise ValueError(f"window_pattern must be a non-empty string of S and L, got {window_pattern!r}")
|
| 53 |
+
self.vocab_size = vocab_size
|
| 54 |
+
self.hidden_size = hidden_size
|
| 55 |
+
self.num_hidden_layers = num_hidden_layers
|
| 56 |
+
self.num_attention_heads = num_attention_heads
|
| 57 |
+
self.num_key_value_heads = num_key_value_heads
|
| 58 |
+
self.max_position_embeddings = max_position_embeddings
|
| 59 |
+
self.window_pattern = window_pattern.upper()
|
| 60 |
+
self.rotary_base = rotary_base
|
| 61 |
+
self.qk_scale = qk_scale
|
| 62 |
+
self.logit_softcap = logit_softcap
|
| 63 |
+
self.value_embed_gate_channels = value_embed_gate_channels
|
| 64 |
+
self.value_embed_gate_range = value_embed_gate_range
|
| 65 |
+
self.smear_channels = smear_channels
|
| 66 |
+
self.use_cache = use_cache
|
| 67 |
+
self.release = release
|
| 68 |
+
kwargs.setdefault("tie_word_embeddings", False)
|
| 69 |
+
super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
|
| 70 |
+
|
| 71 |
+
@property
|
| 72 |
+
def head_dim(self) -> int:
|
| 73 |
+
return self.hidden_size // self.num_attention_heads
|
| 74 |
+
|
| 75 |
+
@property
|
| 76 |
+
def intermediate_size(self) -> int:
|
| 77 |
+
return 4 * self.hidden_size
|
| 78 |
+
|
| 79 |
+
@property
|
| 80 |
+
def window_sizes(self) -> list[int]:
|
| 81 |
+
"""Left attention window of each layer, in tokens."""
|
| 82 |
+
long_window = self.max_position_embeddings
|
| 83 |
+
short_window = -(-long_window // 4 // 128) * 128 # a quarter of the context, rounded up to a multiple of 128 (2048 -> 512)
|
| 84 |
+
pattern = self.window_pattern
|
| 85 |
+
sizes = [short_window if pattern[i % len(pattern)] == "S" else long_window for i in range(self.num_hidden_layers)]
|
| 86 |
+
sizes[-1] = long_window
|
| 87 |
+
return sizes
|
| 88 |
+
|
| 89 |
+
def has_value_embedding(self, layer_idx: int) -> bool:
|
| 90 |
+
"""Value embeddings sit on alternating layers, always including the last one."""
|
| 91 |
+
return layer_idx % 2 == (self.num_hidden_layers - 1) % 2
|
| 92 |
+
|
| 93 |
+
@property
|
| 94 |
+
def backout_layer(self) -> int:
|
| 95 |
+
"""The residual stream after this layer is partly subtracted before the final norm."""
|
| 96 |
+
return self.num_hidden_layers // 2
|
268m-g072-ai-r0.01/generation_config.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 32759,
|
| 4 |
+
"eos_token_id": 32759,
|
| 5 |
+
"output_attentions": false,
|
| 6 |
+
"output_hidden_states": false,
|
| 7 |
+
"pad_token_id": 32759,
|
| 8 |
+
"transformers_version": "5.17.0",
|
| 9 |
+
"use_cache": true
|
| 10 |
+
}
|
268m-g072-ai-r0.01/modeling_wildai.py
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The WildAI language models (a nanochat GPT variant) for Hugging Face ``transformers``.
|
| 2 |
+
|
| 3 |
+
This file is copied verbatim into every released model repository and loaded with ``trust_remote_code=True``.
|
| 4 |
+
It depends only on ``torch`` and ``transformers`` and reproduces the training code's forward pass operation by
|
| 5 |
+
operation, so a model loaded in bfloat16 computes the same function the paper evaluated:
|
| 6 |
+
|
| 7 |
+
- token embedding followed by a parameter-free RMSNorm, then a learned "smear" that mixes each position's embedding
|
| 8 |
+
with the previous token's;
|
| 9 |
+
- per-layer scalars that rescale the residual stream and blend the initial embedding back in before every block;
|
| 10 |
+
- attention with rotary embeddings applied before a per-head RMSNorm of queries and keys (both scaled by 1.2),
|
| 11 |
+
a left sliding window per layer, and gated value embeddings added to the values on alternating layers;
|
| 12 |
+
- a squared-ReLU MLP, parameter-free pre-norms, no biases;
|
| 13 |
+
- "backout": part of the residual stream after the middle layer is subtracted before the final norm;
|
| 14 |
+
- untied output head with logits soft-capped by ``15 * tanh(logits / 15)``.
|
| 15 |
+
|
| 16 |
+
Attention uses PyTorch's scaled_dot_product_attention with explicit window masks, so no FlashAttention install is
|
| 17 |
+
needed. Generation uses the standard ``DynamicCache``.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
from dataclasses import dataclass
|
| 23 |
+
from typing import ClassVar
|
| 24 |
+
|
| 25 |
+
import torch
|
| 26 |
+
import torch.nn.functional as F
|
| 27 |
+
from torch import nn
|
| 28 |
+
from transformers import GenerationMixin, PreTrainedModel
|
| 29 |
+
from transformers.cache_utils import Cache, DynamicCache
|
| 30 |
+
from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
|
| 31 |
+
|
| 32 |
+
from .configuration_wildai import WildAIConfig
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def rms_norm(x: torch.Tensor) -> torch.Tensor:
|
| 36 |
+
"""Parameter-free RMSNorm over the last dimension with PyTorch's default epsilon, as in training."""
|
| 37 |
+
return F.rms_norm(x, (x.size(-1),))
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def apply_rotary(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
|
| 41 |
+
"""Rotate channel pairs (i, i + d/2) of ``x`` (batch, time, heads, head_dim), with the training code's sign convention."""
|
| 42 |
+
d = x.shape[-1] // 2
|
| 43 |
+
x1, x2 = x[..., :d], x[..., d:]
|
| 44 |
+
return torch.cat([x1 * cos + x2 * sin, x1 * (-sin) + x2 * cos], dim=-1)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
@dataclass(frozen=True)
|
| 48 |
+
class WindowMask:
|
| 49 |
+
"""Arguments for scaled_dot_product_attention that implement one attention window."""
|
| 50 |
+
|
| 51 |
+
mask: torch.Tensor | None
|
| 52 |
+
is_causal: bool
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def window_masks(
|
| 56 |
+
window_sizes: list[int], query_len: int, past_len: int, padding_mask: torch.Tensor | None, device: torch.device
|
| 57 |
+
) -> dict[int, WindowMask]:
|
| 58 |
+
"""Masks for each distinct left window. Query i (absolute index past_len + i) sees keys j with i - window <= j <= i.
|
| 59 |
+
|
| 60 |
+
``padding_mask`` is the (batch, past_len + query_len) attention mask when it contains padding, else None. A padding
|
| 61 |
+
query is always allowed to see itself so that its (unused) output stays finite.
|
| 62 |
+
"""
|
| 63 |
+
total = past_len + query_len
|
| 64 |
+
q_idx = torch.arange(past_len, total, device=device)[:, None]
|
| 65 |
+
k_idx = torch.arange(total, device=device)[None, :]
|
| 66 |
+
masks: dict[int, WindowMask] = {}
|
| 67 |
+
for window in sorted(set(window_sizes)):
|
| 68 |
+
if padding_mask is None and window >= total - 1:
|
| 69 |
+
if query_len == 1:
|
| 70 |
+
masks[window] = WindowMask(mask=None, is_causal=False)
|
| 71 |
+
continue
|
| 72 |
+
if past_len == 0:
|
| 73 |
+
masks[window] = WindowMask(mask=None, is_causal=True)
|
| 74 |
+
continue
|
| 75 |
+
allowed = (k_idx <= q_idx) & (q_idx - k_idx <= window)
|
| 76 |
+
if padding_mask is not None:
|
| 77 |
+
allowed = (allowed & padding_mask[:, None, None, :].bool()) | (k_idx == q_idx)
|
| 78 |
+
masks[window] = WindowMask(mask=allowed, is_causal=False)
|
| 79 |
+
return masks
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class WildAIAttention(nn.Module):
|
| 83 |
+
def __init__(self, config: WildAIConfig, layer_idx: int) -> None:
|
| 84 |
+
super().__init__()
|
| 85 |
+
self.layer_idx = layer_idx
|
| 86 |
+
self.num_heads = config.num_attention_heads
|
| 87 |
+
self.num_kv_heads = config.num_key_value_heads
|
| 88 |
+
self.head_dim = config.head_dim
|
| 89 |
+
self.qk_scale = config.qk_scale
|
| 90 |
+
self.window = config.window_sizes[layer_idx]
|
| 91 |
+
hidden, kv_dim = config.hidden_size, config.num_key_value_heads * config.head_dim
|
| 92 |
+
self.q_proj = nn.Linear(hidden, config.num_attention_heads * config.head_dim, bias=False)
|
| 93 |
+
self.k_proj = nn.Linear(hidden, kv_dim, bias=False)
|
| 94 |
+
self.v_proj = nn.Linear(hidden, kv_dim, bias=False)
|
| 95 |
+
self.o_proj = nn.Linear(hidden, hidden, bias=False)
|
| 96 |
+
self.value_embed: nn.Embedding | None = None
|
| 97 |
+
self.value_embed_gate: nn.Linear | None = None
|
| 98 |
+
if config.has_value_embedding(layer_idx):
|
| 99 |
+
self.value_embed = nn.Embedding(config.vocab_size, kv_dim)
|
| 100 |
+
self.value_embed_gate = nn.Linear(config.value_embed_gate_channels, config.num_key_value_heads, bias=False)
|
| 101 |
+
self.gate_channels = config.value_embed_gate_channels
|
| 102 |
+
self.gate_range = config.value_embed_gate_range
|
| 103 |
+
|
| 104 |
+
def forward(
|
| 105 |
+
self,
|
| 106 |
+
x: torch.Tensor,
|
| 107 |
+
input_ids: torch.LongTensor,
|
| 108 |
+
rotary: tuple[torch.Tensor, torch.Tensor],
|
| 109 |
+
mask: WindowMask,
|
| 110 |
+
past_key_values: Cache | None,
|
| 111 |
+
) -> torch.Tensor:
|
| 112 |
+
batch, time, _ = x.shape
|
| 113 |
+
q = self.q_proj(x).view(batch, time, self.num_heads, self.head_dim)
|
| 114 |
+
k = self.k_proj(x).view(batch, time, self.num_kv_heads, self.head_dim)
|
| 115 |
+
v = self.v_proj(x).view(batch, time, self.num_kv_heads, self.head_dim)
|
| 116 |
+
if self.value_embed is not None:
|
| 117 |
+
ve = self.value_embed(input_ids).to(x.dtype).view(batch, time, self.num_kv_heads, self.head_dim)
|
| 118 |
+
gate = self.gate_range * torch.sigmoid(self.value_embed_gate(x[..., : self.gate_channels]))
|
| 119 |
+
v = v + gate.unsqueeze(-1) * ve
|
| 120 |
+
cos, sin = rotary
|
| 121 |
+
q, k = apply_rotary(q, cos, sin), apply_rotary(k, cos, sin)
|
| 122 |
+
q, k = rms_norm(q) * self.qk_scale, rms_norm(k) * self.qk_scale
|
| 123 |
+
q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
|
| 124 |
+
if past_key_values is not None:
|
| 125 |
+
k, v = past_key_values.update(k, v, self.layer_idx)
|
| 126 |
+
y = F.scaled_dot_product_attention(
|
| 127 |
+
q, k, v, attn_mask=mask.mask, is_causal=mask.is_causal, enable_gqa=self.num_kv_heads != self.num_heads
|
| 128 |
+
)
|
| 129 |
+
return self.o_proj(y.transpose(1, 2).reshape(batch, time, -1))
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
class WildAIMLP(nn.Module):
|
| 133 |
+
def __init__(self, config: WildAIConfig) -> None:
|
| 134 |
+
super().__init__()
|
| 135 |
+
self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
|
| 136 |
+
self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
|
| 137 |
+
|
| 138 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 139 |
+
return self.down_proj(F.relu(self.up_proj(x)).square())
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
class WildAIDecoderLayer(nn.Module):
|
| 143 |
+
def __init__(self, config: WildAIConfig, layer_idx: int) -> None:
|
| 144 |
+
super().__init__()
|
| 145 |
+
self.self_attn = WildAIAttention(config, layer_idx)
|
| 146 |
+
self.mlp = WildAIMLP(config)
|
| 147 |
+
|
| 148 |
+
def forward(
|
| 149 |
+
self,
|
| 150 |
+
x: torch.Tensor,
|
| 151 |
+
input_ids: torch.LongTensor,
|
| 152 |
+
rotary: tuple[torch.Tensor, torch.Tensor],
|
| 153 |
+
mask: WindowMask,
|
| 154 |
+
past_key_values: Cache | None,
|
| 155 |
+
) -> torch.Tensor:
|
| 156 |
+
x = x + self.self_attn(rms_norm(x), input_ids, rotary, mask, past_key_values)
|
| 157 |
+
return x + self.mlp(rms_norm(x))
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
class WildAIPreTrainedModel(PreTrainedModel):
|
| 161 |
+
config_class = WildAIConfig
|
| 162 |
+
base_model_prefix = "model"
|
| 163 |
+
main_input_name = "input_ids"
|
| 164 |
+
_no_split_modules: ClassVar[list[str]] = ["WildAIDecoderLayer"]
|
| 165 |
+
_supports_sdpa = True
|
| 166 |
+
|
| 167 |
+
def _init_weights(self, module: nn.Module) -> None:
|
| 168 |
+
# Released models are always loaded from trained weights; this only gives from_config() models sane values.
|
| 169 |
+
if isinstance(module, (nn.Linear, nn.Embedding)):
|
| 170 |
+
nn.init.normal_(module.weight, mean=0.0, std=0.02)
|
| 171 |
+
elif isinstance(module, WildAIModel):
|
| 172 |
+
with torch.no_grad():
|
| 173 |
+
module.resid_lambdas.fill_(1.0)
|
| 174 |
+
module.x0_lambdas.zero_()
|
| 175 |
+
module.smear_lambda.zero_()
|
| 176 |
+
module.backout_lambda.zero_()
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
class WildAIModel(WildAIPreTrainedModel):
|
| 180 |
+
"""The transformer trunk; returns the final normalised hidden states."""
|
| 181 |
+
|
| 182 |
+
def __init__(self, config: WildAIConfig) -> None:
|
| 183 |
+
super().__init__(config)
|
| 184 |
+
n_layer = config.num_hidden_layers
|
| 185 |
+
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
|
| 186 |
+
self.layers = nn.ModuleList([WildAIDecoderLayer(config, i) for i in range(n_layer)])
|
| 187 |
+
self.resid_lambdas = nn.Parameter(torch.ones(n_layer))
|
| 188 |
+
self.x0_lambdas = nn.Parameter(torch.zeros(n_layer))
|
| 189 |
+
self.smear_gate = nn.Linear(config.smear_channels, 1, bias=False)
|
| 190 |
+
self.smear_lambda = nn.Parameter(torch.zeros(1))
|
| 191 |
+
self.backout_lambda = nn.Parameter(torch.zeros(1))
|
| 192 |
+
self.post_init()
|
| 193 |
+
|
| 194 |
+
def get_input_embeddings(self) -> nn.Embedding:
|
| 195 |
+
return self.embed_tokens
|
| 196 |
+
|
| 197 |
+
def set_input_embeddings(self, value: nn.Embedding) -> None:
|
| 198 |
+
self.embed_tokens = value
|
| 199 |
+
|
| 200 |
+
def rotary(self, position_ids: torch.LongTensor, dtype: torch.dtype) -> tuple[torch.Tensor, torch.Tensor]:
|
| 201 |
+
"""cos/sin tables of shape (batch, time, 1, head_dim / 2), computed in float32 and cast like in training."""
|
| 202 |
+
head_dim = self.config.head_dim
|
| 203 |
+
channels = torch.arange(0, head_dim, 2, dtype=torch.float32, device=position_ids.device)
|
| 204 |
+
inv_freq = 1.0 / (self.config.rotary_base ** (channels / head_dim))
|
| 205 |
+
freqs = position_ids.to(torch.float32)[..., None] * inv_freq
|
| 206 |
+
return freqs.cos().to(dtype)[:, :, None, :], freqs.sin().to(dtype)[:, :, None, :]
|
| 207 |
+
|
| 208 |
+
def smear(self, x: torch.Tensor, previous: torch.Tensor | None, padding_mask: torch.Tensor | None) -> torch.Tensor:
|
| 209 |
+
"""Add a gated copy of the previous position's embedding to every position that has one.
|
| 210 |
+
|
| 211 |
+
``previous`` is the normalised embedding of the token just before this chunk (None at the start of a sequence);
|
| 212 |
+
with left padding, the first real token has a padding token before it and gets nothing.
|
| 213 |
+
"""
|
| 214 |
+
channels = self.config.smear_channels
|
| 215 |
+
lam = self.smear_lambda.to(x.dtype)
|
| 216 |
+
first = x[:, :1]
|
| 217 |
+
if previous is not None:
|
| 218 |
+
first = first + lam * torch.sigmoid(self.smear_gate(first[..., :channels])) * previous
|
| 219 |
+
if x.size(1) == 1:
|
| 220 |
+
return first
|
| 221 |
+
gate = lam * torch.sigmoid(self.smear_gate(x[:, 1:, :channels]))
|
| 222 |
+
if padding_mask is not None:
|
| 223 |
+
gate = gate * padding_mask[:, -x.size(1) : -1, None].to(gate.dtype)
|
| 224 |
+
return torch.cat([first, x[:, 1:] + gate * x[:, :-1]], dim=1)
|
| 225 |
+
|
| 226 |
+
def forward(
|
| 227 |
+
self,
|
| 228 |
+
input_ids: torch.LongTensor,
|
| 229 |
+
attention_mask: torch.Tensor | None = None,
|
| 230 |
+
position_ids: torch.LongTensor | None = None,
|
| 231 |
+
past_key_values: Cache | None = None,
|
| 232 |
+
use_cache: bool | None = None,
|
| 233 |
+
previous_token_ids: torch.LongTensor | None = None,
|
| 234 |
+
output_hidden_states: bool | None = None,
|
| 235 |
+
**kwargs: object,
|
| 236 |
+
) -> BaseModelOutputWithPast:
|
| 237 |
+
"""``previous_token_ids`` (batch, 1): the token before ``input_ids[:, 0]``; required when continuing from a cache."""
|
| 238 |
+
if input_ids is None:
|
| 239 |
+
raise ValueError("input_ids are required: the value embeddings are looked up by token id")
|
| 240 |
+
batch, time = input_ids.shape
|
| 241 |
+
use_cache = self.config.use_cache if use_cache is None else use_cache
|
| 242 |
+
if use_cache and past_key_values is None:
|
| 243 |
+
past_key_values = DynamicCache()
|
| 244 |
+
past_len = past_key_values.get_seq_length() if past_key_values is not None else 0
|
| 245 |
+
if past_len > 0 and previous_token_ids is None:
|
| 246 |
+
raise ValueError("previous_token_ids is required when continuing from past_key_values (the smear needs it)")
|
| 247 |
+
padding_mask = None
|
| 248 |
+
if attention_mask is not None:
|
| 249 |
+
if attention_mask.shape != (batch, past_len + time):
|
| 250 |
+
raise ValueError(f"attention_mask must have shape {(batch, past_len + time)}, got {tuple(attention_mask.shape)}")
|
| 251 |
+
if not bool(attention_mask.all()):
|
| 252 |
+
padding_mask = attention_mask
|
| 253 |
+
if position_ids is None:
|
| 254 |
+
position_ids = torch.arange(past_len, past_len + time, device=input_ids.device).expand(batch, -1)
|
| 255 |
+
|
| 256 |
+
x = rms_norm(self.embed_tokens(input_ids))
|
| 257 |
+
previous = rms_norm(self.embed_tokens(previous_token_ids)) if previous_token_ids is not None else None
|
| 258 |
+
x = self.smear(x, previous, padding_mask)
|
| 259 |
+
rotary = self.rotary(position_ids, x.dtype)
|
| 260 |
+
masks = window_masks(self.config.window_sizes, time, past_len, padding_mask, input_ids.device)
|
| 261 |
+
|
| 262 |
+
x0 = x
|
| 263 |
+
backout = None
|
| 264 |
+
hidden_states: tuple[torch.Tensor, ...] = (x,)
|
| 265 |
+
for i, layer in enumerate(self.layers):
|
| 266 |
+
x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0
|
| 267 |
+
x = layer(x, input_ids, rotary, masks[layer.self_attn.window], past_key_values)
|
| 268 |
+
if i == self.config.backout_layer:
|
| 269 |
+
backout = x
|
| 270 |
+
hidden_states += (x,)
|
| 271 |
+
x = rms_norm(x - self.backout_lambda.to(x.dtype) * backout)
|
| 272 |
+
return BaseModelOutputWithPast(
|
| 273 |
+
last_hidden_state=x,
|
| 274 |
+
past_key_values=past_key_values if use_cache else None,
|
| 275 |
+
hidden_states=hidden_states if output_hidden_states else None,
|
| 276 |
+
)
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
class WildAIForCausalLM(WildAIPreTrainedModel, GenerationMixin):
|
| 280 |
+
def __init__(self, config: WildAIConfig) -> None:
|
| 281 |
+
super().__init__(config)
|
| 282 |
+
self.model = WildAIModel(config)
|
| 283 |
+
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
|
| 284 |
+
self.post_init()
|
| 285 |
+
|
| 286 |
+
def get_input_embeddings(self) -> nn.Embedding:
|
| 287 |
+
return self.model.embed_tokens
|
| 288 |
+
|
| 289 |
+
def set_input_embeddings(self, value: nn.Embedding) -> None:
|
| 290 |
+
self.model.embed_tokens = value
|
| 291 |
+
|
| 292 |
+
def get_output_embeddings(self) -> nn.Linear:
|
| 293 |
+
return self.lm_head
|
| 294 |
+
|
| 295 |
+
def set_output_embeddings(self, value: nn.Linear) -> None:
|
| 296 |
+
self.lm_head = value
|
| 297 |
+
|
| 298 |
+
def forward(
|
| 299 |
+
self,
|
| 300 |
+
input_ids: torch.LongTensor,
|
| 301 |
+
attention_mask: torch.Tensor | None = None,
|
| 302 |
+
position_ids: torch.LongTensor | None = None,
|
| 303 |
+
past_key_values: Cache | None = None,
|
| 304 |
+
labels: torch.LongTensor | None = None,
|
| 305 |
+
use_cache: bool | None = None,
|
| 306 |
+
previous_token_ids: torch.LongTensor | None = None,
|
| 307 |
+
output_hidden_states: bool | None = None,
|
| 308 |
+
logits_to_keep: int = 0,
|
| 309 |
+
return_dict: bool | None = None,
|
| 310 |
+
**kwargs: object,
|
| 311 |
+
) -> CausalLMOutputWithPast | tuple[torch.Tensor, ...]:
|
| 312 |
+
"""Float32 logits after the soft cap. ``labels`` are shifted inside (standard causal-LM loss, -100 ignored)."""
|
| 313 |
+
outputs = self.model(
|
| 314 |
+
input_ids,
|
| 315 |
+
attention_mask=attention_mask,
|
| 316 |
+
position_ids=position_ids,
|
| 317 |
+
past_key_values=past_key_values,
|
| 318 |
+
use_cache=use_cache,
|
| 319 |
+
previous_token_ids=previous_token_ids,
|
| 320 |
+
output_hidden_states=output_hidden_states,
|
| 321 |
+
)
|
| 322 |
+
hidden = outputs.last_hidden_state[:, -logits_to_keep:] if logits_to_keep else outputs.last_hidden_state
|
| 323 |
+
logits = self.lm_head(hidden).float()
|
| 324 |
+
cap = self.config.logit_softcap
|
| 325 |
+
logits = cap * torch.tanh(logits / cap)
|
| 326 |
+
loss = None
|
| 327 |
+
if labels is not None:
|
| 328 |
+
loss = F.cross_entropy(logits[:, :-1].flatten(0, 1), labels[:, 1:].flatten().to(logits.device), ignore_index=-100)
|
| 329 |
+
result = CausalLMOutputWithPast(
|
| 330 |
+
loss=loss, logits=logits, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states
|
| 331 |
+
)
|
| 332 |
+
return result.to_tuple() if return_dict is False else result
|
| 333 |
+
|
| 334 |
+
def prepare_inputs_for_generation(self, input_ids: torch.LongTensor, **kwargs: object) -> dict[str, object]:
|
| 335 |
+
# The smear mixes in the embedding of the token before the first new one; take it from the full sequence.
|
| 336 |
+
model_inputs = super().prepare_inputs_for_generation(input_ids, **kwargs)
|
| 337 |
+
new_len = model_inputs["input_ids"].shape[1]
|
| 338 |
+
if input_ids.shape[1] > new_len:
|
| 339 |
+
model_inputs["previous_token_ids"] = input_ids[:, -new_len - 1 : -new_len]
|
| 340 |
+
return model_inputs
|
268m-g072-ai-r0.01/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
268m-g072-ai-r0.01/tokenizer_config.json
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 3 |
+
"bos_token": "<|bos|>",
|
| 4 |
+
"eos_token": "<|bos|>",
|
| 5 |
+
"model_max_length": 2048,
|
| 6 |
+
"model_input_names": [
|
| 7 |
+
"input_ids",
|
| 8 |
+
"attention_mask"
|
| 9 |
+
],
|
| 10 |
+
"clean_up_tokenization_spaces": false
|
| 11 |
+
}
|
268m-g072-ai-r0.5/LICENSE
ADDED
|
@@ -0,0 +1,438 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Attribution-NonCommercial-ShareAlike 4.0 International
|
| 2 |
+
|
| 3 |
+
=======================================================================
|
| 4 |
+
|
| 5 |
+
Creative Commons Corporation ("Creative Commons") is not a law firm and
|
| 6 |
+
does not provide legal services or legal advice. Distribution of
|
| 7 |
+
Creative Commons public licenses does not create a lawyer-client or
|
| 8 |
+
other relationship. Creative Commons makes its licenses and related
|
| 9 |
+
information available on an "as-is" basis. Creative Commons gives no
|
| 10 |
+
warranties regarding its licenses, any material licensed under their
|
| 11 |
+
terms and conditions, or any related information. Creative Commons
|
| 12 |
+
disclaims all liability for damages resulting from their use to the
|
| 13 |
+
fullest extent possible.
|
| 14 |
+
|
| 15 |
+
Using Creative Commons Public Licenses
|
| 16 |
+
|
| 17 |
+
Creative Commons public licenses provide a standard set of terms and
|
| 18 |
+
conditions that creators and other rights holders may use to share
|
| 19 |
+
original works of authorship and other material subject to copyright
|
| 20 |
+
and certain other rights specified in the public license below. The
|
| 21 |
+
following considerations are for informational purposes only, are not
|
| 22 |
+
exhaustive, and do not form part of our licenses.
|
| 23 |
+
|
| 24 |
+
Considerations for licensors: Our public licenses are
|
| 25 |
+
intended for use by those authorized to give the public
|
| 26 |
+
permission to use material in ways otherwise restricted by
|
| 27 |
+
copyright and certain other rights. Our licenses are
|
| 28 |
+
irrevocable. Licensors should read and understand the terms
|
| 29 |
+
and conditions of the license they choose before applying it.
|
| 30 |
+
Licensors should also secure all rights necessary before
|
| 31 |
+
applying our licenses so that the public can reuse the
|
| 32 |
+
material as expected. Licensors should clearly mark any
|
| 33 |
+
material not subject to the license. This includes other CC-
|
| 34 |
+
licensed material, or material used under an exception or
|
| 35 |
+
limitation to copyright. More considerations for licensors:
|
| 36 |
+
wiki.creativecommons.org/Considerations_for_licensors
|
| 37 |
+
|
| 38 |
+
Considerations for the public: By using one of our public
|
| 39 |
+
licenses, a licensor grants the public permission to use the
|
| 40 |
+
licensed material under specified terms and conditions. If
|
| 41 |
+
the licensor's permission is not necessary for any reason--for
|
| 42 |
+
example, because of any applicable exception or limitation to
|
| 43 |
+
copyright--then that use is not regulated by the license. Our
|
| 44 |
+
licenses grant only permissions under copyright and certain
|
| 45 |
+
other rights that a licensor has authority to grant. Use of
|
| 46 |
+
the licensed material may still be restricted for other
|
| 47 |
+
reasons, including because others have copyright or other
|
| 48 |
+
rights in the material. A licensor may make special requests,
|
| 49 |
+
such as asking that all changes be marked or described.
|
| 50 |
+
Although not required by our licenses, you are encouraged to
|
| 51 |
+
respect those requests where reasonable. More considerations
|
| 52 |
+
for the public:
|
| 53 |
+
wiki.creativecommons.org/Considerations_for_licensees
|
| 54 |
+
|
| 55 |
+
=======================================================================
|
| 56 |
+
|
| 57 |
+
Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International
|
| 58 |
+
Public License
|
| 59 |
+
|
| 60 |
+
By exercising the Licensed Rights (defined below), You accept and agree
|
| 61 |
+
to be bound by the terms and conditions of this Creative Commons
|
| 62 |
+
Attribution-NonCommercial-ShareAlike 4.0 International Public License
|
| 63 |
+
("Public License"). To the extent this Public License may be
|
| 64 |
+
interpreted as a contract, You are granted the Licensed Rights in
|
| 65 |
+
consideration of Your acceptance of these terms and conditions, and the
|
| 66 |
+
Licensor grants You such rights in consideration of benefits the
|
| 67 |
+
Licensor receives from making the Licensed Material available under
|
| 68 |
+
these terms and conditions.
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
Section 1 -- Definitions.
|
| 72 |
+
|
| 73 |
+
a. Adapted Material means material subject to Copyright and Similar
|
| 74 |
+
Rights that is derived from or based upon the Licensed Material
|
| 75 |
+
and in which the Licensed Material is translated, altered,
|
| 76 |
+
arranged, transformed, or otherwise modified in a manner requiring
|
| 77 |
+
permission under the Copyright and Similar Rights held by the
|
| 78 |
+
Licensor. For purposes of this Public License, where the Licensed
|
| 79 |
+
Material is a musical work, performance, or sound recording,
|
| 80 |
+
Adapted Material is always produced where the Licensed Material is
|
| 81 |
+
synched in timed relation with a moving image.
|
| 82 |
+
|
| 83 |
+
b. Adapter's License means the license You apply to Your Copyright
|
| 84 |
+
and Similar Rights in Your contributions to Adapted Material in
|
| 85 |
+
accordance with the terms and conditions of this Public License.
|
| 86 |
+
|
| 87 |
+
c. BY-NC-SA Compatible License means a license listed at
|
| 88 |
+
creativecommons.org/compatiblelicenses, approved by Creative
|
| 89 |
+
Commons as essentially the equivalent of this Public License.
|
| 90 |
+
|
| 91 |
+
d. Copyright and Similar Rights means copyright and/or similar rights
|
| 92 |
+
closely related to copyright including, without limitation,
|
| 93 |
+
performance, broadcast, sound recording, and Sui Generis Database
|
| 94 |
+
Rights, without regard to how the rights are labeled or
|
| 95 |
+
categorized. For purposes of this Public License, the rights
|
| 96 |
+
specified in Section 2(b)(1)-(2) are not Copyright and Similar
|
| 97 |
+
Rights.
|
| 98 |
+
|
| 99 |
+
e. Effective Technological Measures means those measures that, in the
|
| 100 |
+
absence of proper authority, may not be circumvented under laws
|
| 101 |
+
fulfilling obligations under Article 11 of the WIPO Copyright
|
| 102 |
+
Treaty adopted on December 20, 1996, and/or similar international
|
| 103 |
+
agreements.
|
| 104 |
+
|
| 105 |
+
f. Exceptions and Limitations means fair use, fair dealing, and/or
|
| 106 |
+
any other exception or limitation to Copyright and Similar Rights
|
| 107 |
+
that applies to Your use of the Licensed Material.
|
| 108 |
+
|
| 109 |
+
g. License Elements means the license attributes listed in the name
|
| 110 |
+
of a Creative Commons Public License. The License Elements of this
|
| 111 |
+
Public License are Attribution, NonCommercial, and ShareAlike.
|
| 112 |
+
|
| 113 |
+
h. Licensed Material means the artistic or literary work, database,
|
| 114 |
+
or other material to which the Licensor applied this Public
|
| 115 |
+
License.
|
| 116 |
+
|
| 117 |
+
i. Licensed Rights means the rights granted to You subject to the
|
| 118 |
+
terms and conditions of this Public License, which are limited to
|
| 119 |
+
all Copyright and Similar Rights that apply to Your use of the
|
| 120 |
+
Licensed Material and that the Licensor has authority to license.
|
| 121 |
+
|
| 122 |
+
j. Licensor means the individual(s) or entity(ies) granting rights
|
| 123 |
+
under this Public License.
|
| 124 |
+
|
| 125 |
+
k. NonCommercial means not primarily intended for or directed towards
|
| 126 |
+
commercial advantage or monetary compensation. For purposes of
|
| 127 |
+
this Public License, the exchange of the Licensed Material for
|
| 128 |
+
other material subject to Copyright and Similar Rights by digital
|
| 129 |
+
file-sharing or similar means is NonCommercial provided there is
|
| 130 |
+
no payment of monetary compensation in connection with the
|
| 131 |
+
exchange.
|
| 132 |
+
|
| 133 |
+
l. Share means to provide material to the public by any means or
|
| 134 |
+
process that requires permission under the Licensed Rights, such
|
| 135 |
+
as reproduction, public display, public performance, distribution,
|
| 136 |
+
dissemination, communication, or importation, and to make material
|
| 137 |
+
available to the public including in ways that members of the
|
| 138 |
+
public may access the material from a place and at a time
|
| 139 |
+
individually chosen by them.
|
| 140 |
+
|
| 141 |
+
m. Sui Generis Database Rights means rights other than copyright
|
| 142 |
+
resulting from Directive 96/9/EC of the European Parliament and of
|
| 143 |
+
the Council of 11 March 1996 on the legal protection of databases,
|
| 144 |
+
as amended and/or succeeded, as well as other essentially
|
| 145 |
+
equivalent rights anywhere in the world.
|
| 146 |
+
|
| 147 |
+
n. You means the individual or entity exercising the Licensed Rights
|
| 148 |
+
under this Public License. Your has a corresponding meaning.
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
Section 2 -- Scope.
|
| 152 |
+
|
| 153 |
+
a. License grant.
|
| 154 |
+
|
| 155 |
+
1. Subject to the terms and conditions of this Public License,
|
| 156 |
+
the Licensor hereby grants You a worldwide, royalty-free,
|
| 157 |
+
non-sublicensable, non-exclusive, irrevocable license to
|
| 158 |
+
exercise the Licensed Rights in the Licensed Material to:
|
| 159 |
+
|
| 160 |
+
a. reproduce and Share the Licensed Material, in whole or
|
| 161 |
+
in part, for NonCommercial purposes only; and
|
| 162 |
+
|
| 163 |
+
b. produce, reproduce, and Share Adapted Material for
|
| 164 |
+
NonCommercial purposes only.
|
| 165 |
+
|
| 166 |
+
2. Exceptions and Limitations. For the avoidance of doubt, where
|
| 167 |
+
Exceptions and Limitations apply to Your use, this Public
|
| 168 |
+
License does not apply, and You do not need to comply with
|
| 169 |
+
its terms and conditions.
|
| 170 |
+
|
| 171 |
+
3. Term. The term of this Public License is specified in Section
|
| 172 |
+
6(a).
|
| 173 |
+
|
| 174 |
+
4. Media and formats; technical modifications allowed. The
|
| 175 |
+
Licensor authorizes You to exercise the Licensed Rights in
|
| 176 |
+
all media and formats whether now known or hereafter created,
|
| 177 |
+
and to make technical modifications necessary to do so. The
|
| 178 |
+
Licensor waives and/or agrees not to assert any right or
|
| 179 |
+
authority to forbid You from making technical modifications
|
| 180 |
+
necessary to exercise the Licensed Rights, including
|
| 181 |
+
technical modifications necessary to circumvent Effective
|
| 182 |
+
Technological Measures. For purposes of this Public License,
|
| 183 |
+
simply making modifications authorized by this Section 2(a)
|
| 184 |
+
(4) never produces Adapted Material.
|
| 185 |
+
|
| 186 |
+
5. Downstream recipients.
|
| 187 |
+
|
| 188 |
+
a. Offer from the Licensor -- Licensed Material. Every
|
| 189 |
+
recipient of the Licensed Material automatically
|
| 190 |
+
receives an offer from the Licensor to exercise the
|
| 191 |
+
Licensed Rights under the terms and conditions of this
|
| 192 |
+
Public License.
|
| 193 |
+
|
| 194 |
+
b. Additional offer from the Licensor -- Adapted Material.
|
| 195 |
+
Every recipient of Adapted Material from You
|
| 196 |
+
automatically receives an offer from the Licensor to
|
| 197 |
+
exercise the Licensed Rights in the Adapted Material
|
| 198 |
+
under the conditions of the Adapter's License You apply.
|
| 199 |
+
|
| 200 |
+
c. No downstream restrictions. You may not offer or impose
|
| 201 |
+
any additional or different terms or conditions on, or
|
| 202 |
+
apply any Effective Technological Measures to, the
|
| 203 |
+
Licensed Material if doing so restricts exercise of the
|
| 204 |
+
Licensed Rights by any recipient of the Licensed
|
| 205 |
+
Material.
|
| 206 |
+
|
| 207 |
+
6. No endorsement. Nothing in this Public License constitutes or
|
| 208 |
+
may be construed as permission to assert or imply that You
|
| 209 |
+
are, or that Your use of the Licensed Material is, connected
|
| 210 |
+
with, or sponsored, endorsed, or granted official status by,
|
| 211 |
+
the Licensor or others designated to receive attribution as
|
| 212 |
+
provided in Section 3(a)(1)(A)(i).
|
| 213 |
+
|
| 214 |
+
b. Other rights.
|
| 215 |
+
|
| 216 |
+
1. Moral rights, such as the right of integrity, are not
|
| 217 |
+
licensed under this Public License, nor are publicity,
|
| 218 |
+
privacy, and/or other similar personality rights; however, to
|
| 219 |
+
the extent possible, the Licensor waives and/or agrees not to
|
| 220 |
+
assert any such rights held by the Licensor to the limited
|
| 221 |
+
extent necessary to allow You to exercise the Licensed
|
| 222 |
+
Rights, but not otherwise.
|
| 223 |
+
|
| 224 |
+
2. Patent and trademark rights are not licensed under this
|
| 225 |
+
Public License.
|
| 226 |
+
|
| 227 |
+
3. To the extent possible, the Licensor waives any right to
|
| 228 |
+
collect royalties from You for the exercise of the Licensed
|
| 229 |
+
Rights, whether directly or through a collecting society
|
| 230 |
+
under any voluntary or waivable statutory or compulsory
|
| 231 |
+
licensing scheme. In all other cases the Licensor expressly
|
| 232 |
+
reserves any right to collect such royalties, including when
|
| 233 |
+
the Licensed Material is used other than for NonCommercial
|
| 234 |
+
purposes.
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
Section 3 -- License Conditions.
|
| 238 |
+
|
| 239 |
+
Your exercise of the Licensed Rights is expressly made subject to the
|
| 240 |
+
following conditions.
|
| 241 |
+
|
| 242 |
+
a. Attribution.
|
| 243 |
+
|
| 244 |
+
1. If You Share the Licensed Material (including in modified
|
| 245 |
+
form), You must:
|
| 246 |
+
|
| 247 |
+
a. retain the following if it is supplied by the Licensor
|
| 248 |
+
with the Licensed Material:
|
| 249 |
+
|
| 250 |
+
i. identification of the creator(s) of the Licensed
|
| 251 |
+
Material and any others designated to receive
|
| 252 |
+
attribution, in any reasonable manner requested by
|
| 253 |
+
the Licensor (including by pseudonym if
|
| 254 |
+
designated);
|
| 255 |
+
|
| 256 |
+
ii. a copyright notice;
|
| 257 |
+
|
| 258 |
+
iii. a notice that refers to this Public License;
|
| 259 |
+
|
| 260 |
+
iv. a notice that refers to the disclaimer of
|
| 261 |
+
warranties;
|
| 262 |
+
|
| 263 |
+
v. a URI or hyperlink to the Licensed Material to the
|
| 264 |
+
extent reasonably practicable;
|
| 265 |
+
|
| 266 |
+
b. indicate if You modified the Licensed Material and
|
| 267 |
+
retain an indication of any previous modifications; and
|
| 268 |
+
|
| 269 |
+
c. indicate the Licensed Material is licensed under this
|
| 270 |
+
Public License, and include the text of, or the URI or
|
| 271 |
+
hyperlink to, this Public License.
|
| 272 |
+
|
| 273 |
+
2. You may satisfy the conditions in Section 3(a)(1) in any
|
| 274 |
+
reasonable manner based on the medium, means, and context in
|
| 275 |
+
which You Share the Licensed Material. For example, it may be
|
| 276 |
+
reasonable to satisfy the conditions by providing a URI or
|
| 277 |
+
hyperlink to a resource that includes the required
|
| 278 |
+
information.
|
| 279 |
+
3. If requested by the Licensor, You must remove any of the
|
| 280 |
+
information required by Section 3(a)(1)(A) to the extent
|
| 281 |
+
reasonably practicable.
|
| 282 |
+
|
| 283 |
+
b. ShareAlike.
|
| 284 |
+
|
| 285 |
+
In addition to the conditions in Section 3(a), if You Share
|
| 286 |
+
Adapted Material You produce, the following conditions also apply.
|
| 287 |
+
|
| 288 |
+
1. The Adapter's License You apply must be a Creative Commons
|
| 289 |
+
license with the same License Elements, this version or
|
| 290 |
+
later, or a BY-NC-SA Compatible License.
|
| 291 |
+
|
| 292 |
+
2. You must include the text of, or the URI or hyperlink to, the
|
| 293 |
+
Adapter's License You apply. You may satisfy this condition
|
| 294 |
+
in any reasonable manner based on the medium, means, and
|
| 295 |
+
context in which You Share Adapted Material.
|
| 296 |
+
|
| 297 |
+
3. You may not offer or impose any additional or different terms
|
| 298 |
+
or conditions on, or apply any Effective Technological
|
| 299 |
+
Measures to, Adapted Material that restrict exercise of the
|
| 300 |
+
rights granted under the Adapter's License You apply.
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
Section 4 -- Sui Generis Database Rights.
|
| 304 |
+
|
| 305 |
+
Where the Licensed Rights include Sui Generis Database Rights that
|
| 306 |
+
apply to Your use of the Licensed Material:
|
| 307 |
+
|
| 308 |
+
a. for the avoidance of doubt, Section 2(a)(1) grants You the right
|
| 309 |
+
to extract, reuse, reproduce, and Share all or a substantial
|
| 310 |
+
portion of the contents of the database for NonCommercial purposes
|
| 311 |
+
only;
|
| 312 |
+
|
| 313 |
+
b. if You include all or a substantial portion of the database
|
| 314 |
+
contents in a database in which You have Sui Generis Database
|
| 315 |
+
Rights, then the database in which You have Sui Generis Database
|
| 316 |
+
Rights (but not its individual contents) is Adapted Material,
|
| 317 |
+
including for purposes of Section 3(b); and
|
| 318 |
+
|
| 319 |
+
c. You must comply with the conditions in Section 3(a) if You Share
|
| 320 |
+
all or a substantial portion of the contents of the database.
|
| 321 |
+
|
| 322 |
+
For the avoidance of doubt, this Section 4 supplements and does not
|
| 323 |
+
replace Your obligations under this Public License where the Licensed
|
| 324 |
+
Rights include other Copyright and Similar Rights.
|
| 325 |
+
|
| 326 |
+
|
| 327 |
+
Section 5 -- Disclaimer of Warranties and Limitation of Liability.
|
| 328 |
+
|
| 329 |
+
a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
|
| 330 |
+
EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
|
| 331 |
+
AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
|
| 332 |
+
ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
|
| 333 |
+
IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
|
| 334 |
+
WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
|
| 335 |
+
PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
|
| 336 |
+
ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
|
| 337 |
+
KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
|
| 338 |
+
ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
|
| 339 |
+
|
| 340 |
+
b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
|
| 341 |
+
TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
|
| 342 |
+
NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
|
| 343 |
+
INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
|
| 344 |
+
COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
|
| 345 |
+
USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
|
| 346 |
+
ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
|
| 347 |
+
DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
|
| 348 |
+
IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
|
| 349 |
+
|
| 350 |
+
c. The disclaimer of warranties and limitation of liability provided
|
| 351 |
+
above shall be interpreted in a manner that, to the extent
|
| 352 |
+
possible, most closely approximates an absolute disclaimer and
|
| 353 |
+
waiver of all liability.
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
Section 6 -- Term and Termination.
|
| 357 |
+
|
| 358 |
+
a. This Public License applies for the term of the Copyright and
|
| 359 |
+
Similar Rights licensed here. However, if You fail to comply with
|
| 360 |
+
this Public License, then Your rights under this Public License
|
| 361 |
+
terminate automatically.
|
| 362 |
+
|
| 363 |
+
b. Where Your right to use the Licensed Material has terminated under
|
| 364 |
+
Section 6(a), it reinstates:
|
| 365 |
+
|
| 366 |
+
1. automatically as of the date the violation is cured, provided
|
| 367 |
+
it is cured within 30 days of Your discovery of the
|
| 368 |
+
violation; or
|
| 369 |
+
|
| 370 |
+
2. upon express reinstatement by the Licensor.
|
| 371 |
+
|
| 372 |
+
For the avoidance of doubt, this Section 6(b) does not affect any
|
| 373 |
+
right the Licensor may have to seek remedies for Your violations
|
| 374 |
+
of this Public License.
|
| 375 |
+
|
| 376 |
+
c. For the avoidance of doubt, the Licensor may also offer the
|
| 377 |
+
Licensed Material under separate terms or conditions or stop
|
| 378 |
+
distributing the Licensed Material at any time; however, doing so
|
| 379 |
+
will not terminate this Public License.
|
| 380 |
+
|
| 381 |
+
d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
|
| 382 |
+
License.
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
Section 7 -- Other Terms and Conditions.
|
| 386 |
+
|
| 387 |
+
a. The Licensor shall not be bound by any additional or different
|
| 388 |
+
terms or conditions communicated by You unless expressly agreed.
|
| 389 |
+
|
| 390 |
+
b. Any arrangements, understandings, or agreements regarding the
|
| 391 |
+
Licensed Material not stated herein are separate from and
|
| 392 |
+
independent of the terms and conditions of this Public License.
|
| 393 |
+
|
| 394 |
+
|
| 395 |
+
Section 8 -- Interpretation.
|
| 396 |
+
|
| 397 |
+
a. For the avoidance of doubt, this Public License does not, and
|
| 398 |
+
shall not be interpreted to, reduce, limit, restrict, or impose
|
| 399 |
+
conditions on any use of the Licensed Material that could lawfully
|
| 400 |
+
be made without permission under this Public License.
|
| 401 |
+
|
| 402 |
+
b. To the extent possible, if any provision of this Public License is
|
| 403 |
+
deemed unenforceable, it shall be automatically reformed to the
|
| 404 |
+
minimum extent necessary to make it enforceable. If the provision
|
| 405 |
+
cannot be reformed, it shall be severed from this Public License
|
| 406 |
+
without affecting the enforceability of the remaining terms and
|
| 407 |
+
conditions.
|
| 408 |
+
|
| 409 |
+
c. No term or condition of this Public License will be waived and no
|
| 410 |
+
failure to comply consented to unless expressly agreed to by the
|
| 411 |
+
Licensor.
|
| 412 |
+
|
| 413 |
+
d. Nothing in this Public License constitutes or may be interpreted
|
| 414 |
+
as a limitation upon, or waiver of, any privileges and immunities
|
| 415 |
+
that apply to the Licensor or You, including from the legal
|
| 416 |
+
processes of any jurisdiction or authority.
|
| 417 |
+
|
| 418 |
+
=======================================================================
|
| 419 |
+
|
| 420 |
+
Creative Commons is not a party to its public
|
| 421 |
+
licenses. Notwithstanding, Creative Commons may elect to apply one of
|
| 422 |
+
its public licenses to material it publishes and in those instances
|
| 423 |
+
will be considered the “Licensor.” The text of the Creative Commons
|
| 424 |
+
public licenses is dedicated to the public domain under the CC0 Public
|
| 425 |
+
Domain Dedication. Except for the limited purpose of indicating that
|
| 426 |
+
material is shared under a Creative Commons public license or as
|
| 427 |
+
otherwise permitted by the Creative Commons policies published at
|
| 428 |
+
creativecommons.org/policies, Creative Commons does not authorize the
|
| 429 |
+
use of the trademark "Creative Commons" or any other trademark or logo
|
| 430 |
+
of Creative Commons without its prior written consent including,
|
| 431 |
+
without limitation, in connection with any unauthorized modifications
|
| 432 |
+
to any of its public licenses or any other arrangements,
|
| 433 |
+
understandings, or agreements concerning use of licensed material. For
|
| 434 |
+
the avoidance of doubt, this paragraph does not form part of the
|
| 435 |
+
public licenses.
|
| 436 |
+
|
| 437 |
+
Creative Commons may be contacted at creativecommons.org.
|
| 438 |
+
|
268m-g072-ai-r0.5/README.md
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: cc-by-nc-sa-4.0
|
| 3 |
+
library_name: transformers
|
| 4 |
+
pipeline_tag: text-generation
|
| 5 |
+
language:
|
| 6 |
+
- en
|
| 7 |
+
datasets:
|
| 8 |
+
- pangram/WildAI
|
| 9 |
+
tags:
|
| 10 |
+
- wildai
|
| 11 |
+
- nanochat
|
| 12 |
+
- scaling-laws
|
| 13 |
+
- ai-generated-text
|
| 14 |
+
- pretraining-data
|
| 15 |
+
---
|
| 16 |
+
|
| 17 |
+
# 268m-g072-ai-r0.5
|
| 18 |
+
|
| 19 |
+
> **License: [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/).** The WildAI code, models and dataset are released under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license: you may use and adapt them for non-commercial purposes, with attribution, and must share adaptations under the same license.
|
| 20 |
+
|
| 21 |
+
A 268M decoder-only language model trained from scratch for [*How Much Is an AI Token Worth? Scaling Laws for Wild AI-Generated Web Text*](https://arxiv.org/abs/2609.40295), one of the paper's models measuring how wild AI-generated web text in pretraining data changes a model's loss. It is a research artifact for studying scaling laws, not an assistant: it has had no instruction tuning or safety training.
|
| 22 |
+
|
| 23 |
+
## Training data
|
| 24 |
+
|
| 25 |
+
The human documents of `268m-g072-control` plus wild AI-generated web documents: 1.17B human and 595.5M AI tokens trained on, an AI-to-human token ratio r = 0.511 (design 0.5), so 33.8% of the training tokens are AI-generated.
|
| 26 |
+
|
| 27 |
+
| | |
|
| 28 |
+
|---|---|
|
| 29 |
+
| Parameters (N, input embedding included, value embeddings excluded) | 268.4M |
|
| 30 |
+
| Layers / width | 16 / 1024 |
|
| 31 |
+
| Training tokens | 1.76B (3,360 optimizer steps) |
|
| 32 |
+
| Human / AI tokens | 1.17B / 595.5M |
|
| 33 |
+
| Group / arm / seed | g072 / ai / 1337 |
|
| 34 |
+
| Used in the paper for | fitting the scaling laws (726 models, 19.9M to 268M) |
|
| 35 |
+
|
| 36 |
+
## Held-out loss
|
| 37 |
+
|
| 38 |
+
| Set | Bits per byte | Text |
|
| 39 |
+
|---|---|---|
|
| 40 |
+
| C4 | 0.9886 | C4 validation text (human-written, pre-2022) |
|
| 41 |
+
| FW22 | 0.9718 | held-out FineWeb documents crawled before 2022 |
|
| 42 |
+
| Paloma | 1.3650 | macro average over 16 Paloma sources |
|
| 43 |
+
| FW26 | 0.8379 | held-out 2026 FineWeb-style crawl (22.3% AI tokens) |
|
| 44 |
+
| FW26-H | 0.9036 | the human-labeled documents of FW26 |
|
| 45 |
+
| FW26-AI | 0.5955 | the AI-labeled documents of FW26 |
|
| 46 |
+
| Cosmo | 0.6115 | Cosmopedia (AI-generated textbooks and stories) |
|
| 47 |
+
|
| 48 |
+
<details><summary>Paloma sources</summary>
|
| 49 |
+
|
| 50 |
+
| Source | Bits per byte |
|
| 51 |
+
|---|---|
|
| 52 |
+
| 4chan | 1.4036 |
|
| 53 |
+
| c4 | 1.0071 |
|
| 54 |
+
| c4_100_domains | 0.9913 |
|
| 55 |
+
| dolma | 1.2579 |
|
| 56 |
+
| falcon_refinedweb | 1.1046 |
|
| 57 |
+
| gab | 1.9059 |
|
| 58 |
+
| m2d2_s2orc | 1.1707 |
|
| 59 |
+
| m2d2_wikipedia | 1.0715 |
|
| 60 |
+
| manosphere | 1.3865 |
|
| 61 |
+
| mc4_en | 1.0931 |
|
| 62 |
+
| programming | 1.9630 |
|
| 63 |
+
| ptb | 1.2283 |
|
| 64 |
+
| reddit_100 | 1.2455 |
|
| 65 |
+
| redpajama | 1.4269 |
|
| 66 |
+
| twitteraae | 2.5183 |
|
| 67 |
+
| wikitext_103 | 1.0652 |
|
| 68 |
+
|
| 69 |
+
</details>
|
| 70 |
+
|
| 71 |
+
## Usage
|
| 72 |
+
|
| 73 |
+
```python
|
| 74 |
+
import torch
|
| 75 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 76 |
+
|
| 77 |
+
tokenizer = AutoTokenizer.from_pretrained("pangram/WildAI-models", subfolder="268m-g072-ai-r0.5", trust_remote_code=True)
|
| 78 |
+
model = AutoModelForCausalLM.from_pretrained("pangram/WildAI-models", subfolder="268m-g072-ai-r0.5", trust_remote_code=True, dtype=torch.bfloat16)
|
| 79 |
+
|
| 80 |
+
inputs = tokenizer("The history of the printing press", return_tensors="pt")
|
| 81 |
+
output = model.generate(**inputs, max_new_tokens=40, do_sample=False)
|
| 82 |
+
print(tokenizer.decode(output[0], skip_special_tokens=True))
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
The tokenizer prepends `<|bos|>` to every text, as in training; pass `add_special_tokens=False` to skip it. The model predicts `<|bos|>` where a document ends, and generation stops there. Weights are stored in bfloat16, the precision the paper's losses were computed in; `dtype=torch.float32` computes in float32 instead.
|
| 86 |
+
|
| 87 |
+
## Architecture
|
| 88 |
+
|
| 89 |
+
The nanochat GPT recipe: rotary embeddings, RMSNorm without weights (also on queries and keys), squared-ReLU MLP, attention windows in a repeating short-short-short-long pattern (512 tokens, then the full 2,048-token context; the last layer always full), gated value embeddings on alternating layers, a previous-token smear, per-layer residual scalars, a mid-network backout, untied input and output embeddings and logits soft-capped at 15. Vocabulary 32,768 tokens, context 2,048. The code ships with the weights (`trust_remote_code=True`) and matches the training code's forward pass.
|
| 90 |
+
|
| 91 |
+
## License
|
| 92 |
+
|
| 93 |
+
**License: [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/).** The WildAI code, models and dataset are released under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license: you may use and adapt them for non-commercial purposes, with attribution, and must share adaptations under the same license. The full text is in `LICENSE`.
|
| 94 |
+
|
| 95 |
+
## Citation
|
| 96 |
+
|
| 97 |
+
```bibtex
|
| 98 |
+
@article{russell2026wildai,
|
| 99 |
+
title = {How Much Is an AI Token Worth? Scaling Laws for Wild AI-Generated Web Text},
|
| 100 |
+
author = {Russell, Jenna and Glickenhaus, Ben and Thai, Katherine and Wieting, John and Iyyer, Mohit and Spero, Max and Emi, Bradley},
|
| 101 |
+
journal = {arXiv preprint arXiv:2609.40295},
|
| 102 |
+
year = {2026},
|
| 103 |
+
url = {https://arxiv.org/abs/2609.40295}
|
| 104 |
+
}
|
| 105 |
+
```
|
268m-g072-ai-r0.5/config.json
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"WildAIForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"auto_map": {
|
| 6 |
+
"AutoConfig": "configuration_wildai.WildAIConfig",
|
| 7 |
+
"AutoModelForCausalLM": "modeling_wildai.WildAIForCausalLM"
|
| 8 |
+
},
|
| 9 |
+
"bos_token_id": 32759,
|
| 10 |
+
"dtype": "bfloat16",
|
| 11 |
+
"eos_token_id": 32759,
|
| 12 |
+
"hidden_size": 1024,
|
| 13 |
+
"logit_softcap": 15.0,
|
| 14 |
+
"max_position_embeddings": 2048,
|
| 15 |
+
"model_type": "wildai",
|
| 16 |
+
"num_attention_heads": 8,
|
| 17 |
+
"num_hidden_layers": 16,
|
| 18 |
+
"num_key_value_heads": 8,
|
| 19 |
+
"pad_token_id": 32759,
|
| 20 |
+
"qk_scale": 1.2,
|
| 21 |
+
"release": {
|
| 22 |
+
"added_ratio": 0.5,
|
| 23 |
+
"ai_ratio": 0.5106759508252973,
|
| 24 |
+
"ai_tokens": 595502084,
|
| 25 |
+
"arm": "ai",
|
| 26 |
+
"depth": 16,
|
| 27 |
+
"group": "g072",
|
| 28 |
+
"human_tokens": 1166105596,
|
| 29 |
+
"n_params": 268436282,
|
| 30 |
+
"name": "268m-g072-ai-r0.5",
|
| 31 |
+
"seed": 1337,
|
| 32 |
+
"size": "268m",
|
| 33 |
+
"split": "fit",
|
| 34 |
+
"steps": 3360,
|
| 35 |
+
"total_tokens": 1761607680
|
| 36 |
+
},
|
| 37 |
+
"rotary_base": 100000.0,
|
| 38 |
+
"smear_channels": 24,
|
| 39 |
+
"tie_word_embeddings": false,
|
| 40 |
+
"transformers_version": "5.17.0",
|
| 41 |
+
"use_cache": true,
|
| 42 |
+
"value_embed_gate_channels": 12,
|
| 43 |
+
"value_embed_gate_range": 3.0,
|
| 44 |
+
"vocab_size": 32768,
|
| 45 |
+
"window_pattern": "SSSL"
|
| 46 |
+
}
|
268m-g072-ai-r0.5/configuration_wildai.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Configuration of the WildAI language models (a nanochat GPT variant) for Hugging Face ``transformers``.
|
| 2 |
+
|
| 3 |
+
This file is copied verbatim into every released model repository and loaded with ``trust_remote_code=True``.
|
| 4 |
+
It depends only on ``transformers``.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from typing import ClassVar
|
| 10 |
+
|
| 11 |
+
from transformers import PretrainedConfig
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class WildAIConfig(PretrainedConfig):
|
| 15 |
+
"""Architecture of a WildAI model. Only width and depth vary across the released models.
|
| 16 |
+
|
| 17 |
+
Every layer uses causal attention limited to a left window: a query attends to itself and the ``window`` tokens before
|
| 18 |
+
it. ``window_pattern`` is tiled over the layers ("S" = a quarter of the context rounded up to a multiple of 128 tokens,
|
| 19 |
+
"L" = the full context) and the last layer always uses "L".
|
| 20 |
+
|
| 21 |
+
``release`` holds the model's entry in the release catalog (name, token counts, AI ratio, ...); it does not affect
|
| 22 |
+
the forward pass.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
model_type = "wildai"
|
| 26 |
+
keys_to_ignore_at_inference: ClassVar[list[str]] = ["past_key_values"]
|
| 27 |
+
|
| 28 |
+
def __init__(
|
| 29 |
+
self,
|
| 30 |
+
vocab_size: int = 32768,
|
| 31 |
+
hidden_size: int = 768,
|
| 32 |
+
num_hidden_layers: int = 12,
|
| 33 |
+
num_attention_heads: int = 6,
|
| 34 |
+
num_key_value_heads: int = 6,
|
| 35 |
+
max_position_embeddings: int = 2048,
|
| 36 |
+
window_pattern: str = "SSSL",
|
| 37 |
+
rotary_base: float = 100000.0,
|
| 38 |
+
qk_scale: float = 1.2,
|
| 39 |
+
logit_softcap: float = 15.0,
|
| 40 |
+
value_embed_gate_channels: int = 12,
|
| 41 |
+
value_embed_gate_range: float = 3.0,
|
| 42 |
+
smear_channels: int = 24,
|
| 43 |
+
bos_token_id: int = 32759,
|
| 44 |
+
eos_token_id: int = 32759,
|
| 45 |
+
use_cache: bool = True,
|
| 46 |
+
release: dict[str, str | int | float | None] | None = None,
|
| 47 |
+
**kwargs: object,
|
| 48 |
+
) -> None:
|
| 49 |
+
if hidden_size % num_attention_heads != 0 or num_attention_heads % num_key_value_heads != 0:
|
| 50 |
+
raise ValueError("hidden_size must divide into num_attention_heads, and those into num_key_value_heads")
|
| 51 |
+
if not window_pattern or set(window_pattern.upper()) - {"S", "L"}:
|
| 52 |
+
raise ValueError(f"window_pattern must be a non-empty string of S and L, got {window_pattern!r}")
|
| 53 |
+
self.vocab_size = vocab_size
|
| 54 |
+
self.hidden_size = hidden_size
|
| 55 |
+
self.num_hidden_layers = num_hidden_layers
|
| 56 |
+
self.num_attention_heads = num_attention_heads
|
| 57 |
+
self.num_key_value_heads = num_key_value_heads
|
| 58 |
+
self.max_position_embeddings = max_position_embeddings
|
| 59 |
+
self.window_pattern = window_pattern.upper()
|
| 60 |
+
self.rotary_base = rotary_base
|
| 61 |
+
self.qk_scale = qk_scale
|
| 62 |
+
self.logit_softcap = logit_softcap
|
| 63 |
+
self.value_embed_gate_channels = value_embed_gate_channels
|
| 64 |
+
self.value_embed_gate_range = value_embed_gate_range
|
| 65 |
+
self.smear_channels = smear_channels
|
| 66 |
+
self.use_cache = use_cache
|
| 67 |
+
self.release = release
|
| 68 |
+
kwargs.setdefault("tie_word_embeddings", False)
|
| 69 |
+
super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
|
| 70 |
+
|
| 71 |
+
@property
|
| 72 |
+
def head_dim(self) -> int:
|
| 73 |
+
return self.hidden_size // self.num_attention_heads
|
| 74 |
+
|
| 75 |
+
@property
|
| 76 |
+
def intermediate_size(self) -> int:
|
| 77 |
+
return 4 * self.hidden_size
|
| 78 |
+
|
| 79 |
+
@property
|
| 80 |
+
def window_sizes(self) -> list[int]:
|
| 81 |
+
"""Left attention window of each layer, in tokens."""
|
| 82 |
+
long_window = self.max_position_embeddings
|
| 83 |
+
short_window = -(-long_window // 4 // 128) * 128 # a quarter of the context, rounded up to a multiple of 128 (2048 -> 512)
|
| 84 |
+
pattern = self.window_pattern
|
| 85 |
+
sizes = [short_window if pattern[i % len(pattern)] == "S" else long_window for i in range(self.num_hidden_layers)]
|
| 86 |
+
sizes[-1] = long_window
|
| 87 |
+
return sizes
|
| 88 |
+
|
| 89 |
+
def has_value_embedding(self, layer_idx: int) -> bool:
|
| 90 |
+
"""Value embeddings sit on alternating layers, always including the last one."""
|
| 91 |
+
return layer_idx % 2 == (self.num_hidden_layers - 1) % 2
|
| 92 |
+
|
| 93 |
+
@property
|
| 94 |
+
def backout_layer(self) -> int:
|
| 95 |
+
"""The residual stream after this layer is partly subtracted before the final norm."""
|
| 96 |
+
return self.num_hidden_layers // 2
|
268m-g072-ai-r0.5/generation_config.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 32759,
|
| 4 |
+
"eos_token_id": 32759,
|
| 5 |
+
"output_attentions": false,
|
| 6 |
+
"output_hidden_states": false,
|
| 7 |
+
"pad_token_id": 32759,
|
| 8 |
+
"transformers_version": "5.17.0",
|
| 9 |
+
"use_cache": true
|
| 10 |
+
}
|
268m-g072-ai-r0.5/modeling_wildai.py
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The WildAI language models (a nanochat GPT variant) for Hugging Face ``transformers``.
|
| 2 |
+
|
| 3 |
+
This file is copied verbatim into every released model repository and loaded with ``trust_remote_code=True``.
|
| 4 |
+
It depends only on ``torch`` and ``transformers`` and reproduces the training code's forward pass operation by
|
| 5 |
+
operation, so a model loaded in bfloat16 computes the same function the paper evaluated:
|
| 6 |
+
|
| 7 |
+
- token embedding followed by a parameter-free RMSNorm, then a learned "smear" that mixes each position's embedding
|
| 8 |
+
with the previous token's;
|
| 9 |
+
- per-layer scalars that rescale the residual stream and blend the initial embedding back in before every block;
|
| 10 |
+
- attention with rotary embeddings applied before a per-head RMSNorm of queries and keys (both scaled by 1.2),
|
| 11 |
+
a left sliding window per layer, and gated value embeddings added to the values on alternating layers;
|
| 12 |
+
- a squared-ReLU MLP, parameter-free pre-norms, no biases;
|
| 13 |
+
- "backout": part of the residual stream after the middle layer is subtracted before the final norm;
|
| 14 |
+
- untied output head with logits soft-capped by ``15 * tanh(logits / 15)``.
|
| 15 |
+
|
| 16 |
+
Attention uses PyTorch's scaled_dot_product_attention with explicit window masks, so no FlashAttention install is
|
| 17 |
+
needed. Generation uses the standard ``DynamicCache``.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
from dataclasses import dataclass
|
| 23 |
+
from typing import ClassVar
|
| 24 |
+
|
| 25 |
+
import torch
|
| 26 |
+
import torch.nn.functional as F
|
| 27 |
+
from torch import nn
|
| 28 |
+
from transformers import GenerationMixin, PreTrainedModel
|
| 29 |
+
from transformers.cache_utils import Cache, DynamicCache
|
| 30 |
+
from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
|
| 31 |
+
|
| 32 |
+
from .configuration_wildai import WildAIConfig
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def rms_norm(x: torch.Tensor) -> torch.Tensor:
|
| 36 |
+
"""Parameter-free RMSNorm over the last dimension with PyTorch's default epsilon, as in training."""
|
| 37 |
+
return F.rms_norm(x, (x.size(-1),))
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def apply_rotary(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
|
| 41 |
+
"""Rotate channel pairs (i, i + d/2) of ``x`` (batch, time, heads, head_dim), with the training code's sign convention."""
|
| 42 |
+
d = x.shape[-1] // 2
|
| 43 |
+
x1, x2 = x[..., :d], x[..., d:]
|
| 44 |
+
return torch.cat([x1 * cos + x2 * sin, x1 * (-sin) + x2 * cos], dim=-1)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
@dataclass(frozen=True)
|
| 48 |
+
class WindowMask:
|
| 49 |
+
"""Arguments for scaled_dot_product_attention that implement one attention window."""
|
| 50 |
+
|
| 51 |
+
mask: torch.Tensor | None
|
| 52 |
+
is_causal: bool
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def window_masks(
|
| 56 |
+
window_sizes: list[int], query_len: int, past_len: int, padding_mask: torch.Tensor | None, device: torch.device
|
| 57 |
+
) -> dict[int, WindowMask]:
|
| 58 |
+
"""Masks for each distinct left window. Query i (absolute index past_len + i) sees keys j with i - window <= j <= i.
|
| 59 |
+
|
| 60 |
+
``padding_mask`` is the (batch, past_len + query_len) attention mask when it contains padding, else None. A padding
|
| 61 |
+
query is always allowed to see itself so that its (unused) output stays finite.
|
| 62 |
+
"""
|
| 63 |
+
total = past_len + query_len
|
| 64 |
+
q_idx = torch.arange(past_len, total, device=device)[:, None]
|
| 65 |
+
k_idx = torch.arange(total, device=device)[None, :]
|
| 66 |
+
masks: dict[int, WindowMask] = {}
|
| 67 |
+
for window in sorted(set(window_sizes)):
|
| 68 |
+
if padding_mask is None and window >= total - 1:
|
| 69 |
+
if query_len == 1:
|
| 70 |
+
masks[window] = WindowMask(mask=None, is_causal=False)
|
| 71 |
+
continue
|
| 72 |
+
if past_len == 0:
|
| 73 |
+
masks[window] = WindowMask(mask=None, is_causal=True)
|
| 74 |
+
continue
|
| 75 |
+
allowed = (k_idx <= q_idx) & (q_idx - k_idx <= window)
|
| 76 |
+
if padding_mask is not None:
|
| 77 |
+
allowed = (allowed & padding_mask[:, None, None, :].bool()) | (k_idx == q_idx)
|
| 78 |
+
masks[window] = WindowMask(mask=allowed, is_causal=False)
|
| 79 |
+
return masks
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class WildAIAttention(nn.Module):
|
| 83 |
+
def __init__(self, config: WildAIConfig, layer_idx: int) -> None:
|
| 84 |
+
super().__init__()
|
| 85 |
+
self.layer_idx = layer_idx
|
| 86 |
+
self.num_heads = config.num_attention_heads
|
| 87 |
+
self.num_kv_heads = config.num_key_value_heads
|
| 88 |
+
self.head_dim = config.head_dim
|
| 89 |
+
self.qk_scale = config.qk_scale
|
| 90 |
+
self.window = config.window_sizes[layer_idx]
|
| 91 |
+
hidden, kv_dim = config.hidden_size, config.num_key_value_heads * config.head_dim
|
| 92 |
+
self.q_proj = nn.Linear(hidden, config.num_attention_heads * config.head_dim, bias=False)
|
| 93 |
+
self.k_proj = nn.Linear(hidden, kv_dim, bias=False)
|
| 94 |
+
self.v_proj = nn.Linear(hidden, kv_dim, bias=False)
|
| 95 |
+
self.o_proj = nn.Linear(hidden, hidden, bias=False)
|
| 96 |
+
self.value_embed: nn.Embedding | None = None
|
| 97 |
+
self.value_embed_gate: nn.Linear | None = None
|
| 98 |
+
if config.has_value_embedding(layer_idx):
|
| 99 |
+
self.value_embed = nn.Embedding(config.vocab_size, kv_dim)
|
| 100 |
+
self.value_embed_gate = nn.Linear(config.value_embed_gate_channels, config.num_key_value_heads, bias=False)
|
| 101 |
+
self.gate_channels = config.value_embed_gate_channels
|
| 102 |
+
self.gate_range = config.value_embed_gate_range
|
| 103 |
+
|
| 104 |
+
def forward(
|
| 105 |
+
self,
|
| 106 |
+
x: torch.Tensor,
|
| 107 |
+
input_ids: torch.LongTensor,
|
| 108 |
+
rotary: tuple[torch.Tensor, torch.Tensor],
|
| 109 |
+
mask: WindowMask,
|
| 110 |
+
past_key_values: Cache | None,
|
| 111 |
+
) -> torch.Tensor:
|
| 112 |
+
batch, time, _ = x.shape
|
| 113 |
+
q = self.q_proj(x).view(batch, time, self.num_heads, self.head_dim)
|
| 114 |
+
k = self.k_proj(x).view(batch, time, self.num_kv_heads, self.head_dim)
|
| 115 |
+
v = self.v_proj(x).view(batch, time, self.num_kv_heads, self.head_dim)
|
| 116 |
+
if self.value_embed is not None:
|
| 117 |
+
ve = self.value_embed(input_ids).to(x.dtype).view(batch, time, self.num_kv_heads, self.head_dim)
|
| 118 |
+
gate = self.gate_range * torch.sigmoid(self.value_embed_gate(x[..., : self.gate_channels]))
|
| 119 |
+
v = v + gate.unsqueeze(-1) * ve
|
| 120 |
+
cos, sin = rotary
|
| 121 |
+
q, k = apply_rotary(q, cos, sin), apply_rotary(k, cos, sin)
|
| 122 |
+
q, k = rms_norm(q) * self.qk_scale, rms_norm(k) * self.qk_scale
|
| 123 |
+
q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
|
| 124 |
+
if past_key_values is not None:
|
| 125 |
+
k, v = past_key_values.update(k, v, self.layer_idx)
|
| 126 |
+
y = F.scaled_dot_product_attention(
|
| 127 |
+
q, k, v, attn_mask=mask.mask, is_causal=mask.is_causal, enable_gqa=self.num_kv_heads != self.num_heads
|
| 128 |
+
)
|
| 129 |
+
return self.o_proj(y.transpose(1, 2).reshape(batch, time, -1))
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
class WildAIMLP(nn.Module):
|
| 133 |
+
def __init__(self, config: WildAIConfig) -> None:
|
| 134 |
+
super().__init__()
|
| 135 |
+
self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
|
| 136 |
+
self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
|
| 137 |
+
|
| 138 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 139 |
+
return self.down_proj(F.relu(self.up_proj(x)).square())
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
class WildAIDecoderLayer(nn.Module):
|
| 143 |
+
def __init__(self, config: WildAIConfig, layer_idx: int) -> None:
|
| 144 |
+
super().__init__()
|
| 145 |
+
self.self_attn = WildAIAttention(config, layer_idx)
|
| 146 |
+
self.mlp = WildAIMLP(config)
|
| 147 |
+
|
| 148 |
+
def forward(
|
| 149 |
+
self,
|
| 150 |
+
x: torch.Tensor,
|
| 151 |
+
input_ids: torch.LongTensor,
|
| 152 |
+
rotary: tuple[torch.Tensor, torch.Tensor],
|
| 153 |
+
mask: WindowMask,
|
| 154 |
+
past_key_values: Cache | None,
|
| 155 |
+
) -> torch.Tensor:
|
| 156 |
+
x = x + self.self_attn(rms_norm(x), input_ids, rotary, mask, past_key_values)
|
| 157 |
+
return x + self.mlp(rms_norm(x))
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
class WildAIPreTrainedModel(PreTrainedModel):
|
| 161 |
+
config_class = WildAIConfig
|
| 162 |
+
base_model_prefix = "model"
|
| 163 |
+
main_input_name = "input_ids"
|
| 164 |
+
_no_split_modules: ClassVar[list[str]] = ["WildAIDecoderLayer"]
|
| 165 |
+
_supports_sdpa = True
|
| 166 |
+
|
| 167 |
+
def _init_weights(self, module: nn.Module) -> None:
|
| 168 |
+
# Released models are always loaded from trained weights; this only gives from_config() models sane values.
|
| 169 |
+
if isinstance(module, (nn.Linear, nn.Embedding)):
|
| 170 |
+
nn.init.normal_(module.weight, mean=0.0, std=0.02)
|
| 171 |
+
elif isinstance(module, WildAIModel):
|
| 172 |
+
with torch.no_grad():
|
| 173 |
+
module.resid_lambdas.fill_(1.0)
|
| 174 |
+
module.x0_lambdas.zero_()
|
| 175 |
+
module.smear_lambda.zero_()
|
| 176 |
+
module.backout_lambda.zero_()
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
class WildAIModel(WildAIPreTrainedModel):
|
| 180 |
+
"""The transformer trunk; returns the final normalised hidden states."""
|
| 181 |
+
|
| 182 |
+
def __init__(self, config: WildAIConfig) -> None:
|
| 183 |
+
super().__init__(config)
|
| 184 |
+
n_layer = config.num_hidden_layers
|
| 185 |
+
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
|
| 186 |
+
self.layers = nn.ModuleList([WildAIDecoderLayer(config, i) for i in range(n_layer)])
|
| 187 |
+
self.resid_lambdas = nn.Parameter(torch.ones(n_layer))
|
| 188 |
+
self.x0_lambdas = nn.Parameter(torch.zeros(n_layer))
|
| 189 |
+
self.smear_gate = nn.Linear(config.smear_channels, 1, bias=False)
|
| 190 |
+
self.smear_lambda = nn.Parameter(torch.zeros(1))
|
| 191 |
+
self.backout_lambda = nn.Parameter(torch.zeros(1))
|
| 192 |
+
self.post_init()
|
| 193 |
+
|
| 194 |
+
def get_input_embeddings(self) -> nn.Embedding:
|
| 195 |
+
return self.embed_tokens
|
| 196 |
+
|
| 197 |
+
def set_input_embeddings(self, value: nn.Embedding) -> None:
|
| 198 |
+
self.embed_tokens = value
|
| 199 |
+
|
| 200 |
+
def rotary(self, position_ids: torch.LongTensor, dtype: torch.dtype) -> tuple[torch.Tensor, torch.Tensor]:
|
| 201 |
+
"""cos/sin tables of shape (batch, time, 1, head_dim / 2), computed in float32 and cast like in training."""
|
| 202 |
+
head_dim = self.config.head_dim
|
| 203 |
+
channels = torch.arange(0, head_dim, 2, dtype=torch.float32, device=position_ids.device)
|
| 204 |
+
inv_freq = 1.0 / (self.config.rotary_base ** (channels / head_dim))
|
| 205 |
+
freqs = position_ids.to(torch.float32)[..., None] * inv_freq
|
| 206 |
+
return freqs.cos().to(dtype)[:, :, None, :], freqs.sin().to(dtype)[:, :, None, :]
|
| 207 |
+
|
| 208 |
+
def smear(self, x: torch.Tensor, previous: torch.Tensor | None, padding_mask: torch.Tensor | None) -> torch.Tensor:
|
| 209 |
+
"""Add a gated copy of the previous position's embedding to every position that has one.
|
| 210 |
+
|
| 211 |
+
``previous`` is the normalised embedding of the token just before this chunk (None at the start of a sequence);
|
| 212 |
+
with left padding, the first real token has a padding token before it and gets nothing.
|
| 213 |
+
"""
|
| 214 |
+
channels = self.config.smear_channels
|
| 215 |
+
lam = self.smear_lambda.to(x.dtype)
|
| 216 |
+
first = x[:, :1]
|
| 217 |
+
if previous is not None:
|
| 218 |
+
first = first + lam * torch.sigmoid(self.smear_gate(first[..., :channels])) * previous
|
| 219 |
+
if x.size(1) == 1:
|
| 220 |
+
return first
|
| 221 |
+
gate = lam * torch.sigmoid(self.smear_gate(x[:, 1:, :channels]))
|
| 222 |
+
if padding_mask is not None:
|
| 223 |
+
gate = gate * padding_mask[:, -x.size(1) : -1, None].to(gate.dtype)
|
| 224 |
+
return torch.cat([first, x[:, 1:] + gate * x[:, :-1]], dim=1)
|
| 225 |
+
|
| 226 |
+
def forward(
|
| 227 |
+
self,
|
| 228 |
+
input_ids: torch.LongTensor,
|
| 229 |
+
attention_mask: torch.Tensor | None = None,
|
| 230 |
+
position_ids: torch.LongTensor | None = None,
|
| 231 |
+
past_key_values: Cache | None = None,
|
| 232 |
+
use_cache: bool | None = None,
|
| 233 |
+
previous_token_ids: torch.LongTensor | None = None,
|
| 234 |
+
output_hidden_states: bool | None = None,
|
| 235 |
+
**kwargs: object,
|
| 236 |
+
) -> BaseModelOutputWithPast:
|
| 237 |
+
"""``previous_token_ids`` (batch, 1): the token before ``input_ids[:, 0]``; required when continuing from a cache."""
|
| 238 |
+
if input_ids is None:
|
| 239 |
+
raise ValueError("input_ids are required: the value embeddings are looked up by token id")
|
| 240 |
+
batch, time = input_ids.shape
|
| 241 |
+
use_cache = self.config.use_cache if use_cache is None else use_cache
|
| 242 |
+
if use_cache and past_key_values is None:
|
| 243 |
+
past_key_values = DynamicCache()
|
| 244 |
+
past_len = past_key_values.get_seq_length() if past_key_values is not None else 0
|
| 245 |
+
if past_len > 0 and previous_token_ids is None:
|
| 246 |
+
raise ValueError("previous_token_ids is required when continuing from past_key_values (the smear needs it)")
|
| 247 |
+
padding_mask = None
|
| 248 |
+
if attention_mask is not None:
|
| 249 |
+
if attention_mask.shape != (batch, past_len + time):
|
| 250 |
+
raise ValueError(f"attention_mask must have shape {(batch, past_len + time)}, got {tuple(attention_mask.shape)}")
|
| 251 |
+
if not bool(attention_mask.all()):
|
| 252 |
+
padding_mask = attention_mask
|
| 253 |
+
if position_ids is None:
|
| 254 |
+
position_ids = torch.arange(past_len, past_len + time, device=input_ids.device).expand(batch, -1)
|
| 255 |
+
|
| 256 |
+
x = rms_norm(self.embed_tokens(input_ids))
|
| 257 |
+
previous = rms_norm(self.embed_tokens(previous_token_ids)) if previous_token_ids is not None else None
|
| 258 |
+
x = self.smear(x, previous, padding_mask)
|
| 259 |
+
rotary = self.rotary(position_ids, x.dtype)
|
| 260 |
+
masks = window_masks(self.config.window_sizes, time, past_len, padding_mask, input_ids.device)
|
| 261 |
+
|
| 262 |
+
x0 = x
|
| 263 |
+
backout = None
|
| 264 |
+
hidden_states: tuple[torch.Tensor, ...] = (x,)
|
| 265 |
+
for i, layer in enumerate(self.layers):
|
| 266 |
+
x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0
|
| 267 |
+
x = layer(x, input_ids, rotary, masks[layer.self_attn.window], past_key_values)
|
| 268 |
+
if i == self.config.backout_layer:
|
| 269 |
+
backout = x
|
| 270 |
+
hidden_states += (x,)
|
| 271 |
+
x = rms_norm(x - self.backout_lambda.to(x.dtype) * backout)
|
| 272 |
+
return BaseModelOutputWithPast(
|
| 273 |
+
last_hidden_state=x,
|
| 274 |
+
past_key_values=past_key_values if use_cache else None,
|
| 275 |
+
hidden_states=hidden_states if output_hidden_states else None,
|
| 276 |
+
)
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
class WildAIForCausalLM(WildAIPreTrainedModel, GenerationMixin):
|
| 280 |
+
def __init__(self, config: WildAIConfig) -> None:
|
| 281 |
+
super().__init__(config)
|
| 282 |
+
self.model = WildAIModel(config)
|
| 283 |
+
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
|
| 284 |
+
self.post_init()
|
| 285 |
+
|
| 286 |
+
def get_input_embeddings(self) -> nn.Embedding:
|
| 287 |
+
return self.model.embed_tokens
|
| 288 |
+
|
| 289 |
+
def set_input_embeddings(self, value: nn.Embedding) -> None:
|
| 290 |
+
self.model.embed_tokens = value
|
| 291 |
+
|
| 292 |
+
def get_output_embeddings(self) -> nn.Linear:
|
| 293 |
+
return self.lm_head
|
| 294 |
+
|
| 295 |
+
def set_output_embeddings(self, value: nn.Linear) -> None:
|
| 296 |
+
self.lm_head = value
|
| 297 |
+
|
| 298 |
+
def forward(
|
| 299 |
+
self,
|
| 300 |
+
input_ids: torch.LongTensor,
|
| 301 |
+
attention_mask: torch.Tensor | None = None,
|
| 302 |
+
position_ids: torch.LongTensor | None = None,
|
| 303 |
+
past_key_values: Cache | None = None,
|
| 304 |
+
labels: torch.LongTensor | None = None,
|
| 305 |
+
use_cache: bool | None = None,
|
| 306 |
+
previous_token_ids: torch.LongTensor | None = None,
|
| 307 |
+
output_hidden_states: bool | None = None,
|
| 308 |
+
logits_to_keep: int = 0,
|
| 309 |
+
return_dict: bool | None = None,
|
| 310 |
+
**kwargs: object,
|
| 311 |
+
) -> CausalLMOutputWithPast | tuple[torch.Tensor, ...]:
|
| 312 |
+
"""Float32 logits after the soft cap. ``labels`` are shifted inside (standard causal-LM loss, -100 ignored)."""
|
| 313 |
+
outputs = self.model(
|
| 314 |
+
input_ids,
|
| 315 |
+
attention_mask=attention_mask,
|
| 316 |
+
position_ids=position_ids,
|
| 317 |
+
past_key_values=past_key_values,
|
| 318 |
+
use_cache=use_cache,
|
| 319 |
+
previous_token_ids=previous_token_ids,
|
| 320 |
+
output_hidden_states=output_hidden_states,
|
| 321 |
+
)
|
| 322 |
+
hidden = outputs.last_hidden_state[:, -logits_to_keep:] if logits_to_keep else outputs.last_hidden_state
|
| 323 |
+
logits = self.lm_head(hidden).float()
|
| 324 |
+
cap = self.config.logit_softcap
|
| 325 |
+
logits = cap * torch.tanh(logits / cap)
|
| 326 |
+
loss = None
|
| 327 |
+
if labels is not None:
|
| 328 |
+
loss = F.cross_entropy(logits[:, :-1].flatten(0, 1), labels[:, 1:].flatten().to(logits.device), ignore_index=-100)
|
| 329 |
+
result = CausalLMOutputWithPast(
|
| 330 |
+
loss=loss, logits=logits, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states
|
| 331 |
+
)
|
| 332 |
+
return result.to_tuple() if return_dict is False else result
|
| 333 |
+
|
| 334 |
+
def prepare_inputs_for_generation(self, input_ids: torch.LongTensor, **kwargs: object) -> dict[str, object]:
|
| 335 |
+
# The smear mixes in the embedding of the token before the first new one; take it from the full sequence.
|
| 336 |
+
model_inputs = super().prepare_inputs_for_generation(input_ids, **kwargs)
|
| 337 |
+
new_len = model_inputs["input_ids"].shape[1]
|
| 338 |
+
if input_ids.shape[1] > new_len:
|
| 339 |
+
model_inputs["previous_token_ids"] = input_ids[:, -new_len - 1 : -new_len]
|
| 340 |
+
return model_inputs
|
268m-g072-ai-r0.5/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
268m-g072-ai-r0.5/tokenizer_config.json
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 3 |
+
"bos_token": "<|bos|>",
|
| 4 |
+
"eos_token": "<|bos|>",
|
| 5 |
+
"model_max_length": 2048,
|
| 6 |
+
"model_input_names": [
|
| 7 |
+
"input_ids",
|
| 8 |
+
"attention_mask"
|
| 9 |
+
],
|
| 10 |
+
"clean_up_tokenization_spaces": false
|
| 11 |
+
}
|
268m-g072-human-r0.005/LICENSE
ADDED
|
@@ -0,0 +1,438 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Attribution-NonCommercial-ShareAlike 4.0 International
|
| 2 |
+
|
| 3 |
+
=======================================================================
|
| 4 |
+
|
| 5 |
+
Creative Commons Corporation ("Creative Commons") is not a law firm and
|
| 6 |
+
does not provide legal services or legal advice. Distribution of
|
| 7 |
+
Creative Commons public licenses does not create a lawyer-client or
|
| 8 |
+
other relationship. Creative Commons makes its licenses and related
|
| 9 |
+
information available on an "as-is" basis. Creative Commons gives no
|
| 10 |
+
warranties regarding its licenses, any material licensed under their
|
| 11 |
+
terms and conditions, or any related information. Creative Commons
|
| 12 |
+
disclaims all liability for damages resulting from their use to the
|
| 13 |
+
fullest extent possible.
|
| 14 |
+
|
| 15 |
+
Using Creative Commons Public Licenses
|
| 16 |
+
|
| 17 |
+
Creative Commons public licenses provide a standard set of terms and
|
| 18 |
+
conditions that creators and other rights holders may use to share
|
| 19 |
+
original works of authorship and other material subject to copyright
|
| 20 |
+
and certain other rights specified in the public license below. The
|
| 21 |
+
following considerations are for informational purposes only, are not
|
| 22 |
+
exhaustive, and do not form part of our licenses.
|
| 23 |
+
|
| 24 |
+
Considerations for licensors: Our public licenses are
|
| 25 |
+
intended for use by those authorized to give the public
|
| 26 |
+
permission to use material in ways otherwise restricted by
|
| 27 |
+
copyright and certain other rights. Our licenses are
|
| 28 |
+
irrevocable. Licensors should read and understand the terms
|
| 29 |
+
and conditions of the license they choose before applying it.
|
| 30 |
+
Licensors should also secure all rights necessary before
|
| 31 |
+
applying our licenses so that the public can reuse the
|
| 32 |
+
material as expected. Licensors should clearly mark any
|
| 33 |
+
material not subject to the license. This includes other CC-
|
| 34 |
+
licensed material, or material used under an exception or
|
| 35 |
+
limitation to copyright. More considerations for licensors:
|
| 36 |
+
wiki.creativecommons.org/Considerations_for_licensors
|
| 37 |
+
|
| 38 |
+
Considerations for the public: By using one of our public
|
| 39 |
+
licenses, a licensor grants the public permission to use the
|
| 40 |
+
licensed material under specified terms and conditions. If
|
| 41 |
+
the licensor's permission is not necessary for any reason--for
|
| 42 |
+
example, because of any applicable exception or limitation to
|
| 43 |
+
copyright--then that use is not regulated by the license. Our
|
| 44 |
+
licenses grant only permissions under copyright and certain
|
| 45 |
+
other rights that a licensor has authority to grant. Use of
|
| 46 |
+
the licensed material may still be restricted for other
|
| 47 |
+
reasons, including because others have copyright or other
|
| 48 |
+
rights in the material. A licensor may make special requests,
|
| 49 |
+
such as asking that all changes be marked or described.
|
| 50 |
+
Although not required by our licenses, you are encouraged to
|
| 51 |
+
respect those requests where reasonable. More considerations
|
| 52 |
+
for the public:
|
| 53 |
+
wiki.creativecommons.org/Considerations_for_licensees
|
| 54 |
+
|
| 55 |
+
=======================================================================
|
| 56 |
+
|
| 57 |
+
Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International
|
| 58 |
+
Public License
|
| 59 |
+
|
| 60 |
+
By exercising the Licensed Rights (defined below), You accept and agree
|
| 61 |
+
to be bound by the terms and conditions of this Creative Commons
|
| 62 |
+
Attribution-NonCommercial-ShareAlike 4.0 International Public License
|
| 63 |
+
("Public License"). To the extent this Public License may be
|
| 64 |
+
interpreted as a contract, You are granted the Licensed Rights in
|
| 65 |
+
consideration of Your acceptance of these terms and conditions, and the
|
| 66 |
+
Licensor grants You such rights in consideration of benefits the
|
| 67 |
+
Licensor receives from making the Licensed Material available under
|
| 68 |
+
these terms and conditions.
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
Section 1 -- Definitions.
|
| 72 |
+
|
| 73 |
+
a. Adapted Material means material subject to Copyright and Similar
|
| 74 |
+
Rights that is derived from or based upon the Licensed Material
|
| 75 |
+
and in which the Licensed Material is translated, altered,
|
| 76 |
+
arranged, transformed, or otherwise modified in a manner requiring
|
| 77 |
+
permission under the Copyright and Similar Rights held by the
|
| 78 |
+
Licensor. For purposes of this Public License, where the Licensed
|
| 79 |
+
Material is a musical work, performance, or sound recording,
|
| 80 |
+
Adapted Material is always produced where the Licensed Material is
|
| 81 |
+
synched in timed relation with a moving image.
|
| 82 |
+
|
| 83 |
+
b. Adapter's License means the license You apply to Your Copyright
|
| 84 |
+
and Similar Rights in Your contributions to Adapted Material in
|
| 85 |
+
accordance with the terms and conditions of this Public License.
|
| 86 |
+
|
| 87 |
+
c. BY-NC-SA Compatible License means a license listed at
|
| 88 |
+
creativecommons.org/compatiblelicenses, approved by Creative
|
| 89 |
+
Commons as essentially the equivalent of this Public License.
|
| 90 |
+
|
| 91 |
+
d. Copyright and Similar Rights means copyright and/or similar rights
|
| 92 |
+
closely related to copyright including, without limitation,
|
| 93 |
+
performance, broadcast, sound recording, and Sui Generis Database
|
| 94 |
+
Rights, without regard to how the rights are labeled or
|
| 95 |
+
categorized. For purposes of this Public License, the rights
|
| 96 |
+
specified in Section 2(b)(1)-(2) are not Copyright and Similar
|
| 97 |
+
Rights.
|
| 98 |
+
|
| 99 |
+
e. Effective Technological Measures means those measures that, in the
|
| 100 |
+
absence of proper authority, may not be circumvented under laws
|
| 101 |
+
fulfilling obligations under Article 11 of the WIPO Copyright
|
| 102 |
+
Treaty adopted on December 20, 1996, and/or similar international
|
| 103 |
+
agreements.
|
| 104 |
+
|
| 105 |
+
f. Exceptions and Limitations means fair use, fair dealing, and/or
|
| 106 |
+
any other exception or limitation to Copyright and Similar Rights
|
| 107 |
+
that applies to Your use of the Licensed Material.
|
| 108 |
+
|
| 109 |
+
g. License Elements means the license attributes listed in the name
|
| 110 |
+
of a Creative Commons Public License. The License Elements of this
|
| 111 |
+
Public License are Attribution, NonCommercial, and ShareAlike.
|
| 112 |
+
|
| 113 |
+
h. Licensed Material means the artistic or literary work, database,
|
| 114 |
+
or other material to which the Licensor applied this Public
|
| 115 |
+
License.
|
| 116 |
+
|
| 117 |
+
i. Licensed Rights means the rights granted to You subject to the
|
| 118 |
+
terms and conditions of this Public License, which are limited to
|
| 119 |
+
all Copyright and Similar Rights that apply to Your use of the
|
| 120 |
+
Licensed Material and that the Licensor has authority to license.
|
| 121 |
+
|
| 122 |
+
j. Licensor means the individual(s) or entity(ies) granting rights
|
| 123 |
+
under this Public License.
|
| 124 |
+
|
| 125 |
+
k. NonCommercial means not primarily intended for or directed towards
|
| 126 |
+
commercial advantage or monetary compensation. For purposes of
|
| 127 |
+
this Public License, the exchange of the Licensed Material for
|
| 128 |
+
other material subject to Copyright and Similar Rights by digital
|
| 129 |
+
file-sharing or similar means is NonCommercial provided there is
|
| 130 |
+
no payment of monetary compensation in connection with the
|
| 131 |
+
exchange.
|
| 132 |
+
|
| 133 |
+
l. Share means to provide material to the public by any means or
|
| 134 |
+
process that requires permission under the Licensed Rights, such
|
| 135 |
+
as reproduction, public display, public performance, distribution,
|
| 136 |
+
dissemination, communication, or importation, and to make material
|
| 137 |
+
available to the public including in ways that members of the
|
| 138 |
+
public may access the material from a place and at a time
|
| 139 |
+
individually chosen by them.
|
| 140 |
+
|
| 141 |
+
m. Sui Generis Database Rights means rights other than copyright
|
| 142 |
+
resulting from Directive 96/9/EC of the European Parliament and of
|
| 143 |
+
the Council of 11 March 1996 on the legal protection of databases,
|
| 144 |
+
as amended and/or succeeded, as well as other essentially
|
| 145 |
+
equivalent rights anywhere in the world.
|
| 146 |
+
|
| 147 |
+
n. You means the individual or entity exercising the Licensed Rights
|
| 148 |
+
under this Public License. Your has a corresponding meaning.
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
Section 2 -- Scope.
|
| 152 |
+
|
| 153 |
+
a. License grant.
|
| 154 |
+
|
| 155 |
+
1. Subject to the terms and conditions of this Public License,
|
| 156 |
+
the Licensor hereby grants You a worldwide, royalty-free,
|
| 157 |
+
non-sublicensable, non-exclusive, irrevocable license to
|
| 158 |
+
exercise the Licensed Rights in the Licensed Material to:
|
| 159 |
+
|
| 160 |
+
a. reproduce and Share the Licensed Material, in whole or
|
| 161 |
+
in part, for NonCommercial purposes only; and
|
| 162 |
+
|
| 163 |
+
b. produce, reproduce, and Share Adapted Material for
|
| 164 |
+
NonCommercial purposes only.
|
| 165 |
+
|
| 166 |
+
2. Exceptions and Limitations. For the avoidance of doubt, where
|
| 167 |
+
Exceptions and Limitations apply to Your use, this Public
|
| 168 |
+
License does not apply, and You do not need to comply with
|
| 169 |
+
its terms and conditions.
|
| 170 |
+
|
| 171 |
+
3. Term. The term of this Public License is specified in Section
|
| 172 |
+
6(a).
|
| 173 |
+
|
| 174 |
+
4. Media and formats; technical modifications allowed. The
|
| 175 |
+
Licensor authorizes You to exercise the Licensed Rights in
|
| 176 |
+
all media and formats whether now known or hereafter created,
|
| 177 |
+
and to make technical modifications necessary to do so. The
|
| 178 |
+
Licensor waives and/or agrees not to assert any right or
|
| 179 |
+
authority to forbid You from making technical modifications
|
| 180 |
+
necessary to exercise the Licensed Rights, including
|
| 181 |
+
technical modifications necessary to circumvent Effective
|
| 182 |
+
Technological Measures. For purposes of this Public License,
|
| 183 |
+
simply making modifications authorized by this Section 2(a)
|
| 184 |
+
(4) never produces Adapted Material.
|
| 185 |
+
|
| 186 |
+
5. Downstream recipients.
|
| 187 |
+
|
| 188 |
+
a. Offer from the Licensor -- Licensed Material. Every
|
| 189 |
+
recipient of the Licensed Material automatically
|
| 190 |
+
receives an offer from the Licensor to exercise the
|
| 191 |
+
Licensed Rights under the terms and conditions of this
|
| 192 |
+
Public License.
|
| 193 |
+
|
| 194 |
+
b. Additional offer from the Licensor -- Adapted Material.
|
| 195 |
+
Every recipient of Adapted Material from You
|
| 196 |
+
automatically receives an offer from the Licensor to
|
| 197 |
+
exercise the Licensed Rights in the Adapted Material
|
| 198 |
+
under the conditions of the Adapter's License You apply.
|
| 199 |
+
|
| 200 |
+
c. No downstream restrictions. You may not offer or impose
|
| 201 |
+
any additional or different terms or conditions on, or
|
| 202 |
+
apply any Effective Technological Measures to, the
|
| 203 |
+
Licensed Material if doing so restricts exercise of the
|
| 204 |
+
Licensed Rights by any recipient of the Licensed
|
| 205 |
+
Material.
|
| 206 |
+
|
| 207 |
+
6. No endorsement. Nothing in this Public License constitutes or
|
| 208 |
+
may be construed as permission to assert or imply that You
|
| 209 |
+
are, or that Your use of the Licensed Material is, connected
|
| 210 |
+
with, or sponsored, endorsed, or granted official status by,
|
| 211 |
+
the Licensor or others designated to receive attribution as
|
| 212 |
+
provided in Section 3(a)(1)(A)(i).
|
| 213 |
+
|
| 214 |
+
b. Other rights.
|
| 215 |
+
|
| 216 |
+
1. Moral rights, such as the right of integrity, are not
|
| 217 |
+
licensed under this Public License, nor are publicity,
|
| 218 |
+
privacy, and/or other similar personality rights; however, to
|
| 219 |
+
the extent possible, the Licensor waives and/or agrees not to
|
| 220 |
+
assert any such rights held by the Licensor to the limited
|
| 221 |
+
extent necessary to allow You to exercise the Licensed
|
| 222 |
+
Rights, but not otherwise.
|
| 223 |
+
|
| 224 |
+
2. Patent and trademark rights are not licensed under this
|
| 225 |
+
Public License.
|
| 226 |
+
|
| 227 |
+
3. To the extent possible, the Licensor waives any right to
|
| 228 |
+
collect royalties from You for the exercise of the Licensed
|
| 229 |
+
Rights, whether directly or through a collecting society
|
| 230 |
+
under any voluntary or waivable statutory or compulsory
|
| 231 |
+
licensing scheme. In all other cases the Licensor expressly
|
| 232 |
+
reserves any right to collect such royalties, including when
|
| 233 |
+
the Licensed Material is used other than for NonCommercial
|
| 234 |
+
purposes.
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
Section 3 -- License Conditions.
|
| 238 |
+
|
| 239 |
+
Your exercise of the Licensed Rights is expressly made subject to the
|
| 240 |
+
following conditions.
|
| 241 |
+
|
| 242 |
+
a. Attribution.
|
| 243 |
+
|
| 244 |
+
1. If You Share the Licensed Material (including in modified
|
| 245 |
+
form), You must:
|
| 246 |
+
|
| 247 |
+
a. retain the following if it is supplied by the Licensor
|
| 248 |
+
with the Licensed Material:
|
| 249 |
+
|
| 250 |
+
i. identification of the creator(s) of the Licensed
|
| 251 |
+
Material and any others designated to receive
|
| 252 |
+
attribution, in any reasonable manner requested by
|
| 253 |
+
the Licensor (including by pseudonym if
|
| 254 |
+
designated);
|
| 255 |
+
|
| 256 |
+
ii. a copyright notice;
|
| 257 |
+
|
| 258 |
+
iii. a notice that refers to this Public License;
|
| 259 |
+
|
| 260 |
+
iv. a notice that refers to the disclaimer of
|
| 261 |
+
warranties;
|
| 262 |
+
|
| 263 |
+
v. a URI or hyperlink to the Licensed Material to the
|
| 264 |
+
extent reasonably practicable;
|
| 265 |
+
|
| 266 |
+
b. indicate if You modified the Licensed Material and
|
| 267 |
+
retain an indication of any previous modifications; and
|
| 268 |
+
|
| 269 |
+
c. indicate the Licensed Material is licensed under this
|
| 270 |
+
Public License, and include the text of, or the URI or
|
| 271 |
+
hyperlink to, this Public License.
|
| 272 |
+
|
| 273 |
+
2. You may satisfy the conditions in Section 3(a)(1) in any
|
| 274 |
+
reasonable manner based on the medium, means, and context in
|
| 275 |
+
which You Share the Licensed Material. For example, it may be
|
| 276 |
+
reasonable to satisfy the conditions by providing a URI or
|
| 277 |
+
hyperlink to a resource that includes the required
|
| 278 |
+
information.
|
| 279 |
+
3. If requested by the Licensor, You must remove any of the
|
| 280 |
+
information required by Section 3(a)(1)(A) to the extent
|
| 281 |
+
reasonably practicable.
|
| 282 |
+
|
| 283 |
+
b. ShareAlike.
|
| 284 |
+
|
| 285 |
+
In addition to the conditions in Section 3(a), if You Share
|
| 286 |
+
Adapted Material You produce, the following conditions also apply.
|
| 287 |
+
|
| 288 |
+
1. The Adapter's License You apply must be a Creative Commons
|
| 289 |
+
license with the same License Elements, this version or
|
| 290 |
+
later, or a BY-NC-SA Compatible License.
|
| 291 |
+
|
| 292 |
+
2. You must include the text of, or the URI or hyperlink to, the
|
| 293 |
+
Adapter's License You apply. You may satisfy this condition
|
| 294 |
+
in any reasonable manner based on the medium, means, and
|
| 295 |
+
context in which You Share Adapted Material.
|
| 296 |
+
|
| 297 |
+
3. You may not offer or impose any additional or different terms
|
| 298 |
+
or conditions on, or apply any Effective Technological
|
| 299 |
+
Measures to, Adapted Material that restrict exercise of the
|
| 300 |
+
rights granted under the Adapter's License You apply.
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
Section 4 -- Sui Generis Database Rights.
|
| 304 |
+
|
| 305 |
+
Where the Licensed Rights include Sui Generis Database Rights that
|
| 306 |
+
apply to Your use of the Licensed Material:
|
| 307 |
+
|
| 308 |
+
a. for the avoidance of doubt, Section 2(a)(1) grants You the right
|
| 309 |
+
to extract, reuse, reproduce, and Share all or a substantial
|
| 310 |
+
portion of the contents of the database for NonCommercial purposes
|
| 311 |
+
only;
|
| 312 |
+
|
| 313 |
+
b. if You include all or a substantial portion of the database
|
| 314 |
+
contents in a database in which You have Sui Generis Database
|
| 315 |
+
Rights, then the database in which You have Sui Generis Database
|
| 316 |
+
Rights (but not its individual contents) is Adapted Material,
|
| 317 |
+
including for purposes of Section 3(b); and
|
| 318 |
+
|
| 319 |
+
c. You must comply with the conditions in Section 3(a) if You Share
|
| 320 |
+
all or a substantial portion of the contents of the database.
|
| 321 |
+
|
| 322 |
+
For the avoidance of doubt, this Section 4 supplements and does not
|
| 323 |
+
replace Your obligations under this Public License where the Licensed
|
| 324 |
+
Rights include other Copyright and Similar Rights.
|
| 325 |
+
|
| 326 |
+
|
| 327 |
+
Section 5 -- Disclaimer of Warranties and Limitation of Liability.
|
| 328 |
+
|
| 329 |
+
a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
|
| 330 |
+
EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
|
| 331 |
+
AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
|
| 332 |
+
ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
|
| 333 |
+
IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
|
| 334 |
+
WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
|
| 335 |
+
PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
|
| 336 |
+
ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
|
| 337 |
+
KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
|
| 338 |
+
ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
|
| 339 |
+
|
| 340 |
+
b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
|
| 341 |
+
TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
|
| 342 |
+
NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
|
| 343 |
+
INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
|
| 344 |
+
COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
|
| 345 |
+
USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
|
| 346 |
+
ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
|
| 347 |
+
DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
|
| 348 |
+
IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
|
| 349 |
+
|
| 350 |
+
c. The disclaimer of warranties and limitation of liability provided
|
| 351 |
+
above shall be interpreted in a manner that, to the extent
|
| 352 |
+
possible, most closely approximates an absolute disclaimer and
|
| 353 |
+
waiver of all liability.
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
Section 6 -- Term and Termination.
|
| 357 |
+
|
| 358 |
+
a. This Public License applies for the term of the Copyright and
|
| 359 |
+
Similar Rights licensed here. However, if You fail to comply with
|
| 360 |
+
this Public License, then Your rights under this Public License
|
| 361 |
+
terminate automatically.
|
| 362 |
+
|
| 363 |
+
b. Where Your right to use the Licensed Material has terminated under
|
| 364 |
+
Section 6(a), it reinstates:
|
| 365 |
+
|
| 366 |
+
1. automatically as of the date the violation is cured, provided
|
| 367 |
+
it is cured within 30 days of Your discovery of the
|
| 368 |
+
violation; or
|
| 369 |
+
|
| 370 |
+
2. upon express reinstatement by the Licensor.
|
| 371 |
+
|
| 372 |
+
For the avoidance of doubt, this Section 6(b) does not affect any
|
| 373 |
+
right the Licensor may have to seek remedies for Your violations
|
| 374 |
+
of this Public License.
|
| 375 |
+
|
| 376 |
+
c. For the avoidance of doubt, the Licensor may also offer the
|
| 377 |
+
Licensed Material under separate terms or conditions or stop
|
| 378 |
+
distributing the Licensed Material at any time; however, doing so
|
| 379 |
+
will not terminate this Public License.
|
| 380 |
+
|
| 381 |
+
d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
|
| 382 |
+
License.
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
Section 7 -- Other Terms and Conditions.
|
| 386 |
+
|
| 387 |
+
a. The Licensor shall not be bound by any additional or different
|
| 388 |
+
terms or conditions communicated by You unless expressly agreed.
|
| 389 |
+
|
| 390 |
+
b. Any arrangements, understandings, or agreements regarding the
|
| 391 |
+
Licensed Material not stated herein are separate from and
|
| 392 |
+
independent of the terms and conditions of this Public License.
|
| 393 |
+
|
| 394 |
+
|
| 395 |
+
Section 8 -- Interpretation.
|
| 396 |
+
|
| 397 |
+
a. For the avoidance of doubt, this Public License does not, and
|
| 398 |
+
shall not be interpreted to, reduce, limit, restrict, or impose
|
| 399 |
+
conditions on any use of the Licensed Material that could lawfully
|
| 400 |
+
be made without permission under this Public License.
|
| 401 |
+
|
| 402 |
+
b. To the extent possible, if any provision of this Public License is
|
| 403 |
+
deemed unenforceable, it shall be automatically reformed to the
|
| 404 |
+
minimum extent necessary to make it enforceable. If the provision
|
| 405 |
+
cannot be reformed, it shall be severed from this Public License
|
| 406 |
+
without affecting the enforceability of the remaining terms and
|
| 407 |
+
conditions.
|
| 408 |
+
|
| 409 |
+
c. No term or condition of this Public License will be waived and no
|
| 410 |
+
failure to comply consented to unless expressly agreed to by the
|
| 411 |
+
Licensor.
|
| 412 |
+
|
| 413 |
+
d. Nothing in this Public License constitutes or may be interpreted
|
| 414 |
+
as a limitation upon, or waiver of, any privileges and immunities
|
| 415 |
+
that apply to the Licensor or You, including from the legal
|
| 416 |
+
processes of any jurisdiction or authority.
|
| 417 |
+
|
| 418 |
+
=======================================================================
|
| 419 |
+
|
| 420 |
+
Creative Commons is not a party to its public
|
| 421 |
+
licenses. Notwithstanding, Creative Commons may elect to apply one of
|
| 422 |
+
its public licenses to material it publishes and in those instances
|
| 423 |
+
will be considered the “Licensor.” The text of the Creative Commons
|
| 424 |
+
public licenses is dedicated to the public domain under the CC0 Public
|
| 425 |
+
Domain Dedication. Except for the limited purpose of indicating that
|
| 426 |
+
material is shared under a Creative Commons public license or as
|
| 427 |
+
otherwise permitted by the Creative Commons policies published at
|
| 428 |
+
creativecommons.org/policies, Creative Commons does not authorize the
|
| 429 |
+
use of the trademark "Creative Commons" or any other trademark or logo
|
| 430 |
+
of Creative Commons without its prior written consent including,
|
| 431 |
+
without limitation, in connection with any unauthorized modifications
|
| 432 |
+
to any of its public licenses or any other arrangements,
|
| 433 |
+
understandings, or agreements concerning use of licensed material. For
|
| 434 |
+
the avoidance of doubt, this paragraph does not form part of the
|
| 435 |
+
public licenses.
|
| 436 |
+
|
| 437 |
+
Creative Commons may be contacted at creativecommons.org.
|
| 438 |
+
|
268m-g072-human-r0.005/README.md
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: cc-by-nc-sa-4.0
|
| 3 |
+
library_name: transformers
|
| 4 |
+
pipeline_tag: text-generation
|
| 5 |
+
language:
|
| 6 |
+
- en
|
| 7 |
+
datasets:
|
| 8 |
+
- pangram/WildAI
|
| 9 |
+
tags:
|
| 10 |
+
- wildai
|
| 11 |
+
- nanochat
|
| 12 |
+
- scaling-laws
|
| 13 |
+
- ai-generated-text
|
| 14 |
+
- pretraining-data
|
| 15 |
+
---
|
| 16 |
+
|
| 17 |
+
# 268m-g072-human-r0.005
|
| 18 |
+
|
| 19 |
+
> **License: [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/).** The WildAI code, models and dataset are released under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license: you may use and adapt them for non-commercial purposes, with attribution, and must share adaptations under the same license.
|
| 20 |
+
|
| 21 |
+
A 268M decoder-only language model trained from scratch for [*How Much Is an AI Token Worth? Scaling Laws for Wild AI-Generated Web Text*](https://arxiv.org/abs/2609.40295), one of the paper's models measuring how wild AI-generated web text in pretraining data changes a model's loss. It is a research artifact for studying scaling laws, not an assistant: it has had no instruction tuning or safety training.
|
| 22 |
+
|
| 23 |
+
## Training data
|
| 24 |
+
|
| 25 |
+
The human documents of `268m-g072-control` plus fresh human-written web documents (0.005 times the control's tokens): 1.18B human tokens, the human-text counterpart of adding AI text. No AI text.
|
| 26 |
+
|
| 27 |
+
| | |
|
| 28 |
+
|---|---|
|
| 29 |
+
| Parameters (N, input embedding included, value embeddings excluded) | 268.4M |
|
| 30 |
+
| Layers / width | 16 / 1024 |
|
| 31 |
+
| Training tokens | 1.18B (2,251 optimizer steps) |
|
| 32 |
+
| Human / AI tokens | 1.18B / 0.0M |
|
| 33 |
+
| Group / arm / seed | g072 / human / 1337 |
|
| 34 |
+
| Used in the paper for | fitting the scaling laws (726 models, 19.9M to 268M) |
|
| 35 |
+
|
| 36 |
+
## Held-out loss
|
| 37 |
+
|
| 38 |
+
| Set | Bits per byte | Text |
|
| 39 |
+
|---|---|---|
|
| 40 |
+
| C4 | 0.9966 | C4 validation text (human-written, pre-2022) |
|
| 41 |
+
| FW22 | 0.9811 | held-out FineWeb documents crawled before 2022 |
|
| 42 |
+
| Paloma | 1.3630 | macro average over 16 Paloma sources |
|
| 43 |
+
| FW26 | 0.8697 | held-out 2026 FineWeb-style crawl (22.3% AI tokens) |
|
| 44 |
+
| FW26-H | 0.9139 | the human-labeled documents of FW26 |
|
| 45 |
+
| FW26-AI | 0.7070 | the AI-labeled documents of FW26 |
|
| 46 |
+
| Cosmo | 0.7050 | Cosmopedia (AI-generated textbooks and stories) |
|
| 47 |
+
|
| 48 |
+
<details><summary>Paloma sources</summary>
|
| 49 |
+
|
| 50 |
+
| Source | Bits per byte |
|
| 51 |
+
|---|---|
|
| 52 |
+
| 4chan | 1.3835 |
|
| 53 |
+
| c4 | 1.0147 |
|
| 54 |
+
| c4_100_domains | 0.9970 |
|
| 55 |
+
| dolma | 1.2555 |
|
| 56 |
+
| falcon_refinedweb | 1.1076 |
|
| 57 |
+
| gab | 1.9004 |
|
| 58 |
+
| m2d2_s2orc | 1.1690 |
|
| 59 |
+
| m2d2_wikipedia | 1.0736 |
|
| 60 |
+
| manosphere | 1.3908 |
|
| 61 |
+
| mc4_en | 1.1000 |
|
| 62 |
+
| programming | 1.9443 |
|
| 63 |
+
| ptb | 1.2327 |
|
| 64 |
+
| reddit_100 | 1.2526 |
|
| 65 |
+
| redpajama | 1.4196 |
|
| 66 |
+
| twitteraae | 2.4920 |
|
| 67 |
+
| wikitext_103 | 1.0751 |
|
| 68 |
+
|
| 69 |
+
</details>
|
| 70 |
+
|
| 71 |
+
## Usage
|
| 72 |
+
|
| 73 |
+
```python
|
| 74 |
+
import torch
|
| 75 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 76 |
+
|
| 77 |
+
tokenizer = AutoTokenizer.from_pretrained("pangram/WildAI-models", subfolder="268m-g072-human-r0.005", trust_remote_code=True)
|
| 78 |
+
model = AutoModelForCausalLM.from_pretrained("pangram/WildAI-models", subfolder="268m-g072-human-r0.005", trust_remote_code=True, dtype=torch.bfloat16)
|
| 79 |
+
|
| 80 |
+
inputs = tokenizer("The history of the printing press", return_tensors="pt")
|
| 81 |
+
output = model.generate(**inputs, max_new_tokens=40, do_sample=False)
|
| 82 |
+
print(tokenizer.decode(output[0], skip_special_tokens=True))
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
The tokenizer prepends `<|bos|>` to every text, as in training; pass `add_special_tokens=False` to skip it. The model predicts `<|bos|>` where a document ends, and generation stops there. Weights are stored in bfloat16, the precision the paper's losses were computed in; `dtype=torch.float32` computes in float32 instead.
|
| 86 |
+
|
| 87 |
+
## Architecture
|
| 88 |
+
|
| 89 |
+
The nanochat GPT recipe: rotary embeddings, RMSNorm without weights (also on queries and keys), squared-ReLU MLP, attention windows in a repeating short-short-short-long pattern (512 tokens, then the full 2,048-token context; the last layer always full), gated value embeddings on alternating layers, a previous-token smear, per-layer residual scalars, a mid-network backout, untied input and output embeddings and logits soft-capped at 15. Vocabulary 32,768 tokens, context 2,048. The code ships with the weights (`trust_remote_code=True`) and matches the training code's forward pass.
|
| 90 |
+
|
| 91 |
+
## License
|
| 92 |
+
|
| 93 |
+
**License: [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/).** The WildAI code, models and dataset are released under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license: you may use and adapt them for non-commercial purposes, with attribution, and must share adaptations under the same license. The full text is in `LICENSE`.
|
| 94 |
+
|
| 95 |
+
## Citation
|
| 96 |
+
|
| 97 |
+
```bibtex
|
| 98 |
+
@article{russell2026wildai,
|
| 99 |
+
title = {How Much Is an AI Token Worth? Scaling Laws for Wild AI-Generated Web Text},
|
| 100 |
+
author = {Russell, Jenna and Glickenhaus, Ben and Thai, Katherine and Wieting, John and Iyyer, Mohit and Spero, Max and Emi, Bradley},
|
| 101 |
+
journal = {arXiv preprint arXiv:2609.40295},
|
| 102 |
+
year = {2026},
|
| 103 |
+
url = {https://arxiv.org/abs/2609.40295}
|
| 104 |
+
}
|
| 105 |
+
```
|
268m-g072-human-r0.005/config.json
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"WildAIForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"auto_map": {
|
| 6 |
+
"AutoConfig": "configuration_wildai.WildAIConfig",
|
| 7 |
+
"AutoModelForCausalLM": "modeling_wildai.WildAIForCausalLM"
|
| 8 |
+
},
|
| 9 |
+
"bos_token_id": 32759,
|
| 10 |
+
"dtype": "bfloat16",
|
| 11 |
+
"eos_token_id": 32759,
|
| 12 |
+
"hidden_size": 1024,
|
| 13 |
+
"logit_softcap": 15.0,
|
| 14 |
+
"max_position_embeddings": 2048,
|
| 15 |
+
"model_type": "wildai",
|
| 16 |
+
"num_attention_heads": 8,
|
| 17 |
+
"num_hidden_layers": 16,
|
| 18 |
+
"num_key_value_heads": 8,
|
| 19 |
+
"pad_token_id": 32759,
|
| 20 |
+
"qk_scale": 1.2,
|
| 21 |
+
"release": {
|
| 22 |
+
"added_ratio": 0.005,
|
| 23 |
+
"ai_ratio": 0.0,
|
| 24 |
+
"ai_tokens": 0,
|
| 25 |
+
"arm": "human",
|
| 26 |
+
"depth": 16,
|
| 27 |
+
"group": "g072",
|
| 28 |
+
"human_tokens": 1180172288,
|
| 29 |
+
"n_params": 268436282,
|
| 30 |
+
"name": "268m-g072-human-r0.005",
|
| 31 |
+
"seed": 1337,
|
| 32 |
+
"size": "268m",
|
| 33 |
+
"split": "fit",
|
| 34 |
+
"steps": 2251,
|
| 35 |
+
"total_tokens": 1180172288
|
| 36 |
+
},
|
| 37 |
+
"rotary_base": 100000.0,
|
| 38 |
+
"smear_channels": 24,
|
| 39 |
+
"tie_word_embeddings": false,
|
| 40 |
+
"transformers_version": "5.17.0",
|
| 41 |
+
"use_cache": true,
|
| 42 |
+
"value_embed_gate_channels": 12,
|
| 43 |
+
"value_embed_gate_range": 3.0,
|
| 44 |
+
"vocab_size": 32768,
|
| 45 |
+
"window_pattern": "SSSL"
|
| 46 |
+
}
|
268m-g072-human-r0.005/configuration_wildai.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Configuration of the WildAI language models (a nanochat GPT variant) for Hugging Face ``transformers``.
|
| 2 |
+
|
| 3 |
+
This file is copied verbatim into every released model repository and loaded with ``trust_remote_code=True``.
|
| 4 |
+
It depends only on ``transformers``.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from typing import ClassVar
|
| 10 |
+
|
| 11 |
+
from transformers import PretrainedConfig
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class WildAIConfig(PretrainedConfig):
|
| 15 |
+
"""Architecture of a WildAI model. Only width and depth vary across the released models.
|
| 16 |
+
|
| 17 |
+
Every layer uses causal attention limited to a left window: a query attends to itself and the ``window`` tokens before
|
| 18 |
+
it. ``window_pattern`` is tiled over the layers ("S" = a quarter of the context rounded up to a multiple of 128 tokens,
|
| 19 |
+
"L" = the full context) and the last layer always uses "L".
|
| 20 |
+
|
| 21 |
+
``release`` holds the model's entry in the release catalog (name, token counts, AI ratio, ...); it does not affect
|
| 22 |
+
the forward pass.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
model_type = "wildai"
|
| 26 |
+
keys_to_ignore_at_inference: ClassVar[list[str]] = ["past_key_values"]
|
| 27 |
+
|
| 28 |
+
def __init__(
|
| 29 |
+
self,
|
| 30 |
+
vocab_size: int = 32768,
|
| 31 |
+
hidden_size: int = 768,
|
| 32 |
+
num_hidden_layers: int = 12,
|
| 33 |
+
num_attention_heads: int = 6,
|
| 34 |
+
num_key_value_heads: int = 6,
|
| 35 |
+
max_position_embeddings: int = 2048,
|
| 36 |
+
window_pattern: str = "SSSL",
|
| 37 |
+
rotary_base: float = 100000.0,
|
| 38 |
+
qk_scale: float = 1.2,
|
| 39 |
+
logit_softcap: float = 15.0,
|
| 40 |
+
value_embed_gate_channels: int = 12,
|
| 41 |
+
value_embed_gate_range: float = 3.0,
|
| 42 |
+
smear_channels: int = 24,
|
| 43 |
+
bos_token_id: int = 32759,
|
| 44 |
+
eos_token_id: int = 32759,
|
| 45 |
+
use_cache: bool = True,
|
| 46 |
+
release: dict[str, str | int | float | None] | None = None,
|
| 47 |
+
**kwargs: object,
|
| 48 |
+
) -> None:
|
| 49 |
+
if hidden_size % num_attention_heads != 0 or num_attention_heads % num_key_value_heads != 0:
|
| 50 |
+
raise ValueError("hidden_size must divide into num_attention_heads, and those into num_key_value_heads")
|
| 51 |
+
if not window_pattern or set(window_pattern.upper()) - {"S", "L"}:
|
| 52 |
+
raise ValueError(f"window_pattern must be a non-empty string of S and L, got {window_pattern!r}")
|
| 53 |
+
self.vocab_size = vocab_size
|
| 54 |
+
self.hidden_size = hidden_size
|
| 55 |
+
self.num_hidden_layers = num_hidden_layers
|
| 56 |
+
self.num_attention_heads = num_attention_heads
|
| 57 |
+
self.num_key_value_heads = num_key_value_heads
|
| 58 |
+
self.max_position_embeddings = max_position_embeddings
|
| 59 |
+
self.window_pattern = window_pattern.upper()
|
| 60 |
+
self.rotary_base = rotary_base
|
| 61 |
+
self.qk_scale = qk_scale
|
| 62 |
+
self.logit_softcap = logit_softcap
|
| 63 |
+
self.value_embed_gate_channels = value_embed_gate_channels
|
| 64 |
+
self.value_embed_gate_range = value_embed_gate_range
|
| 65 |
+
self.smear_channels = smear_channels
|
| 66 |
+
self.use_cache = use_cache
|
| 67 |
+
self.release = release
|
| 68 |
+
kwargs.setdefault("tie_word_embeddings", False)
|
| 69 |
+
super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
|
| 70 |
+
|
| 71 |
+
@property
|
| 72 |
+
def head_dim(self) -> int:
|
| 73 |
+
return self.hidden_size // self.num_attention_heads
|
| 74 |
+
|
| 75 |
+
@property
|
| 76 |
+
def intermediate_size(self) -> int:
|
| 77 |
+
return 4 * self.hidden_size
|
| 78 |
+
|
| 79 |
+
@property
|
| 80 |
+
def window_sizes(self) -> list[int]:
|
| 81 |
+
"""Left attention window of each layer, in tokens."""
|
| 82 |
+
long_window = self.max_position_embeddings
|
| 83 |
+
short_window = -(-long_window // 4 // 128) * 128 # a quarter of the context, rounded up to a multiple of 128 (2048 -> 512)
|
| 84 |
+
pattern = self.window_pattern
|
| 85 |
+
sizes = [short_window if pattern[i % len(pattern)] == "S" else long_window for i in range(self.num_hidden_layers)]
|
| 86 |
+
sizes[-1] = long_window
|
| 87 |
+
return sizes
|
| 88 |
+
|
| 89 |
+
def has_value_embedding(self, layer_idx: int) -> bool:
|
| 90 |
+
"""Value embeddings sit on alternating layers, always including the last one."""
|
| 91 |
+
return layer_idx % 2 == (self.num_hidden_layers - 1) % 2
|
| 92 |
+
|
| 93 |
+
@property
|
| 94 |
+
def backout_layer(self) -> int:
|
| 95 |
+
"""The residual stream after this layer is partly subtracted before the final norm."""
|
| 96 |
+
return self.num_hidden_layers // 2
|
268m-g072-human-r0.005/generation_config.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 32759,
|
| 4 |
+
"eos_token_id": 32759,
|
| 5 |
+
"output_attentions": false,
|
| 6 |
+
"output_hidden_states": false,
|
| 7 |
+
"pad_token_id": 32759,
|
| 8 |
+
"transformers_version": "5.17.0",
|
| 9 |
+
"use_cache": true
|
| 10 |
+
}
|
268m-g072-human-r0.005/modeling_wildai.py
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The WildAI language models (a nanochat GPT variant) for Hugging Face ``transformers``.
|
| 2 |
+
|
| 3 |
+
This file is copied verbatim into every released model repository and loaded with ``trust_remote_code=True``.
|
| 4 |
+
It depends only on ``torch`` and ``transformers`` and reproduces the training code's forward pass operation by
|
| 5 |
+
operation, so a model loaded in bfloat16 computes the same function the paper evaluated:
|
| 6 |
+
|
| 7 |
+
- token embedding followed by a parameter-free RMSNorm, then a learned "smear" that mixes each position's embedding
|
| 8 |
+
with the previous token's;
|
| 9 |
+
- per-layer scalars that rescale the residual stream and blend the initial embedding back in before every block;
|
| 10 |
+
- attention with rotary embeddings applied before a per-head RMSNorm of queries and keys (both scaled by 1.2),
|
| 11 |
+
a left sliding window per layer, and gated value embeddings added to the values on alternating layers;
|
| 12 |
+
- a squared-ReLU MLP, parameter-free pre-norms, no biases;
|
| 13 |
+
- "backout": part of the residual stream after the middle layer is subtracted before the final norm;
|
| 14 |
+
- untied output head with logits soft-capped by ``15 * tanh(logits / 15)``.
|
| 15 |
+
|
| 16 |
+
Attention uses PyTorch's scaled_dot_product_attention with explicit window masks, so no FlashAttention install is
|
| 17 |
+
needed. Generation uses the standard ``DynamicCache``.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
from dataclasses import dataclass
|
| 23 |
+
from typing import ClassVar
|
| 24 |
+
|
| 25 |
+
import torch
|
| 26 |
+
import torch.nn.functional as F
|
| 27 |
+
from torch import nn
|
| 28 |
+
from transformers import GenerationMixin, PreTrainedModel
|
| 29 |
+
from transformers.cache_utils import Cache, DynamicCache
|
| 30 |
+
from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
|
| 31 |
+
|
| 32 |
+
from .configuration_wildai import WildAIConfig
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def rms_norm(x: torch.Tensor) -> torch.Tensor:
|
| 36 |
+
"""Parameter-free RMSNorm over the last dimension with PyTorch's default epsilon, as in training."""
|
| 37 |
+
return F.rms_norm(x, (x.size(-1),))
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def apply_rotary(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
|
| 41 |
+
"""Rotate channel pairs (i, i + d/2) of ``x`` (batch, time, heads, head_dim), with the training code's sign convention."""
|
| 42 |
+
d = x.shape[-1] // 2
|
| 43 |
+
x1, x2 = x[..., :d], x[..., d:]
|
| 44 |
+
return torch.cat([x1 * cos + x2 * sin, x1 * (-sin) + x2 * cos], dim=-1)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
@dataclass(frozen=True)
|
| 48 |
+
class WindowMask:
|
| 49 |
+
"""Arguments for scaled_dot_product_attention that implement one attention window."""
|
| 50 |
+
|
| 51 |
+
mask: torch.Tensor | None
|
| 52 |
+
is_causal: bool
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def window_masks(
|
| 56 |
+
window_sizes: list[int], query_len: int, past_len: int, padding_mask: torch.Tensor | None, device: torch.device
|
| 57 |
+
) -> dict[int, WindowMask]:
|
| 58 |
+
"""Masks for each distinct left window. Query i (absolute index past_len + i) sees keys j with i - window <= j <= i.
|
| 59 |
+
|
| 60 |
+
``padding_mask`` is the (batch, past_len + query_len) attention mask when it contains padding, else None. A padding
|
| 61 |
+
query is always allowed to see itself so that its (unused) output stays finite.
|
| 62 |
+
"""
|
| 63 |
+
total = past_len + query_len
|
| 64 |
+
q_idx = torch.arange(past_len, total, device=device)[:, None]
|
| 65 |
+
k_idx = torch.arange(total, device=device)[None, :]
|
| 66 |
+
masks: dict[int, WindowMask] = {}
|
| 67 |
+
for window in sorted(set(window_sizes)):
|
| 68 |
+
if padding_mask is None and window >= total - 1:
|
| 69 |
+
if query_len == 1:
|
| 70 |
+
masks[window] = WindowMask(mask=None, is_causal=False)
|
| 71 |
+
continue
|
| 72 |
+
if past_len == 0:
|
| 73 |
+
masks[window] = WindowMask(mask=None, is_causal=True)
|
| 74 |
+
continue
|
| 75 |
+
allowed = (k_idx <= q_idx) & (q_idx - k_idx <= window)
|
| 76 |
+
if padding_mask is not None:
|
| 77 |
+
allowed = (allowed & padding_mask[:, None, None, :].bool()) | (k_idx == q_idx)
|
| 78 |
+
masks[window] = WindowMask(mask=allowed, is_causal=False)
|
| 79 |
+
return masks
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class WildAIAttention(nn.Module):
|
| 83 |
+
def __init__(self, config: WildAIConfig, layer_idx: int) -> None:
|
| 84 |
+
super().__init__()
|
| 85 |
+
self.layer_idx = layer_idx
|
| 86 |
+
self.num_heads = config.num_attention_heads
|
| 87 |
+
self.num_kv_heads = config.num_key_value_heads
|
| 88 |
+
self.head_dim = config.head_dim
|
| 89 |
+
self.qk_scale = config.qk_scale
|
| 90 |
+
self.window = config.window_sizes[layer_idx]
|
| 91 |
+
hidden, kv_dim = config.hidden_size, config.num_key_value_heads * config.head_dim
|
| 92 |
+
self.q_proj = nn.Linear(hidden, config.num_attention_heads * config.head_dim, bias=False)
|
| 93 |
+
self.k_proj = nn.Linear(hidden, kv_dim, bias=False)
|
| 94 |
+
self.v_proj = nn.Linear(hidden, kv_dim, bias=False)
|
| 95 |
+
self.o_proj = nn.Linear(hidden, hidden, bias=False)
|
| 96 |
+
self.value_embed: nn.Embedding | None = None
|
| 97 |
+
self.value_embed_gate: nn.Linear | None = None
|
| 98 |
+
if config.has_value_embedding(layer_idx):
|
| 99 |
+
self.value_embed = nn.Embedding(config.vocab_size, kv_dim)
|
| 100 |
+
self.value_embed_gate = nn.Linear(config.value_embed_gate_channels, config.num_key_value_heads, bias=False)
|
| 101 |
+
self.gate_channels = config.value_embed_gate_channels
|
| 102 |
+
self.gate_range = config.value_embed_gate_range
|
| 103 |
+
|
| 104 |
+
def forward(
|
| 105 |
+
self,
|
| 106 |
+
x: torch.Tensor,
|
| 107 |
+
input_ids: torch.LongTensor,
|
| 108 |
+
rotary: tuple[torch.Tensor, torch.Tensor],
|
| 109 |
+
mask: WindowMask,
|
| 110 |
+
past_key_values: Cache | None,
|
| 111 |
+
) -> torch.Tensor:
|
| 112 |
+
batch, time, _ = x.shape
|
| 113 |
+
q = self.q_proj(x).view(batch, time, self.num_heads, self.head_dim)
|
| 114 |
+
k = self.k_proj(x).view(batch, time, self.num_kv_heads, self.head_dim)
|
| 115 |
+
v = self.v_proj(x).view(batch, time, self.num_kv_heads, self.head_dim)
|
| 116 |
+
if self.value_embed is not None:
|
| 117 |
+
ve = self.value_embed(input_ids).to(x.dtype).view(batch, time, self.num_kv_heads, self.head_dim)
|
| 118 |
+
gate = self.gate_range * torch.sigmoid(self.value_embed_gate(x[..., : self.gate_channels]))
|
| 119 |
+
v = v + gate.unsqueeze(-1) * ve
|
| 120 |
+
cos, sin = rotary
|
| 121 |
+
q, k = apply_rotary(q, cos, sin), apply_rotary(k, cos, sin)
|
| 122 |
+
q, k = rms_norm(q) * self.qk_scale, rms_norm(k) * self.qk_scale
|
| 123 |
+
q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
|
| 124 |
+
if past_key_values is not None:
|
| 125 |
+
k, v = past_key_values.update(k, v, self.layer_idx)
|
| 126 |
+
y = F.scaled_dot_product_attention(
|
| 127 |
+
q, k, v, attn_mask=mask.mask, is_causal=mask.is_causal, enable_gqa=self.num_kv_heads != self.num_heads
|
| 128 |
+
)
|
| 129 |
+
return self.o_proj(y.transpose(1, 2).reshape(batch, time, -1))
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
class WildAIMLP(nn.Module):
|
| 133 |
+
def __init__(self, config: WildAIConfig) -> None:
|
| 134 |
+
super().__init__()
|
| 135 |
+
self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
|
| 136 |
+
self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
|
| 137 |
+
|
| 138 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 139 |
+
return self.down_proj(F.relu(self.up_proj(x)).square())
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
class WildAIDecoderLayer(nn.Module):
|
| 143 |
+
def __init__(self, config: WildAIConfig, layer_idx: int) -> None:
|
| 144 |
+
super().__init__()
|
| 145 |
+
self.self_attn = WildAIAttention(config, layer_idx)
|
| 146 |
+
self.mlp = WildAIMLP(config)
|
| 147 |
+
|
| 148 |
+
def forward(
|
| 149 |
+
self,
|
| 150 |
+
x: torch.Tensor,
|
| 151 |
+
input_ids: torch.LongTensor,
|
| 152 |
+
rotary: tuple[torch.Tensor, torch.Tensor],
|
| 153 |
+
mask: WindowMask,
|
| 154 |
+
past_key_values: Cache | None,
|
| 155 |
+
) -> torch.Tensor:
|
| 156 |
+
x = x + self.self_attn(rms_norm(x), input_ids, rotary, mask, past_key_values)
|
| 157 |
+
return x + self.mlp(rms_norm(x))
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
class WildAIPreTrainedModel(PreTrainedModel):
|
| 161 |
+
config_class = WildAIConfig
|
| 162 |
+
base_model_prefix = "model"
|
| 163 |
+
main_input_name = "input_ids"
|
| 164 |
+
_no_split_modules: ClassVar[list[str]] = ["WildAIDecoderLayer"]
|
| 165 |
+
_supports_sdpa = True
|
| 166 |
+
|
| 167 |
+
def _init_weights(self, module: nn.Module) -> None:
|
| 168 |
+
# Released models are always loaded from trained weights; this only gives from_config() models sane values.
|
| 169 |
+
if isinstance(module, (nn.Linear, nn.Embedding)):
|
| 170 |
+
nn.init.normal_(module.weight, mean=0.0, std=0.02)
|
| 171 |
+
elif isinstance(module, WildAIModel):
|
| 172 |
+
with torch.no_grad():
|
| 173 |
+
module.resid_lambdas.fill_(1.0)
|
| 174 |
+
module.x0_lambdas.zero_()
|
| 175 |
+
module.smear_lambda.zero_()
|
| 176 |
+
module.backout_lambda.zero_()
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
class WildAIModel(WildAIPreTrainedModel):
|
| 180 |
+
"""The transformer trunk; returns the final normalised hidden states."""
|
| 181 |
+
|
| 182 |
+
def __init__(self, config: WildAIConfig) -> None:
|
| 183 |
+
super().__init__(config)
|
| 184 |
+
n_layer = config.num_hidden_layers
|
| 185 |
+
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
|
| 186 |
+
self.layers = nn.ModuleList([WildAIDecoderLayer(config, i) for i in range(n_layer)])
|
| 187 |
+
self.resid_lambdas = nn.Parameter(torch.ones(n_layer))
|
| 188 |
+
self.x0_lambdas = nn.Parameter(torch.zeros(n_layer))
|
| 189 |
+
self.smear_gate = nn.Linear(config.smear_channels, 1, bias=False)
|
| 190 |
+
self.smear_lambda = nn.Parameter(torch.zeros(1))
|
| 191 |
+
self.backout_lambda = nn.Parameter(torch.zeros(1))
|
| 192 |
+
self.post_init()
|
| 193 |
+
|
| 194 |
+
def get_input_embeddings(self) -> nn.Embedding:
|
| 195 |
+
return self.embed_tokens
|
| 196 |
+
|
| 197 |
+
def set_input_embeddings(self, value: nn.Embedding) -> None:
|
| 198 |
+
self.embed_tokens = value
|
| 199 |
+
|
| 200 |
+
def rotary(self, position_ids: torch.LongTensor, dtype: torch.dtype) -> tuple[torch.Tensor, torch.Tensor]:
|
| 201 |
+
"""cos/sin tables of shape (batch, time, 1, head_dim / 2), computed in float32 and cast like in training."""
|
| 202 |
+
head_dim = self.config.head_dim
|
| 203 |
+
channels = torch.arange(0, head_dim, 2, dtype=torch.float32, device=position_ids.device)
|
| 204 |
+
inv_freq = 1.0 / (self.config.rotary_base ** (channels / head_dim))
|
| 205 |
+
freqs = position_ids.to(torch.float32)[..., None] * inv_freq
|
| 206 |
+
return freqs.cos().to(dtype)[:, :, None, :], freqs.sin().to(dtype)[:, :, None, :]
|
| 207 |
+
|
| 208 |
+
def smear(self, x: torch.Tensor, previous: torch.Tensor | None, padding_mask: torch.Tensor | None) -> torch.Tensor:
|
| 209 |
+
"""Add a gated copy of the previous position's embedding to every position that has one.
|
| 210 |
+
|
| 211 |
+
``previous`` is the normalised embedding of the token just before this chunk (None at the start of a sequence);
|
| 212 |
+
with left padding, the first real token has a padding token before it and gets nothing.
|
| 213 |
+
"""
|
| 214 |
+
channels = self.config.smear_channels
|
| 215 |
+
lam = self.smear_lambda.to(x.dtype)
|
| 216 |
+
first = x[:, :1]
|
| 217 |
+
if previous is not None:
|
| 218 |
+
first = first + lam * torch.sigmoid(self.smear_gate(first[..., :channels])) * previous
|
| 219 |
+
if x.size(1) == 1:
|
| 220 |
+
return first
|
| 221 |
+
gate = lam * torch.sigmoid(self.smear_gate(x[:, 1:, :channels]))
|
| 222 |
+
if padding_mask is not None:
|
| 223 |
+
gate = gate * padding_mask[:, -x.size(1) : -1, None].to(gate.dtype)
|
| 224 |
+
return torch.cat([first, x[:, 1:] + gate * x[:, :-1]], dim=1)
|
| 225 |
+
|
| 226 |
+
def forward(
|
| 227 |
+
self,
|
| 228 |
+
input_ids: torch.LongTensor,
|
| 229 |
+
attention_mask: torch.Tensor | None = None,
|
| 230 |
+
position_ids: torch.LongTensor | None = None,
|
| 231 |
+
past_key_values: Cache | None = None,
|
| 232 |
+
use_cache: bool | None = None,
|
| 233 |
+
previous_token_ids: torch.LongTensor | None = None,
|
| 234 |
+
output_hidden_states: bool | None = None,
|
| 235 |
+
**kwargs: object,
|
| 236 |
+
) -> BaseModelOutputWithPast:
|
| 237 |
+
"""``previous_token_ids`` (batch, 1): the token before ``input_ids[:, 0]``; required when continuing from a cache."""
|
| 238 |
+
if input_ids is None:
|
| 239 |
+
raise ValueError("input_ids are required: the value embeddings are looked up by token id")
|
| 240 |
+
batch, time = input_ids.shape
|
| 241 |
+
use_cache = self.config.use_cache if use_cache is None else use_cache
|
| 242 |
+
if use_cache and past_key_values is None:
|
| 243 |
+
past_key_values = DynamicCache()
|
| 244 |
+
past_len = past_key_values.get_seq_length() if past_key_values is not None else 0
|
| 245 |
+
if past_len > 0 and previous_token_ids is None:
|
| 246 |
+
raise ValueError("previous_token_ids is required when continuing from past_key_values (the smear needs it)")
|
| 247 |
+
padding_mask = None
|
| 248 |
+
if attention_mask is not None:
|
| 249 |
+
if attention_mask.shape != (batch, past_len + time):
|
| 250 |
+
raise ValueError(f"attention_mask must have shape {(batch, past_len + time)}, got {tuple(attention_mask.shape)}")
|
| 251 |
+
if not bool(attention_mask.all()):
|
| 252 |
+
padding_mask = attention_mask
|
| 253 |
+
if position_ids is None:
|
| 254 |
+
position_ids = torch.arange(past_len, past_len + time, device=input_ids.device).expand(batch, -1)
|
| 255 |
+
|
| 256 |
+
x = rms_norm(self.embed_tokens(input_ids))
|
| 257 |
+
previous = rms_norm(self.embed_tokens(previous_token_ids)) if previous_token_ids is not None else None
|
| 258 |
+
x = self.smear(x, previous, padding_mask)
|
| 259 |
+
rotary = self.rotary(position_ids, x.dtype)
|
| 260 |
+
masks = window_masks(self.config.window_sizes, time, past_len, padding_mask, input_ids.device)
|
| 261 |
+
|
| 262 |
+
x0 = x
|
| 263 |
+
backout = None
|
| 264 |
+
hidden_states: tuple[torch.Tensor, ...] = (x,)
|
| 265 |
+
for i, layer in enumerate(self.layers):
|
| 266 |
+
x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0
|
| 267 |
+
x = layer(x, input_ids, rotary, masks[layer.self_attn.window], past_key_values)
|
| 268 |
+
if i == self.config.backout_layer:
|
| 269 |
+
backout = x
|
| 270 |
+
hidden_states += (x,)
|
| 271 |
+
x = rms_norm(x - self.backout_lambda.to(x.dtype) * backout)
|
| 272 |
+
return BaseModelOutputWithPast(
|
| 273 |
+
last_hidden_state=x,
|
| 274 |
+
past_key_values=past_key_values if use_cache else None,
|
| 275 |
+
hidden_states=hidden_states if output_hidden_states else None,
|
| 276 |
+
)
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
class WildAIForCausalLM(WildAIPreTrainedModel, GenerationMixin):
|
| 280 |
+
def __init__(self, config: WildAIConfig) -> None:
|
| 281 |
+
super().__init__(config)
|
| 282 |
+
self.model = WildAIModel(config)
|
| 283 |
+
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
|
| 284 |
+
self.post_init()
|
| 285 |
+
|
| 286 |
+
def get_input_embeddings(self) -> nn.Embedding:
|
| 287 |
+
return self.model.embed_tokens
|
| 288 |
+
|
| 289 |
+
def set_input_embeddings(self, value: nn.Embedding) -> None:
|
| 290 |
+
self.model.embed_tokens = value
|
| 291 |
+
|
| 292 |
+
def get_output_embeddings(self) -> nn.Linear:
|
| 293 |
+
return self.lm_head
|
| 294 |
+
|
| 295 |
+
def set_output_embeddings(self, value: nn.Linear) -> None:
|
| 296 |
+
self.lm_head = value
|
| 297 |
+
|
| 298 |
+
def forward(
|
| 299 |
+
self,
|
| 300 |
+
input_ids: torch.LongTensor,
|
| 301 |
+
attention_mask: torch.Tensor | None = None,
|
| 302 |
+
position_ids: torch.LongTensor | None = None,
|
| 303 |
+
past_key_values: Cache | None = None,
|
| 304 |
+
labels: torch.LongTensor | None = None,
|
| 305 |
+
use_cache: bool | None = None,
|
| 306 |
+
previous_token_ids: torch.LongTensor | None = None,
|
| 307 |
+
output_hidden_states: bool | None = None,
|
| 308 |
+
logits_to_keep: int = 0,
|
| 309 |
+
return_dict: bool | None = None,
|
| 310 |
+
**kwargs: object,
|
| 311 |
+
) -> CausalLMOutputWithPast | tuple[torch.Tensor, ...]:
|
| 312 |
+
"""Float32 logits after the soft cap. ``labels`` are shifted inside (standard causal-LM loss, -100 ignored)."""
|
| 313 |
+
outputs = self.model(
|
| 314 |
+
input_ids,
|
| 315 |
+
attention_mask=attention_mask,
|
| 316 |
+
position_ids=position_ids,
|
| 317 |
+
past_key_values=past_key_values,
|
| 318 |
+
use_cache=use_cache,
|
| 319 |
+
previous_token_ids=previous_token_ids,
|
| 320 |
+
output_hidden_states=output_hidden_states,
|
| 321 |
+
)
|
| 322 |
+
hidden = outputs.last_hidden_state[:, -logits_to_keep:] if logits_to_keep else outputs.last_hidden_state
|
| 323 |
+
logits = self.lm_head(hidden).float()
|
| 324 |
+
cap = self.config.logit_softcap
|
| 325 |
+
logits = cap * torch.tanh(logits / cap)
|
| 326 |
+
loss = None
|
| 327 |
+
if labels is not None:
|
| 328 |
+
loss = F.cross_entropy(logits[:, :-1].flatten(0, 1), labels[:, 1:].flatten().to(logits.device), ignore_index=-100)
|
| 329 |
+
result = CausalLMOutputWithPast(
|
| 330 |
+
loss=loss, logits=logits, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states
|
| 331 |
+
)
|
| 332 |
+
return result.to_tuple() if return_dict is False else result
|
| 333 |
+
|
| 334 |
+
def prepare_inputs_for_generation(self, input_ids: torch.LongTensor, **kwargs: object) -> dict[str, object]:
|
| 335 |
+
# The smear mixes in the embedding of the token before the first new one; take it from the full sequence.
|
| 336 |
+
model_inputs = super().prepare_inputs_for_generation(input_ids, **kwargs)
|
| 337 |
+
new_len = model_inputs["input_ids"].shape[1]
|
| 338 |
+
if input_ids.shape[1] > new_len:
|
| 339 |
+
model_inputs["previous_token_ids"] = input_ids[:, -new_len - 1 : -new_len]
|
| 340 |
+
return model_inputs
|
268m-g072-human-r0.005/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
268m-g072-human-r0.005/tokenizer_config.json
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 3 |
+
"bos_token": "<|bos|>",
|
| 4 |
+
"eos_token": "<|bos|>",
|
| 5 |
+
"model_max_length": 2048,
|
| 6 |
+
"model_input_names": [
|
| 7 |
+
"input_ids",
|
| 8 |
+
"attention_mask"
|
| 9 |
+
],
|
| 10 |
+
"clean_up_tokenization_spaces": false
|
| 11 |
+
}
|
268m-g073-ai-r0.05/LICENSE
ADDED
|
@@ -0,0 +1,438 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Attribution-NonCommercial-ShareAlike 4.0 International
|
| 2 |
+
|
| 3 |
+
=======================================================================
|
| 4 |
+
|
| 5 |
+
Creative Commons Corporation ("Creative Commons") is not a law firm and
|
| 6 |
+
does not provide legal services or legal advice. Distribution of
|
| 7 |
+
Creative Commons public licenses does not create a lawyer-client or
|
| 8 |
+
other relationship. Creative Commons makes its licenses and related
|
| 9 |
+
information available on an "as-is" basis. Creative Commons gives no
|
| 10 |
+
warranties regarding its licenses, any material licensed under their
|
| 11 |
+
terms and conditions, or any related information. Creative Commons
|
| 12 |
+
disclaims all liability for damages resulting from their use to the
|
| 13 |
+
fullest extent possible.
|
| 14 |
+
|
| 15 |
+
Using Creative Commons Public Licenses
|
| 16 |
+
|
| 17 |
+
Creative Commons public licenses provide a standard set of terms and
|
| 18 |
+
conditions that creators and other rights holders may use to share
|
| 19 |
+
original works of authorship and other material subject to copyright
|
| 20 |
+
and certain other rights specified in the public license below. The
|
| 21 |
+
following considerations are for informational purposes only, are not
|
| 22 |
+
exhaustive, and do not form part of our licenses.
|
| 23 |
+
|
| 24 |
+
Considerations for licensors: Our public licenses are
|
| 25 |
+
intended for use by those authorized to give the public
|
| 26 |
+
permission to use material in ways otherwise restricted by
|
| 27 |
+
copyright and certain other rights. Our licenses are
|
| 28 |
+
irrevocable. Licensors should read and understand the terms
|
| 29 |
+
and conditions of the license they choose before applying it.
|
| 30 |
+
Licensors should also secure all rights necessary before
|
| 31 |
+
applying our licenses so that the public can reuse the
|
| 32 |
+
material as expected. Licensors should clearly mark any
|
| 33 |
+
material not subject to the license. This includes other CC-
|
| 34 |
+
licensed material, or material used under an exception or
|
| 35 |
+
limitation to copyright. More considerations for licensors:
|
| 36 |
+
wiki.creativecommons.org/Considerations_for_licensors
|
| 37 |
+
|
| 38 |
+
Considerations for the public: By using one of our public
|
| 39 |
+
licenses, a licensor grants the public permission to use the
|
| 40 |
+
licensed material under specified terms and conditions. If
|
| 41 |
+
the licensor's permission is not necessary for any reason--for
|
| 42 |
+
example, because of any applicable exception or limitation to
|
| 43 |
+
copyright--then that use is not regulated by the license. Our
|
| 44 |
+
licenses grant only permissions under copyright and certain
|
| 45 |
+
other rights that a licensor has authority to grant. Use of
|
| 46 |
+
the licensed material may still be restricted for other
|
| 47 |
+
reasons, including because others have copyright or other
|
| 48 |
+
rights in the material. A licensor may make special requests,
|
| 49 |
+
such as asking that all changes be marked or described.
|
| 50 |
+
Although not required by our licenses, you are encouraged to
|
| 51 |
+
respect those requests where reasonable. More considerations
|
| 52 |
+
for the public:
|
| 53 |
+
wiki.creativecommons.org/Considerations_for_licensees
|
| 54 |
+
|
| 55 |
+
=======================================================================
|
| 56 |
+
|
| 57 |
+
Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International
|
| 58 |
+
Public License
|
| 59 |
+
|
| 60 |
+
By exercising the Licensed Rights (defined below), You accept and agree
|
| 61 |
+
to be bound by the terms and conditions of this Creative Commons
|
| 62 |
+
Attribution-NonCommercial-ShareAlike 4.0 International Public License
|
| 63 |
+
("Public License"). To the extent this Public License may be
|
| 64 |
+
interpreted as a contract, You are granted the Licensed Rights in
|
| 65 |
+
consideration of Your acceptance of these terms and conditions, and the
|
| 66 |
+
Licensor grants You such rights in consideration of benefits the
|
| 67 |
+
Licensor receives from making the Licensed Material available under
|
| 68 |
+
these terms and conditions.
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
Section 1 -- Definitions.
|
| 72 |
+
|
| 73 |
+
a. Adapted Material means material subject to Copyright and Similar
|
| 74 |
+
Rights that is derived from or based upon the Licensed Material
|
| 75 |
+
and in which the Licensed Material is translated, altered,
|
| 76 |
+
arranged, transformed, or otherwise modified in a manner requiring
|
| 77 |
+
permission under the Copyright and Similar Rights held by the
|
| 78 |
+
Licensor. For purposes of this Public License, where the Licensed
|
| 79 |
+
Material is a musical work, performance, or sound recording,
|
| 80 |
+
Adapted Material is always produced where the Licensed Material is
|
| 81 |
+
synched in timed relation with a moving image.
|
| 82 |
+
|
| 83 |
+
b. Adapter's License means the license You apply to Your Copyright
|
| 84 |
+
and Similar Rights in Your contributions to Adapted Material in
|
| 85 |
+
accordance with the terms and conditions of this Public License.
|
| 86 |
+
|
| 87 |
+
c. BY-NC-SA Compatible License means a license listed at
|
| 88 |
+
creativecommons.org/compatiblelicenses, approved by Creative
|
| 89 |
+
Commons as essentially the equivalent of this Public License.
|
| 90 |
+
|
| 91 |
+
d. Copyright and Similar Rights means copyright and/or similar rights
|
| 92 |
+
closely related to copyright including, without limitation,
|
| 93 |
+
performance, broadcast, sound recording, and Sui Generis Database
|
| 94 |
+
Rights, without regard to how the rights are labeled or
|
| 95 |
+
categorized. For purposes of this Public License, the rights
|
| 96 |
+
specified in Section 2(b)(1)-(2) are not Copyright and Similar
|
| 97 |
+
Rights.
|
| 98 |
+
|
| 99 |
+
e. Effective Technological Measures means those measures that, in the
|
| 100 |
+
absence of proper authority, may not be circumvented under laws
|
| 101 |
+
fulfilling obligations under Article 11 of the WIPO Copyright
|
| 102 |
+
Treaty adopted on December 20, 1996, and/or similar international
|
| 103 |
+
agreements.
|
| 104 |
+
|
| 105 |
+
f. Exceptions and Limitations means fair use, fair dealing, and/or
|
| 106 |
+
any other exception or limitation to Copyright and Similar Rights
|
| 107 |
+
that applies to Your use of the Licensed Material.
|
| 108 |
+
|
| 109 |
+
g. License Elements means the license attributes listed in the name
|
| 110 |
+
of a Creative Commons Public License. The License Elements of this
|
| 111 |
+
Public License are Attribution, NonCommercial, and ShareAlike.
|
| 112 |
+
|
| 113 |
+
h. Licensed Material means the artistic or literary work, database,
|
| 114 |
+
or other material to which the Licensor applied this Public
|
| 115 |
+
License.
|
| 116 |
+
|
| 117 |
+
i. Licensed Rights means the rights granted to You subject to the
|
| 118 |
+
terms and conditions of this Public License, which are limited to
|
| 119 |
+
all Copyright and Similar Rights that apply to Your use of the
|
| 120 |
+
Licensed Material and that the Licensor has authority to license.
|
| 121 |
+
|
| 122 |
+
j. Licensor means the individual(s) or entity(ies) granting rights
|
| 123 |
+
under this Public License.
|
| 124 |
+
|
| 125 |
+
k. NonCommercial means not primarily intended for or directed towards
|
| 126 |
+
commercial advantage or monetary compensation. For purposes of
|
| 127 |
+
this Public License, the exchange of the Licensed Material for
|
| 128 |
+
other material subject to Copyright and Similar Rights by digital
|
| 129 |
+
file-sharing or similar means is NonCommercial provided there is
|
| 130 |
+
no payment of monetary compensation in connection with the
|
| 131 |
+
exchange.
|
| 132 |
+
|
| 133 |
+
l. Share means to provide material to the public by any means or
|
| 134 |
+
process that requires permission under the Licensed Rights, such
|
| 135 |
+
as reproduction, public display, public performance, distribution,
|
| 136 |
+
dissemination, communication, or importation, and to make material
|
| 137 |
+
available to the public including in ways that members of the
|
| 138 |
+
public may access the material from a place and at a time
|
| 139 |
+
individually chosen by them.
|
| 140 |
+
|
| 141 |
+
m. Sui Generis Database Rights means rights other than copyright
|
| 142 |
+
resulting from Directive 96/9/EC of the European Parliament and of
|
| 143 |
+
the Council of 11 March 1996 on the legal protection of databases,
|
| 144 |
+
as amended and/or succeeded, as well as other essentially
|
| 145 |
+
equivalent rights anywhere in the world.
|
| 146 |
+
|
| 147 |
+
n. You means the individual or entity exercising the Licensed Rights
|
| 148 |
+
under this Public License. Your has a corresponding meaning.
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
Section 2 -- Scope.
|
| 152 |
+
|
| 153 |
+
a. License grant.
|
| 154 |
+
|
| 155 |
+
1. Subject to the terms and conditions of this Public License,
|
| 156 |
+
the Licensor hereby grants You a worldwide, royalty-free,
|
| 157 |
+
non-sublicensable, non-exclusive, irrevocable license to
|
| 158 |
+
exercise the Licensed Rights in the Licensed Material to:
|
| 159 |
+
|
| 160 |
+
a. reproduce and Share the Licensed Material, in whole or
|
| 161 |
+
in part, for NonCommercial purposes only; and
|
| 162 |
+
|
| 163 |
+
b. produce, reproduce, and Share Adapted Material for
|
| 164 |
+
NonCommercial purposes only.
|
| 165 |
+
|
| 166 |
+
2. Exceptions and Limitations. For the avoidance of doubt, where
|
| 167 |
+
Exceptions and Limitations apply to Your use, this Public
|
| 168 |
+
License does not apply, and You do not need to comply with
|
| 169 |
+
its terms and conditions.
|
| 170 |
+
|
| 171 |
+
3. Term. The term of this Public License is specified in Section
|
| 172 |
+
6(a).
|
| 173 |
+
|
| 174 |
+
4. Media and formats; technical modifications allowed. The
|
| 175 |
+
Licensor authorizes You to exercise the Licensed Rights in
|
| 176 |
+
all media and formats whether now known or hereafter created,
|
| 177 |
+
and to make technical modifications necessary to do so. The
|
| 178 |
+
Licensor waives and/or agrees not to assert any right or
|
| 179 |
+
authority to forbid You from making technical modifications
|
| 180 |
+
necessary to exercise the Licensed Rights, including
|
| 181 |
+
technical modifications necessary to circumvent Effective
|
| 182 |
+
Technological Measures. For purposes of this Public License,
|
| 183 |
+
simply making modifications authorized by this Section 2(a)
|
| 184 |
+
(4) never produces Adapted Material.
|
| 185 |
+
|
| 186 |
+
5. Downstream recipients.
|
| 187 |
+
|
| 188 |
+
a. Offer from the Licensor -- Licensed Material. Every
|
| 189 |
+
recipient of the Licensed Material automatically
|
| 190 |
+
receives an offer from the Licensor to exercise the
|
| 191 |
+
Licensed Rights under the terms and conditions of this
|
| 192 |
+
Public License.
|
| 193 |
+
|
| 194 |
+
b. Additional offer from the Licensor -- Adapted Material.
|
| 195 |
+
Every recipient of Adapted Material from You
|
| 196 |
+
automatically receives an offer from the Licensor to
|
| 197 |
+
exercise the Licensed Rights in the Adapted Material
|
| 198 |
+
under the conditions of the Adapter's License You apply.
|
| 199 |
+
|
| 200 |
+
c. No downstream restrictions. You may not offer or impose
|
| 201 |
+
any additional or different terms or conditions on, or
|
| 202 |
+
apply any Effective Technological Measures to, the
|
| 203 |
+
Licensed Material if doing so restricts exercise of the
|
| 204 |
+
Licensed Rights by any recipient of the Licensed
|
| 205 |
+
Material.
|
| 206 |
+
|
| 207 |
+
6. No endorsement. Nothing in this Public License constitutes or
|
| 208 |
+
may be construed as permission to assert or imply that You
|
| 209 |
+
are, or that Your use of the Licensed Material is, connected
|
| 210 |
+
with, or sponsored, endorsed, or granted official status by,
|
| 211 |
+
the Licensor or others designated to receive attribution as
|
| 212 |
+
provided in Section 3(a)(1)(A)(i).
|
| 213 |
+
|
| 214 |
+
b. Other rights.
|
| 215 |
+
|
| 216 |
+
1. Moral rights, such as the right of integrity, are not
|
| 217 |
+
licensed under this Public License, nor are publicity,
|
| 218 |
+
privacy, and/or other similar personality rights; however, to
|
| 219 |
+
the extent possible, the Licensor waives and/or agrees not to
|
| 220 |
+
assert any such rights held by the Licensor to the limited
|
| 221 |
+
extent necessary to allow You to exercise the Licensed
|
| 222 |
+
Rights, but not otherwise.
|
| 223 |
+
|
| 224 |
+
2. Patent and trademark rights are not licensed under this
|
| 225 |
+
Public License.
|
| 226 |
+
|
| 227 |
+
3. To the extent possible, the Licensor waives any right to
|
| 228 |
+
collect royalties from You for the exercise of the Licensed
|
| 229 |
+
Rights, whether directly or through a collecting society
|
| 230 |
+
under any voluntary or waivable statutory or compulsory
|
| 231 |
+
licensing scheme. In all other cases the Licensor expressly
|
| 232 |
+
reserves any right to collect such royalties, including when
|
| 233 |
+
the Licensed Material is used other than for NonCommercial
|
| 234 |
+
purposes.
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
Section 3 -- License Conditions.
|
| 238 |
+
|
| 239 |
+
Your exercise of the Licensed Rights is expressly made subject to the
|
| 240 |
+
following conditions.
|
| 241 |
+
|
| 242 |
+
a. Attribution.
|
| 243 |
+
|
| 244 |
+
1. If You Share the Licensed Material (including in modified
|
| 245 |
+
form), You must:
|
| 246 |
+
|
| 247 |
+
a. retain the following if it is supplied by the Licensor
|
| 248 |
+
with the Licensed Material:
|
| 249 |
+
|
| 250 |
+
i. identification of the creator(s) of the Licensed
|
| 251 |
+
Material and any others designated to receive
|
| 252 |
+
attribution, in any reasonable manner requested by
|
| 253 |
+
the Licensor (including by pseudonym if
|
| 254 |
+
designated);
|
| 255 |
+
|
| 256 |
+
ii. a copyright notice;
|
| 257 |
+
|
| 258 |
+
iii. a notice that refers to this Public License;
|
| 259 |
+
|
| 260 |
+
iv. a notice that refers to the disclaimer of
|
| 261 |
+
warranties;
|
| 262 |
+
|
| 263 |
+
v. a URI or hyperlink to the Licensed Material to the
|
| 264 |
+
extent reasonably practicable;
|
| 265 |
+
|
| 266 |
+
b. indicate if You modified the Licensed Material and
|
| 267 |
+
retain an indication of any previous modifications; and
|
| 268 |
+
|
| 269 |
+
c. indicate the Licensed Material is licensed under this
|
| 270 |
+
Public License, and include the text of, or the URI or
|
| 271 |
+
hyperlink to, this Public License.
|
| 272 |
+
|
| 273 |
+
2. You may satisfy the conditions in Section 3(a)(1) in any
|
| 274 |
+
reasonable manner based on the medium, means, and context in
|
| 275 |
+
which You Share the Licensed Material. For example, it may be
|
| 276 |
+
reasonable to satisfy the conditions by providing a URI or
|
| 277 |
+
hyperlink to a resource that includes the required
|
| 278 |
+
information.
|
| 279 |
+
3. If requested by the Licensor, You must remove any of the
|
| 280 |
+
information required by Section 3(a)(1)(A) to the extent
|
| 281 |
+
reasonably practicable.
|
| 282 |
+
|
| 283 |
+
b. ShareAlike.
|
| 284 |
+
|
| 285 |
+
In addition to the conditions in Section 3(a), if You Share
|
| 286 |
+
Adapted Material You produce, the following conditions also apply.
|
| 287 |
+
|
| 288 |
+
1. The Adapter's License You apply must be a Creative Commons
|
| 289 |
+
license with the same License Elements, this version or
|
| 290 |
+
later, or a BY-NC-SA Compatible License.
|
| 291 |
+
|
| 292 |
+
2. You must include the text of, or the URI or hyperlink to, the
|
| 293 |
+
Adapter's License You apply. You may satisfy this condition
|
| 294 |
+
in any reasonable manner based on the medium, means, and
|
| 295 |
+
context in which You Share Adapted Material.
|
| 296 |
+
|
| 297 |
+
3. You may not offer or impose any additional or different terms
|
| 298 |
+
or conditions on, or apply any Effective Technological
|
| 299 |
+
Measures to, Adapted Material that restrict exercise of the
|
| 300 |
+
rights granted under the Adapter's License You apply.
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
Section 4 -- Sui Generis Database Rights.
|
| 304 |
+
|
| 305 |
+
Where the Licensed Rights include Sui Generis Database Rights that
|
| 306 |
+
apply to Your use of the Licensed Material:
|
| 307 |
+
|
| 308 |
+
a. for the avoidance of doubt, Section 2(a)(1) grants You the right
|
| 309 |
+
to extract, reuse, reproduce, and Share all or a substantial
|
| 310 |
+
portion of the contents of the database for NonCommercial purposes
|
| 311 |
+
only;
|
| 312 |
+
|
| 313 |
+
b. if You include all or a substantial portion of the database
|
| 314 |
+
contents in a database in which You have Sui Generis Database
|
| 315 |
+
Rights, then the database in which You have Sui Generis Database
|
| 316 |
+
Rights (but not its individual contents) is Adapted Material,
|
| 317 |
+
including for purposes of Section 3(b); and
|
| 318 |
+
|
| 319 |
+
c. You must comply with the conditions in Section 3(a) if You Share
|
| 320 |
+
all or a substantial portion of the contents of the database.
|
| 321 |
+
|
| 322 |
+
For the avoidance of doubt, this Section 4 supplements and does not
|
| 323 |
+
replace Your obligations under this Public License where the Licensed
|
| 324 |
+
Rights include other Copyright and Similar Rights.
|
| 325 |
+
|
| 326 |
+
|
| 327 |
+
Section 5 -- Disclaimer of Warranties and Limitation of Liability.
|
| 328 |
+
|
| 329 |
+
a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
|
| 330 |
+
EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
|
| 331 |
+
AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
|
| 332 |
+
ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
|
| 333 |
+
IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
|
| 334 |
+
WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
|
| 335 |
+
PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
|
| 336 |
+
ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
|
| 337 |
+
KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
|
| 338 |
+
ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
|
| 339 |
+
|
| 340 |
+
b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
|
| 341 |
+
TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
|
| 342 |
+
NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
|
| 343 |
+
INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
|
| 344 |
+
COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
|
| 345 |
+
USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
|
| 346 |
+
ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
|
| 347 |
+
DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
|
| 348 |
+
IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
|
| 349 |
+
|
| 350 |
+
c. The disclaimer of warranties and limitation of liability provided
|
| 351 |
+
above shall be interpreted in a manner that, to the extent
|
| 352 |
+
possible, most closely approximates an absolute disclaimer and
|
| 353 |
+
waiver of all liability.
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
Section 6 -- Term and Termination.
|
| 357 |
+
|
| 358 |
+
a. This Public License applies for the term of the Copyright and
|
| 359 |
+
Similar Rights licensed here. However, if You fail to comply with
|
| 360 |
+
this Public License, then Your rights under this Public License
|
| 361 |
+
terminate automatically.
|
| 362 |
+
|
| 363 |
+
b. Where Your right to use the Licensed Material has terminated under
|
| 364 |
+
Section 6(a), it reinstates:
|
| 365 |
+
|
| 366 |
+
1. automatically as of the date the violation is cured, provided
|
| 367 |
+
it is cured within 30 days of Your discovery of the
|
| 368 |
+
violation; or
|
| 369 |
+
|
| 370 |
+
2. upon express reinstatement by the Licensor.
|
| 371 |
+
|
| 372 |
+
For the avoidance of doubt, this Section 6(b) does not affect any
|
| 373 |
+
right the Licensor may have to seek remedies for Your violations
|
| 374 |
+
of this Public License.
|
| 375 |
+
|
| 376 |
+
c. For the avoidance of doubt, the Licensor may also offer the
|
| 377 |
+
Licensed Material under separate terms or conditions or stop
|
| 378 |
+
distributing the Licensed Material at any time; however, doing so
|
| 379 |
+
will not terminate this Public License.
|
| 380 |
+
|
| 381 |
+
d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
|
| 382 |
+
License.
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
Section 7 -- Other Terms and Conditions.
|
| 386 |
+
|
| 387 |
+
a. The Licensor shall not be bound by any additional or different
|
| 388 |
+
terms or conditions communicated by You unless expressly agreed.
|
| 389 |
+
|
| 390 |
+
b. Any arrangements, understandings, or agreements regarding the
|
| 391 |
+
Licensed Material not stated herein are separate from and
|
| 392 |
+
independent of the terms and conditions of this Public License.
|
| 393 |
+
|
| 394 |
+
|
| 395 |
+
Section 8 -- Interpretation.
|
| 396 |
+
|
| 397 |
+
a. For the avoidance of doubt, this Public License does not, and
|
| 398 |
+
shall not be interpreted to, reduce, limit, restrict, or impose
|
| 399 |
+
conditions on any use of the Licensed Material that could lawfully
|
| 400 |
+
be made without permission under this Public License.
|
| 401 |
+
|
| 402 |
+
b. To the extent possible, if any provision of this Public License is
|
| 403 |
+
deemed unenforceable, it shall be automatically reformed to the
|
| 404 |
+
minimum extent necessary to make it enforceable. If the provision
|
| 405 |
+
cannot be reformed, it shall be severed from this Public License
|
| 406 |
+
without affecting the enforceability of the remaining terms and
|
| 407 |
+
conditions.
|
| 408 |
+
|
| 409 |
+
c. No term or condition of this Public License will be waived and no
|
| 410 |
+
failure to comply consented to unless expressly agreed to by the
|
| 411 |
+
Licensor.
|
| 412 |
+
|
| 413 |
+
d. Nothing in this Public License constitutes or may be interpreted
|
| 414 |
+
as a limitation upon, or waiver of, any privileges and immunities
|
| 415 |
+
that apply to the Licensor or You, including from the legal
|
| 416 |
+
processes of any jurisdiction or authority.
|
| 417 |
+
|
| 418 |
+
=======================================================================
|
| 419 |
+
|
| 420 |
+
Creative Commons is not a party to its public
|
| 421 |
+
licenses. Notwithstanding, Creative Commons may elect to apply one of
|
| 422 |
+
its public licenses to material it publishes and in those instances
|
| 423 |
+
will be considered the “Licensor.” The text of the Creative Commons
|
| 424 |
+
public licenses is dedicated to the public domain under the CC0 Public
|
| 425 |
+
Domain Dedication. Except for the limited purpose of indicating that
|
| 426 |
+
material is shared under a Creative Commons public license or as
|
| 427 |
+
otherwise permitted by the Creative Commons policies published at
|
| 428 |
+
creativecommons.org/policies, Creative Commons does not authorize the
|
| 429 |
+
use of the trademark "Creative Commons" or any other trademark or logo
|
| 430 |
+
of Creative Commons without its prior written consent including,
|
| 431 |
+
without limitation, in connection with any unauthorized modifications
|
| 432 |
+
to any of its public licenses or any other arrangements,
|
| 433 |
+
understandings, or agreements concerning use of licensed material. For
|
| 434 |
+
the avoidance of doubt, this paragraph does not form part of the
|
| 435 |
+
public licenses.
|
| 436 |
+
|
| 437 |
+
Creative Commons may be contacted at creativecommons.org.
|
| 438 |
+
|
268m-g073-ai-r0.05/README.md
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: cc-by-nc-sa-4.0
|
| 3 |
+
library_name: transformers
|
| 4 |
+
pipeline_tag: text-generation
|
| 5 |
+
language:
|
| 6 |
+
- en
|
| 7 |
+
datasets:
|
| 8 |
+
- pangram/WildAI
|
| 9 |
+
tags:
|
| 10 |
+
- wildai
|
| 11 |
+
- nanochat
|
| 12 |
+
- scaling-laws
|
| 13 |
+
- ai-generated-text
|
| 14 |
+
- pretraining-data
|
| 15 |
+
---
|
| 16 |
+
|
| 17 |
+
# 268m-g073-ai-r0.05
|
| 18 |
+
|
| 19 |
+
> **License: [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/).** The WildAI code, models and dataset are released under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license: you may use and adapt them for non-commercial purposes, with attribution, and must share adaptations under the same license.
|
| 20 |
+
|
| 21 |
+
A 268M decoder-only language model trained from scratch for [*How Much Is an AI Token Worth? Scaling Laws for Wild AI-Generated Web Text*](https://arxiv.org/abs/2609.40295), one of the paper's models measuring how wild AI-generated web text in pretraining data changes a model's loss. It is a research artifact for studying scaling laws, not an assistant: it has had no instruction tuning or safety training.
|
| 22 |
+
|
| 23 |
+
## Training data
|
| 24 |
+
|
| 25 |
+
The human documents of `268m-g073-control` plus wild AI-generated web documents: 1.17B human and 59.4M AI tokens trained on, an AI-to-human token ratio r = 0.0506 (design 0.05), so 4.8% of the training tokens are AI-generated.
|
| 26 |
+
|
| 27 |
+
| | |
|
| 28 |
+
|---|---|
|
| 29 |
+
| Parameters (N, input embedding included, value embeddings excluded) | 268.4M |
|
| 30 |
+
| Layers / width | 16 / 1024 |
|
| 31 |
+
| Training tokens | 1.23B (2,352 optimizer steps) |
|
| 32 |
+
| Human / AI tokens | 1.17B / 59.4M |
|
| 33 |
+
| Group / arm / seed | g073 / ai / 271828 |
|
| 34 |
+
| Used in the paper for | fitting the scaling laws (726 models, 19.9M to 268M) |
|
| 35 |
+
|
| 36 |
+
## Held-out loss
|
| 37 |
+
|
| 38 |
+
| Set | Bits per byte | Text |
|
| 39 |
+
|---|---|---|
|
| 40 |
+
| C4 | 0.9939 | C4 validation text (human-written, pre-2022) |
|
| 41 |
+
| FW22 | 0.9782 | held-out FineWeb documents crawled before 2022 |
|
| 42 |
+
| Paloma | 1.3697 | macro average over 16 Paloma sources |
|
| 43 |
+
| FW26 | 0.8579 | held-out 2026 FineWeb-style crawl (22.3% AI tokens) |
|
| 44 |
+
| FW26-H | 0.9104 | the human-labeled documents of FW26 |
|
| 45 |
+
| FW26-AI | 0.6643 | the AI-labeled documents of FW26 |
|
| 46 |
+
| Cosmo | 0.6705 | Cosmopedia (AI-generated textbooks and stories) |
|
| 47 |
+
|
| 48 |
+
<details><summary>Paloma sources</summary>
|
| 49 |
+
|
| 50 |
+
| Source | Bits per byte |
|
| 51 |
+
|---|---|
|
| 52 |
+
| 4chan | 1.3876 |
|
| 53 |
+
| c4 | 1.0131 |
|
| 54 |
+
| c4_100_domains | 0.9956 |
|
| 55 |
+
| dolma | 1.2619 |
|
| 56 |
+
| falcon_refinedweb | 1.1111 |
|
| 57 |
+
| gab | 1.8985 |
|
| 58 |
+
| m2d2_s2orc | 1.1756 |
|
| 59 |
+
| m2d2_wikipedia | 1.0838 |
|
| 60 |
+
| manosphere | 1.4042 |
|
| 61 |
+
| mc4_en | 1.1029 |
|
| 62 |
+
| programming | 1.9801 |
|
| 63 |
+
| ptb | 1.2424 |
|
| 64 |
+
| reddit_100 | 1.2479 |
|
| 65 |
+
| redpajama | 1.4305 |
|
| 66 |
+
| twitteraae | 2.4997 |
|
| 67 |
+
| wikitext_103 | 1.0805 |
|
| 68 |
+
|
| 69 |
+
</details>
|
| 70 |
+
|
| 71 |
+
## Usage
|
| 72 |
+
|
| 73 |
+
```python
|
| 74 |
+
import torch
|
| 75 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 76 |
+
|
| 77 |
+
tokenizer = AutoTokenizer.from_pretrained("pangram/WildAI-models", subfolder="268m-g073-ai-r0.05", trust_remote_code=True)
|
| 78 |
+
model = AutoModelForCausalLM.from_pretrained("pangram/WildAI-models", subfolder="268m-g073-ai-r0.05", trust_remote_code=True, dtype=torch.bfloat16)
|
| 79 |
+
|
| 80 |
+
inputs = tokenizer("The history of the printing press", return_tensors="pt")
|
| 81 |
+
output = model.generate(**inputs, max_new_tokens=40, do_sample=False)
|
| 82 |
+
print(tokenizer.decode(output[0], skip_special_tokens=True))
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
The tokenizer prepends `<|bos|>` to every text, as in training; pass `add_special_tokens=False` to skip it. The model predicts `<|bos|>` where a document ends, and generation stops there. Weights are stored in bfloat16, the precision the paper's losses were computed in; `dtype=torch.float32` computes in float32 instead.
|
| 86 |
+
|
| 87 |
+
## Architecture
|
| 88 |
+
|
| 89 |
+
The nanochat GPT recipe: rotary embeddings, RMSNorm without weights (also on queries and keys), squared-ReLU MLP, attention windows in a repeating short-short-short-long pattern (512 tokens, then the full 2,048-token context; the last layer always full), gated value embeddings on alternating layers, a previous-token smear, per-layer residual scalars, a mid-network backout, untied input and output embeddings and logits soft-capped at 15. Vocabulary 32,768 tokens, context 2,048. The code ships with the weights (`trust_remote_code=True`) and matches the training code's forward pass.
|
| 90 |
+
|
| 91 |
+
## License
|
| 92 |
+
|
| 93 |
+
**License: [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/).** The WildAI code, models and dataset are released under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license: you may use and adapt them for non-commercial purposes, with attribution, and must share adaptations under the same license. The full text is in `LICENSE`.
|
| 94 |
+
|
| 95 |
+
## Citation
|
| 96 |
+
|
| 97 |
+
```bibtex
|
| 98 |
+
@article{russell2026wildai,
|
| 99 |
+
title = {How Much Is an AI Token Worth? Scaling Laws for Wild AI-Generated Web Text},
|
| 100 |
+
author = {Russell, Jenna and Glickenhaus, Ben and Thai, Katherine and Wieting, John and Iyyer, Mohit and Spero, Max and Emi, Bradley},
|
| 101 |
+
journal = {arXiv preprint arXiv:2609.40295},
|
| 102 |
+
year = {2026},
|
| 103 |
+
url = {https://arxiv.org/abs/2609.40295}
|
| 104 |
+
}
|
| 105 |
+
```
|
268m-g073-ai-r0.05/configuration_wildai.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Configuration of the WildAI language models (a nanochat GPT variant) for Hugging Face ``transformers``.
|
| 2 |
+
|
| 3 |
+
This file is copied verbatim into every released model repository and loaded with ``trust_remote_code=True``.
|
| 4 |
+
It depends only on ``transformers``.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from typing import ClassVar
|
| 10 |
+
|
| 11 |
+
from transformers import PretrainedConfig
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class WildAIConfig(PretrainedConfig):
|
| 15 |
+
"""Architecture of a WildAI model. Only width and depth vary across the released models.
|
| 16 |
+
|
| 17 |
+
Every layer uses causal attention limited to a left window: a query attends to itself and the ``window`` tokens before
|
| 18 |
+
it. ``window_pattern`` is tiled over the layers ("S" = a quarter of the context rounded up to a multiple of 128 tokens,
|
| 19 |
+
"L" = the full context) and the last layer always uses "L".
|
| 20 |
+
|
| 21 |
+
``release`` holds the model's entry in the release catalog (name, token counts, AI ratio, ...); it does not affect
|
| 22 |
+
the forward pass.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
model_type = "wildai"
|
| 26 |
+
keys_to_ignore_at_inference: ClassVar[list[str]] = ["past_key_values"]
|
| 27 |
+
|
| 28 |
+
def __init__(
|
| 29 |
+
self,
|
| 30 |
+
vocab_size: int = 32768,
|
| 31 |
+
hidden_size: int = 768,
|
| 32 |
+
num_hidden_layers: int = 12,
|
| 33 |
+
num_attention_heads: int = 6,
|
| 34 |
+
num_key_value_heads: int = 6,
|
| 35 |
+
max_position_embeddings: int = 2048,
|
| 36 |
+
window_pattern: str = "SSSL",
|
| 37 |
+
rotary_base: float = 100000.0,
|
| 38 |
+
qk_scale: float = 1.2,
|
| 39 |
+
logit_softcap: float = 15.0,
|
| 40 |
+
value_embed_gate_channels: int = 12,
|
| 41 |
+
value_embed_gate_range: float = 3.0,
|
| 42 |
+
smear_channels: int = 24,
|
| 43 |
+
bos_token_id: int = 32759,
|
| 44 |
+
eos_token_id: int = 32759,
|
| 45 |
+
use_cache: bool = True,
|
| 46 |
+
release: dict[str, str | int | float | None] | None = None,
|
| 47 |
+
**kwargs: object,
|
| 48 |
+
) -> None:
|
| 49 |
+
if hidden_size % num_attention_heads != 0 or num_attention_heads % num_key_value_heads != 0:
|
| 50 |
+
raise ValueError("hidden_size must divide into num_attention_heads, and those into num_key_value_heads")
|
| 51 |
+
if not window_pattern or set(window_pattern.upper()) - {"S", "L"}:
|
| 52 |
+
raise ValueError(f"window_pattern must be a non-empty string of S and L, got {window_pattern!r}")
|
| 53 |
+
self.vocab_size = vocab_size
|
| 54 |
+
self.hidden_size = hidden_size
|
| 55 |
+
self.num_hidden_layers = num_hidden_layers
|
| 56 |
+
self.num_attention_heads = num_attention_heads
|
| 57 |
+
self.num_key_value_heads = num_key_value_heads
|
| 58 |
+
self.max_position_embeddings = max_position_embeddings
|
| 59 |
+
self.window_pattern = window_pattern.upper()
|
| 60 |
+
self.rotary_base = rotary_base
|
| 61 |
+
self.qk_scale = qk_scale
|
| 62 |
+
self.logit_softcap = logit_softcap
|
| 63 |
+
self.value_embed_gate_channels = value_embed_gate_channels
|
| 64 |
+
self.value_embed_gate_range = value_embed_gate_range
|
| 65 |
+
self.smear_channels = smear_channels
|
| 66 |
+
self.use_cache = use_cache
|
| 67 |
+
self.release = release
|
| 68 |
+
kwargs.setdefault("tie_word_embeddings", False)
|
| 69 |
+
super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
|
| 70 |
+
|
| 71 |
+
@property
|
| 72 |
+
def head_dim(self) -> int:
|
| 73 |
+
return self.hidden_size // self.num_attention_heads
|
| 74 |
+
|
| 75 |
+
@property
|
| 76 |
+
def intermediate_size(self) -> int:
|
| 77 |
+
return 4 * self.hidden_size
|
| 78 |
+
|
| 79 |
+
@property
|
| 80 |
+
def window_sizes(self) -> list[int]:
|
| 81 |
+
"""Left attention window of each layer, in tokens."""
|
| 82 |
+
long_window = self.max_position_embeddings
|
| 83 |
+
short_window = -(-long_window // 4 // 128) * 128 # a quarter of the context, rounded up to a multiple of 128 (2048 -> 512)
|
| 84 |
+
pattern = self.window_pattern
|
| 85 |
+
sizes = [short_window if pattern[i % len(pattern)] == "S" else long_window for i in range(self.num_hidden_layers)]
|
| 86 |
+
sizes[-1] = long_window
|
| 87 |
+
return sizes
|
| 88 |
+
|
| 89 |
+
def has_value_embedding(self, layer_idx: int) -> bool:
|
| 90 |
+
"""Value embeddings sit on alternating layers, always including the last one."""
|
| 91 |
+
return layer_idx % 2 == (self.num_hidden_layers - 1) % 2
|
| 92 |
+
|
| 93 |
+
@property
|
| 94 |
+
def backout_layer(self) -> int:
|
| 95 |
+
"""The residual stream after this layer is partly subtracted before the final norm."""
|
| 96 |
+
return self.num_hidden_layers // 2
|
268m-g073-ai-r0.05/generation_config.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 32759,
|
| 4 |
+
"eos_token_id": 32759,
|
| 5 |
+
"output_attentions": false,
|
| 6 |
+
"output_hidden_states": false,
|
| 7 |
+
"pad_token_id": 32759,
|
| 8 |
+
"transformers_version": "5.17.0",
|
| 9 |
+
"use_cache": true
|
| 10 |
+
}
|
268m-g073-ai-r0.05/modeling_wildai.py
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The WildAI language models (a nanochat GPT variant) for Hugging Face ``transformers``.
|
| 2 |
+
|
| 3 |
+
This file is copied verbatim into every released model repository and loaded with ``trust_remote_code=True``.
|
| 4 |
+
It depends only on ``torch`` and ``transformers`` and reproduces the training code's forward pass operation by
|
| 5 |
+
operation, so a model loaded in bfloat16 computes the same function the paper evaluated:
|
| 6 |
+
|
| 7 |
+
- token embedding followed by a parameter-free RMSNorm, then a learned "smear" that mixes each position's embedding
|
| 8 |
+
with the previous token's;
|
| 9 |
+
- per-layer scalars that rescale the residual stream and blend the initial embedding back in before every block;
|
| 10 |
+
- attention with rotary embeddings applied before a per-head RMSNorm of queries and keys (both scaled by 1.2),
|
| 11 |
+
a left sliding window per layer, and gated value embeddings added to the values on alternating layers;
|
| 12 |
+
- a squared-ReLU MLP, parameter-free pre-norms, no biases;
|
| 13 |
+
- "backout": part of the residual stream after the middle layer is subtracted before the final norm;
|
| 14 |
+
- untied output head with logits soft-capped by ``15 * tanh(logits / 15)``.
|
| 15 |
+
|
| 16 |
+
Attention uses PyTorch's scaled_dot_product_attention with explicit window masks, so no FlashAttention install is
|
| 17 |
+
needed. Generation uses the standard ``DynamicCache``.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
from dataclasses import dataclass
|
| 23 |
+
from typing import ClassVar
|
| 24 |
+
|
| 25 |
+
import torch
|
| 26 |
+
import torch.nn.functional as F
|
| 27 |
+
from torch import nn
|
| 28 |
+
from transformers import GenerationMixin, PreTrainedModel
|
| 29 |
+
from transformers.cache_utils import Cache, DynamicCache
|
| 30 |
+
from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
|
| 31 |
+
|
| 32 |
+
from .configuration_wildai import WildAIConfig
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def rms_norm(x: torch.Tensor) -> torch.Tensor:
|
| 36 |
+
"""Parameter-free RMSNorm over the last dimension with PyTorch's default epsilon, as in training."""
|
| 37 |
+
return F.rms_norm(x, (x.size(-1),))
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def apply_rotary(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
|
| 41 |
+
"""Rotate channel pairs (i, i + d/2) of ``x`` (batch, time, heads, head_dim), with the training code's sign convention."""
|
| 42 |
+
d = x.shape[-1] // 2
|
| 43 |
+
x1, x2 = x[..., :d], x[..., d:]
|
| 44 |
+
return torch.cat([x1 * cos + x2 * sin, x1 * (-sin) + x2 * cos], dim=-1)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
@dataclass(frozen=True)
|
| 48 |
+
class WindowMask:
|
| 49 |
+
"""Arguments for scaled_dot_product_attention that implement one attention window."""
|
| 50 |
+
|
| 51 |
+
mask: torch.Tensor | None
|
| 52 |
+
is_causal: bool
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def window_masks(
|
| 56 |
+
window_sizes: list[int], query_len: int, past_len: int, padding_mask: torch.Tensor | None, device: torch.device
|
| 57 |
+
) -> dict[int, WindowMask]:
|
| 58 |
+
"""Masks for each distinct left window. Query i (absolute index past_len + i) sees keys j with i - window <= j <= i.
|
| 59 |
+
|
| 60 |
+
``padding_mask`` is the (batch, past_len + query_len) attention mask when it contains padding, else None. A padding
|
| 61 |
+
query is always allowed to see itself so that its (unused) output stays finite.
|
| 62 |
+
"""
|
| 63 |
+
total = past_len + query_len
|
| 64 |
+
q_idx = torch.arange(past_len, total, device=device)[:, None]
|
| 65 |
+
k_idx = torch.arange(total, device=device)[None, :]
|
| 66 |
+
masks: dict[int, WindowMask] = {}
|
| 67 |
+
for window in sorted(set(window_sizes)):
|
| 68 |
+
if padding_mask is None and window >= total - 1:
|
| 69 |
+
if query_len == 1:
|
| 70 |
+
masks[window] = WindowMask(mask=None, is_causal=False)
|
| 71 |
+
continue
|
| 72 |
+
if past_len == 0:
|
| 73 |
+
masks[window] = WindowMask(mask=None, is_causal=True)
|
| 74 |
+
continue
|
| 75 |
+
allowed = (k_idx <= q_idx) & (q_idx - k_idx <= window)
|
| 76 |
+
if padding_mask is not None:
|
| 77 |
+
allowed = (allowed & padding_mask[:, None, None, :].bool()) | (k_idx == q_idx)
|
| 78 |
+
masks[window] = WindowMask(mask=allowed, is_causal=False)
|
| 79 |
+
return masks
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class WildAIAttention(nn.Module):
|
| 83 |
+
def __init__(self, config: WildAIConfig, layer_idx: int) -> None:
|
| 84 |
+
super().__init__()
|
| 85 |
+
self.layer_idx = layer_idx
|
| 86 |
+
self.num_heads = config.num_attention_heads
|
| 87 |
+
self.num_kv_heads = config.num_key_value_heads
|
| 88 |
+
self.head_dim = config.head_dim
|
| 89 |
+
self.qk_scale = config.qk_scale
|
| 90 |
+
self.window = config.window_sizes[layer_idx]
|
| 91 |
+
hidden, kv_dim = config.hidden_size, config.num_key_value_heads * config.head_dim
|
| 92 |
+
self.q_proj = nn.Linear(hidden, config.num_attention_heads * config.head_dim, bias=False)
|
| 93 |
+
self.k_proj = nn.Linear(hidden, kv_dim, bias=False)
|
| 94 |
+
self.v_proj = nn.Linear(hidden, kv_dim, bias=False)
|
| 95 |
+
self.o_proj = nn.Linear(hidden, hidden, bias=False)
|
| 96 |
+
self.value_embed: nn.Embedding | None = None
|
| 97 |
+
self.value_embed_gate: nn.Linear | None = None
|
| 98 |
+
if config.has_value_embedding(layer_idx):
|
| 99 |
+
self.value_embed = nn.Embedding(config.vocab_size, kv_dim)
|
| 100 |
+
self.value_embed_gate = nn.Linear(config.value_embed_gate_channels, config.num_key_value_heads, bias=False)
|
| 101 |
+
self.gate_channels = config.value_embed_gate_channels
|
| 102 |
+
self.gate_range = config.value_embed_gate_range
|
| 103 |
+
|
| 104 |
+
def forward(
|
| 105 |
+
self,
|
| 106 |
+
x: torch.Tensor,
|
| 107 |
+
input_ids: torch.LongTensor,
|
| 108 |
+
rotary: tuple[torch.Tensor, torch.Tensor],
|
| 109 |
+
mask: WindowMask,
|
| 110 |
+
past_key_values: Cache | None,
|
| 111 |
+
) -> torch.Tensor:
|
| 112 |
+
batch, time, _ = x.shape
|
| 113 |
+
q = self.q_proj(x).view(batch, time, self.num_heads, self.head_dim)
|
| 114 |
+
k = self.k_proj(x).view(batch, time, self.num_kv_heads, self.head_dim)
|
| 115 |
+
v = self.v_proj(x).view(batch, time, self.num_kv_heads, self.head_dim)
|
| 116 |
+
if self.value_embed is not None:
|
| 117 |
+
ve = self.value_embed(input_ids).to(x.dtype).view(batch, time, self.num_kv_heads, self.head_dim)
|
| 118 |
+
gate = self.gate_range * torch.sigmoid(self.value_embed_gate(x[..., : self.gate_channels]))
|
| 119 |
+
v = v + gate.unsqueeze(-1) * ve
|
| 120 |
+
cos, sin = rotary
|
| 121 |
+
q, k = apply_rotary(q, cos, sin), apply_rotary(k, cos, sin)
|
| 122 |
+
q, k = rms_norm(q) * self.qk_scale, rms_norm(k) * self.qk_scale
|
| 123 |
+
q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
|
| 124 |
+
if past_key_values is not None:
|
| 125 |
+
k, v = past_key_values.update(k, v, self.layer_idx)
|
| 126 |
+
y = F.scaled_dot_product_attention(
|
| 127 |
+
q, k, v, attn_mask=mask.mask, is_causal=mask.is_causal, enable_gqa=self.num_kv_heads != self.num_heads
|
| 128 |
+
)
|
| 129 |
+
return self.o_proj(y.transpose(1, 2).reshape(batch, time, -1))
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
class WildAIMLP(nn.Module):
|
| 133 |
+
def __init__(self, config: WildAIConfig) -> None:
|
| 134 |
+
super().__init__()
|
| 135 |
+
self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
|
| 136 |
+
self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
|
| 137 |
+
|
| 138 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 139 |
+
return self.down_proj(F.relu(self.up_proj(x)).square())
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
class WildAIDecoderLayer(nn.Module):
|
| 143 |
+
def __init__(self, config: WildAIConfig, layer_idx: int) -> None:
|
| 144 |
+
super().__init__()
|
| 145 |
+
self.self_attn = WildAIAttention(config, layer_idx)
|
| 146 |
+
self.mlp = WildAIMLP(config)
|
| 147 |
+
|
| 148 |
+
def forward(
|
| 149 |
+
self,
|
| 150 |
+
x: torch.Tensor,
|
| 151 |
+
input_ids: torch.LongTensor,
|
| 152 |
+
rotary: tuple[torch.Tensor, torch.Tensor],
|
| 153 |
+
mask: WindowMask,
|
| 154 |
+
past_key_values: Cache | None,
|
| 155 |
+
) -> torch.Tensor:
|
| 156 |
+
x = x + self.self_attn(rms_norm(x), input_ids, rotary, mask, past_key_values)
|
| 157 |
+
return x + self.mlp(rms_norm(x))
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
class WildAIPreTrainedModel(PreTrainedModel):
|
| 161 |
+
config_class = WildAIConfig
|
| 162 |
+
base_model_prefix = "model"
|
| 163 |
+
main_input_name = "input_ids"
|
| 164 |
+
_no_split_modules: ClassVar[list[str]] = ["WildAIDecoderLayer"]
|
| 165 |
+
_supports_sdpa = True
|
| 166 |
+
|
| 167 |
+
def _init_weights(self, module: nn.Module) -> None:
|
| 168 |
+
# Released models are always loaded from trained weights; this only gives from_config() models sane values.
|
| 169 |
+
if isinstance(module, (nn.Linear, nn.Embedding)):
|
| 170 |
+
nn.init.normal_(module.weight, mean=0.0, std=0.02)
|
| 171 |
+
elif isinstance(module, WildAIModel):
|
| 172 |
+
with torch.no_grad():
|
| 173 |
+
module.resid_lambdas.fill_(1.0)
|
| 174 |
+
module.x0_lambdas.zero_()
|
| 175 |
+
module.smear_lambda.zero_()
|
| 176 |
+
module.backout_lambda.zero_()
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
class WildAIModel(WildAIPreTrainedModel):
|
| 180 |
+
"""The transformer trunk; returns the final normalised hidden states."""
|
| 181 |
+
|
| 182 |
+
def __init__(self, config: WildAIConfig) -> None:
|
| 183 |
+
super().__init__(config)
|
| 184 |
+
n_layer = config.num_hidden_layers
|
| 185 |
+
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
|
| 186 |
+
self.layers = nn.ModuleList([WildAIDecoderLayer(config, i) for i in range(n_layer)])
|
| 187 |
+
self.resid_lambdas = nn.Parameter(torch.ones(n_layer))
|
| 188 |
+
self.x0_lambdas = nn.Parameter(torch.zeros(n_layer))
|
| 189 |
+
self.smear_gate = nn.Linear(config.smear_channels, 1, bias=False)
|
| 190 |
+
self.smear_lambda = nn.Parameter(torch.zeros(1))
|
| 191 |
+
self.backout_lambda = nn.Parameter(torch.zeros(1))
|
| 192 |
+
self.post_init()
|
| 193 |
+
|
| 194 |
+
def get_input_embeddings(self) -> nn.Embedding:
|
| 195 |
+
return self.embed_tokens
|
| 196 |
+
|
| 197 |
+
def set_input_embeddings(self, value: nn.Embedding) -> None:
|
| 198 |
+
self.embed_tokens = value
|
| 199 |
+
|
| 200 |
+
def rotary(self, position_ids: torch.LongTensor, dtype: torch.dtype) -> tuple[torch.Tensor, torch.Tensor]:
|
| 201 |
+
"""cos/sin tables of shape (batch, time, 1, head_dim / 2), computed in float32 and cast like in training."""
|
| 202 |
+
head_dim = self.config.head_dim
|
| 203 |
+
channels = torch.arange(0, head_dim, 2, dtype=torch.float32, device=position_ids.device)
|
| 204 |
+
inv_freq = 1.0 / (self.config.rotary_base ** (channels / head_dim))
|
| 205 |
+
freqs = position_ids.to(torch.float32)[..., None] * inv_freq
|
| 206 |
+
return freqs.cos().to(dtype)[:, :, None, :], freqs.sin().to(dtype)[:, :, None, :]
|
| 207 |
+
|
| 208 |
+
def smear(self, x: torch.Tensor, previous: torch.Tensor | None, padding_mask: torch.Tensor | None) -> torch.Tensor:
|
| 209 |
+
"""Add a gated copy of the previous position's embedding to every position that has one.
|
| 210 |
+
|
| 211 |
+
``previous`` is the normalised embedding of the token just before this chunk (None at the start of a sequence);
|
| 212 |
+
with left padding, the first real token has a padding token before it and gets nothing.
|
| 213 |
+
"""
|
| 214 |
+
channels = self.config.smear_channels
|
| 215 |
+
lam = self.smear_lambda.to(x.dtype)
|
| 216 |
+
first = x[:, :1]
|
| 217 |
+
if previous is not None:
|
| 218 |
+
first = first + lam * torch.sigmoid(self.smear_gate(first[..., :channels])) * previous
|
| 219 |
+
if x.size(1) == 1:
|
| 220 |
+
return first
|
| 221 |
+
gate = lam * torch.sigmoid(self.smear_gate(x[:, 1:, :channels]))
|
| 222 |
+
if padding_mask is not None:
|
| 223 |
+
gate = gate * padding_mask[:, -x.size(1) : -1, None].to(gate.dtype)
|
| 224 |
+
return torch.cat([first, x[:, 1:] + gate * x[:, :-1]], dim=1)
|
| 225 |
+
|
| 226 |
+
def forward(
|
| 227 |
+
self,
|
| 228 |
+
input_ids: torch.LongTensor,
|
| 229 |
+
attention_mask: torch.Tensor | None = None,
|
| 230 |
+
position_ids: torch.LongTensor | None = None,
|
| 231 |
+
past_key_values: Cache | None = None,
|
| 232 |
+
use_cache: bool | None = None,
|
| 233 |
+
previous_token_ids: torch.LongTensor | None = None,
|
| 234 |
+
output_hidden_states: bool | None = None,
|
| 235 |
+
**kwargs: object,
|
| 236 |
+
) -> BaseModelOutputWithPast:
|
| 237 |
+
"""``previous_token_ids`` (batch, 1): the token before ``input_ids[:, 0]``; required when continuing from a cache."""
|
| 238 |
+
if input_ids is None:
|
| 239 |
+
raise ValueError("input_ids are required: the value embeddings are looked up by token id")
|
| 240 |
+
batch, time = input_ids.shape
|
| 241 |
+
use_cache = self.config.use_cache if use_cache is None else use_cache
|
| 242 |
+
if use_cache and past_key_values is None:
|
| 243 |
+
past_key_values = DynamicCache()
|
| 244 |
+
past_len = past_key_values.get_seq_length() if past_key_values is not None else 0
|
| 245 |
+
if past_len > 0 and previous_token_ids is None:
|
| 246 |
+
raise ValueError("previous_token_ids is required when continuing from past_key_values (the smear needs it)")
|
| 247 |
+
padding_mask = None
|
| 248 |
+
if attention_mask is not None:
|
| 249 |
+
if attention_mask.shape != (batch, past_len + time):
|
| 250 |
+
raise ValueError(f"attention_mask must have shape {(batch, past_len + time)}, got {tuple(attention_mask.shape)}")
|
| 251 |
+
if not bool(attention_mask.all()):
|
| 252 |
+
padding_mask = attention_mask
|
| 253 |
+
if position_ids is None:
|
| 254 |
+
position_ids = torch.arange(past_len, past_len + time, device=input_ids.device).expand(batch, -1)
|
| 255 |
+
|
| 256 |
+
x = rms_norm(self.embed_tokens(input_ids))
|
| 257 |
+
previous = rms_norm(self.embed_tokens(previous_token_ids)) if previous_token_ids is not None else None
|
| 258 |
+
x = self.smear(x, previous, padding_mask)
|
| 259 |
+
rotary = self.rotary(position_ids, x.dtype)
|
| 260 |
+
masks = window_masks(self.config.window_sizes, time, past_len, padding_mask, input_ids.device)
|
| 261 |
+
|
| 262 |
+
x0 = x
|
| 263 |
+
backout = None
|
| 264 |
+
hidden_states: tuple[torch.Tensor, ...] = (x,)
|
| 265 |
+
for i, layer in enumerate(self.layers):
|
| 266 |
+
x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0
|
| 267 |
+
x = layer(x, input_ids, rotary, masks[layer.self_attn.window], past_key_values)
|
| 268 |
+
if i == self.config.backout_layer:
|
| 269 |
+
backout = x
|
| 270 |
+
hidden_states += (x,)
|
| 271 |
+
x = rms_norm(x - self.backout_lambda.to(x.dtype) * backout)
|
| 272 |
+
return BaseModelOutputWithPast(
|
| 273 |
+
last_hidden_state=x,
|
| 274 |
+
past_key_values=past_key_values if use_cache else None,
|
| 275 |
+
hidden_states=hidden_states if output_hidden_states else None,
|
| 276 |
+
)
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
class WildAIForCausalLM(WildAIPreTrainedModel, GenerationMixin):
|
| 280 |
+
def __init__(self, config: WildAIConfig) -> None:
|
| 281 |
+
super().__init__(config)
|
| 282 |
+
self.model = WildAIModel(config)
|
| 283 |
+
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
|
| 284 |
+
self.post_init()
|
| 285 |
+
|
| 286 |
+
def get_input_embeddings(self) -> nn.Embedding:
|
| 287 |
+
return self.model.embed_tokens
|
| 288 |
+
|
| 289 |
+
def set_input_embeddings(self, value: nn.Embedding) -> None:
|
| 290 |
+
self.model.embed_tokens = value
|
| 291 |
+
|
| 292 |
+
def get_output_embeddings(self) -> nn.Linear:
|
| 293 |
+
return self.lm_head
|
| 294 |
+
|
| 295 |
+
def set_output_embeddings(self, value: nn.Linear) -> None:
|
| 296 |
+
self.lm_head = value
|
| 297 |
+
|
| 298 |
+
def forward(
|
| 299 |
+
self,
|
| 300 |
+
input_ids: torch.LongTensor,
|
| 301 |
+
attention_mask: torch.Tensor | None = None,
|
| 302 |
+
position_ids: torch.LongTensor | None = None,
|
| 303 |
+
past_key_values: Cache | None = None,
|
| 304 |
+
labels: torch.LongTensor | None = None,
|
| 305 |
+
use_cache: bool | None = None,
|
| 306 |
+
previous_token_ids: torch.LongTensor | None = None,
|
| 307 |
+
output_hidden_states: bool | None = None,
|
| 308 |
+
logits_to_keep: int = 0,
|
| 309 |
+
return_dict: bool | None = None,
|
| 310 |
+
**kwargs: object,
|
| 311 |
+
) -> CausalLMOutputWithPast | tuple[torch.Tensor, ...]:
|
| 312 |
+
"""Float32 logits after the soft cap. ``labels`` are shifted inside (standard causal-LM loss, -100 ignored)."""
|
| 313 |
+
outputs = self.model(
|
| 314 |
+
input_ids,
|
| 315 |
+
attention_mask=attention_mask,
|
| 316 |
+
position_ids=position_ids,
|
| 317 |
+
past_key_values=past_key_values,
|
| 318 |
+
use_cache=use_cache,
|
| 319 |
+
previous_token_ids=previous_token_ids,
|
| 320 |
+
output_hidden_states=output_hidden_states,
|
| 321 |
+
)
|
| 322 |
+
hidden = outputs.last_hidden_state[:, -logits_to_keep:] if logits_to_keep else outputs.last_hidden_state
|
| 323 |
+
logits = self.lm_head(hidden).float()
|
| 324 |
+
cap = self.config.logit_softcap
|
| 325 |
+
logits = cap * torch.tanh(logits / cap)
|
| 326 |
+
loss = None
|
| 327 |
+
if labels is not None:
|
| 328 |
+
loss = F.cross_entropy(logits[:, :-1].flatten(0, 1), labels[:, 1:].flatten().to(logits.device), ignore_index=-100)
|
| 329 |
+
result = CausalLMOutputWithPast(
|
| 330 |
+
loss=loss, logits=logits, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states
|
| 331 |
+
)
|
| 332 |
+
return result.to_tuple() if return_dict is False else result
|
| 333 |
+
|
| 334 |
+
def prepare_inputs_for_generation(self, input_ids: torch.LongTensor, **kwargs: object) -> dict[str, object]:
|
| 335 |
+
# The smear mixes in the embedding of the token before the first new one; take it from the full sequence.
|
| 336 |
+
model_inputs = super().prepare_inputs_for_generation(input_ids, **kwargs)
|
| 337 |
+
new_len = model_inputs["input_ids"].shape[1]
|
| 338 |
+
if input_ids.shape[1] > new_len:
|
| 339 |
+
model_inputs["previous_token_ids"] = input_ids[:, -new_len - 1 : -new_len]
|
| 340 |
+
return model_inputs
|
268m-g073-ai-r0.05/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
268m-g073-human-r0.025/LICENSE
ADDED
|
@@ -0,0 +1,438 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Attribution-NonCommercial-ShareAlike 4.0 International
|
| 2 |
+
|
| 3 |
+
=======================================================================
|
| 4 |
+
|
| 5 |
+
Creative Commons Corporation ("Creative Commons") is not a law firm and
|
| 6 |
+
does not provide legal services or legal advice. Distribution of
|
| 7 |
+
Creative Commons public licenses does not create a lawyer-client or
|
| 8 |
+
other relationship. Creative Commons makes its licenses and related
|
| 9 |
+
information available on an "as-is" basis. Creative Commons gives no
|
| 10 |
+
warranties regarding its licenses, any material licensed under their
|
| 11 |
+
terms and conditions, or any related information. Creative Commons
|
| 12 |
+
disclaims all liability for damages resulting from their use to the
|
| 13 |
+
fullest extent possible.
|
| 14 |
+
|
| 15 |
+
Using Creative Commons Public Licenses
|
| 16 |
+
|
| 17 |
+
Creative Commons public licenses provide a standard set of terms and
|
| 18 |
+
conditions that creators and other rights holders may use to share
|
| 19 |
+
original works of authorship and other material subject to copyright
|
| 20 |
+
and certain other rights specified in the public license below. The
|
| 21 |
+
following considerations are for informational purposes only, are not
|
| 22 |
+
exhaustive, and do not form part of our licenses.
|
| 23 |
+
|
| 24 |
+
Considerations for licensors: Our public licenses are
|
| 25 |
+
intended for use by those authorized to give the public
|
| 26 |
+
permission to use material in ways otherwise restricted by
|
| 27 |
+
copyright and certain other rights. Our licenses are
|
| 28 |
+
irrevocable. Licensors should read and understand the terms
|
| 29 |
+
and conditions of the license they choose before applying it.
|
| 30 |
+
Licensors should also secure all rights necessary before
|
| 31 |
+
applying our licenses so that the public can reuse the
|
| 32 |
+
material as expected. Licensors should clearly mark any
|
| 33 |
+
material not subject to the license. This includes other CC-
|
| 34 |
+
licensed material, or material used under an exception or
|
| 35 |
+
limitation to copyright. More considerations for licensors:
|
| 36 |
+
wiki.creativecommons.org/Considerations_for_licensors
|
| 37 |
+
|
| 38 |
+
Considerations for the public: By using one of our public
|
| 39 |
+
licenses, a licensor grants the public permission to use the
|
| 40 |
+
licensed material under specified terms and conditions. If
|
| 41 |
+
the licensor's permission is not necessary for any reason--for
|
| 42 |
+
example, because of any applicable exception or limitation to
|
| 43 |
+
copyright--then that use is not regulated by the license. Our
|
| 44 |
+
licenses grant only permissions under copyright and certain
|
| 45 |
+
other rights that a licensor has authority to grant. Use of
|
| 46 |
+
the licensed material may still be restricted for other
|
| 47 |
+
reasons, including because others have copyright or other
|
| 48 |
+
rights in the material. A licensor may make special requests,
|
| 49 |
+
such as asking that all changes be marked or described.
|
| 50 |
+
Although not required by our licenses, you are encouraged to
|
| 51 |
+
respect those requests where reasonable. More considerations
|
| 52 |
+
for the public:
|
| 53 |
+
wiki.creativecommons.org/Considerations_for_licensees
|
| 54 |
+
|
| 55 |
+
=======================================================================
|
| 56 |
+
|
| 57 |
+
Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International
|
| 58 |
+
Public License
|
| 59 |
+
|
| 60 |
+
By exercising the Licensed Rights (defined below), You accept and agree
|
| 61 |
+
to be bound by the terms and conditions of this Creative Commons
|
| 62 |
+
Attribution-NonCommercial-ShareAlike 4.0 International Public License
|
| 63 |
+
("Public License"). To the extent this Public License may be
|
| 64 |
+
interpreted as a contract, You are granted the Licensed Rights in
|
| 65 |
+
consideration of Your acceptance of these terms and conditions, and the
|
| 66 |
+
Licensor grants You such rights in consideration of benefits the
|
| 67 |
+
Licensor receives from making the Licensed Material available under
|
| 68 |
+
these terms and conditions.
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
Section 1 -- Definitions.
|
| 72 |
+
|
| 73 |
+
a. Adapted Material means material subject to Copyright and Similar
|
| 74 |
+
Rights that is derived from or based upon the Licensed Material
|
| 75 |
+
and in which the Licensed Material is translated, altered,
|
| 76 |
+
arranged, transformed, or otherwise modified in a manner requiring
|
| 77 |
+
permission under the Copyright and Similar Rights held by the
|
| 78 |
+
Licensor. For purposes of this Public License, where the Licensed
|
| 79 |
+
Material is a musical work, performance, or sound recording,
|
| 80 |
+
Adapted Material is always produced where the Licensed Material is
|
| 81 |
+
synched in timed relation with a moving image.
|
| 82 |
+
|
| 83 |
+
b. Adapter's License means the license You apply to Your Copyright
|
| 84 |
+
and Similar Rights in Your contributions to Adapted Material in
|
| 85 |
+
accordance with the terms and conditions of this Public License.
|
| 86 |
+
|
| 87 |
+
c. BY-NC-SA Compatible License means a license listed at
|
| 88 |
+
creativecommons.org/compatiblelicenses, approved by Creative
|
| 89 |
+
Commons as essentially the equivalent of this Public License.
|
| 90 |
+
|
| 91 |
+
d. Copyright and Similar Rights means copyright and/or similar rights
|
| 92 |
+
closely related to copyright including, without limitation,
|
| 93 |
+
performance, broadcast, sound recording, and Sui Generis Database
|
| 94 |
+
Rights, without regard to how the rights are labeled or
|
| 95 |
+
categorized. For purposes of this Public License, the rights
|
| 96 |
+
specified in Section 2(b)(1)-(2) are not Copyright and Similar
|
| 97 |
+
Rights.
|
| 98 |
+
|
| 99 |
+
e. Effective Technological Measures means those measures that, in the
|
| 100 |
+
absence of proper authority, may not be circumvented under laws
|
| 101 |
+
fulfilling obligations under Article 11 of the WIPO Copyright
|
| 102 |
+
Treaty adopted on December 20, 1996, and/or similar international
|
| 103 |
+
agreements.
|
| 104 |
+
|
| 105 |
+
f. Exceptions and Limitations means fair use, fair dealing, and/or
|
| 106 |
+
any other exception or limitation to Copyright and Similar Rights
|
| 107 |
+
that applies to Your use of the Licensed Material.
|
| 108 |
+
|
| 109 |
+
g. License Elements means the license attributes listed in the name
|
| 110 |
+
of a Creative Commons Public License. The License Elements of this
|
| 111 |
+
Public License are Attribution, NonCommercial, and ShareAlike.
|
| 112 |
+
|
| 113 |
+
h. Licensed Material means the artistic or literary work, database,
|
| 114 |
+
or other material to which the Licensor applied this Public
|
| 115 |
+
License.
|
| 116 |
+
|
| 117 |
+
i. Licensed Rights means the rights granted to You subject to the
|
| 118 |
+
terms and conditions of this Public License, which are limited to
|
| 119 |
+
all Copyright and Similar Rights that apply to Your use of the
|
| 120 |
+
Licensed Material and that the Licensor has authority to license.
|
| 121 |
+
|
| 122 |
+
j. Licensor means the individual(s) or entity(ies) granting rights
|
| 123 |
+
under this Public License.
|
| 124 |
+
|
| 125 |
+
k. NonCommercial means not primarily intended for or directed towards
|
| 126 |
+
commercial advantage or monetary compensation. For purposes of
|
| 127 |
+
this Public License, the exchange of the Licensed Material for
|
| 128 |
+
other material subject to Copyright and Similar Rights by digital
|
| 129 |
+
file-sharing or similar means is NonCommercial provided there is
|
| 130 |
+
no payment of monetary compensation in connection with the
|
| 131 |
+
exchange.
|
| 132 |
+
|
| 133 |
+
l. Share means to provide material to the public by any means or
|
| 134 |
+
process that requires permission under the Licensed Rights, such
|
| 135 |
+
as reproduction, public display, public performance, distribution,
|
| 136 |
+
dissemination, communication, or importation, and to make material
|
| 137 |
+
available to the public including in ways that members of the
|
| 138 |
+
public may access the material from a place and at a time
|
| 139 |
+
individually chosen by them.
|
| 140 |
+
|
| 141 |
+
m. Sui Generis Database Rights means rights other than copyright
|
| 142 |
+
resulting from Directive 96/9/EC of the European Parliament and of
|
| 143 |
+
the Council of 11 March 1996 on the legal protection of databases,
|
| 144 |
+
as amended and/or succeeded, as well as other essentially
|
| 145 |
+
equivalent rights anywhere in the world.
|
| 146 |
+
|
| 147 |
+
n. You means the individual or entity exercising the Licensed Rights
|
| 148 |
+
under this Public License. Your has a corresponding meaning.
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
Section 2 -- Scope.
|
| 152 |
+
|
| 153 |
+
a. License grant.
|
| 154 |
+
|
| 155 |
+
1. Subject to the terms and conditions of this Public License,
|
| 156 |
+
the Licensor hereby grants You a worldwide, royalty-free,
|
| 157 |
+
non-sublicensable, non-exclusive, irrevocable license to
|
| 158 |
+
exercise the Licensed Rights in the Licensed Material to:
|
| 159 |
+
|
| 160 |
+
a. reproduce and Share the Licensed Material, in whole or
|
| 161 |
+
in part, for NonCommercial purposes only; and
|
| 162 |
+
|
| 163 |
+
b. produce, reproduce, and Share Adapted Material for
|
| 164 |
+
NonCommercial purposes only.
|
| 165 |
+
|
| 166 |
+
2. Exceptions and Limitations. For the avoidance of doubt, where
|
| 167 |
+
Exceptions and Limitations apply to Your use, this Public
|
| 168 |
+
License does not apply, and You do not need to comply with
|
| 169 |
+
its terms and conditions.
|
| 170 |
+
|
| 171 |
+
3. Term. The term of this Public License is specified in Section
|
| 172 |
+
6(a).
|
| 173 |
+
|
| 174 |
+
4. Media and formats; technical modifications allowed. The
|
| 175 |
+
Licensor authorizes You to exercise the Licensed Rights in
|
| 176 |
+
all media and formats whether now known or hereafter created,
|
| 177 |
+
and to make technical modifications necessary to do so. The
|
| 178 |
+
Licensor waives and/or agrees not to assert any right or
|
| 179 |
+
authority to forbid You from making technical modifications
|
| 180 |
+
necessary to exercise the Licensed Rights, including
|
| 181 |
+
technical modifications necessary to circumvent Effective
|
| 182 |
+
Technological Measures. For purposes of this Public License,
|
| 183 |
+
simply making modifications authorized by this Section 2(a)
|
| 184 |
+
(4) never produces Adapted Material.
|
| 185 |
+
|
| 186 |
+
5. Downstream recipients.
|
| 187 |
+
|
| 188 |
+
a. Offer from the Licensor -- Licensed Material. Every
|
| 189 |
+
recipient of the Licensed Material automatically
|
| 190 |
+
receives an offer from the Licensor to exercise the
|
| 191 |
+
Licensed Rights under the terms and conditions of this
|
| 192 |
+
Public License.
|
| 193 |
+
|
| 194 |
+
b. Additional offer from the Licensor -- Adapted Material.
|
| 195 |
+
Every recipient of Adapted Material from You
|
| 196 |
+
automatically receives an offer from the Licensor to
|
| 197 |
+
exercise the Licensed Rights in the Adapted Material
|
| 198 |
+
under the conditions of the Adapter's License You apply.
|
| 199 |
+
|
| 200 |
+
c. No downstream restrictions. You may not offer or impose
|
| 201 |
+
any additional or different terms or conditions on, or
|
| 202 |
+
apply any Effective Technological Measures to, the
|
| 203 |
+
Licensed Material if doing so restricts exercise of the
|
| 204 |
+
Licensed Rights by any recipient of the Licensed
|
| 205 |
+
Material.
|
| 206 |
+
|
| 207 |
+
6. No endorsement. Nothing in this Public License constitutes or
|
| 208 |
+
may be construed as permission to assert or imply that You
|
| 209 |
+
are, or that Your use of the Licensed Material is, connected
|
| 210 |
+
with, or sponsored, endorsed, or granted official status by,
|
| 211 |
+
the Licensor or others designated to receive attribution as
|
| 212 |
+
provided in Section 3(a)(1)(A)(i).
|
| 213 |
+
|
| 214 |
+
b. Other rights.
|
| 215 |
+
|
| 216 |
+
1. Moral rights, such as the right of integrity, are not
|
| 217 |
+
licensed under this Public License, nor are publicity,
|
| 218 |
+
privacy, and/or other similar personality rights; however, to
|
| 219 |
+
the extent possible, the Licensor waives and/or agrees not to
|
| 220 |
+
assert any such rights held by the Licensor to the limited
|
| 221 |
+
extent necessary to allow You to exercise the Licensed
|
| 222 |
+
Rights, but not otherwise.
|
| 223 |
+
|
| 224 |
+
2. Patent and trademark rights are not licensed under this
|
| 225 |
+
Public License.
|
| 226 |
+
|
| 227 |
+
3. To the extent possible, the Licensor waives any right to
|
| 228 |
+
collect royalties from You for the exercise of the Licensed
|
| 229 |
+
Rights, whether directly or through a collecting society
|
| 230 |
+
under any voluntary or waivable statutory or compulsory
|
| 231 |
+
licensing scheme. In all other cases the Licensor expressly
|
| 232 |
+
reserves any right to collect such royalties, including when
|
| 233 |
+
the Licensed Material is used other than for NonCommercial
|
| 234 |
+
purposes.
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
Section 3 -- License Conditions.
|
| 238 |
+
|
| 239 |
+
Your exercise of the Licensed Rights is expressly made subject to the
|
| 240 |
+
following conditions.
|
| 241 |
+
|
| 242 |
+
a. Attribution.
|
| 243 |
+
|
| 244 |
+
1. If You Share the Licensed Material (including in modified
|
| 245 |
+
form), You must:
|
| 246 |
+
|
| 247 |
+
a. retain the following if it is supplied by the Licensor
|
| 248 |
+
with the Licensed Material:
|
| 249 |
+
|
| 250 |
+
i. identification of the creator(s) of the Licensed
|
| 251 |
+
Material and any others designated to receive
|
| 252 |
+
attribution, in any reasonable manner requested by
|
| 253 |
+
the Licensor (including by pseudonym if
|
| 254 |
+
designated);
|
| 255 |
+
|
| 256 |
+
ii. a copyright notice;
|
| 257 |
+
|
| 258 |
+
iii. a notice that refers to this Public License;
|
| 259 |
+
|
| 260 |
+
iv. a notice that refers to the disclaimer of
|
| 261 |
+
warranties;
|
| 262 |
+
|
| 263 |
+
v. a URI or hyperlink to the Licensed Material to the
|
| 264 |
+
extent reasonably practicable;
|
| 265 |
+
|
| 266 |
+
b. indicate if You modified the Licensed Material and
|
| 267 |
+
retain an indication of any previous modifications; and
|
| 268 |
+
|
| 269 |
+
c. indicate the Licensed Material is licensed under this
|
| 270 |
+
Public License, and include the text of, or the URI or
|
| 271 |
+
hyperlink to, this Public License.
|
| 272 |
+
|
| 273 |
+
2. You may satisfy the conditions in Section 3(a)(1) in any
|
| 274 |
+
reasonable manner based on the medium, means, and context in
|
| 275 |
+
which You Share the Licensed Material. For example, it may be
|
| 276 |
+
reasonable to satisfy the conditions by providing a URI or
|
| 277 |
+
hyperlink to a resource that includes the required
|
| 278 |
+
information.
|
| 279 |
+
3. If requested by the Licensor, You must remove any of the
|
| 280 |
+
information required by Section 3(a)(1)(A) to the extent
|
| 281 |
+
reasonably practicable.
|
| 282 |
+
|
| 283 |
+
b. ShareAlike.
|
| 284 |
+
|
| 285 |
+
In addition to the conditions in Section 3(a), if You Share
|
| 286 |
+
Adapted Material You produce, the following conditions also apply.
|
| 287 |
+
|
| 288 |
+
1. The Adapter's License You apply must be a Creative Commons
|
| 289 |
+
license with the same License Elements, this version or
|
| 290 |
+
later, or a BY-NC-SA Compatible License.
|
| 291 |
+
|
| 292 |
+
2. You must include the text of, or the URI or hyperlink to, the
|
| 293 |
+
Adapter's License You apply. You may satisfy this condition
|
| 294 |
+
in any reasonable manner based on the medium, means, and
|
| 295 |
+
context in which You Share Adapted Material.
|
| 296 |
+
|
| 297 |
+
3. You may not offer or impose any additional or different terms
|
| 298 |
+
or conditions on, or apply any Effective Technological
|
| 299 |
+
Measures to, Adapted Material that restrict exercise of the
|
| 300 |
+
rights granted under the Adapter's License You apply.
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
Section 4 -- Sui Generis Database Rights.
|
| 304 |
+
|
| 305 |
+
Where the Licensed Rights include Sui Generis Database Rights that
|
| 306 |
+
apply to Your use of the Licensed Material:
|
| 307 |
+
|
| 308 |
+
a. for the avoidance of doubt, Section 2(a)(1) grants You the right
|
| 309 |
+
to extract, reuse, reproduce, and Share all or a substantial
|
| 310 |
+
portion of the contents of the database for NonCommercial purposes
|
| 311 |
+
only;
|
| 312 |
+
|
| 313 |
+
b. if You include all or a substantial portion of the database
|
| 314 |
+
contents in a database in which You have Sui Generis Database
|
| 315 |
+
Rights, then the database in which You have Sui Generis Database
|
| 316 |
+
Rights (but not its individual contents) is Adapted Material,
|
| 317 |
+
including for purposes of Section 3(b); and
|
| 318 |
+
|
| 319 |
+
c. You must comply with the conditions in Section 3(a) if You Share
|
| 320 |
+
all or a substantial portion of the contents of the database.
|
| 321 |
+
|
| 322 |
+
For the avoidance of doubt, this Section 4 supplements and does not
|
| 323 |
+
replace Your obligations under this Public License where the Licensed
|
| 324 |
+
Rights include other Copyright and Similar Rights.
|
| 325 |
+
|
| 326 |
+
|
| 327 |
+
Section 5 -- Disclaimer of Warranties and Limitation of Liability.
|
| 328 |
+
|
| 329 |
+
a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
|
| 330 |
+
EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
|
| 331 |
+
AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
|
| 332 |
+
ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
|
| 333 |
+
IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
|
| 334 |
+
WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
|
| 335 |
+
PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
|
| 336 |
+
ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
|
| 337 |
+
KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
|
| 338 |
+
ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
|
| 339 |
+
|
| 340 |
+
b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
|
| 341 |
+
TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
|
| 342 |
+
NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
|
| 343 |
+
INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
|
| 344 |
+
COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
|
| 345 |
+
USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
|
| 346 |
+
ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
|
| 347 |
+
DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
|
| 348 |
+
IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
|
| 349 |
+
|
| 350 |
+
c. The disclaimer of warranties and limitation of liability provided
|
| 351 |
+
above shall be interpreted in a manner that, to the extent
|
| 352 |
+
possible, most closely approximates an absolute disclaimer and
|
| 353 |
+
waiver of all liability.
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
Section 6 -- Term and Termination.
|
| 357 |
+
|
| 358 |
+
a. This Public License applies for the term of the Copyright and
|
| 359 |
+
Similar Rights licensed here. However, if You fail to comply with
|
| 360 |
+
this Public License, then Your rights under this Public License
|
| 361 |
+
terminate automatically.
|
| 362 |
+
|
| 363 |
+
b. Where Your right to use the Licensed Material has terminated under
|
| 364 |
+
Section 6(a), it reinstates:
|
| 365 |
+
|
| 366 |
+
1. automatically as of the date the violation is cured, provided
|
| 367 |
+
it is cured within 30 days of Your discovery of the
|
| 368 |
+
violation; or
|
| 369 |
+
|
| 370 |
+
2. upon express reinstatement by the Licensor.
|
| 371 |
+
|
| 372 |
+
For the avoidance of doubt, this Section 6(b) does not affect any
|
| 373 |
+
right the Licensor may have to seek remedies for Your violations
|
| 374 |
+
of this Public License.
|
| 375 |
+
|
| 376 |
+
c. For the avoidance of doubt, the Licensor may also offer the
|
| 377 |
+
Licensed Material under separate terms or conditions or stop
|
| 378 |
+
distributing the Licensed Material at any time; however, doing so
|
| 379 |
+
will not terminate this Public License.
|
| 380 |
+
|
| 381 |
+
d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
|
| 382 |
+
License.
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
Section 7 -- Other Terms and Conditions.
|
| 386 |
+
|
| 387 |
+
a. The Licensor shall not be bound by any additional or different
|
| 388 |
+
terms or conditions communicated by You unless expressly agreed.
|
| 389 |
+
|
| 390 |
+
b. Any arrangements, understandings, or agreements regarding the
|
| 391 |
+
Licensed Material not stated herein are separate from and
|
| 392 |
+
independent of the terms and conditions of this Public License.
|
| 393 |
+
|
| 394 |
+
|
| 395 |
+
Section 8 -- Interpretation.
|
| 396 |
+
|
| 397 |
+
a. For the avoidance of doubt, this Public License does not, and
|
| 398 |
+
shall not be interpreted to, reduce, limit, restrict, or impose
|
| 399 |
+
conditions on any use of the Licensed Material that could lawfully
|
| 400 |
+
be made without permission under this Public License.
|
| 401 |
+
|
| 402 |
+
b. To the extent possible, if any provision of this Public License is
|
| 403 |
+
deemed unenforceable, it shall be automatically reformed to the
|
| 404 |
+
minimum extent necessary to make it enforceable. If the provision
|
| 405 |
+
cannot be reformed, it shall be severed from this Public License
|
| 406 |
+
without affecting the enforceability of the remaining terms and
|
| 407 |
+
conditions.
|
| 408 |
+
|
| 409 |
+
c. No term or condition of this Public License will be waived and no
|
| 410 |
+
failure to comply consented to unless expressly agreed to by the
|
| 411 |
+
Licensor.
|
| 412 |
+
|
| 413 |
+
d. Nothing in this Public License constitutes or may be interpreted
|
| 414 |
+
as a limitation upon, or waiver of, any privileges and immunities
|
| 415 |
+
that apply to the Licensor or You, including from the legal
|
| 416 |
+
processes of any jurisdiction or authority.
|
| 417 |
+
|
| 418 |
+
=======================================================================
|
| 419 |
+
|
| 420 |
+
Creative Commons is not a party to its public
|
| 421 |
+
licenses. Notwithstanding, Creative Commons may elect to apply one of
|
| 422 |
+
its public licenses to material it publishes and in those instances
|
| 423 |
+
will be considered the “Licensor.” The text of the Creative Commons
|
| 424 |
+
public licenses is dedicated to the public domain under the CC0 Public
|
| 425 |
+
Domain Dedication. Except for the limited purpose of indicating that
|
| 426 |
+
material is shared under a Creative Commons public license or as
|
| 427 |
+
otherwise permitted by the Creative Commons policies published at
|
| 428 |
+
creativecommons.org/policies, Creative Commons does not authorize the
|
| 429 |
+
use of the trademark "Creative Commons" or any other trademark or logo
|
| 430 |
+
of Creative Commons without its prior written consent including,
|
| 431 |
+
without limitation, in connection with any unauthorized modifications
|
| 432 |
+
to any of its public licenses or any other arrangements,
|
| 433 |
+
understandings, or agreements concerning use of licensed material. For
|
| 434 |
+
the avoidance of doubt, this paragraph does not form part of the
|
| 435 |
+
public licenses.
|
| 436 |
+
|
| 437 |
+
Creative Commons may be contacted at creativecommons.org.
|
| 438 |
+
|
268m-g073-human-r0.025/README.md
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: cc-by-nc-sa-4.0
|
| 3 |
+
library_name: transformers
|
| 4 |
+
pipeline_tag: text-generation
|
| 5 |
+
language:
|
| 6 |
+
- en
|
| 7 |
+
datasets:
|
| 8 |
+
- pangram/WildAI
|
| 9 |
+
tags:
|
| 10 |
+
- wildai
|
| 11 |
+
- nanochat
|
| 12 |
+
- scaling-laws
|
| 13 |
+
- ai-generated-text
|
| 14 |
+
- pretraining-data
|
| 15 |
+
---
|
| 16 |
+
|
| 17 |
+
# 268m-g073-human-r0.025
|
| 18 |
+
|
| 19 |
+
> **License: [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/).** The WildAI code, models and dataset are released under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license: you may use and adapt them for non-commercial purposes, with attribution, and must share adaptations under the same license.
|
| 20 |
+
|
| 21 |
+
A 268M decoder-only language model trained from scratch for [*How Much Is an AI Token Worth? Scaling Laws for Wild AI-Generated Web Text*](https://arxiv.org/abs/2609.40295), one of the paper's models measuring how wild AI-generated web text in pretraining data changes a model's loss. It is a research artifact for studying scaling laws, not an assistant: it has had no instruction tuning or safety training.
|
| 22 |
+
|
| 23 |
+
## Training data
|
| 24 |
+
|
| 25 |
+
The human documents of `268m-g073-control` plus fresh human-written web documents (0.025 times the control's tokens): 1.20B human tokens, the human-text counterpart of adding AI text. No AI text.
|
| 26 |
+
|
| 27 |
+
| | |
|
| 28 |
+
|---|---|
|
| 29 |
+
| Parameters (N, input embedding included, value embeddings excluded) | 268.4M |
|
| 30 |
+
| Layers / width | 16 / 1024 |
|
| 31 |
+
| Training tokens | 1.20B (2,296 optimizer steps) |
|
| 32 |
+
| Human / AI tokens | 1.20B / 0.0M |
|
| 33 |
+
| Group / arm / seed | g073 / human / 271828 |
|
| 34 |
+
| Used in the paper for | fitting the scaling laws (726 models, 19.9M to 268M) |
|
| 35 |
+
|
| 36 |
+
## Held-out loss
|
| 37 |
+
|
| 38 |
+
| Set | Bits per byte | Text |
|
| 39 |
+
|---|---|---|
|
| 40 |
+
| C4 | 0.9943 | C4 validation text (human-written, pre-2022) |
|
| 41 |
+
| FW22 | 0.9786 | held-out FineWeb documents crawled before 2022 |
|
| 42 |
+
| Paloma | 1.3691 | macro average over 16 Paloma sources |
|
| 43 |
+
| FW26 | 0.8671 | held-out 2026 FineWeb-style crawl (22.3% AI tokens) |
|
| 44 |
+
| FW26-H | 0.9115 | the human-labeled documents of FW26 |
|
| 45 |
+
| FW26-AI | 0.7037 | the AI-labeled documents of FW26 |
|
| 46 |
+
| Cosmo | 0.7007 | Cosmopedia (AI-generated textbooks and stories) |
|
| 47 |
+
|
| 48 |
+
<details><summary>Paloma sources</summary>
|
| 49 |
+
|
| 50 |
+
| Source | Bits per byte |
|
| 51 |
+
|---|---|
|
| 52 |
+
| 4chan | 1.3955 |
|
| 53 |
+
| c4 | 1.0138 |
|
| 54 |
+
| c4_100_domains | 0.9954 |
|
| 55 |
+
| dolma | 1.2625 |
|
| 56 |
+
| falcon_refinedweb | 1.1099 |
|
| 57 |
+
| gab | 1.8929 |
|
| 58 |
+
| m2d2_s2orc | 1.1779 |
|
| 59 |
+
| m2d2_wikipedia | 1.0818 |
|
| 60 |
+
| manosphere | 1.3913 |
|
| 61 |
+
| mc4_en | 1.1036 |
|
| 62 |
+
| programming | 1.9928 |
|
| 63 |
+
| ptb | 1.2345 |
|
| 64 |
+
| reddit_100 | 1.2453 |
|
| 65 |
+
| redpajama | 1.4332 |
|
| 66 |
+
| twitteraae | 2.4938 |
|
| 67 |
+
| wikitext_103 | 1.0820 |
|
| 68 |
+
|
| 69 |
+
</details>
|
| 70 |
+
|
| 71 |
+
## Usage
|
| 72 |
+
|
| 73 |
+
```python
|
| 74 |
+
import torch
|
| 75 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 76 |
+
|
| 77 |
+
tokenizer = AutoTokenizer.from_pretrained("pangram/WildAI-models", subfolder="268m-g073-human-r0.025", trust_remote_code=True)
|
| 78 |
+
model = AutoModelForCausalLM.from_pretrained("pangram/WildAI-models", subfolder="268m-g073-human-r0.025", trust_remote_code=True, dtype=torch.bfloat16)
|
| 79 |
+
|
| 80 |
+
inputs = tokenizer("The history of the printing press", return_tensors="pt")
|
| 81 |
+
output = model.generate(**inputs, max_new_tokens=40, do_sample=False)
|
| 82 |
+
print(tokenizer.decode(output[0], skip_special_tokens=True))
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
The tokenizer prepends `<|bos|>` to every text, as in training; pass `add_special_tokens=False` to skip it. The model predicts `<|bos|>` where a document ends, and generation stops there. Weights are stored in bfloat16, the precision the paper's losses were computed in; `dtype=torch.float32` computes in float32 instead.
|
| 86 |
+
|
| 87 |
+
## Architecture
|
| 88 |
+
|
| 89 |
+
The nanochat GPT recipe: rotary embeddings, RMSNorm without weights (also on queries and keys), squared-ReLU MLP, attention windows in a repeating short-short-short-long pattern (512 tokens, then the full 2,048-token context; the last layer always full), gated value embeddings on alternating layers, a previous-token smear, per-layer residual scalars, a mid-network backout, untied input and output embeddings and logits soft-capped at 15. Vocabulary 32,768 tokens, context 2,048. The code ships with the weights (`trust_remote_code=True`) and matches the training code's forward pass.
|
| 90 |
+
|
| 91 |
+
## License
|
| 92 |
+
|
| 93 |
+
**License: [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/).** The WildAI code, models and dataset are released under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license: you may use and adapt them for non-commercial purposes, with attribution, and must share adaptations under the same license. The full text is in `LICENSE`.
|
| 94 |
+
|
| 95 |
+
## Citation
|
| 96 |
+
|
| 97 |
+
```bibtex
|
| 98 |
+
@article{russell2026wildai,
|
| 99 |
+
title = {How Much Is an AI Token Worth? Scaling Laws for Wild AI-Generated Web Text},
|
| 100 |
+
author = {Russell, Jenna and Glickenhaus, Ben and Thai, Katherine and Wieting, John and Iyyer, Mohit and Spero, Max and Emi, Bradley},
|
| 101 |
+
journal = {arXiv preprint arXiv:2609.40295},
|
| 102 |
+
year = {2026},
|
| 103 |
+
url = {https://arxiv.org/abs/2609.40295}
|
| 104 |
+
}
|
| 105 |
+
```
|
268m-g073-human-r0.025/config.json
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"WildAIForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"auto_map": {
|
| 6 |
+
"AutoConfig": "configuration_wildai.WildAIConfig",
|
| 7 |
+
"AutoModelForCausalLM": "modeling_wildai.WildAIForCausalLM"
|
| 8 |
+
},
|
| 9 |
+
"bos_token_id": 32759,
|
| 10 |
+
"dtype": "bfloat16",
|
| 11 |
+
"eos_token_id": 32759,
|
| 12 |
+
"hidden_size": 1024,
|
| 13 |
+
"logit_softcap": 15.0,
|
| 14 |
+
"max_position_embeddings": 2048,
|
| 15 |
+
"model_type": "wildai",
|
| 16 |
+
"num_attention_heads": 8,
|
| 17 |
+
"num_hidden_layers": 16,
|
| 18 |
+
"num_key_value_heads": 8,
|
| 19 |
+
"pad_token_id": 32759,
|
| 20 |
+
"qk_scale": 1.2,
|
| 21 |
+
"release": {
|
| 22 |
+
"added_ratio": 0.025,
|
| 23 |
+
"ai_ratio": 0.0,
|
| 24 |
+
"ai_tokens": 0,
|
| 25 |
+
"arm": "human",
|
| 26 |
+
"depth": 16,
|
| 27 |
+
"group": "g073",
|
| 28 |
+
"human_tokens": 1203765248,
|
| 29 |
+
"n_params": 268436282,
|
| 30 |
+
"name": "268m-g073-human-r0.025",
|
| 31 |
+
"seed": 271828,
|
| 32 |
+
"size": "268m",
|
| 33 |
+
"split": "fit",
|
| 34 |
+
"steps": 2296,
|
| 35 |
+
"total_tokens": 1203765248
|
| 36 |
+
},
|
| 37 |
+
"rotary_base": 100000.0,
|
| 38 |
+
"smear_channels": 24,
|
| 39 |
+
"tie_word_embeddings": false,
|
| 40 |
+
"transformers_version": "5.17.0",
|
| 41 |
+
"use_cache": true,
|
| 42 |
+
"value_embed_gate_channels": 12,
|
| 43 |
+
"value_embed_gate_range": 3.0,
|
| 44 |
+
"vocab_size": 32768,
|
| 45 |
+
"window_pattern": "SSSL"
|
| 46 |
+
}
|
268m-g073-human-r0.025/configuration_wildai.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Configuration of the WildAI language models (a nanochat GPT variant) for Hugging Face ``transformers``.
|
| 2 |
+
|
| 3 |
+
This file is copied verbatim into every released model repository and loaded with ``trust_remote_code=True``.
|
| 4 |
+
It depends only on ``transformers``.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from typing import ClassVar
|
| 10 |
+
|
| 11 |
+
from transformers import PretrainedConfig
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class WildAIConfig(PretrainedConfig):
|
| 15 |
+
"""Architecture of a WildAI model. Only width and depth vary across the released models.
|
| 16 |
+
|
| 17 |
+
Every layer uses causal attention limited to a left window: a query attends to itself and the ``window`` tokens before
|
| 18 |
+
it. ``window_pattern`` is tiled over the layers ("S" = a quarter of the context rounded up to a multiple of 128 tokens,
|
| 19 |
+
"L" = the full context) and the last layer always uses "L".
|
| 20 |
+
|
| 21 |
+
``release`` holds the model's entry in the release catalog (name, token counts, AI ratio, ...); it does not affect
|
| 22 |
+
the forward pass.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
model_type = "wildai"
|
| 26 |
+
keys_to_ignore_at_inference: ClassVar[list[str]] = ["past_key_values"]
|
| 27 |
+
|
| 28 |
+
def __init__(
|
| 29 |
+
self,
|
| 30 |
+
vocab_size: int = 32768,
|
| 31 |
+
hidden_size: int = 768,
|
| 32 |
+
num_hidden_layers: int = 12,
|
| 33 |
+
num_attention_heads: int = 6,
|
| 34 |
+
num_key_value_heads: int = 6,
|
| 35 |
+
max_position_embeddings: int = 2048,
|
| 36 |
+
window_pattern: str = "SSSL",
|
| 37 |
+
rotary_base: float = 100000.0,
|
| 38 |
+
qk_scale: float = 1.2,
|
| 39 |
+
logit_softcap: float = 15.0,
|
| 40 |
+
value_embed_gate_channels: int = 12,
|
| 41 |
+
value_embed_gate_range: float = 3.0,
|
| 42 |
+
smear_channels: int = 24,
|
| 43 |
+
bos_token_id: int = 32759,
|
| 44 |
+
eos_token_id: int = 32759,
|
| 45 |
+
use_cache: bool = True,
|
| 46 |
+
release: dict[str, str | int | float | None] | None = None,
|
| 47 |
+
**kwargs: object,
|
| 48 |
+
) -> None:
|
| 49 |
+
if hidden_size % num_attention_heads != 0 or num_attention_heads % num_key_value_heads != 0:
|
| 50 |
+
raise ValueError("hidden_size must divide into num_attention_heads, and those into num_key_value_heads")
|
| 51 |
+
if not window_pattern or set(window_pattern.upper()) - {"S", "L"}:
|
| 52 |
+
raise ValueError(f"window_pattern must be a non-empty string of S and L, got {window_pattern!r}")
|
| 53 |
+
self.vocab_size = vocab_size
|
| 54 |
+
self.hidden_size = hidden_size
|
| 55 |
+
self.num_hidden_layers = num_hidden_layers
|
| 56 |
+
self.num_attention_heads = num_attention_heads
|
| 57 |
+
self.num_key_value_heads = num_key_value_heads
|
| 58 |
+
self.max_position_embeddings = max_position_embeddings
|
| 59 |
+
self.window_pattern = window_pattern.upper()
|
| 60 |
+
self.rotary_base = rotary_base
|
| 61 |
+
self.qk_scale = qk_scale
|
| 62 |
+
self.logit_softcap = logit_softcap
|
| 63 |
+
self.value_embed_gate_channels = value_embed_gate_channels
|
| 64 |
+
self.value_embed_gate_range = value_embed_gate_range
|
| 65 |
+
self.smear_channels = smear_channels
|
| 66 |
+
self.use_cache = use_cache
|
| 67 |
+
self.release = release
|
| 68 |
+
kwargs.setdefault("tie_word_embeddings", False)
|
| 69 |
+
super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
|
| 70 |
+
|
| 71 |
+
@property
|
| 72 |
+
def head_dim(self) -> int:
|
| 73 |
+
return self.hidden_size // self.num_attention_heads
|
| 74 |
+
|
| 75 |
+
@property
|
| 76 |
+
def intermediate_size(self) -> int:
|
| 77 |
+
return 4 * self.hidden_size
|
| 78 |
+
|
| 79 |
+
@property
|
| 80 |
+
def window_sizes(self) -> list[int]:
|
| 81 |
+
"""Left attention window of each layer, in tokens."""
|
| 82 |
+
long_window = self.max_position_embeddings
|
| 83 |
+
short_window = -(-long_window // 4 // 128) * 128 # a quarter of the context, rounded up to a multiple of 128 (2048 -> 512)
|
| 84 |
+
pattern = self.window_pattern
|
| 85 |
+
sizes = [short_window if pattern[i % len(pattern)] == "S" else long_window for i in range(self.num_hidden_layers)]
|
| 86 |
+
sizes[-1] = long_window
|
| 87 |
+
return sizes
|
| 88 |
+
|
| 89 |
+
def has_value_embedding(self, layer_idx: int) -> bool:
|
| 90 |
+
"""Value embeddings sit on alternating layers, always including the last one."""
|
| 91 |
+
return layer_idx % 2 == (self.num_hidden_layers - 1) % 2
|
| 92 |
+
|
| 93 |
+
@property
|
| 94 |
+
def backout_layer(self) -> int:
|
| 95 |
+
"""The residual stream after this layer is partly subtracted before the final norm."""
|
| 96 |
+
return self.num_hidden_layers // 2
|
268m-g073-human-r0.025/generation_config.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 32759,
|
| 4 |
+
"eos_token_id": 32759,
|
| 5 |
+
"output_attentions": false,
|
| 6 |
+
"output_hidden_states": false,
|
| 7 |
+
"pad_token_id": 32759,
|
| 8 |
+
"transformers_version": "5.17.0",
|
| 9 |
+
"use_cache": true
|
| 10 |
+
}
|
268m-g073-human-r0.025/modeling_wildai.py
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The WildAI language models (a nanochat GPT variant) for Hugging Face ``transformers``.
|
| 2 |
+
|
| 3 |
+
This file is copied verbatim into every released model repository and loaded with ``trust_remote_code=True``.
|
| 4 |
+
It depends only on ``torch`` and ``transformers`` and reproduces the training code's forward pass operation by
|
| 5 |
+
operation, so a model loaded in bfloat16 computes the same function the paper evaluated:
|
| 6 |
+
|
| 7 |
+
- token embedding followed by a parameter-free RMSNorm, then a learned "smear" that mixes each position's embedding
|
| 8 |
+
with the previous token's;
|
| 9 |
+
- per-layer scalars that rescale the residual stream and blend the initial embedding back in before every block;
|
| 10 |
+
- attention with rotary embeddings applied before a per-head RMSNorm of queries and keys (both scaled by 1.2),
|
| 11 |
+
a left sliding window per layer, and gated value embeddings added to the values on alternating layers;
|
| 12 |
+
- a squared-ReLU MLP, parameter-free pre-norms, no biases;
|
| 13 |
+
- "backout": part of the residual stream after the middle layer is subtracted before the final norm;
|
| 14 |
+
- untied output head with logits soft-capped by ``15 * tanh(logits / 15)``.
|
| 15 |
+
|
| 16 |
+
Attention uses PyTorch's scaled_dot_product_attention with explicit window masks, so no FlashAttention install is
|
| 17 |
+
needed. Generation uses the standard ``DynamicCache``.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
from dataclasses import dataclass
|
| 23 |
+
from typing import ClassVar
|
| 24 |
+
|
| 25 |
+
import torch
|
| 26 |
+
import torch.nn.functional as F
|
| 27 |
+
from torch import nn
|
| 28 |
+
from transformers import GenerationMixin, PreTrainedModel
|
| 29 |
+
from transformers.cache_utils import Cache, DynamicCache
|
| 30 |
+
from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
|
| 31 |
+
|
| 32 |
+
from .configuration_wildai import WildAIConfig
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def rms_norm(x: torch.Tensor) -> torch.Tensor:
|
| 36 |
+
"""Parameter-free RMSNorm over the last dimension with PyTorch's default epsilon, as in training."""
|
| 37 |
+
return F.rms_norm(x, (x.size(-1),))
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def apply_rotary(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
|
| 41 |
+
"""Rotate channel pairs (i, i + d/2) of ``x`` (batch, time, heads, head_dim), with the training code's sign convention."""
|
| 42 |
+
d = x.shape[-1] // 2
|
| 43 |
+
x1, x2 = x[..., :d], x[..., d:]
|
| 44 |
+
return torch.cat([x1 * cos + x2 * sin, x1 * (-sin) + x2 * cos], dim=-1)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
@dataclass(frozen=True)
|
| 48 |
+
class WindowMask:
|
| 49 |
+
"""Arguments for scaled_dot_product_attention that implement one attention window."""
|
| 50 |
+
|
| 51 |
+
mask: torch.Tensor | None
|
| 52 |
+
is_causal: bool
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def window_masks(
|
| 56 |
+
window_sizes: list[int], query_len: int, past_len: int, padding_mask: torch.Tensor | None, device: torch.device
|
| 57 |
+
) -> dict[int, WindowMask]:
|
| 58 |
+
"""Masks for each distinct left window. Query i (absolute index past_len + i) sees keys j with i - window <= j <= i.
|
| 59 |
+
|
| 60 |
+
``padding_mask`` is the (batch, past_len + query_len) attention mask when it contains padding, else None. A padding
|
| 61 |
+
query is always allowed to see itself so that its (unused) output stays finite.
|
| 62 |
+
"""
|
| 63 |
+
total = past_len + query_len
|
| 64 |
+
q_idx = torch.arange(past_len, total, device=device)[:, None]
|
| 65 |
+
k_idx = torch.arange(total, device=device)[None, :]
|
| 66 |
+
masks: dict[int, WindowMask] = {}
|
| 67 |
+
for window in sorted(set(window_sizes)):
|
| 68 |
+
if padding_mask is None and window >= total - 1:
|
| 69 |
+
if query_len == 1:
|
| 70 |
+
masks[window] = WindowMask(mask=None, is_causal=False)
|
| 71 |
+
continue
|
| 72 |
+
if past_len == 0:
|
| 73 |
+
masks[window] = WindowMask(mask=None, is_causal=True)
|
| 74 |
+
continue
|
| 75 |
+
allowed = (k_idx <= q_idx) & (q_idx - k_idx <= window)
|
| 76 |
+
if padding_mask is not None:
|
| 77 |
+
allowed = (allowed & padding_mask[:, None, None, :].bool()) | (k_idx == q_idx)
|
| 78 |
+
masks[window] = WindowMask(mask=allowed, is_causal=False)
|
| 79 |
+
return masks
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class WildAIAttention(nn.Module):
|
| 83 |
+
def __init__(self, config: WildAIConfig, layer_idx: int) -> None:
|
| 84 |
+
super().__init__()
|
| 85 |
+
self.layer_idx = layer_idx
|
| 86 |
+
self.num_heads = config.num_attention_heads
|
| 87 |
+
self.num_kv_heads = config.num_key_value_heads
|
| 88 |
+
self.head_dim = config.head_dim
|
| 89 |
+
self.qk_scale = config.qk_scale
|
| 90 |
+
self.window = config.window_sizes[layer_idx]
|
| 91 |
+
hidden, kv_dim = config.hidden_size, config.num_key_value_heads * config.head_dim
|
| 92 |
+
self.q_proj = nn.Linear(hidden, config.num_attention_heads * config.head_dim, bias=False)
|
| 93 |
+
self.k_proj = nn.Linear(hidden, kv_dim, bias=False)
|
| 94 |
+
self.v_proj = nn.Linear(hidden, kv_dim, bias=False)
|
| 95 |
+
self.o_proj = nn.Linear(hidden, hidden, bias=False)
|
| 96 |
+
self.value_embed: nn.Embedding | None = None
|
| 97 |
+
self.value_embed_gate: nn.Linear | None = None
|
| 98 |
+
if config.has_value_embedding(layer_idx):
|
| 99 |
+
self.value_embed = nn.Embedding(config.vocab_size, kv_dim)
|
| 100 |
+
self.value_embed_gate = nn.Linear(config.value_embed_gate_channels, config.num_key_value_heads, bias=False)
|
| 101 |
+
self.gate_channels = config.value_embed_gate_channels
|
| 102 |
+
self.gate_range = config.value_embed_gate_range
|
| 103 |
+
|
| 104 |
+
def forward(
|
| 105 |
+
self,
|
| 106 |
+
x: torch.Tensor,
|
| 107 |
+
input_ids: torch.LongTensor,
|
| 108 |
+
rotary: tuple[torch.Tensor, torch.Tensor],
|
| 109 |
+
mask: WindowMask,
|
| 110 |
+
past_key_values: Cache | None,
|
| 111 |
+
) -> torch.Tensor:
|
| 112 |
+
batch, time, _ = x.shape
|
| 113 |
+
q = self.q_proj(x).view(batch, time, self.num_heads, self.head_dim)
|
| 114 |
+
k = self.k_proj(x).view(batch, time, self.num_kv_heads, self.head_dim)
|
| 115 |
+
v = self.v_proj(x).view(batch, time, self.num_kv_heads, self.head_dim)
|
| 116 |
+
if self.value_embed is not None:
|
| 117 |
+
ve = self.value_embed(input_ids).to(x.dtype).view(batch, time, self.num_kv_heads, self.head_dim)
|
| 118 |
+
gate = self.gate_range * torch.sigmoid(self.value_embed_gate(x[..., : self.gate_channels]))
|
| 119 |
+
v = v + gate.unsqueeze(-1) * ve
|
| 120 |
+
cos, sin = rotary
|
| 121 |
+
q, k = apply_rotary(q, cos, sin), apply_rotary(k, cos, sin)
|
| 122 |
+
q, k = rms_norm(q) * self.qk_scale, rms_norm(k) * self.qk_scale
|
| 123 |
+
q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
|
| 124 |
+
if past_key_values is not None:
|
| 125 |
+
k, v = past_key_values.update(k, v, self.layer_idx)
|
| 126 |
+
y = F.scaled_dot_product_attention(
|
| 127 |
+
q, k, v, attn_mask=mask.mask, is_causal=mask.is_causal, enable_gqa=self.num_kv_heads != self.num_heads
|
| 128 |
+
)
|
| 129 |
+
return self.o_proj(y.transpose(1, 2).reshape(batch, time, -1))
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
class WildAIMLP(nn.Module):
|
| 133 |
+
def __init__(self, config: WildAIConfig) -> None:
|
| 134 |
+
super().__init__()
|
| 135 |
+
self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
|
| 136 |
+
self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
|
| 137 |
+
|
| 138 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 139 |
+
return self.down_proj(F.relu(self.up_proj(x)).square())
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
class WildAIDecoderLayer(nn.Module):
|
| 143 |
+
def __init__(self, config: WildAIConfig, layer_idx: int) -> None:
|
| 144 |
+
super().__init__()
|
| 145 |
+
self.self_attn = WildAIAttention(config, layer_idx)
|
| 146 |
+
self.mlp = WildAIMLP(config)
|
| 147 |
+
|
| 148 |
+
def forward(
|
| 149 |
+
self,
|
| 150 |
+
x: torch.Tensor,
|
| 151 |
+
input_ids: torch.LongTensor,
|
| 152 |
+
rotary: tuple[torch.Tensor, torch.Tensor],
|
| 153 |
+
mask: WindowMask,
|
| 154 |
+
past_key_values: Cache | None,
|
| 155 |
+
) -> torch.Tensor:
|
| 156 |
+
x = x + self.self_attn(rms_norm(x), input_ids, rotary, mask, past_key_values)
|
| 157 |
+
return x + self.mlp(rms_norm(x))
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
class WildAIPreTrainedModel(PreTrainedModel):
|
| 161 |
+
config_class = WildAIConfig
|
| 162 |
+
base_model_prefix = "model"
|
| 163 |
+
main_input_name = "input_ids"
|
| 164 |
+
_no_split_modules: ClassVar[list[str]] = ["WildAIDecoderLayer"]
|
| 165 |
+
_supports_sdpa = True
|
| 166 |
+
|
| 167 |
+
def _init_weights(self, module: nn.Module) -> None:
|
| 168 |
+
# Released models are always loaded from trained weights; this only gives from_config() models sane values.
|
| 169 |
+
if isinstance(module, (nn.Linear, nn.Embedding)):
|
| 170 |
+
nn.init.normal_(module.weight, mean=0.0, std=0.02)
|
| 171 |
+
elif isinstance(module, WildAIModel):
|
| 172 |
+
with torch.no_grad():
|
| 173 |
+
module.resid_lambdas.fill_(1.0)
|
| 174 |
+
module.x0_lambdas.zero_()
|
| 175 |
+
module.smear_lambda.zero_()
|
| 176 |
+
module.backout_lambda.zero_()
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
class WildAIModel(WildAIPreTrainedModel):
|
| 180 |
+
"""The transformer trunk; returns the final normalised hidden states."""
|
| 181 |
+
|
| 182 |
+
def __init__(self, config: WildAIConfig) -> None:
|
| 183 |
+
super().__init__(config)
|
| 184 |
+
n_layer = config.num_hidden_layers
|
| 185 |
+
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
|
| 186 |
+
self.layers = nn.ModuleList([WildAIDecoderLayer(config, i) for i in range(n_layer)])
|
| 187 |
+
self.resid_lambdas = nn.Parameter(torch.ones(n_layer))
|
| 188 |
+
self.x0_lambdas = nn.Parameter(torch.zeros(n_layer))
|
| 189 |
+
self.smear_gate = nn.Linear(config.smear_channels, 1, bias=False)
|
| 190 |
+
self.smear_lambda = nn.Parameter(torch.zeros(1))
|
| 191 |
+
self.backout_lambda = nn.Parameter(torch.zeros(1))
|
| 192 |
+
self.post_init()
|
| 193 |
+
|
| 194 |
+
def get_input_embeddings(self) -> nn.Embedding:
|
| 195 |
+
return self.embed_tokens
|
| 196 |
+
|
| 197 |
+
def set_input_embeddings(self, value: nn.Embedding) -> None:
|
| 198 |
+
self.embed_tokens = value
|
| 199 |
+
|
| 200 |
+
def rotary(self, position_ids: torch.LongTensor, dtype: torch.dtype) -> tuple[torch.Tensor, torch.Tensor]:
|
| 201 |
+
"""cos/sin tables of shape (batch, time, 1, head_dim / 2), computed in float32 and cast like in training."""
|
| 202 |
+
head_dim = self.config.head_dim
|
| 203 |
+
channels = torch.arange(0, head_dim, 2, dtype=torch.float32, device=position_ids.device)
|
| 204 |
+
inv_freq = 1.0 / (self.config.rotary_base ** (channels / head_dim))
|
| 205 |
+
freqs = position_ids.to(torch.float32)[..., None] * inv_freq
|
| 206 |
+
return freqs.cos().to(dtype)[:, :, None, :], freqs.sin().to(dtype)[:, :, None, :]
|
| 207 |
+
|
| 208 |
+
def smear(self, x: torch.Tensor, previous: torch.Tensor | None, padding_mask: torch.Tensor | None) -> torch.Tensor:
|
| 209 |
+
"""Add a gated copy of the previous position's embedding to every position that has one.
|
| 210 |
+
|
| 211 |
+
``previous`` is the normalised embedding of the token just before this chunk (None at the start of a sequence);
|
| 212 |
+
with left padding, the first real token has a padding token before it and gets nothing.
|
| 213 |
+
"""
|
| 214 |
+
channels = self.config.smear_channels
|
| 215 |
+
lam = self.smear_lambda.to(x.dtype)
|
| 216 |
+
first = x[:, :1]
|
| 217 |
+
if previous is not None:
|
| 218 |
+
first = first + lam * torch.sigmoid(self.smear_gate(first[..., :channels])) * previous
|
| 219 |
+
if x.size(1) == 1:
|
| 220 |
+
return first
|
| 221 |
+
gate = lam * torch.sigmoid(self.smear_gate(x[:, 1:, :channels]))
|
| 222 |
+
if padding_mask is not None:
|
| 223 |
+
gate = gate * padding_mask[:, -x.size(1) : -1, None].to(gate.dtype)
|
| 224 |
+
return torch.cat([first, x[:, 1:] + gate * x[:, :-1]], dim=1)
|
| 225 |
+
|
| 226 |
+
def forward(
|
| 227 |
+
self,
|
| 228 |
+
input_ids: torch.LongTensor,
|
| 229 |
+
attention_mask: torch.Tensor | None = None,
|
| 230 |
+
position_ids: torch.LongTensor | None = None,
|
| 231 |
+
past_key_values: Cache | None = None,
|
| 232 |
+
use_cache: bool | None = None,
|
| 233 |
+
previous_token_ids: torch.LongTensor | None = None,
|
| 234 |
+
output_hidden_states: bool | None = None,
|
| 235 |
+
**kwargs: object,
|
| 236 |
+
) -> BaseModelOutputWithPast:
|
| 237 |
+
"""``previous_token_ids`` (batch, 1): the token before ``input_ids[:, 0]``; required when continuing from a cache."""
|
| 238 |
+
if input_ids is None:
|
| 239 |
+
raise ValueError("input_ids are required: the value embeddings are looked up by token id")
|
| 240 |
+
batch, time = input_ids.shape
|
| 241 |
+
use_cache = self.config.use_cache if use_cache is None else use_cache
|
| 242 |
+
if use_cache and past_key_values is None:
|
| 243 |
+
past_key_values = DynamicCache()
|
| 244 |
+
past_len = past_key_values.get_seq_length() if past_key_values is not None else 0
|
| 245 |
+
if past_len > 0 and previous_token_ids is None:
|
| 246 |
+
raise ValueError("previous_token_ids is required when continuing from past_key_values (the smear needs it)")
|
| 247 |
+
padding_mask = None
|
| 248 |
+
if attention_mask is not None:
|
| 249 |
+
if attention_mask.shape != (batch, past_len + time):
|
| 250 |
+
raise ValueError(f"attention_mask must have shape {(batch, past_len + time)}, got {tuple(attention_mask.shape)}")
|
| 251 |
+
if not bool(attention_mask.all()):
|
| 252 |
+
padding_mask = attention_mask
|
| 253 |
+
if position_ids is None:
|
| 254 |
+
position_ids = torch.arange(past_len, past_len + time, device=input_ids.device).expand(batch, -1)
|
| 255 |
+
|
| 256 |
+
x = rms_norm(self.embed_tokens(input_ids))
|
| 257 |
+
previous = rms_norm(self.embed_tokens(previous_token_ids)) if previous_token_ids is not None else None
|
| 258 |
+
x = self.smear(x, previous, padding_mask)
|
| 259 |
+
rotary = self.rotary(position_ids, x.dtype)
|
| 260 |
+
masks = window_masks(self.config.window_sizes, time, past_len, padding_mask, input_ids.device)
|
| 261 |
+
|
| 262 |
+
x0 = x
|
| 263 |
+
backout = None
|
| 264 |
+
hidden_states: tuple[torch.Tensor, ...] = (x,)
|
| 265 |
+
for i, layer in enumerate(self.layers):
|
| 266 |
+
x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0
|
| 267 |
+
x = layer(x, input_ids, rotary, masks[layer.self_attn.window], past_key_values)
|
| 268 |
+
if i == self.config.backout_layer:
|
| 269 |
+
backout = x
|
| 270 |
+
hidden_states += (x,)
|
| 271 |
+
x = rms_norm(x - self.backout_lambda.to(x.dtype) * backout)
|
| 272 |
+
return BaseModelOutputWithPast(
|
| 273 |
+
last_hidden_state=x,
|
| 274 |
+
past_key_values=past_key_values if use_cache else None,
|
| 275 |
+
hidden_states=hidden_states if output_hidden_states else None,
|
| 276 |
+
)
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
class WildAIForCausalLM(WildAIPreTrainedModel, GenerationMixin):
|
| 280 |
+
def __init__(self, config: WildAIConfig) -> None:
|
| 281 |
+
super().__init__(config)
|
| 282 |
+
self.model = WildAIModel(config)
|
| 283 |
+
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
|
| 284 |
+
self.post_init()
|
| 285 |
+
|
| 286 |
+
def get_input_embeddings(self) -> nn.Embedding:
|
| 287 |
+
return self.model.embed_tokens
|
| 288 |
+
|
| 289 |
+
def set_input_embeddings(self, value: nn.Embedding) -> None:
|
| 290 |
+
self.model.embed_tokens = value
|
| 291 |
+
|
| 292 |
+
def get_output_embeddings(self) -> nn.Linear:
|
| 293 |
+
return self.lm_head
|
| 294 |
+
|
| 295 |
+
def set_output_embeddings(self, value: nn.Linear) -> None:
|
| 296 |
+
self.lm_head = value
|
| 297 |
+
|
| 298 |
+
def forward(
|
| 299 |
+
self,
|
| 300 |
+
input_ids: torch.LongTensor,
|
| 301 |
+
attention_mask: torch.Tensor | None = None,
|
| 302 |
+
position_ids: torch.LongTensor | None = None,
|
| 303 |
+
past_key_values: Cache | None = None,
|
| 304 |
+
labels: torch.LongTensor | None = None,
|
| 305 |
+
use_cache: bool | None = None,
|
| 306 |
+
previous_token_ids: torch.LongTensor | None = None,
|
| 307 |
+
output_hidden_states: bool | None = None,
|
| 308 |
+
logits_to_keep: int = 0,
|
| 309 |
+
return_dict: bool | None = None,
|
| 310 |
+
**kwargs: object,
|
| 311 |
+
) -> CausalLMOutputWithPast | tuple[torch.Tensor, ...]:
|
| 312 |
+
"""Float32 logits after the soft cap. ``labels`` are shifted inside (standard causal-LM loss, -100 ignored)."""
|
| 313 |
+
outputs = self.model(
|
| 314 |
+
input_ids,
|
| 315 |
+
attention_mask=attention_mask,
|
| 316 |
+
position_ids=position_ids,
|
| 317 |
+
past_key_values=past_key_values,
|
| 318 |
+
use_cache=use_cache,
|
| 319 |
+
previous_token_ids=previous_token_ids,
|
| 320 |
+
output_hidden_states=output_hidden_states,
|
| 321 |
+
)
|
| 322 |
+
hidden = outputs.last_hidden_state[:, -logits_to_keep:] if logits_to_keep else outputs.last_hidden_state
|
| 323 |
+
logits = self.lm_head(hidden).float()
|
| 324 |
+
cap = self.config.logit_softcap
|
| 325 |
+
logits = cap * torch.tanh(logits / cap)
|
| 326 |
+
loss = None
|
| 327 |
+
if labels is not None:
|
| 328 |
+
loss = F.cross_entropy(logits[:, :-1].flatten(0, 1), labels[:, 1:].flatten().to(logits.device), ignore_index=-100)
|
| 329 |
+
result = CausalLMOutputWithPast(
|
| 330 |
+
loss=loss, logits=logits, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states
|
| 331 |
+
)
|
| 332 |
+
return result.to_tuple() if return_dict is False else result
|
| 333 |
+
|
| 334 |
+
def prepare_inputs_for_generation(self, input_ids: torch.LongTensor, **kwargs: object) -> dict[str, object]:
|
| 335 |
+
# The smear mixes in the embedding of the token before the first new one; take it from the full sequence.
|
| 336 |
+
model_inputs = super().prepare_inputs_for_generation(input_ids, **kwargs)
|
| 337 |
+
new_len = model_inputs["input_ids"].shape[1]
|
| 338 |
+
if input_ids.shape[1] > new_len:
|
| 339 |
+
model_inputs["previous_token_ids"] = input_ids[:, -new_len - 1 : -new_len]
|
| 340 |
+
return model_inputs
|
268m-g073-human-r0.025/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
268m-g073-human-r0.025/tokenizer_config.json
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 3 |
+
"bos_token": "<|bos|>",
|
| 4 |
+
"eos_token": "<|bos|>",
|
| 5 |
+
"model_max_length": 2048,
|
| 6 |
+
"model_input_names": [
|
| 7 |
+
"input_ids",
|
| 8 |
+
"attention_mask"
|
| 9 |
+
],
|
| 10 |
+
"clean_up_tokenization_spaces": false
|
| 11 |
+
}
|
268m-g073-human-r0.05/LICENSE
ADDED
|
@@ -0,0 +1,438 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Attribution-NonCommercial-ShareAlike 4.0 International
|
| 2 |
+
|
| 3 |
+
=======================================================================
|
| 4 |
+
|
| 5 |
+
Creative Commons Corporation ("Creative Commons") is not a law firm and
|
| 6 |
+
does not provide legal services or legal advice. Distribution of
|
| 7 |
+
Creative Commons public licenses does not create a lawyer-client or
|
| 8 |
+
other relationship. Creative Commons makes its licenses and related
|
| 9 |
+
information available on an "as-is" basis. Creative Commons gives no
|
| 10 |
+
warranties regarding its licenses, any material licensed under their
|
| 11 |
+
terms and conditions, or any related information. Creative Commons
|
| 12 |
+
disclaims all liability for damages resulting from their use to the
|
| 13 |
+
fullest extent possible.
|
| 14 |
+
|
| 15 |
+
Using Creative Commons Public Licenses
|
| 16 |
+
|
| 17 |
+
Creative Commons public licenses provide a standard set of terms and
|
| 18 |
+
conditions that creators and other rights holders may use to share
|
| 19 |
+
original works of authorship and other material subject to copyright
|
| 20 |
+
and certain other rights specified in the public license below. The
|
| 21 |
+
following considerations are for informational purposes only, are not
|
| 22 |
+
exhaustive, and do not form part of our licenses.
|
| 23 |
+
|
| 24 |
+
Considerations for licensors: Our public licenses are
|
| 25 |
+
intended for use by those authorized to give the public
|
| 26 |
+
permission to use material in ways otherwise restricted by
|
| 27 |
+
copyright and certain other rights. Our licenses are
|
| 28 |
+
irrevocable. Licensors should read and understand the terms
|
| 29 |
+
and conditions of the license they choose before applying it.
|
| 30 |
+
Licensors should also secure all rights necessary before
|
| 31 |
+
applying our licenses so that the public can reuse the
|
| 32 |
+
material as expected. Licensors should clearly mark any
|
| 33 |
+
material not subject to the license. This includes other CC-
|
| 34 |
+
licensed material, or material used under an exception or
|
| 35 |
+
limitation to copyright. More considerations for licensors:
|
| 36 |
+
wiki.creativecommons.org/Considerations_for_licensors
|
| 37 |
+
|
| 38 |
+
Considerations for the public: By using one of our public
|
| 39 |
+
licenses, a licensor grants the public permission to use the
|
| 40 |
+
licensed material under specified terms and conditions. If
|
| 41 |
+
the licensor's permission is not necessary for any reason--for
|
| 42 |
+
example, because of any applicable exception or limitation to
|
| 43 |
+
copyright--then that use is not regulated by the license. Our
|
| 44 |
+
licenses grant only permissions under copyright and certain
|
| 45 |
+
other rights that a licensor has authority to grant. Use of
|
| 46 |
+
the licensed material may still be restricted for other
|
| 47 |
+
reasons, including because others have copyright or other
|
| 48 |
+
rights in the material. A licensor may make special requests,
|
| 49 |
+
such as asking that all changes be marked or described.
|
| 50 |
+
Although not required by our licenses, you are encouraged to
|
| 51 |
+
respect those requests where reasonable. More considerations
|
| 52 |
+
for the public:
|
| 53 |
+
wiki.creativecommons.org/Considerations_for_licensees
|
| 54 |
+
|
| 55 |
+
=======================================================================
|
| 56 |
+
|
| 57 |
+
Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International
|
| 58 |
+
Public License
|
| 59 |
+
|
| 60 |
+
By exercising the Licensed Rights (defined below), You accept and agree
|
| 61 |
+
to be bound by the terms and conditions of this Creative Commons
|
| 62 |
+
Attribution-NonCommercial-ShareAlike 4.0 International Public License
|
| 63 |
+
("Public License"). To the extent this Public License may be
|
| 64 |
+
interpreted as a contract, You are granted the Licensed Rights in
|
| 65 |
+
consideration of Your acceptance of these terms and conditions, and the
|
| 66 |
+
Licensor grants You such rights in consideration of benefits the
|
| 67 |
+
Licensor receives from making the Licensed Material available under
|
| 68 |
+
these terms and conditions.
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
Section 1 -- Definitions.
|
| 72 |
+
|
| 73 |
+
a. Adapted Material means material subject to Copyright and Similar
|
| 74 |
+
Rights that is derived from or based upon the Licensed Material
|
| 75 |
+
and in which the Licensed Material is translated, altered,
|
| 76 |
+
arranged, transformed, or otherwise modified in a manner requiring
|
| 77 |
+
permission under the Copyright and Similar Rights held by the
|
| 78 |
+
Licensor. For purposes of this Public License, where the Licensed
|
| 79 |
+
Material is a musical work, performance, or sound recording,
|
| 80 |
+
Adapted Material is always produced where the Licensed Material is
|
| 81 |
+
synched in timed relation with a moving image.
|
| 82 |
+
|
| 83 |
+
b. Adapter's License means the license You apply to Your Copyright
|
| 84 |
+
and Similar Rights in Your contributions to Adapted Material in
|
| 85 |
+
accordance with the terms and conditions of this Public License.
|
| 86 |
+
|
| 87 |
+
c. BY-NC-SA Compatible License means a license listed at
|
| 88 |
+
creativecommons.org/compatiblelicenses, approved by Creative
|
| 89 |
+
Commons as essentially the equivalent of this Public License.
|
| 90 |
+
|
| 91 |
+
d. Copyright and Similar Rights means copyright and/or similar rights
|
| 92 |
+
closely related to copyright including, without limitation,
|
| 93 |
+
performance, broadcast, sound recording, and Sui Generis Database
|
| 94 |
+
Rights, without regard to how the rights are labeled or
|
| 95 |
+
categorized. For purposes of this Public License, the rights
|
| 96 |
+
specified in Section 2(b)(1)-(2) are not Copyright and Similar
|
| 97 |
+
Rights.
|
| 98 |
+
|
| 99 |
+
e. Effective Technological Measures means those measures that, in the
|
| 100 |
+
absence of proper authority, may not be circumvented under laws
|
| 101 |
+
fulfilling obligations under Article 11 of the WIPO Copyright
|
| 102 |
+
Treaty adopted on December 20, 1996, and/or similar international
|
| 103 |
+
agreements.
|
| 104 |
+
|
| 105 |
+
f. Exceptions and Limitations means fair use, fair dealing, and/or
|
| 106 |
+
any other exception or limitation to Copyright and Similar Rights
|
| 107 |
+
that applies to Your use of the Licensed Material.
|
| 108 |
+
|
| 109 |
+
g. License Elements means the license attributes listed in the name
|
| 110 |
+
of a Creative Commons Public License. The License Elements of this
|
| 111 |
+
Public License are Attribution, NonCommercial, and ShareAlike.
|
| 112 |
+
|
| 113 |
+
h. Licensed Material means the artistic or literary work, database,
|
| 114 |
+
or other material to which the Licensor applied this Public
|
| 115 |
+
License.
|
| 116 |
+
|
| 117 |
+
i. Licensed Rights means the rights granted to You subject to the
|
| 118 |
+
terms and conditions of this Public License, which are limited to
|
| 119 |
+
all Copyright and Similar Rights that apply to Your use of the
|
| 120 |
+
Licensed Material and that the Licensor has authority to license.
|
| 121 |
+
|
| 122 |
+
j. Licensor means the individual(s) or entity(ies) granting rights
|
| 123 |
+
under this Public License.
|
| 124 |
+
|
| 125 |
+
k. NonCommercial means not primarily intended for or directed towards
|
| 126 |
+
commercial advantage or monetary compensation. For purposes of
|
| 127 |
+
this Public License, the exchange of the Licensed Material for
|
| 128 |
+
other material subject to Copyright and Similar Rights by digital
|
| 129 |
+
file-sharing or similar means is NonCommercial provided there is
|
| 130 |
+
no payment of monetary compensation in connection with the
|
| 131 |
+
exchange.
|
| 132 |
+
|
| 133 |
+
l. Share means to provide material to the public by any means or
|
| 134 |
+
process that requires permission under the Licensed Rights, such
|
| 135 |
+
as reproduction, public display, public performance, distribution,
|
| 136 |
+
dissemination, communication, or importation, and to make material
|
| 137 |
+
available to the public including in ways that members of the
|
| 138 |
+
public may access the material from a place and at a time
|
| 139 |
+
individually chosen by them.
|
| 140 |
+
|
| 141 |
+
m. Sui Generis Database Rights means rights other than copyright
|
| 142 |
+
resulting from Directive 96/9/EC of the European Parliament and of
|
| 143 |
+
the Council of 11 March 1996 on the legal protection of databases,
|
| 144 |
+
as amended and/or succeeded, as well as other essentially
|
| 145 |
+
equivalent rights anywhere in the world.
|
| 146 |
+
|
| 147 |
+
n. You means the individual or entity exercising the Licensed Rights
|
| 148 |
+
under this Public License. Your has a corresponding meaning.
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
Section 2 -- Scope.
|
| 152 |
+
|
| 153 |
+
a. License grant.
|
| 154 |
+
|
| 155 |
+
1. Subject to the terms and conditions of this Public License,
|
| 156 |
+
the Licensor hereby grants You a worldwide, royalty-free,
|
| 157 |
+
non-sublicensable, non-exclusive, irrevocable license to
|
| 158 |
+
exercise the Licensed Rights in the Licensed Material to:
|
| 159 |
+
|
| 160 |
+
a. reproduce and Share the Licensed Material, in whole or
|
| 161 |
+
in part, for NonCommercial purposes only; and
|
| 162 |
+
|
| 163 |
+
b. produce, reproduce, and Share Adapted Material for
|
| 164 |
+
NonCommercial purposes only.
|
| 165 |
+
|
| 166 |
+
2. Exceptions and Limitations. For the avoidance of doubt, where
|
| 167 |
+
Exceptions and Limitations apply to Your use, this Public
|
| 168 |
+
License does not apply, and You do not need to comply with
|
| 169 |
+
its terms and conditions.
|
| 170 |
+
|
| 171 |
+
3. Term. The term of this Public License is specified in Section
|
| 172 |
+
6(a).
|
| 173 |
+
|
| 174 |
+
4. Media and formats; technical modifications allowed. The
|
| 175 |
+
Licensor authorizes You to exercise the Licensed Rights in
|
| 176 |
+
all media and formats whether now known or hereafter created,
|
| 177 |
+
and to make technical modifications necessary to do so. The
|
| 178 |
+
Licensor waives and/or agrees not to assert any right or
|
| 179 |
+
authority to forbid You from making technical modifications
|
| 180 |
+
necessary to exercise the Licensed Rights, including
|
| 181 |
+
technical modifications necessary to circumvent Effective
|
| 182 |
+
Technological Measures. For purposes of this Public License,
|
| 183 |
+
simply making modifications authorized by this Section 2(a)
|
| 184 |
+
(4) never produces Adapted Material.
|
| 185 |
+
|
| 186 |
+
5. Downstream recipients.
|
| 187 |
+
|
| 188 |
+
a. Offer from the Licensor -- Licensed Material. Every
|
| 189 |
+
recipient of the Licensed Material automatically
|
| 190 |
+
receives an offer from the Licensor to exercise the
|
| 191 |
+
Licensed Rights under the terms and conditions of this
|
| 192 |
+
Public License.
|
| 193 |
+
|
| 194 |
+
b. Additional offer from the Licensor -- Adapted Material.
|
| 195 |
+
Every recipient of Adapted Material from You
|
| 196 |
+
automatically receives an offer from the Licensor to
|
| 197 |
+
exercise the Licensed Rights in the Adapted Material
|
| 198 |
+
under the conditions of the Adapter's License You apply.
|
| 199 |
+
|
| 200 |
+
c. No downstream restrictions. You may not offer or impose
|
| 201 |
+
any additional or different terms or conditions on, or
|
| 202 |
+
apply any Effective Technological Measures to, the
|
| 203 |
+
Licensed Material if doing so restricts exercise of the
|
| 204 |
+
Licensed Rights by any recipient of the Licensed
|
| 205 |
+
Material.
|
| 206 |
+
|
| 207 |
+
6. No endorsement. Nothing in this Public License constitutes or
|
| 208 |
+
may be construed as permission to assert or imply that You
|
| 209 |
+
are, or that Your use of the Licensed Material is, connected
|
| 210 |
+
with, or sponsored, endorsed, or granted official status by,
|
| 211 |
+
the Licensor or others designated to receive attribution as
|
| 212 |
+
provided in Section 3(a)(1)(A)(i).
|
| 213 |
+
|
| 214 |
+
b. Other rights.
|
| 215 |
+
|
| 216 |
+
1. Moral rights, such as the right of integrity, are not
|
| 217 |
+
licensed under this Public License, nor are publicity,
|
| 218 |
+
privacy, and/or other similar personality rights; however, to
|
| 219 |
+
the extent possible, the Licensor waives and/or agrees not to
|
| 220 |
+
assert any such rights held by the Licensor to the limited
|
| 221 |
+
extent necessary to allow You to exercise the Licensed
|
| 222 |
+
Rights, but not otherwise.
|
| 223 |
+
|
| 224 |
+
2. Patent and trademark rights are not licensed under this
|
| 225 |
+
Public License.
|
| 226 |
+
|
| 227 |
+
3. To the extent possible, the Licensor waives any right to
|
| 228 |
+
collect royalties from You for the exercise of the Licensed
|
| 229 |
+
Rights, whether directly or through a collecting society
|
| 230 |
+
under any voluntary or waivable statutory or compulsory
|
| 231 |
+
licensing scheme. In all other cases the Licensor expressly
|
| 232 |
+
reserves any right to collect such royalties, including when
|
| 233 |
+
the Licensed Material is used other than for NonCommercial
|
| 234 |
+
purposes.
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
Section 3 -- License Conditions.
|
| 238 |
+
|
| 239 |
+
Your exercise of the Licensed Rights is expressly made subject to the
|
| 240 |
+
following conditions.
|
| 241 |
+
|
| 242 |
+
a. Attribution.
|
| 243 |
+
|
| 244 |
+
1. If You Share the Licensed Material (including in modified
|
| 245 |
+
form), You must:
|
| 246 |
+
|
| 247 |
+
a. retain the following if it is supplied by the Licensor
|
| 248 |
+
with the Licensed Material:
|
| 249 |
+
|
| 250 |
+
i. identification of the creator(s) of the Licensed
|
| 251 |
+
Material and any others designated to receive
|
| 252 |
+
attribution, in any reasonable manner requested by
|
| 253 |
+
the Licensor (including by pseudonym if
|
| 254 |
+
designated);
|
| 255 |
+
|
| 256 |
+
ii. a copyright notice;
|
| 257 |
+
|
| 258 |
+
iii. a notice that refers to this Public License;
|
| 259 |
+
|
| 260 |
+
iv. a notice that refers to the disclaimer of
|
| 261 |
+
warranties;
|
| 262 |
+
|
| 263 |
+
v. a URI or hyperlink to the Licensed Material to the
|
| 264 |
+
extent reasonably practicable;
|
| 265 |
+
|
| 266 |
+
b. indicate if You modified the Licensed Material and
|
| 267 |
+
retain an indication of any previous modifications; and
|
| 268 |
+
|
| 269 |
+
c. indicate the Licensed Material is licensed under this
|
| 270 |
+
Public License, and include the text of, or the URI or
|
| 271 |
+
hyperlink to, this Public License.
|
| 272 |
+
|
| 273 |
+
2. You may satisfy the conditions in Section 3(a)(1) in any
|
| 274 |
+
reasonable manner based on the medium, means, and context in
|
| 275 |
+
which You Share the Licensed Material. For example, it may be
|
| 276 |
+
reasonable to satisfy the conditions by providing a URI or
|
| 277 |
+
hyperlink to a resource that includes the required
|
| 278 |
+
information.
|
| 279 |
+
3. If requested by the Licensor, You must remove any of the
|
| 280 |
+
information required by Section 3(a)(1)(A) to the extent
|
| 281 |
+
reasonably practicable.
|
| 282 |
+
|
| 283 |
+
b. ShareAlike.
|
| 284 |
+
|
| 285 |
+
In addition to the conditions in Section 3(a), if You Share
|
| 286 |
+
Adapted Material You produce, the following conditions also apply.
|
| 287 |
+
|
| 288 |
+
1. The Adapter's License You apply must be a Creative Commons
|
| 289 |
+
license with the same License Elements, this version or
|
| 290 |
+
later, or a BY-NC-SA Compatible License.
|
| 291 |
+
|
| 292 |
+
2. You must include the text of, or the URI or hyperlink to, the
|
| 293 |
+
Adapter's License You apply. You may satisfy this condition
|
| 294 |
+
in any reasonable manner based on the medium, means, and
|
| 295 |
+
context in which You Share Adapted Material.
|
| 296 |
+
|
| 297 |
+
3. You may not offer or impose any additional or different terms
|
| 298 |
+
or conditions on, or apply any Effective Technological
|
| 299 |
+
Measures to, Adapted Material that restrict exercise of the
|
| 300 |
+
rights granted under the Adapter's License You apply.
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
Section 4 -- Sui Generis Database Rights.
|
| 304 |
+
|
| 305 |
+
Where the Licensed Rights include Sui Generis Database Rights that
|
| 306 |
+
apply to Your use of the Licensed Material:
|
| 307 |
+
|
| 308 |
+
a. for the avoidance of doubt, Section 2(a)(1) grants You the right
|
| 309 |
+
to extract, reuse, reproduce, and Share all or a substantial
|
| 310 |
+
portion of the contents of the database for NonCommercial purposes
|
| 311 |
+
only;
|
| 312 |
+
|
| 313 |
+
b. if You include all or a substantial portion of the database
|
| 314 |
+
contents in a database in which You have Sui Generis Database
|
| 315 |
+
Rights, then the database in which You have Sui Generis Database
|
| 316 |
+
Rights (but not its individual contents) is Adapted Material,
|
| 317 |
+
including for purposes of Section 3(b); and
|
| 318 |
+
|
| 319 |
+
c. You must comply with the conditions in Section 3(a) if You Share
|
| 320 |
+
all or a substantial portion of the contents of the database.
|
| 321 |
+
|
| 322 |
+
For the avoidance of doubt, this Section 4 supplements and does not
|
| 323 |
+
replace Your obligations under this Public License where the Licensed
|
| 324 |
+
Rights include other Copyright and Similar Rights.
|
| 325 |
+
|
| 326 |
+
|
| 327 |
+
Section 5 -- Disclaimer of Warranties and Limitation of Liability.
|
| 328 |
+
|
| 329 |
+
a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
|
| 330 |
+
EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
|
| 331 |
+
AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
|
| 332 |
+
ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
|
| 333 |
+
IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
|
| 334 |
+
WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
|
| 335 |
+
PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
|
| 336 |
+
ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
|
| 337 |
+
KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
|
| 338 |
+
ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
|
| 339 |
+
|
| 340 |
+
b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
|
| 341 |
+
TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
|
| 342 |
+
NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
|
| 343 |
+
INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
|
| 344 |
+
COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
|
| 345 |
+
USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
|
| 346 |
+
ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
|
| 347 |
+
DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
|
| 348 |
+
IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
|
| 349 |
+
|
| 350 |
+
c. The disclaimer of warranties and limitation of liability provided
|
| 351 |
+
above shall be interpreted in a manner that, to the extent
|
| 352 |
+
possible, most closely approximates an absolute disclaimer and
|
| 353 |
+
waiver of all liability.
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
Section 6 -- Term and Termination.
|
| 357 |
+
|
| 358 |
+
a. This Public License applies for the term of the Copyright and
|
| 359 |
+
Similar Rights licensed here. However, if You fail to comply with
|
| 360 |
+
this Public License, then Your rights under this Public License
|
| 361 |
+
terminate automatically.
|
| 362 |
+
|
| 363 |
+
b. Where Your right to use the Licensed Material has terminated under
|
| 364 |
+
Section 6(a), it reinstates:
|
| 365 |
+
|
| 366 |
+
1. automatically as of the date the violation is cured, provided
|
| 367 |
+
it is cured within 30 days of Your discovery of the
|
| 368 |
+
violation; or
|
| 369 |
+
|
| 370 |
+
2. upon express reinstatement by the Licensor.
|
| 371 |
+
|
| 372 |
+
For the avoidance of doubt, this Section 6(b) does not affect any
|
| 373 |
+
right the Licensor may have to seek remedies for Your violations
|
| 374 |
+
of this Public License.
|
| 375 |
+
|
| 376 |
+
c. For the avoidance of doubt, the Licensor may also offer the
|
| 377 |
+
Licensed Material under separate terms or conditions or stop
|
| 378 |
+
distributing the Licensed Material at any time; however, doing so
|
| 379 |
+
will not terminate this Public License.
|
| 380 |
+
|
| 381 |
+
d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
|
| 382 |
+
License.
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
Section 7 -- Other Terms and Conditions.
|
| 386 |
+
|
| 387 |
+
a. The Licensor shall not be bound by any additional or different
|
| 388 |
+
terms or conditions communicated by You unless expressly agreed.
|
| 389 |
+
|
| 390 |
+
b. Any arrangements, understandings, or agreements regarding the
|
| 391 |
+
Licensed Material not stated herein are separate from and
|
| 392 |
+
independent of the terms and conditions of this Public License.
|
| 393 |
+
|
| 394 |
+
|
| 395 |
+
Section 8 -- Interpretation.
|
| 396 |
+
|
| 397 |
+
a. For the avoidance of doubt, this Public License does not, and
|
| 398 |
+
shall not be interpreted to, reduce, limit, restrict, or impose
|
| 399 |
+
conditions on any use of the Licensed Material that could lawfully
|
| 400 |
+
be made without permission under this Public License.
|
| 401 |
+
|
| 402 |
+
b. To the extent possible, if any provision of this Public License is
|
| 403 |
+
deemed unenforceable, it shall be automatically reformed to the
|
| 404 |
+
minimum extent necessary to make it enforceable. If the provision
|
| 405 |
+
cannot be reformed, it shall be severed from this Public License
|
| 406 |
+
without affecting the enforceability of the remaining terms and
|
| 407 |
+
conditions.
|
| 408 |
+
|
| 409 |
+
c. No term or condition of this Public License will be waived and no
|
| 410 |
+
failure to comply consented to unless expressly agreed to by the
|
| 411 |
+
Licensor.
|
| 412 |
+
|
| 413 |
+
d. Nothing in this Public License constitutes or may be interpreted
|
| 414 |
+
as a limitation upon, or waiver of, any privileges and immunities
|
| 415 |
+
that apply to the Licensor or You, including from the legal
|
| 416 |
+
processes of any jurisdiction or authority.
|
| 417 |
+
|
| 418 |
+
=======================================================================
|
| 419 |
+
|
| 420 |
+
Creative Commons is not a party to its public
|
| 421 |
+
licenses. Notwithstanding, Creative Commons may elect to apply one of
|
| 422 |
+
its public licenses to material it publishes and in those instances
|
| 423 |
+
will be considered the “Licensor.” The text of the Creative Commons
|
| 424 |
+
public licenses is dedicated to the public domain under the CC0 Public
|
| 425 |
+
Domain Dedication. Except for the limited purpose of indicating that
|
| 426 |
+
material is shared under a Creative Commons public license or as
|
| 427 |
+
otherwise permitted by the Creative Commons policies published at
|
| 428 |
+
creativecommons.org/policies, Creative Commons does not authorize the
|
| 429 |
+
use of the trademark "Creative Commons" or any other trademark or logo
|
| 430 |
+
of Creative Commons without its prior written consent including,
|
| 431 |
+
without limitation, in connection with any unauthorized modifications
|
| 432 |
+
to any of its public licenses or any other arrangements,
|
| 433 |
+
understandings, or agreements concerning use of licensed material. For
|
| 434 |
+
the avoidance of doubt, this paragraph does not form part of the
|
| 435 |
+
public licenses.
|
| 436 |
+
|
| 437 |
+
Creative Commons may be contacted at creativecommons.org.
|
| 438 |
+
|
268m-g073-human-r0.05/README.md
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: cc-by-nc-sa-4.0
|
| 3 |
+
library_name: transformers
|
| 4 |
+
pipeline_tag: text-generation
|
| 5 |
+
language:
|
| 6 |
+
- en
|
| 7 |
+
datasets:
|
| 8 |
+
- pangram/WildAI
|
| 9 |
+
tags:
|
| 10 |
+
- wildai
|
| 11 |
+
- nanochat
|
| 12 |
+
- scaling-laws
|
| 13 |
+
- ai-generated-text
|
| 14 |
+
- pretraining-data
|
| 15 |
+
---
|
| 16 |
+
|
| 17 |
+
# 268m-g073-human-r0.05
|
| 18 |
+
|
| 19 |
+
> **License: [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/).** The WildAI code, models and dataset are released under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license: you may use and adapt them for non-commercial purposes, with attribution, and must share adaptations under the same license.
|
| 20 |
+
|
| 21 |
+
A 268M decoder-only language model trained from scratch for [*How Much Is an AI Token Worth? Scaling Laws for Wild AI-Generated Web Text*](https://arxiv.org/abs/2609.40295), one of the paper's models measuring how wild AI-generated web text in pretraining data changes a model's loss. It is a research artifact for studying scaling laws, not an assistant: it has had no instruction tuning or safety training.
|
| 22 |
+
|
| 23 |
+
## Training data
|
| 24 |
+
|
| 25 |
+
The human documents of `268m-g073-control` plus fresh human-written web documents (0.05 times the control's tokens): 1.23B human tokens, the human-text counterpart of adding AI text. No AI text.
|
| 26 |
+
|
| 27 |
+
| | |
|
| 28 |
+
|---|---|
|
| 29 |
+
| Parameters (N, input embedding included, value embeddings excluded) | 268.4M |
|
| 30 |
+
| Layers / width | 16 / 1024 |
|
| 31 |
+
| Training tokens | 1.23B (2,352 optimizer steps) |
|
| 32 |
+
| Human / AI tokens | 1.23B / 0.0M |
|
| 33 |
+
| Group / arm / seed | g073 / human / 271828 |
|
| 34 |
+
| Used in the paper for | fitting the scaling laws (726 models, 19.9M to 268M) |
|
| 35 |
+
|
| 36 |
+
## Held-out loss
|
| 37 |
+
|
| 38 |
+
| Set | Bits per byte | Text |
|
| 39 |
+
|---|---|---|
|
| 40 |
+
| C4 | 0.9933 | C4 validation text (human-written, pre-2022) |
|
| 41 |
+
| FW22 | 0.9776 | held-out FineWeb documents crawled before 2022 |
|
| 42 |
+
| Paloma | 1.3671 | macro average over 16 Paloma sources |
|
| 43 |
+
| FW26 | 0.8659 | held-out 2026 FineWeb-style crawl (22.3% AI tokens) |
|
| 44 |
+
| FW26-H | 0.9102 | the human-labeled documents of FW26 |
|
| 45 |
+
| FW26-AI | 0.7028 | the AI-labeled documents of FW26 |
|
| 46 |
+
| Cosmo | 0.7026 | Cosmopedia (AI-generated textbooks and stories) |
|
| 47 |
+
|
| 48 |
+
<details><summary>Paloma sources</summary>
|
| 49 |
+
|
| 50 |
+
| Source | Bits per byte |
|
| 51 |
+
|---|---|
|
| 52 |
+
| 4chan | 1.4051 |
|
| 53 |
+
| c4 | 1.0126 |
|
| 54 |
+
| c4_100_domains | 0.9946 |
|
| 55 |
+
| dolma | 1.2604 |
|
| 56 |
+
| falcon_refinedweb | 1.1104 |
|
| 57 |
+
| gab | 1.8878 |
|
| 58 |
+
| m2d2_s2orc | 1.1769 |
|
| 59 |
+
| m2d2_wikipedia | 1.0828 |
|
| 60 |
+
| manosphere | 1.3912 |
|
| 61 |
+
| mc4_en | 1.1020 |
|
| 62 |
+
| programming | 1.9758 |
|
| 63 |
+
| ptb | 1.2299 |
|
| 64 |
+
| reddit_100 | 1.2457 |
|
| 65 |
+
| redpajama | 1.4304 |
|
| 66 |
+
| twitteraae | 2.4905 |
|
| 67 |
+
| wikitext_103 | 1.0775 |
|
| 68 |
+
|
| 69 |
+
</details>
|
| 70 |
+
|
| 71 |
+
## Usage
|
| 72 |
+
|
| 73 |
+
```python
|
| 74 |
+
import torch
|
| 75 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 76 |
+
|
| 77 |
+
tokenizer = AutoTokenizer.from_pretrained("pangram/WildAI-models", subfolder="268m-g073-human-r0.05", trust_remote_code=True)
|
| 78 |
+
model = AutoModelForCausalLM.from_pretrained("pangram/WildAI-models", subfolder="268m-g073-human-r0.05", trust_remote_code=True, dtype=torch.bfloat16)
|
| 79 |
+
|
| 80 |
+
inputs = tokenizer("The history of the printing press", return_tensors="pt")
|
| 81 |
+
output = model.generate(**inputs, max_new_tokens=40, do_sample=False)
|
| 82 |
+
print(tokenizer.decode(output[0], skip_special_tokens=True))
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
The tokenizer prepends `<|bos|>` to every text, as in training; pass `add_special_tokens=False` to skip it. The model predicts `<|bos|>` where a document ends, and generation stops there. Weights are stored in bfloat16, the precision the paper's losses were computed in; `dtype=torch.float32` computes in float32 instead.
|
| 86 |
+
|
| 87 |
+
## Architecture
|
| 88 |
+
|
| 89 |
+
The nanochat GPT recipe: rotary embeddings, RMSNorm without weights (also on queries and keys), squared-ReLU MLP, attention windows in a repeating short-short-short-long pattern (512 tokens, then the full 2,048-token context; the last layer always full), gated value embeddings on alternating layers, a previous-token smear, per-layer residual scalars, a mid-network backout, untied input and output embeddings and logits soft-capped at 15. Vocabulary 32,768 tokens, context 2,048. The code ships with the weights (`trust_remote_code=True`) and matches the training code's forward pass.
|
| 90 |
+
|
| 91 |
+
## License
|
| 92 |
+
|
| 93 |
+
**License: [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/).** The WildAI code, models and dataset are released under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license: you may use and adapt them for non-commercial purposes, with attribution, and must share adaptations under the same license. The full text is in `LICENSE`.
|
| 94 |
+
|
| 95 |
+
## Citation
|
| 96 |
+
|
| 97 |
+
```bibtex
|
| 98 |
+
@article{russell2026wildai,
|
| 99 |
+
title = {How Much Is an AI Token Worth? Scaling Laws for Wild AI-Generated Web Text},
|
| 100 |
+
author = {Russell, Jenna and Glickenhaus, Ben and Thai, Katherine and Wieting, John and Iyyer, Mohit and Spero, Max and Emi, Bradley},
|
| 101 |
+
journal = {arXiv preprint arXiv:2609.40295},
|
| 102 |
+
year = {2026},
|
| 103 |
+
url = {https://arxiv.org/abs/2609.40295}
|
| 104 |
+
}
|
| 105 |
+
```
|
268m-g073-human-r0.05/config.json
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"WildAIForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"auto_map": {
|
| 6 |
+
"AutoConfig": "configuration_wildai.WildAIConfig",
|
| 7 |
+
"AutoModelForCausalLM": "modeling_wildai.WildAIForCausalLM"
|
| 8 |
+
},
|
| 9 |
+
"bos_token_id": 32759,
|
| 10 |
+
"dtype": "bfloat16",
|
| 11 |
+
"eos_token_id": 32759,
|
| 12 |
+
"hidden_size": 1024,
|
| 13 |
+
"logit_softcap": 15.0,
|
| 14 |
+
"max_position_embeddings": 2048,
|
| 15 |
+
"model_type": "wildai",
|
| 16 |
+
"num_attention_heads": 8,
|
| 17 |
+
"num_hidden_layers": 16,
|
| 18 |
+
"num_key_value_heads": 8,
|
| 19 |
+
"pad_token_id": 32759,
|
| 20 |
+
"qk_scale": 1.2,
|
| 21 |
+
"release": {
|
| 22 |
+
"added_ratio": 0.05,
|
| 23 |
+
"ai_ratio": 0.0,
|
| 24 |
+
"ai_tokens": 0,
|
| 25 |
+
"arm": "human",
|
| 26 |
+
"depth": 16,
|
| 27 |
+
"group": "g073",
|
| 28 |
+
"human_tokens": 1233125376,
|
| 29 |
+
"n_params": 268436282,
|
| 30 |
+
"name": "268m-g073-human-r0.05",
|
| 31 |
+
"seed": 271828,
|
| 32 |
+
"size": "268m",
|
| 33 |
+
"split": "fit",
|
| 34 |
+
"steps": 2352,
|
| 35 |
+
"total_tokens": 1233125376
|
| 36 |
+
},
|
| 37 |
+
"rotary_base": 100000.0,
|
| 38 |
+
"smear_channels": 24,
|
| 39 |
+
"tie_word_embeddings": false,
|
| 40 |
+
"transformers_version": "5.17.0",
|
| 41 |
+
"use_cache": true,
|
| 42 |
+
"value_embed_gate_channels": 12,
|
| 43 |
+
"value_embed_gate_range": 3.0,
|
| 44 |
+
"vocab_size": 32768,
|
| 45 |
+
"window_pattern": "SSSL"
|
| 46 |
+
}
|
268m-g073-human-r0.05/configuration_wildai.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Configuration of the WildAI language models (a nanochat GPT variant) for Hugging Face ``transformers``.
|
| 2 |
+
|
| 3 |
+
This file is copied verbatim into every released model repository and loaded with ``trust_remote_code=True``.
|
| 4 |
+
It depends only on ``transformers``.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from typing import ClassVar
|
| 10 |
+
|
| 11 |
+
from transformers import PretrainedConfig
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class WildAIConfig(PretrainedConfig):
|
| 15 |
+
"""Architecture of a WildAI model. Only width and depth vary across the released models.
|
| 16 |
+
|
| 17 |
+
Every layer uses causal attention limited to a left window: a query attends to itself and the ``window`` tokens before
|
| 18 |
+
it. ``window_pattern`` is tiled over the layers ("S" = a quarter of the context rounded up to a multiple of 128 tokens,
|
| 19 |
+
"L" = the full context) and the last layer always uses "L".
|
| 20 |
+
|
| 21 |
+
``release`` holds the model's entry in the release catalog (name, token counts, AI ratio, ...); it does not affect
|
| 22 |
+
the forward pass.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
model_type = "wildai"
|
| 26 |
+
keys_to_ignore_at_inference: ClassVar[list[str]] = ["past_key_values"]
|
| 27 |
+
|
| 28 |
+
def __init__(
|
| 29 |
+
self,
|
| 30 |
+
vocab_size: int = 32768,
|
| 31 |
+
hidden_size: int = 768,
|
| 32 |
+
num_hidden_layers: int = 12,
|
| 33 |
+
num_attention_heads: int = 6,
|
| 34 |
+
num_key_value_heads: int = 6,
|
| 35 |
+
max_position_embeddings: int = 2048,
|
| 36 |
+
window_pattern: str = "SSSL",
|
| 37 |
+
rotary_base: float = 100000.0,
|
| 38 |
+
qk_scale: float = 1.2,
|
| 39 |
+
logit_softcap: float = 15.0,
|
| 40 |
+
value_embed_gate_channels: int = 12,
|
| 41 |
+
value_embed_gate_range: float = 3.0,
|
| 42 |
+
smear_channels: int = 24,
|
| 43 |
+
bos_token_id: int = 32759,
|
| 44 |
+
eos_token_id: int = 32759,
|
| 45 |
+
use_cache: bool = True,
|
| 46 |
+
release: dict[str, str | int | float | None] | None = None,
|
| 47 |
+
**kwargs: object,
|
| 48 |
+
) -> None:
|
| 49 |
+
if hidden_size % num_attention_heads != 0 or num_attention_heads % num_key_value_heads != 0:
|
| 50 |
+
raise ValueError("hidden_size must divide into num_attention_heads, and those into num_key_value_heads")
|
| 51 |
+
if not window_pattern or set(window_pattern.upper()) - {"S", "L"}:
|
| 52 |
+
raise ValueError(f"window_pattern must be a non-empty string of S and L, got {window_pattern!r}")
|
| 53 |
+
self.vocab_size = vocab_size
|
| 54 |
+
self.hidden_size = hidden_size
|
| 55 |
+
self.num_hidden_layers = num_hidden_layers
|
| 56 |
+
self.num_attention_heads = num_attention_heads
|
| 57 |
+
self.num_key_value_heads = num_key_value_heads
|
| 58 |
+
self.max_position_embeddings = max_position_embeddings
|
| 59 |
+
self.window_pattern = window_pattern.upper()
|
| 60 |
+
self.rotary_base = rotary_base
|
| 61 |
+
self.qk_scale = qk_scale
|
| 62 |
+
self.logit_softcap = logit_softcap
|
| 63 |
+
self.value_embed_gate_channels = value_embed_gate_channels
|
| 64 |
+
self.value_embed_gate_range = value_embed_gate_range
|
| 65 |
+
self.smear_channels = smear_channels
|
| 66 |
+
self.use_cache = use_cache
|
| 67 |
+
self.release = release
|
| 68 |
+
kwargs.setdefault("tie_word_embeddings", False)
|
| 69 |
+
super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
|
| 70 |
+
|
| 71 |
+
@property
|
| 72 |
+
def head_dim(self) -> int:
|
| 73 |
+
return self.hidden_size // self.num_attention_heads
|
| 74 |
+
|
| 75 |
+
@property
|
| 76 |
+
def intermediate_size(self) -> int:
|
| 77 |
+
return 4 * self.hidden_size
|
| 78 |
+
|
| 79 |
+
@property
|
| 80 |
+
def window_sizes(self) -> list[int]:
|
| 81 |
+
"""Left attention window of each layer, in tokens."""
|
| 82 |
+
long_window = self.max_position_embeddings
|
| 83 |
+
short_window = -(-long_window // 4 // 128) * 128 # a quarter of the context, rounded up to a multiple of 128 (2048 -> 512)
|
| 84 |
+
pattern = self.window_pattern
|
| 85 |
+
sizes = [short_window if pattern[i % len(pattern)] == "S" else long_window for i in range(self.num_hidden_layers)]
|
| 86 |
+
sizes[-1] = long_window
|
| 87 |
+
return sizes
|
| 88 |
+
|
| 89 |
+
def has_value_embedding(self, layer_idx: int) -> bool:
|
| 90 |
+
"""Value embeddings sit on alternating layers, always including the last one."""
|
| 91 |
+
return layer_idx % 2 == (self.num_hidden_layers - 1) % 2
|
| 92 |
+
|
| 93 |
+
@property
|
| 94 |
+
def backout_layer(self) -> int:
|
| 95 |
+
"""The residual stream after this layer is partly subtracted before the final norm."""
|
| 96 |
+
return self.num_hidden_layers // 2
|
268m-g073-human-r0.05/generation_config.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 32759,
|
| 4 |
+
"eos_token_id": 32759,
|
| 5 |
+
"output_attentions": false,
|
| 6 |
+
"output_hidden_states": false,
|
| 7 |
+
"pad_token_id": 32759,
|
| 8 |
+
"transformers_version": "5.17.0",
|
| 9 |
+
"use_cache": true
|
| 10 |
+
}
|
268m-g073-human-r0.05/modeling_wildai.py
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The WildAI language models (a nanochat GPT variant) for Hugging Face ``transformers``.
|
| 2 |
+
|
| 3 |
+
This file is copied verbatim into every released model repository and loaded with ``trust_remote_code=True``.
|
| 4 |
+
It depends only on ``torch`` and ``transformers`` and reproduces the training code's forward pass operation by
|
| 5 |
+
operation, so a model loaded in bfloat16 computes the same function the paper evaluated:
|
| 6 |
+
|
| 7 |
+
- token embedding followed by a parameter-free RMSNorm, then a learned "smear" that mixes each position's embedding
|
| 8 |
+
with the previous token's;
|
| 9 |
+
- per-layer scalars that rescale the residual stream and blend the initial embedding back in before every block;
|
| 10 |
+
- attention with rotary embeddings applied before a per-head RMSNorm of queries and keys (both scaled by 1.2),
|
| 11 |
+
a left sliding window per layer, and gated value embeddings added to the values on alternating layers;
|
| 12 |
+
- a squared-ReLU MLP, parameter-free pre-norms, no biases;
|
| 13 |
+
- "backout": part of the residual stream after the middle layer is subtracted before the final norm;
|
| 14 |
+
- untied output head with logits soft-capped by ``15 * tanh(logits / 15)``.
|
| 15 |
+
|
| 16 |
+
Attention uses PyTorch's scaled_dot_product_attention with explicit window masks, so no FlashAttention install is
|
| 17 |
+
needed. Generation uses the standard ``DynamicCache``.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
from dataclasses import dataclass
|
| 23 |
+
from typing import ClassVar
|
| 24 |
+
|
| 25 |
+
import torch
|
| 26 |
+
import torch.nn.functional as F
|
| 27 |
+
from torch import nn
|
| 28 |
+
from transformers import GenerationMixin, PreTrainedModel
|
| 29 |
+
from transformers.cache_utils import Cache, DynamicCache
|
| 30 |
+
from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
|
| 31 |
+
|
| 32 |
+
from .configuration_wildai import WildAIConfig
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def rms_norm(x: torch.Tensor) -> torch.Tensor:
|
| 36 |
+
"""Parameter-free RMSNorm over the last dimension with PyTorch's default epsilon, as in training."""
|
| 37 |
+
return F.rms_norm(x, (x.size(-1),))
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def apply_rotary(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
|
| 41 |
+
"""Rotate channel pairs (i, i + d/2) of ``x`` (batch, time, heads, head_dim), with the training code's sign convention."""
|
| 42 |
+
d = x.shape[-1] // 2
|
| 43 |
+
x1, x2 = x[..., :d], x[..., d:]
|
| 44 |
+
return torch.cat([x1 * cos + x2 * sin, x1 * (-sin) + x2 * cos], dim=-1)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
@dataclass(frozen=True)
|
| 48 |
+
class WindowMask:
|
| 49 |
+
"""Arguments for scaled_dot_product_attention that implement one attention window."""
|
| 50 |
+
|
| 51 |
+
mask: torch.Tensor | None
|
| 52 |
+
is_causal: bool
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def window_masks(
|
| 56 |
+
window_sizes: list[int], query_len: int, past_len: int, padding_mask: torch.Tensor | None, device: torch.device
|
| 57 |
+
) -> dict[int, WindowMask]:
|
| 58 |
+
"""Masks for each distinct left window. Query i (absolute index past_len + i) sees keys j with i - window <= j <= i.
|
| 59 |
+
|
| 60 |
+
``padding_mask`` is the (batch, past_len + query_len) attention mask when it contains padding, else None. A padding
|
| 61 |
+
query is always allowed to see itself so that its (unused) output stays finite.
|
| 62 |
+
"""
|
| 63 |
+
total = past_len + query_len
|
| 64 |
+
q_idx = torch.arange(past_len, total, device=device)[:, None]
|
| 65 |
+
k_idx = torch.arange(total, device=device)[None, :]
|
| 66 |
+
masks: dict[int, WindowMask] = {}
|
| 67 |
+
for window in sorted(set(window_sizes)):
|
| 68 |
+
if padding_mask is None and window >= total - 1:
|
| 69 |
+
if query_len == 1:
|
| 70 |
+
masks[window] = WindowMask(mask=None, is_causal=False)
|
| 71 |
+
continue
|
| 72 |
+
if past_len == 0:
|
| 73 |
+
masks[window] = WindowMask(mask=None, is_causal=True)
|
| 74 |
+
continue
|
| 75 |
+
allowed = (k_idx <= q_idx) & (q_idx - k_idx <= window)
|
| 76 |
+
if padding_mask is not None:
|
| 77 |
+
allowed = (allowed & padding_mask[:, None, None, :].bool()) | (k_idx == q_idx)
|
| 78 |
+
masks[window] = WindowMask(mask=allowed, is_causal=False)
|
| 79 |
+
return masks
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class WildAIAttention(nn.Module):
|
| 83 |
+
def __init__(self, config: WildAIConfig, layer_idx: int) -> None:
|
| 84 |
+
super().__init__()
|
| 85 |
+
self.layer_idx = layer_idx
|
| 86 |
+
self.num_heads = config.num_attention_heads
|
| 87 |
+
self.num_kv_heads = config.num_key_value_heads
|
| 88 |
+
self.head_dim = config.head_dim
|
| 89 |
+
self.qk_scale = config.qk_scale
|
| 90 |
+
self.window = config.window_sizes[layer_idx]
|
| 91 |
+
hidden, kv_dim = config.hidden_size, config.num_key_value_heads * config.head_dim
|
| 92 |
+
self.q_proj = nn.Linear(hidden, config.num_attention_heads * config.head_dim, bias=False)
|
| 93 |
+
self.k_proj = nn.Linear(hidden, kv_dim, bias=False)
|
| 94 |
+
self.v_proj = nn.Linear(hidden, kv_dim, bias=False)
|
| 95 |
+
self.o_proj = nn.Linear(hidden, hidden, bias=False)
|
| 96 |
+
self.value_embed: nn.Embedding | None = None
|
| 97 |
+
self.value_embed_gate: nn.Linear | None = None
|
| 98 |
+
if config.has_value_embedding(layer_idx):
|
| 99 |
+
self.value_embed = nn.Embedding(config.vocab_size, kv_dim)
|
| 100 |
+
self.value_embed_gate = nn.Linear(config.value_embed_gate_channels, config.num_key_value_heads, bias=False)
|
| 101 |
+
self.gate_channels = config.value_embed_gate_channels
|
| 102 |
+
self.gate_range = config.value_embed_gate_range
|
| 103 |
+
|
| 104 |
+
def forward(
|
| 105 |
+
self,
|
| 106 |
+
x: torch.Tensor,
|
| 107 |
+
input_ids: torch.LongTensor,
|
| 108 |
+
rotary: tuple[torch.Tensor, torch.Tensor],
|
| 109 |
+
mask: WindowMask,
|
| 110 |
+
past_key_values: Cache | None,
|
| 111 |
+
) -> torch.Tensor:
|
| 112 |
+
batch, time, _ = x.shape
|
| 113 |
+
q = self.q_proj(x).view(batch, time, self.num_heads, self.head_dim)
|
| 114 |
+
k = self.k_proj(x).view(batch, time, self.num_kv_heads, self.head_dim)
|
| 115 |
+
v = self.v_proj(x).view(batch, time, self.num_kv_heads, self.head_dim)
|
| 116 |
+
if self.value_embed is not None:
|
| 117 |
+
ve = self.value_embed(input_ids).to(x.dtype).view(batch, time, self.num_kv_heads, self.head_dim)
|
| 118 |
+
gate = self.gate_range * torch.sigmoid(self.value_embed_gate(x[..., : self.gate_channels]))
|
| 119 |
+
v = v + gate.unsqueeze(-1) * ve
|
| 120 |
+
cos, sin = rotary
|
| 121 |
+
q, k = apply_rotary(q, cos, sin), apply_rotary(k, cos, sin)
|
| 122 |
+
q, k = rms_norm(q) * self.qk_scale, rms_norm(k) * self.qk_scale
|
| 123 |
+
q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
|
| 124 |
+
if past_key_values is not None:
|
| 125 |
+
k, v = past_key_values.update(k, v, self.layer_idx)
|
| 126 |
+
y = F.scaled_dot_product_attention(
|
| 127 |
+
q, k, v, attn_mask=mask.mask, is_causal=mask.is_causal, enable_gqa=self.num_kv_heads != self.num_heads
|
| 128 |
+
)
|
| 129 |
+
return self.o_proj(y.transpose(1, 2).reshape(batch, time, -1))
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
class WildAIMLP(nn.Module):
|
| 133 |
+
def __init__(self, config: WildAIConfig) -> None:
|
| 134 |
+
super().__init__()
|
| 135 |
+
self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
|
| 136 |
+
self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
|
| 137 |
+
|
| 138 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 139 |
+
return self.down_proj(F.relu(self.up_proj(x)).square())
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
class WildAIDecoderLayer(nn.Module):
|
| 143 |
+
def __init__(self, config: WildAIConfig, layer_idx: int) -> None:
|
| 144 |
+
super().__init__()
|
| 145 |
+
self.self_attn = WildAIAttention(config, layer_idx)
|
| 146 |
+
self.mlp = WildAIMLP(config)
|
| 147 |
+
|
| 148 |
+
def forward(
|
| 149 |
+
self,
|
| 150 |
+
x: torch.Tensor,
|
| 151 |
+
input_ids: torch.LongTensor,
|
| 152 |
+
rotary: tuple[torch.Tensor, torch.Tensor],
|
| 153 |
+
mask: WindowMask,
|
| 154 |
+
past_key_values: Cache | None,
|
| 155 |
+
) -> torch.Tensor:
|
| 156 |
+
x = x + self.self_attn(rms_norm(x), input_ids, rotary, mask, past_key_values)
|
| 157 |
+
return x + self.mlp(rms_norm(x))
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
class WildAIPreTrainedModel(PreTrainedModel):
|
| 161 |
+
config_class = WildAIConfig
|
| 162 |
+
base_model_prefix = "model"
|
| 163 |
+
main_input_name = "input_ids"
|
| 164 |
+
_no_split_modules: ClassVar[list[str]] = ["WildAIDecoderLayer"]
|
| 165 |
+
_supports_sdpa = True
|
| 166 |
+
|
| 167 |
+
def _init_weights(self, module: nn.Module) -> None:
|
| 168 |
+
# Released models are always loaded from trained weights; this only gives from_config() models sane values.
|
| 169 |
+
if isinstance(module, (nn.Linear, nn.Embedding)):
|
| 170 |
+
nn.init.normal_(module.weight, mean=0.0, std=0.02)
|
| 171 |
+
elif isinstance(module, WildAIModel):
|
| 172 |
+
with torch.no_grad():
|
| 173 |
+
module.resid_lambdas.fill_(1.0)
|
| 174 |
+
module.x0_lambdas.zero_()
|
| 175 |
+
module.smear_lambda.zero_()
|
| 176 |
+
module.backout_lambda.zero_()
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
class WildAIModel(WildAIPreTrainedModel):
|
| 180 |
+
"""The transformer trunk; returns the final normalised hidden states."""
|
| 181 |
+
|
| 182 |
+
def __init__(self, config: WildAIConfig) -> None:
|
| 183 |
+
super().__init__(config)
|
| 184 |
+
n_layer = config.num_hidden_layers
|
| 185 |
+
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
|
| 186 |
+
self.layers = nn.ModuleList([WildAIDecoderLayer(config, i) for i in range(n_layer)])
|
| 187 |
+
self.resid_lambdas = nn.Parameter(torch.ones(n_layer))
|
| 188 |
+
self.x0_lambdas = nn.Parameter(torch.zeros(n_layer))
|
| 189 |
+
self.smear_gate = nn.Linear(config.smear_channels, 1, bias=False)
|
| 190 |
+
self.smear_lambda = nn.Parameter(torch.zeros(1))
|
| 191 |
+
self.backout_lambda = nn.Parameter(torch.zeros(1))
|
| 192 |
+
self.post_init()
|
| 193 |
+
|
| 194 |
+
def get_input_embeddings(self) -> nn.Embedding:
|
| 195 |
+
return self.embed_tokens
|
| 196 |
+
|
| 197 |
+
def set_input_embeddings(self, value: nn.Embedding) -> None:
|
| 198 |
+
self.embed_tokens = value
|
| 199 |
+
|
| 200 |
+
def rotary(self, position_ids: torch.LongTensor, dtype: torch.dtype) -> tuple[torch.Tensor, torch.Tensor]:
|
| 201 |
+
"""cos/sin tables of shape (batch, time, 1, head_dim / 2), computed in float32 and cast like in training."""
|
| 202 |
+
head_dim = self.config.head_dim
|
| 203 |
+
channels = torch.arange(0, head_dim, 2, dtype=torch.float32, device=position_ids.device)
|
| 204 |
+
inv_freq = 1.0 / (self.config.rotary_base ** (channels / head_dim))
|
| 205 |
+
freqs = position_ids.to(torch.float32)[..., None] * inv_freq
|
| 206 |
+
return freqs.cos().to(dtype)[:, :, None, :], freqs.sin().to(dtype)[:, :, None, :]
|
| 207 |
+
|
| 208 |
+
def smear(self, x: torch.Tensor, previous: torch.Tensor | None, padding_mask: torch.Tensor | None) -> torch.Tensor:
|
| 209 |
+
"""Add a gated copy of the previous position's embedding to every position that has one.
|
| 210 |
+
|
| 211 |
+
``previous`` is the normalised embedding of the token just before this chunk (None at the start of a sequence);
|
| 212 |
+
with left padding, the first real token has a padding token before it and gets nothing.
|
| 213 |
+
"""
|
| 214 |
+
channels = self.config.smear_channels
|
| 215 |
+
lam = self.smear_lambda.to(x.dtype)
|
| 216 |
+
first = x[:, :1]
|
| 217 |
+
if previous is not None:
|
| 218 |
+
first = first + lam * torch.sigmoid(self.smear_gate(first[..., :channels])) * previous
|
| 219 |
+
if x.size(1) == 1:
|
| 220 |
+
return first
|
| 221 |
+
gate = lam * torch.sigmoid(self.smear_gate(x[:, 1:, :channels]))
|
| 222 |
+
if padding_mask is not None:
|
| 223 |
+
gate = gate * padding_mask[:, -x.size(1) : -1, None].to(gate.dtype)
|
| 224 |
+
return torch.cat([first, x[:, 1:] + gate * x[:, :-1]], dim=1)
|
| 225 |
+
|
| 226 |
+
def forward(
|
| 227 |
+
self,
|
| 228 |
+
input_ids: torch.LongTensor,
|
| 229 |
+
attention_mask: torch.Tensor | None = None,
|
| 230 |
+
position_ids: torch.LongTensor | None = None,
|
| 231 |
+
past_key_values: Cache | None = None,
|
| 232 |
+
use_cache: bool | None = None,
|
| 233 |
+
previous_token_ids: torch.LongTensor | None = None,
|
| 234 |
+
output_hidden_states: bool | None = None,
|
| 235 |
+
**kwargs: object,
|
| 236 |
+
) -> BaseModelOutputWithPast:
|
| 237 |
+
"""``previous_token_ids`` (batch, 1): the token before ``input_ids[:, 0]``; required when continuing from a cache."""
|
| 238 |
+
if input_ids is None:
|
| 239 |
+
raise ValueError("input_ids are required: the value embeddings are looked up by token id")
|
| 240 |
+
batch, time = input_ids.shape
|
| 241 |
+
use_cache = self.config.use_cache if use_cache is None else use_cache
|
| 242 |
+
if use_cache and past_key_values is None:
|
| 243 |
+
past_key_values = DynamicCache()
|
| 244 |
+
past_len = past_key_values.get_seq_length() if past_key_values is not None else 0
|
| 245 |
+
if past_len > 0 and previous_token_ids is None:
|
| 246 |
+
raise ValueError("previous_token_ids is required when continuing from past_key_values (the smear needs it)")
|
| 247 |
+
padding_mask = None
|
| 248 |
+
if attention_mask is not None:
|
| 249 |
+
if attention_mask.shape != (batch, past_len + time):
|
| 250 |
+
raise ValueError(f"attention_mask must have shape {(batch, past_len + time)}, got {tuple(attention_mask.shape)}")
|
| 251 |
+
if not bool(attention_mask.all()):
|
| 252 |
+
padding_mask = attention_mask
|
| 253 |
+
if position_ids is None:
|
| 254 |
+
position_ids = torch.arange(past_len, past_len + time, device=input_ids.device).expand(batch, -1)
|
| 255 |
+
|
| 256 |
+
x = rms_norm(self.embed_tokens(input_ids))
|
| 257 |
+
previous = rms_norm(self.embed_tokens(previous_token_ids)) if previous_token_ids is not None else None
|
| 258 |
+
x = self.smear(x, previous, padding_mask)
|
| 259 |
+
rotary = self.rotary(position_ids, x.dtype)
|
| 260 |
+
masks = window_masks(self.config.window_sizes, time, past_len, padding_mask, input_ids.device)
|
| 261 |
+
|
| 262 |
+
x0 = x
|
| 263 |
+
backout = None
|
| 264 |
+
hidden_states: tuple[torch.Tensor, ...] = (x,)
|
| 265 |
+
for i, layer in enumerate(self.layers):
|
| 266 |
+
x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0
|
| 267 |
+
x = layer(x, input_ids, rotary, masks[layer.self_attn.window], past_key_values)
|
| 268 |
+
if i == self.config.backout_layer:
|
| 269 |
+
backout = x
|
| 270 |
+
hidden_states += (x,)
|
| 271 |
+
x = rms_norm(x - self.backout_lambda.to(x.dtype) * backout)
|
| 272 |
+
return BaseModelOutputWithPast(
|
| 273 |
+
last_hidden_state=x,
|
| 274 |
+
past_key_values=past_key_values if use_cache else None,
|
| 275 |
+
hidden_states=hidden_states if output_hidden_states else None,
|
| 276 |
+
)
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
class WildAIForCausalLM(WildAIPreTrainedModel, GenerationMixin):
|
| 280 |
+
def __init__(self, config: WildAIConfig) -> None:
|
| 281 |
+
super().__init__(config)
|
| 282 |
+
self.model = WildAIModel(config)
|
| 283 |
+
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
|
| 284 |
+
self.post_init()
|
| 285 |
+
|
| 286 |
+
def get_input_embeddings(self) -> nn.Embedding:
|
| 287 |
+
return self.model.embed_tokens
|
| 288 |
+
|
| 289 |
+
def set_input_embeddings(self, value: nn.Embedding) -> None:
|
| 290 |
+
self.model.embed_tokens = value
|
| 291 |
+
|
| 292 |
+
def get_output_embeddings(self) -> nn.Linear:
|
| 293 |
+
return self.lm_head
|
| 294 |
+
|
| 295 |
+
def set_output_embeddings(self, value: nn.Linear) -> None:
|
| 296 |
+
self.lm_head = value
|
| 297 |
+
|
| 298 |
+
def forward(
|
| 299 |
+
self,
|
| 300 |
+
input_ids: torch.LongTensor,
|
| 301 |
+
attention_mask: torch.Tensor | None = None,
|
| 302 |
+
position_ids: torch.LongTensor | None = None,
|
| 303 |
+
past_key_values: Cache | None = None,
|
| 304 |
+
labels: torch.LongTensor | None = None,
|
| 305 |
+
use_cache: bool | None = None,
|
| 306 |
+
previous_token_ids: torch.LongTensor | None = None,
|
| 307 |
+
output_hidden_states: bool | None = None,
|
| 308 |
+
logits_to_keep: int = 0,
|
| 309 |
+
return_dict: bool | None = None,
|
| 310 |
+
**kwargs: object,
|
| 311 |
+
) -> CausalLMOutputWithPast | tuple[torch.Tensor, ...]:
|
| 312 |
+
"""Float32 logits after the soft cap. ``labels`` are shifted inside (standard causal-LM loss, -100 ignored)."""
|
| 313 |
+
outputs = self.model(
|
| 314 |
+
input_ids,
|
| 315 |
+
attention_mask=attention_mask,
|
| 316 |
+
position_ids=position_ids,
|
| 317 |
+
past_key_values=past_key_values,
|
| 318 |
+
use_cache=use_cache,
|
| 319 |
+
previous_token_ids=previous_token_ids,
|
| 320 |
+
output_hidden_states=output_hidden_states,
|
| 321 |
+
)
|
| 322 |
+
hidden = outputs.last_hidden_state[:, -logits_to_keep:] if logits_to_keep else outputs.last_hidden_state
|
| 323 |
+
logits = self.lm_head(hidden).float()
|
| 324 |
+
cap = self.config.logit_softcap
|
| 325 |
+
logits = cap * torch.tanh(logits / cap)
|
| 326 |
+
loss = None
|
| 327 |
+
if labels is not None:
|
| 328 |
+
loss = F.cross_entropy(logits[:, :-1].flatten(0, 1), labels[:, 1:].flatten().to(logits.device), ignore_index=-100)
|
| 329 |
+
result = CausalLMOutputWithPast(
|
| 330 |
+
loss=loss, logits=logits, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states
|
| 331 |
+
)
|
| 332 |
+
return result.to_tuple() if return_dict is False else result
|
| 333 |
+
|
| 334 |
+
def prepare_inputs_for_generation(self, input_ids: torch.LongTensor, **kwargs: object) -> dict[str, object]:
|
| 335 |
+
# The smear mixes in the embedding of the token before the first new one; take it from the full sequence.
|
| 336 |
+
model_inputs = super().prepare_inputs_for_generation(input_ids, **kwargs)
|
| 337 |
+
new_len = model_inputs["input_ids"].shape[1]
|
| 338 |
+
if input_ids.shape[1] > new_len:
|
| 339 |
+
model_inputs["previous_token_ids"] = input_ids[:, -new_len - 1 : -new_len]
|
| 340 |
+
return model_inputs
|
268m-g073-human-r0.05/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
268m-g073-human-r0.05/tokenizer_config.json
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 3 |
+
"bos_token": "<|bos|>",
|
| 4 |
+
"eos_token": "<|bos|>",
|
| 5 |
+
"model_max_length": 2048,
|
| 6 |
+
"model_input_names": [
|
| 7 |
+
"input_ids",
|
| 8 |
+
"attention_mask"
|
| 9 |
+
],
|
| 10 |
+
"clean_up_tokenization_spaces": false
|
| 11 |
+
}
|
268m-g074-human-r0.025/LICENSE
ADDED
|
@@ -0,0 +1,438 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Attribution-NonCommercial-ShareAlike 4.0 International
|
| 2 |
+
|
| 3 |
+
=======================================================================
|
| 4 |
+
|
| 5 |
+
Creative Commons Corporation ("Creative Commons") is not a law firm and
|
| 6 |
+
does not provide legal services or legal advice. Distribution of
|
| 7 |
+
Creative Commons public licenses does not create a lawyer-client or
|
| 8 |
+
other relationship. Creative Commons makes its licenses and related
|
| 9 |
+
information available on an "as-is" basis. Creative Commons gives no
|
| 10 |
+
warranties regarding its licenses, any material licensed under their
|
| 11 |
+
terms and conditions, or any related information. Creative Commons
|
| 12 |
+
disclaims all liability for damages resulting from their use to the
|
| 13 |
+
fullest extent possible.
|
| 14 |
+
|
| 15 |
+
Using Creative Commons Public Licenses
|
| 16 |
+
|
| 17 |
+
Creative Commons public licenses provide a standard set of terms and
|
| 18 |
+
conditions that creators and other rights holders may use to share
|
| 19 |
+
original works of authorship and other material subject to copyright
|
| 20 |
+
and certain other rights specified in the public license below. The
|
| 21 |
+
following considerations are for informational purposes only, are not
|
| 22 |
+
exhaustive, and do not form part of our licenses.
|
| 23 |
+
|
| 24 |
+
Considerations for licensors: Our public licenses are
|
| 25 |
+
intended for use by those authorized to give the public
|
| 26 |
+
permission to use material in ways otherwise restricted by
|
| 27 |
+
copyright and certain other rights. Our licenses are
|
| 28 |
+
irrevocable. Licensors should read and understand the terms
|
| 29 |
+
and conditions of the license they choose before applying it.
|
| 30 |
+
Licensors should also secure all rights necessary before
|
| 31 |
+
applying our licenses so that the public can reuse the
|
| 32 |
+
material as expected. Licensors should clearly mark any
|
| 33 |
+
material not subject to the license. This includes other CC-
|
| 34 |
+
licensed material, or material used under an exception or
|
| 35 |
+
limitation to copyright. More considerations for licensors:
|
| 36 |
+
wiki.creativecommons.org/Considerations_for_licensors
|
| 37 |
+
|
| 38 |
+
Considerations for the public: By using one of our public
|
| 39 |
+
licenses, a licensor grants the public permission to use the
|
| 40 |
+
licensed material under specified terms and conditions. If
|
| 41 |
+
the licensor's permission is not necessary for any reason--for
|
| 42 |
+
example, because of any applicable exception or limitation to
|
| 43 |
+
copyright--then that use is not regulated by the license. Our
|
| 44 |
+
licenses grant only permissions under copyright and certain
|
| 45 |
+
other rights that a licensor has authority to grant. Use of
|
| 46 |
+
the licensed material may still be restricted for other
|
| 47 |
+
reasons, including because others have copyright or other
|
| 48 |
+
rights in the material. A licensor may make special requests,
|
| 49 |
+
such as asking that all changes be marked or described.
|
| 50 |
+
Although not required by our licenses, you are encouraged to
|
| 51 |
+
respect those requests where reasonable. More considerations
|
| 52 |
+
for the public:
|
| 53 |
+
wiki.creativecommons.org/Considerations_for_licensees
|
| 54 |
+
|
| 55 |
+
=======================================================================
|
| 56 |
+
|
| 57 |
+
Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International
|
| 58 |
+
Public License
|
| 59 |
+
|
| 60 |
+
By exercising the Licensed Rights (defined below), You accept and agree
|
| 61 |
+
to be bound by the terms and conditions of this Creative Commons
|
| 62 |
+
Attribution-NonCommercial-ShareAlike 4.0 International Public License
|
| 63 |
+
("Public License"). To the extent this Public License may be
|
| 64 |
+
interpreted as a contract, You are granted the Licensed Rights in
|
| 65 |
+
consideration of Your acceptance of these terms and conditions, and the
|
| 66 |
+
Licensor grants You such rights in consideration of benefits the
|
| 67 |
+
Licensor receives from making the Licensed Material available under
|
| 68 |
+
these terms and conditions.
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
Section 1 -- Definitions.
|
| 72 |
+
|
| 73 |
+
a. Adapted Material means material subject to Copyright and Similar
|
| 74 |
+
Rights that is derived from or based upon the Licensed Material
|
| 75 |
+
and in which the Licensed Material is translated, altered,
|
| 76 |
+
arranged, transformed, or otherwise modified in a manner requiring
|
| 77 |
+
permission under the Copyright and Similar Rights held by the
|
| 78 |
+
Licensor. For purposes of this Public License, where the Licensed
|
| 79 |
+
Material is a musical work, performance, or sound recording,
|
| 80 |
+
Adapted Material is always produced where the Licensed Material is
|
| 81 |
+
synched in timed relation with a moving image.
|
| 82 |
+
|
| 83 |
+
b. Adapter's License means the license You apply to Your Copyright
|
| 84 |
+
and Similar Rights in Your contributions to Adapted Material in
|
| 85 |
+
accordance with the terms and conditions of this Public License.
|
| 86 |
+
|
| 87 |
+
c. BY-NC-SA Compatible License means a license listed at
|
| 88 |
+
creativecommons.org/compatiblelicenses, approved by Creative
|
| 89 |
+
Commons as essentially the equivalent of this Public License.
|
| 90 |
+
|
| 91 |
+
d. Copyright and Similar Rights means copyright and/or similar rights
|
| 92 |
+
closely related to copyright including, without limitation,
|
| 93 |
+
performance, broadcast, sound recording, and Sui Generis Database
|
| 94 |
+
Rights, without regard to how the rights are labeled or
|
| 95 |
+
categorized. For purposes of this Public License, the rights
|
| 96 |
+
specified in Section 2(b)(1)-(2) are not Copyright and Similar
|
| 97 |
+
Rights.
|
| 98 |
+
|
| 99 |
+
e. Effective Technological Measures means those measures that, in the
|
| 100 |
+
absence of proper authority, may not be circumvented under laws
|
| 101 |
+
fulfilling obligations under Article 11 of the WIPO Copyright
|
| 102 |
+
Treaty adopted on December 20, 1996, and/or similar international
|
| 103 |
+
agreements.
|
| 104 |
+
|
| 105 |
+
f. Exceptions and Limitations means fair use, fair dealing, and/or
|
| 106 |
+
any other exception or limitation to Copyright and Similar Rights
|
| 107 |
+
that applies to Your use of the Licensed Material.
|
| 108 |
+
|
| 109 |
+
g. License Elements means the license attributes listed in the name
|
| 110 |
+
of a Creative Commons Public License. The License Elements of this
|
| 111 |
+
Public License are Attribution, NonCommercial, and ShareAlike.
|
| 112 |
+
|
| 113 |
+
h. Licensed Material means the artistic or literary work, database,
|
| 114 |
+
or other material to which the Licensor applied this Public
|
| 115 |
+
License.
|
| 116 |
+
|
| 117 |
+
i. Licensed Rights means the rights granted to You subject to the
|
| 118 |
+
terms and conditions of this Public License, which are limited to
|
| 119 |
+
all Copyright and Similar Rights that apply to Your use of the
|
| 120 |
+
Licensed Material and that the Licensor has authority to license.
|
| 121 |
+
|
| 122 |
+
j. Licensor means the individual(s) or entity(ies) granting rights
|
| 123 |
+
under this Public License.
|
| 124 |
+
|
| 125 |
+
k. NonCommercial means not primarily intended for or directed towards
|
| 126 |
+
commercial advantage or monetary compensation. For purposes of
|
| 127 |
+
this Public License, the exchange of the Licensed Material for
|
| 128 |
+
other material subject to Copyright and Similar Rights by digital
|
| 129 |
+
file-sharing or similar means is NonCommercial provided there is
|
| 130 |
+
no payment of monetary compensation in connection with the
|
| 131 |
+
exchange.
|
| 132 |
+
|
| 133 |
+
l. Share means to provide material to the public by any means or
|
| 134 |
+
process that requires permission under the Licensed Rights, such
|
| 135 |
+
as reproduction, public display, public performance, distribution,
|
| 136 |
+
dissemination, communication, or importation, and to make material
|
| 137 |
+
available to the public including in ways that members of the
|
| 138 |
+
public may access the material from a place and at a time
|
| 139 |
+
individually chosen by them.
|
| 140 |
+
|
| 141 |
+
m. Sui Generis Database Rights means rights other than copyright
|
| 142 |
+
resulting from Directive 96/9/EC of the European Parliament and of
|
| 143 |
+
the Council of 11 March 1996 on the legal protection of databases,
|
| 144 |
+
as amended and/or succeeded, as well as other essentially
|
| 145 |
+
equivalent rights anywhere in the world.
|
| 146 |
+
|
| 147 |
+
n. You means the individual or entity exercising the Licensed Rights
|
| 148 |
+
under this Public License. Your has a corresponding meaning.
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
Section 2 -- Scope.
|
| 152 |
+
|
| 153 |
+
a. License grant.
|
| 154 |
+
|
| 155 |
+
1. Subject to the terms and conditions of this Public License,
|
| 156 |
+
the Licensor hereby grants You a worldwide, royalty-free,
|
| 157 |
+
non-sublicensable, non-exclusive, irrevocable license to
|
| 158 |
+
exercise the Licensed Rights in the Licensed Material to:
|
| 159 |
+
|
| 160 |
+
a. reproduce and Share the Licensed Material, in whole or
|
| 161 |
+
in part, for NonCommercial purposes only; and
|
| 162 |
+
|
| 163 |
+
b. produce, reproduce, and Share Adapted Material for
|
| 164 |
+
NonCommercial purposes only.
|
| 165 |
+
|
| 166 |
+
2. Exceptions and Limitations. For the avoidance of doubt, where
|
| 167 |
+
Exceptions and Limitations apply to Your use, this Public
|
| 168 |
+
License does not apply, and You do not need to comply with
|
| 169 |
+
its terms and conditions.
|
| 170 |
+
|
| 171 |
+
3. Term. The term of this Public License is specified in Section
|
| 172 |
+
6(a).
|
| 173 |
+
|
| 174 |
+
4. Media and formats; technical modifications allowed. The
|
| 175 |
+
Licensor authorizes You to exercise the Licensed Rights in
|
| 176 |
+
all media and formats whether now known or hereafter created,
|
| 177 |
+
and to make technical modifications necessary to do so. The
|
| 178 |
+
Licensor waives and/or agrees not to assert any right or
|
| 179 |
+
authority to forbid You from making technical modifications
|
| 180 |
+
necessary to exercise the Licensed Rights, including
|
| 181 |
+
technical modifications necessary to circumvent Effective
|
| 182 |
+
Technological Measures. For purposes of this Public License,
|
| 183 |
+
simply making modifications authorized by this Section 2(a)
|
| 184 |
+
(4) never produces Adapted Material.
|
| 185 |
+
|
| 186 |
+
5. Downstream recipients.
|
| 187 |
+
|
| 188 |
+
a. Offer from the Licensor -- Licensed Material. Every
|
| 189 |
+
recipient of the Licensed Material automatically
|
| 190 |
+
receives an offer from the Licensor to exercise the
|
| 191 |
+
Licensed Rights under the terms and conditions of this
|
| 192 |
+
Public License.
|
| 193 |
+
|
| 194 |
+
b. Additional offer from the Licensor -- Adapted Material.
|
| 195 |
+
Every recipient of Adapted Material from You
|
| 196 |
+
automatically receives an offer from the Licensor to
|
| 197 |
+
exercise the Licensed Rights in the Adapted Material
|
| 198 |
+
under the conditions of the Adapter's License You apply.
|
| 199 |
+
|
| 200 |
+
c. No downstream restrictions. You may not offer or impose
|
| 201 |
+
any additional or different terms or conditions on, or
|
| 202 |
+
apply any Effective Technological Measures to, the
|
| 203 |
+
Licensed Material if doing so restricts exercise of the
|
| 204 |
+
Licensed Rights by any recipient of the Licensed
|
| 205 |
+
Material.
|
| 206 |
+
|
| 207 |
+
6. No endorsement. Nothing in this Public License constitutes or
|
| 208 |
+
may be construed as permission to assert or imply that You
|
| 209 |
+
are, or that Your use of the Licensed Material is, connected
|
| 210 |
+
with, or sponsored, endorsed, or granted official status by,
|
| 211 |
+
the Licensor or others designated to receive attribution as
|
| 212 |
+
provided in Section 3(a)(1)(A)(i).
|
| 213 |
+
|
| 214 |
+
b. Other rights.
|
| 215 |
+
|
| 216 |
+
1. Moral rights, such as the right of integrity, are not
|
| 217 |
+
licensed under this Public License, nor are publicity,
|
| 218 |
+
privacy, and/or other similar personality rights; however, to
|
| 219 |
+
the extent possible, the Licensor waives and/or agrees not to
|
| 220 |
+
assert any such rights held by the Licensor to the limited
|
| 221 |
+
extent necessary to allow You to exercise the Licensed
|
| 222 |
+
Rights, but not otherwise.
|
| 223 |
+
|
| 224 |
+
2. Patent and trademark rights are not licensed under this
|
| 225 |
+
Public License.
|
| 226 |
+
|
| 227 |
+
3. To the extent possible, the Licensor waives any right to
|
| 228 |
+
collect royalties from You for the exercise of the Licensed
|
| 229 |
+
Rights, whether directly or through a collecting society
|
| 230 |
+
under any voluntary or waivable statutory or compulsory
|
| 231 |
+
licensing scheme. In all other cases the Licensor expressly
|
| 232 |
+
reserves any right to collect such royalties, including when
|
| 233 |
+
the Licensed Material is used other than for NonCommercial
|
| 234 |
+
purposes.
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
Section 3 -- License Conditions.
|
| 238 |
+
|
| 239 |
+
Your exercise of the Licensed Rights is expressly made subject to the
|
| 240 |
+
following conditions.
|
| 241 |
+
|
| 242 |
+
a. Attribution.
|
| 243 |
+
|
| 244 |
+
1. If You Share the Licensed Material (including in modified
|
| 245 |
+
form), You must:
|
| 246 |
+
|
| 247 |
+
a. retain the following if it is supplied by the Licensor
|
| 248 |
+
with the Licensed Material:
|
| 249 |
+
|
| 250 |
+
i. identification of the creator(s) of the Licensed
|
| 251 |
+
Material and any others designated to receive
|
| 252 |
+
attribution, in any reasonable manner requested by
|
| 253 |
+
the Licensor (including by pseudonym if
|
| 254 |
+
designated);
|
| 255 |
+
|
| 256 |
+
ii. a copyright notice;
|
| 257 |
+
|
| 258 |
+
iii. a notice that refers to this Public License;
|
| 259 |
+
|
| 260 |
+
iv. a notice that refers to the disclaimer of
|
| 261 |
+
warranties;
|
| 262 |
+
|
| 263 |
+
v. a URI or hyperlink to the Licensed Material to the
|
| 264 |
+
extent reasonably practicable;
|
| 265 |
+
|
| 266 |
+
b. indicate if You modified the Licensed Material and
|
| 267 |
+
retain an indication of any previous modifications; and
|
| 268 |
+
|
| 269 |
+
c. indicate the Licensed Material is licensed under this
|
| 270 |
+
Public License, and include the text of, or the URI or
|
| 271 |
+
hyperlink to, this Public License.
|
| 272 |
+
|
| 273 |
+
2. You may satisfy the conditions in Section 3(a)(1) in any
|
| 274 |
+
reasonable manner based on the medium, means, and context in
|
| 275 |
+
which You Share the Licensed Material. For example, it may be
|
| 276 |
+
reasonable to satisfy the conditions by providing a URI or
|
| 277 |
+
hyperlink to a resource that includes the required
|
| 278 |
+
information.
|
| 279 |
+
3. If requested by the Licensor, You must remove any of the
|
| 280 |
+
information required by Section 3(a)(1)(A) to the extent
|
| 281 |
+
reasonably practicable.
|
| 282 |
+
|
| 283 |
+
b. ShareAlike.
|
| 284 |
+
|
| 285 |
+
In addition to the conditions in Section 3(a), if You Share
|
| 286 |
+
Adapted Material You produce, the following conditions also apply.
|
| 287 |
+
|
| 288 |
+
1. The Adapter's License You apply must be a Creative Commons
|
| 289 |
+
license with the same License Elements, this version or
|
| 290 |
+
later, or a BY-NC-SA Compatible License.
|
| 291 |
+
|
| 292 |
+
2. You must include the text of, or the URI or hyperlink to, the
|
| 293 |
+
Adapter's License You apply. You may satisfy this condition
|
| 294 |
+
in any reasonable manner based on the medium, means, and
|
| 295 |
+
context in which You Share Adapted Material.
|
| 296 |
+
|
| 297 |
+
3. You may not offer or impose any additional or different terms
|
| 298 |
+
or conditions on, or apply any Effective Technological
|
| 299 |
+
Measures to, Adapted Material that restrict exercise of the
|
| 300 |
+
rights granted under the Adapter's License You apply.
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
Section 4 -- Sui Generis Database Rights.
|
| 304 |
+
|
| 305 |
+
Where the Licensed Rights include Sui Generis Database Rights that
|
| 306 |
+
apply to Your use of the Licensed Material:
|
| 307 |
+
|
| 308 |
+
a. for the avoidance of doubt, Section 2(a)(1) grants You the right
|
| 309 |
+
to extract, reuse, reproduce, and Share all or a substantial
|
| 310 |
+
portion of the contents of the database for NonCommercial purposes
|
| 311 |
+
only;
|
| 312 |
+
|
| 313 |
+
b. if You include all or a substantial portion of the database
|
| 314 |
+
contents in a database in which You have Sui Generis Database
|
| 315 |
+
Rights, then the database in which You have Sui Generis Database
|
| 316 |
+
Rights (but not its individual contents) is Adapted Material,
|
| 317 |
+
including for purposes of Section 3(b); and
|
| 318 |
+
|
| 319 |
+
c. You must comply with the conditions in Section 3(a) if You Share
|
| 320 |
+
all or a substantial portion of the contents of the database.
|
| 321 |
+
|
| 322 |
+
For the avoidance of doubt, this Section 4 supplements and does not
|
| 323 |
+
replace Your obligations under this Public License where the Licensed
|
| 324 |
+
Rights include other Copyright and Similar Rights.
|
| 325 |
+
|
| 326 |
+
|
| 327 |
+
Section 5 -- Disclaimer of Warranties and Limitation of Liability.
|
| 328 |
+
|
| 329 |
+
a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
|
| 330 |
+
EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
|
| 331 |
+
AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
|
| 332 |
+
ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
|
| 333 |
+
IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
|
| 334 |
+
WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
|
| 335 |
+
PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
|
| 336 |
+
ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
|
| 337 |
+
KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
|
| 338 |
+
ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
|
| 339 |
+
|
| 340 |
+
b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
|
| 341 |
+
TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
|
| 342 |
+
NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
|
| 343 |
+
INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
|
| 344 |
+
COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
|
| 345 |
+
USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
|
| 346 |
+
ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
|
| 347 |
+
DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
|
| 348 |
+
IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
|
| 349 |
+
|
| 350 |
+
c. The disclaimer of warranties and limitation of liability provided
|
| 351 |
+
above shall be interpreted in a manner that, to the extent
|
| 352 |
+
possible, most closely approximates an absolute disclaimer and
|
| 353 |
+
waiver of all liability.
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
Section 6 -- Term and Termination.
|
| 357 |
+
|
| 358 |
+
a. This Public License applies for the term of the Copyright and
|
| 359 |
+
Similar Rights licensed here. However, if You fail to comply with
|
| 360 |
+
this Public License, then Your rights under this Public License
|
| 361 |
+
terminate automatically.
|
| 362 |
+
|
| 363 |
+
b. Where Your right to use the Licensed Material has terminated under
|
| 364 |
+
Section 6(a), it reinstates:
|
| 365 |
+
|
| 366 |
+
1. automatically as of the date the violation is cured, provided
|
| 367 |
+
it is cured within 30 days of Your discovery of the
|
| 368 |
+
violation; or
|
| 369 |
+
|
| 370 |
+
2. upon express reinstatement by the Licensor.
|
| 371 |
+
|
| 372 |
+
For the avoidance of doubt, this Section 6(b) does not affect any
|
| 373 |
+
right the Licensor may have to seek remedies for Your violations
|
| 374 |
+
of this Public License.
|
| 375 |
+
|
| 376 |
+
c. For the avoidance of doubt, the Licensor may also offer the
|
| 377 |
+
Licensed Material under separate terms or conditions or stop
|
| 378 |
+
distributing the Licensed Material at any time; however, doing so
|
| 379 |
+
will not terminate this Public License.
|
| 380 |
+
|
| 381 |
+
d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
|
| 382 |
+
License.
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
Section 7 -- Other Terms and Conditions.
|
| 386 |
+
|
| 387 |
+
a. The Licensor shall not be bound by any additional or different
|
| 388 |
+
terms or conditions communicated by You unless expressly agreed.
|
| 389 |
+
|
| 390 |
+
b. Any arrangements, understandings, or agreements regarding the
|
| 391 |
+
Licensed Material not stated herein are separate from and
|
| 392 |
+
independent of the terms and conditions of this Public License.
|
| 393 |
+
|
| 394 |
+
|
| 395 |
+
Section 8 -- Interpretation.
|
| 396 |
+
|
| 397 |
+
a. For the avoidance of doubt, this Public License does not, and
|
| 398 |
+
shall not be interpreted to, reduce, limit, restrict, or impose
|
| 399 |
+
conditions on any use of the Licensed Material that could lawfully
|
| 400 |
+
be made without permission under this Public License.
|
| 401 |
+
|
| 402 |
+
b. To the extent possible, if any provision of this Public License is
|
| 403 |
+
deemed unenforceable, it shall be automatically reformed to the
|
| 404 |
+
minimum extent necessary to make it enforceable. If the provision
|
| 405 |
+
cannot be reformed, it shall be severed from this Public License
|
| 406 |
+
without affecting the enforceability of the remaining terms and
|
| 407 |
+
conditions.
|
| 408 |
+
|
| 409 |
+
c. No term or condition of this Public License will be waived and no
|
| 410 |
+
failure to comply consented to unless expressly agreed to by the
|
| 411 |
+
Licensor.
|
| 412 |
+
|
| 413 |
+
d. Nothing in this Public License constitutes or may be interpreted
|
| 414 |
+
as a limitation upon, or waiver of, any privileges and immunities
|
| 415 |
+
that apply to the Licensor or You, including from the legal
|
| 416 |
+
processes of any jurisdiction or authority.
|
| 417 |
+
|
| 418 |
+
=======================================================================
|
| 419 |
+
|
| 420 |
+
Creative Commons is not a party to its public
|
| 421 |
+
licenses. Notwithstanding, Creative Commons may elect to apply one of
|
| 422 |
+
its public licenses to material it publishes and in those instances
|
| 423 |
+
will be considered the “Licensor.” The text of the Creative Commons
|
| 424 |
+
public licenses is dedicated to the public domain under the CC0 Public
|
| 425 |
+
Domain Dedication. Except for the limited purpose of indicating that
|
| 426 |
+
material is shared under a Creative Commons public license or as
|
| 427 |
+
otherwise permitted by the Creative Commons policies published at
|
| 428 |
+
creativecommons.org/policies, Creative Commons does not authorize the
|
| 429 |
+
use of the trademark "Creative Commons" or any other trademark or logo
|
| 430 |
+
of Creative Commons without its prior written consent including,
|
| 431 |
+
without limitation, in connection with any unauthorized modifications
|
| 432 |
+
to any of its public licenses or any other arrangements,
|
| 433 |
+
understandings, or agreements concerning use of licensed material. For
|
| 434 |
+
the avoidance of doubt, this paragraph does not form part of the
|
| 435 |
+
public licenses.
|
| 436 |
+
|
| 437 |
+
Creative Commons may be contacted at creativecommons.org.
|
| 438 |
+
|
268m-g074-human-r0.025/README.md
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: cc-by-nc-sa-4.0
|
| 3 |
+
library_name: transformers
|
| 4 |
+
pipeline_tag: text-generation
|
| 5 |
+
language:
|
| 6 |
+
- en
|
| 7 |
+
datasets:
|
| 8 |
+
- pangram/WildAI
|
| 9 |
+
tags:
|
| 10 |
+
- wildai
|
| 11 |
+
- nanochat
|
| 12 |
+
- scaling-laws
|
| 13 |
+
- ai-generated-text
|
| 14 |
+
- pretraining-data
|
| 15 |
+
---
|
| 16 |
+
|
| 17 |
+
# 268m-g074-human-r0.025
|
| 18 |
+
|
| 19 |
+
> **License: [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/).** The WildAI code, models and dataset are released under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license: you may use and adapt them for non-commercial purposes, with attribution, and must share adaptations under the same license.
|
| 20 |
+
|
| 21 |
+
A 268M decoder-only language model trained from scratch for [*How Much Is an AI Token Worth? Scaling Laws for Wild AI-Generated Web Text*](https://arxiv.org/abs/2609.40295), one of the paper's models measuring how wild AI-generated web text in pretraining data changes a model's loss. It is a research artifact for studying scaling laws, not an assistant: it has had no instruction tuning or safety training.
|
| 22 |
+
|
| 23 |
+
## Training data
|
| 24 |
+
|
| 25 |
+
The human documents of `268m-g074-control` plus fresh human-written web documents (0.025 times the control's tokens): 1.20B human tokens, the human-text counterpart of adding AI text. No AI text.
|
| 26 |
+
|
| 27 |
+
| | |
|
| 28 |
+
|---|---|
|
| 29 |
+
| Parameters (N, input embedding included, value embeddings excluded) | 268.4M |
|
| 30 |
+
| Layers / width | 16 / 1024 |
|
| 31 |
+
| Training tokens | 1.20B (2,296 optimizer steps) |
|
| 32 |
+
| Human / AI tokens | 1.20B / 0.0M |
|
| 33 |
+
| Group / arm / seed | g074 / human / 314159 |
|
| 34 |
+
| Used in the paper for | fitting the scaling laws (726 models, 19.9M to 268M) |
|
| 35 |
+
|
| 36 |
+
## Held-out loss
|
| 37 |
+
|
| 38 |
+
| Set | Bits per byte | Text |
|
| 39 |
+
|---|---|---|
|
| 40 |
+
| C4 | 0.9952 | C4 validation text (human-written, pre-2022) |
|
| 41 |
+
| FW22 | 0.9794 | held-out FineWeb documents crawled before 2022 |
|
| 42 |
+
| Paloma | 1.3628 | macro average over 16 Paloma sources |
|
| 43 |
+
| FW26 | 0.8678 | held-out 2026 FineWeb-style crawl (22.3% AI tokens) |
|
| 44 |
+
| FW26-H | 0.9121 | the human-labeled documents of FW26 |
|
| 45 |
+
| FW26-AI | 0.7047 | the AI-labeled documents of FW26 |
|
| 46 |
+
| Cosmo | 0.7014 | Cosmopedia (AI-generated textbooks and stories) |
|
| 47 |
+
|
| 48 |
+
<details><summary>Paloma sources</summary>
|
| 49 |
+
|
| 50 |
+
| Source | Bits per byte |
|
| 51 |
+
|---|---|
|
| 52 |
+
| 4chan | 1.3848 |
|
| 53 |
+
| c4 | 1.0136 |
|
| 54 |
+
| c4_100_domains | 0.9958 |
|
| 55 |
+
| dolma | 1.2560 |
|
| 56 |
+
| falcon_refinedweb | 1.1085 |
|
| 57 |
+
| gab | 1.8919 |
|
| 58 |
+
| m2d2_s2orc | 1.1726 |
|
| 59 |
+
| m2d2_wikipedia | 1.0771 |
|
| 60 |
+
| manosphere | 1.3856 |
|
| 61 |
+
| mc4_en | 1.1000 |
|
| 62 |
+
| programming | 1.9477 |
|
| 63 |
+
| ptb | 1.2269 |
|
| 64 |
+
| reddit_100 | 1.2471 |
|
| 65 |
+
| redpajama | 1.4254 |
|
| 66 |
+
| twitteraae | 2.4961 |
|
| 67 |
+
| wikitext_103 | 1.0753 |
|
| 68 |
+
|
| 69 |
+
</details>
|
| 70 |
+
|
| 71 |
+
## Usage
|
| 72 |
+
|
| 73 |
+
```python
|
| 74 |
+
import torch
|
| 75 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 76 |
+
|
| 77 |
+
tokenizer = AutoTokenizer.from_pretrained("pangram/WildAI-models", subfolder="268m-g074-human-r0.025", trust_remote_code=True)
|
| 78 |
+
model = AutoModelForCausalLM.from_pretrained("pangram/WildAI-models", subfolder="268m-g074-human-r0.025", trust_remote_code=True, dtype=torch.bfloat16)
|
| 79 |
+
|
| 80 |
+
inputs = tokenizer("The history of the printing press", return_tensors="pt")
|
| 81 |
+
output = model.generate(**inputs, max_new_tokens=40, do_sample=False)
|
| 82 |
+
print(tokenizer.decode(output[0], skip_special_tokens=True))
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
The tokenizer prepends `<|bos|>` to every text, as in training; pass `add_special_tokens=False` to skip it. The model predicts `<|bos|>` where a document ends, and generation stops there. Weights are stored in bfloat16, the precision the paper's losses were computed in; `dtype=torch.float32` computes in float32 instead.
|
| 86 |
+
|
| 87 |
+
## Architecture
|
| 88 |
+
|
| 89 |
+
The nanochat GPT recipe: rotary embeddings, RMSNorm without weights (also on queries and keys), squared-ReLU MLP, attention windows in a repeating short-short-short-long pattern (512 tokens, then the full 2,048-token context; the last layer always full), gated value embeddings on alternating layers, a previous-token smear, per-layer residual scalars, a mid-network backout, untied input and output embeddings and logits soft-capped at 15. Vocabulary 32,768 tokens, context 2,048. The code ships with the weights (`trust_remote_code=True`) and matches the training code's forward pass.
|
| 90 |
+
|
| 91 |
+
## License
|
| 92 |
+
|
| 93 |
+
**License: [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/).** The WildAI code, models and dataset are released under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license: you may use and adapt them for non-commercial purposes, with attribution, and must share adaptations under the same license. The full text is in `LICENSE`.
|
| 94 |
+
|
| 95 |
+
## Citation
|
| 96 |
+
|
| 97 |
+
```bibtex
|
| 98 |
+
@article{russell2026wildai,
|
| 99 |
+
title = {How Much Is an AI Token Worth? Scaling Laws for Wild AI-Generated Web Text},
|
| 100 |
+
author = {Russell, Jenna and Glickenhaus, Ben and Thai, Katherine and Wieting, John and Iyyer, Mohit and Spero, Max and Emi, Bradley},
|
| 101 |
+
journal = {arXiv preprint arXiv:2609.40295},
|
| 102 |
+
year = {2026},
|
| 103 |
+
url = {https://arxiv.org/abs/2609.40295}
|
| 104 |
+
}
|
| 105 |
+
```
|
268m-g074-human-r0.025/config.json
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"WildAIForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"auto_map": {
|
| 6 |
+
"AutoConfig": "configuration_wildai.WildAIConfig",
|
| 7 |
+
"AutoModelForCausalLM": "modeling_wildai.WildAIForCausalLM"
|
| 8 |
+
},
|
| 9 |
+
"bos_token_id": 32759,
|
| 10 |
+
"dtype": "bfloat16",
|
| 11 |
+
"eos_token_id": 32759,
|
| 12 |
+
"hidden_size": 1024,
|
| 13 |
+
"logit_softcap": 15.0,
|
| 14 |
+
"max_position_embeddings": 2048,
|
| 15 |
+
"model_type": "wildai",
|
| 16 |
+
"num_attention_heads": 8,
|
| 17 |
+
"num_hidden_layers": 16,
|
| 18 |
+
"num_key_value_heads": 8,
|
| 19 |
+
"pad_token_id": 32759,
|
| 20 |
+
"qk_scale": 1.2,
|
| 21 |
+
"release": {
|
| 22 |
+
"added_ratio": 0.025,
|
| 23 |
+
"ai_ratio": 0.0,
|
| 24 |
+
"ai_tokens": 0,
|
| 25 |
+
"arm": "human",
|
| 26 |
+
"depth": 16,
|
| 27 |
+
"group": "g074",
|
| 28 |
+
"human_tokens": 1203765248,
|
| 29 |
+
"n_params": 268436282,
|
| 30 |
+
"name": "268m-g074-human-r0.025",
|
| 31 |
+
"seed": 314159,
|
| 32 |
+
"size": "268m",
|
| 33 |
+
"split": "fit",
|
| 34 |
+
"steps": 2296,
|
| 35 |
+
"total_tokens": 1203765248
|
| 36 |
+
},
|
| 37 |
+
"rotary_base": 100000.0,
|
| 38 |
+
"smear_channels": 24,
|
| 39 |
+
"tie_word_embeddings": false,
|
| 40 |
+
"transformers_version": "5.17.0",
|
| 41 |
+
"use_cache": true,
|
| 42 |
+
"value_embed_gate_channels": 12,
|
| 43 |
+
"value_embed_gate_range": 3.0,
|
| 44 |
+
"vocab_size": 32768,
|
| 45 |
+
"window_pattern": "SSSL"
|
| 46 |
+
}
|
268m-g074-human-r0.025/configuration_wildai.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Configuration of the WildAI language models (a nanochat GPT variant) for Hugging Face ``transformers``.
|
| 2 |
+
|
| 3 |
+
This file is copied verbatim into every released model repository and loaded with ``trust_remote_code=True``.
|
| 4 |
+
It depends only on ``transformers``.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from typing import ClassVar
|
| 10 |
+
|
| 11 |
+
from transformers import PretrainedConfig
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class WildAIConfig(PretrainedConfig):
|
| 15 |
+
"""Architecture of a WildAI model. Only width and depth vary across the released models.
|
| 16 |
+
|
| 17 |
+
Every layer uses causal attention limited to a left window: a query attends to itself and the ``window`` tokens before
|
| 18 |
+
it. ``window_pattern`` is tiled over the layers ("S" = a quarter of the context rounded up to a multiple of 128 tokens,
|
| 19 |
+
"L" = the full context) and the last layer always uses "L".
|
| 20 |
+
|
| 21 |
+
``release`` holds the model's entry in the release catalog (name, token counts, AI ratio, ...); it does not affect
|
| 22 |
+
the forward pass.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
model_type = "wildai"
|
| 26 |
+
keys_to_ignore_at_inference: ClassVar[list[str]] = ["past_key_values"]
|
| 27 |
+
|
| 28 |
+
def __init__(
|
| 29 |
+
self,
|
| 30 |
+
vocab_size: int = 32768,
|
| 31 |
+
hidden_size: int = 768,
|
| 32 |
+
num_hidden_layers: int = 12,
|
| 33 |
+
num_attention_heads: int = 6,
|
| 34 |
+
num_key_value_heads: int = 6,
|
| 35 |
+
max_position_embeddings: int = 2048,
|
| 36 |
+
window_pattern: str = "SSSL",
|
| 37 |
+
rotary_base: float = 100000.0,
|
| 38 |
+
qk_scale: float = 1.2,
|
| 39 |
+
logit_softcap: float = 15.0,
|
| 40 |
+
value_embed_gate_channels: int = 12,
|
| 41 |
+
value_embed_gate_range: float = 3.0,
|
| 42 |
+
smear_channels: int = 24,
|
| 43 |
+
bos_token_id: int = 32759,
|
| 44 |
+
eos_token_id: int = 32759,
|
| 45 |
+
use_cache: bool = True,
|
| 46 |
+
release: dict[str, str | int | float | None] | None = None,
|
| 47 |
+
**kwargs: object,
|
| 48 |
+
) -> None:
|
| 49 |
+
if hidden_size % num_attention_heads != 0 or num_attention_heads % num_key_value_heads != 0:
|
| 50 |
+
raise ValueError("hidden_size must divide into num_attention_heads, and those into num_key_value_heads")
|
| 51 |
+
if not window_pattern or set(window_pattern.upper()) - {"S", "L"}:
|
| 52 |
+
raise ValueError(f"window_pattern must be a non-empty string of S and L, got {window_pattern!r}")
|
| 53 |
+
self.vocab_size = vocab_size
|
| 54 |
+
self.hidden_size = hidden_size
|
| 55 |
+
self.num_hidden_layers = num_hidden_layers
|
| 56 |
+
self.num_attention_heads = num_attention_heads
|
| 57 |
+
self.num_key_value_heads = num_key_value_heads
|
| 58 |
+
self.max_position_embeddings = max_position_embeddings
|
| 59 |
+
self.window_pattern = window_pattern.upper()
|
| 60 |
+
self.rotary_base = rotary_base
|
| 61 |
+
self.qk_scale = qk_scale
|
| 62 |
+
self.logit_softcap = logit_softcap
|
| 63 |
+
self.value_embed_gate_channels = value_embed_gate_channels
|
| 64 |
+
self.value_embed_gate_range = value_embed_gate_range
|
| 65 |
+
self.smear_channels = smear_channels
|
| 66 |
+
self.use_cache = use_cache
|
| 67 |
+
self.release = release
|
| 68 |
+
kwargs.setdefault("tie_word_embeddings", False)
|
| 69 |
+
super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
|
| 70 |
+
|
| 71 |
+
@property
|
| 72 |
+
def head_dim(self) -> int:
|
| 73 |
+
return self.hidden_size // self.num_attention_heads
|
| 74 |
+
|
| 75 |
+
@property
|
| 76 |
+
def intermediate_size(self) -> int:
|
| 77 |
+
return 4 * self.hidden_size
|
| 78 |
+
|
| 79 |
+
@property
|
| 80 |
+
def window_sizes(self) -> list[int]:
|
| 81 |
+
"""Left attention window of each layer, in tokens."""
|
| 82 |
+
long_window = self.max_position_embeddings
|
| 83 |
+
short_window = -(-long_window // 4 // 128) * 128 # a quarter of the context, rounded up to a multiple of 128 (2048 -> 512)
|
| 84 |
+
pattern = self.window_pattern
|
| 85 |
+
sizes = [short_window if pattern[i % len(pattern)] == "S" else long_window for i in range(self.num_hidden_layers)]
|
| 86 |
+
sizes[-1] = long_window
|
| 87 |
+
return sizes
|
| 88 |
+
|
| 89 |
+
def has_value_embedding(self, layer_idx: int) -> bool:
|
| 90 |
+
"""Value embeddings sit on alternating layers, always including the last one."""
|
| 91 |
+
return layer_idx % 2 == (self.num_hidden_layers - 1) % 2
|
| 92 |
+
|
| 93 |
+
@property
|
| 94 |
+
def backout_layer(self) -> int:
|
| 95 |
+
"""The residual stream after this layer is partly subtracted before the final norm."""
|
| 96 |
+
return self.num_hidden_layers // 2
|